From 9220bdf89f38c71a076abf403db2eaa842f07c40 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 26 Jul 2026 06:15:22 +0800 Subject: [PATCH 1/4] feat(automation): add durable scheduled run leasing --- .../src-tauri/src/automation_commands.rs | 98 +- .../src-tauri/src/kernel/automation.rs | 12 + .../src-tauri/src/kernel/event_store.rs | 1493 ++++++++++++++++- apps/desktop/src-tauri/src/main.rs | 3 +- scripts/automation-center.test.mjs | 32 + 5 files changed, 1605 insertions(+), 33 deletions(-) diff --git a/apps/desktop/src-tauri/src/automation_commands.rs b/apps/desktop/src-tauri/src/automation_commands.rs index 932d20c..d9548ae 100644 --- a/apps/desktop/src-tauri/src/automation_commands.rs +++ b/apps/desktop/src-tauri/src/automation_commands.rs @@ -1,3 +1,6 @@ +use std::thread; +use std::time::Duration; + use chrono::{DateTime, Utc}; use tauri::State; use uuid::Uuid; @@ -7,11 +10,52 @@ use crate::kernel::automation::{ next_scheduled_at, AutomationDefinition, AutomationDefinitionStatus, AutomationRun, AutomationSchedule, ReviewQueueItemView, }; +use crate::kernel::event_store::EventStore; + +const AUTOMATION_SCHEDULER_INTERVAL: Duration = Duration::from_secs(30); fn store_error(error: impl std::fmt::Display) -> String { error.to_string() } +fn run_automation_scheduler_once(store: &EventStore, now: DateTime) -> Result { + store + .recover_expired_automation_runs(now) + .map_err(store_error)?; + store + .reconcile_automation_agent_runs(now) + .map_err(store_error)?; + let mut queued = 0; + for definition in store.list_automation_definitions().map_err(store_error)? { + let conversation_id = format!("automation:{}", definition.id); + if store + .enqueue_due_automation_agent_run( + definition.id, + now, + "desktop-automation-scheduler".to_string(), + conversation_id, + ) + .map_err(store_error)? + .is_some() + { + queued += 1; + } + } + Ok(queued) +} + +pub fn spawn_automation_scheduler(state: AppState) { + let _ = thread::Builder::new() + .name("ds-agent-automation-scheduler".to_string()) + .spawn(move || loop { + let event_store = state.event_store(); + if let Ok(store) = event_store.lock() { + let _ = run_automation_scheduler_once(&store, Utc::now()); + } + thread::sleep(AUTOMATION_SCHEDULER_INTERVAL); + }); +} + #[tauri::command] pub fn list_automation_definitions( state: State<'_, AppState>, @@ -268,22 +312,42 @@ pub fn run_due_automation_sweep(state: State<'_, AppState>) -> Result, } +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AutomationCheckpointRecord { + pub contract_version: String, + pub revision: u64, + pub definition_revision: u64, + pub agent_run_id: Uuid, + pub lease_generation: u64, + pub checkpoint: AutomationCheckpoint, +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum ReviewQueueItemStatus { diff --git a/apps/desktop/src-tauri/src/kernel/event_store.rs b/apps/desktop/src-tauri/src/kernel/event_store.rs index 112dc6c..167a014 100644 --- a/apps/desktop/src-tauri/src/kernel/event_store.rs +++ b/apps/desktop/src-tauri/src/kernel/event_store.rs @@ -29,8 +29,8 @@ use crate::kernel::agent_run::{ }; use crate::kernel::automation::{ automation_run_transition_allowed, next_scheduled_at, trigger_window_key, AutomationCheckpoint, - AutomationDefinition, AutomationDefinitionStatus, AutomationRun, AutomationRunStatus, - MissedRunPolicy, ReviewQueueItem, + AutomationCheckpointRecord, AutomationDefinition, AutomationDefinitionStatus, AutomationRun, + AutomationRunStatus, MissedRunPolicy, ReviewQueueItem, AUTOMATION_CHECKPOINT_CONTRACT_VERSION, }; use crate::kernel::capability::{CapabilityInvocation, CapabilityInvocationStatus}; use crate::kernel::connectors::catalog::ConnectorSyncHealthSnapshot; @@ -1029,6 +1029,38 @@ pub struct EventStore { conn: Connection, } +pub(crate) struct AutomationRunLease { + run_id: Uuid, + owner_id: String, + fence_token: Uuid, + generation: u64, + lease_expires_at: DateTime, +} + +impl AutomationRunLease { + pub(crate) fn run_id(&self) -> Uuid { + self.run_id + } + + pub(crate) fn owner_id(&self) -> &str { + &self.owner_id + } + + pub(crate) fn generation(&self) -> u64 { + self.generation + } + + pub(crate) fn lease_expires_at(&self) -> DateTime { + self.lease_expires_at + } +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct AutomationRunRecoverySweep { + pub requeued: usize, + pub effect_unknown: usize, +} + pub(crate) struct ConnectorSyncRecoveryClaim { job_id: Uuid, claim_id: Uuid, @@ -1423,7 +1455,13 @@ impl EventStore { run_json TEXT NOT NULL, status TEXT NOT NULL, definition_revision INTEGER NOT NULL DEFAULT 0, + definition_snapshot_json TEXT, claimed_by TEXT, + lease_owner_id TEXT, + lease_fence_token TEXT, + lease_generation INTEGER NOT NULL DEFAULT 0, + lease_expires_at TEXT, + lease_heartbeat_at TEXT, updated_at TEXT NOT NULL, UNIQUE (definition_id, trigger_window_key) ); @@ -1434,6 +1472,11 @@ impl EventStore { CREATE TABLE IF NOT EXISTS automation_checkpoints ( automation_run_id TEXT PRIMARY KEY NOT NULL, checkpoint_json TEXT NOT NULL, + contract_version TEXT, + revision INTEGER NOT NULL DEFAULT 0, + definition_revision INTEGER, + agent_run_id TEXT, + lease_generation INTEGER, updated_at TEXT NOT NULL ); @@ -1676,6 +1719,63 @@ impl EventStore { "definition_revision", "ALTER TABLE automation_runs ADD COLUMN definition_revision INTEGER NOT NULL DEFAULT 0", )?; + for (column, sql) in [ + ( + "definition_snapshot_json", + "ALTER TABLE automation_runs ADD COLUMN definition_snapshot_json TEXT", + ), + ( + "lease_owner_id", + "ALTER TABLE automation_runs ADD COLUMN lease_owner_id TEXT", + ), + ( + "lease_fence_token", + "ALTER TABLE automation_runs ADD COLUMN lease_fence_token TEXT", + ), + ( + "lease_generation", + "ALTER TABLE automation_runs ADD COLUMN lease_generation INTEGER NOT NULL DEFAULT 0", + ), + ( + "lease_expires_at", + "ALTER TABLE automation_runs ADD COLUMN lease_expires_at TEXT", + ), + ( + "lease_heartbeat_at", + "ALTER TABLE automation_runs ADD COLUMN lease_heartbeat_at TEXT", + ), + ] { + ensure_sqlite_column(&self.conn, "automation_runs", column, sql)?; + } + for (column, sql) in [ + ( + "contract_version", + "ALTER TABLE automation_checkpoints ADD COLUMN contract_version TEXT", + ), + ( + "revision", + "ALTER TABLE automation_checkpoints ADD COLUMN revision INTEGER NOT NULL DEFAULT 0", + ), + ( + "definition_revision", + "ALTER TABLE automation_checkpoints ADD COLUMN definition_revision INTEGER", + ), + ( + "agent_run_id", + "ALTER TABLE automation_checkpoints ADD COLUMN agent_run_id TEXT", + ), + ( + "lease_generation", + "ALTER TABLE automation_checkpoints ADD COLUMN lease_generation INTEGER", + ), + ] { + ensure_sqlite_column(&self.conn, "automation_checkpoints", column, sql)?; + } + self.conn.execute( + r#"CREATE INDEX IF NOT EXISTS idx_automation_runs_lease + ON automation_runs (lease_expires_at, status)"#, + [], + )?; artifact::migrate(self)?; computer_use::migrate(self)?; connector_draft::migrate(self)?; @@ -2401,13 +2501,14 @@ impl EventStore { .collect() } + #[cfg(test)] pub fn claim_due_automation_run( &self, definition_id: Uuid, now: DateTime, worker_id: String, ) -> EventStoreResult> { - let transaction = self.conn.unchecked_transaction()?; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; let definition_json = transaction.query_row( "SELECT definition_json FROM automation_definitions WHERE id = ?1", params![definition_id.to_string()], @@ -2451,8 +2552,8 @@ impl EventStore { let inserted = transaction.execute( r#"INSERT OR IGNORE INTO automation_runs (id, definition_id, trigger_window_key, scheduled_for, run_json, status, - definition_revision, claimed_by, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#, + definition_revision, definition_snapshot_json, claimed_by, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#, params![ run.id.to_string(), definition.id.to_string(), @@ -2465,6 +2566,7 @@ impl EventStore { "automation run definition revision is too large".to_string(), ) })?, + definition_json, worker_id, timestamp, ], @@ -2480,7 +2582,7 @@ impl EventStore { scheduler_id: String, conversation_id: String, ) -> EventStoreResult> { - let transaction = self.conn.unchecked_transaction()?; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; let definition_json = transaction.query_row( "SELECT definition_json FROM automation_definitions WHERE id = ?1", params![definition_id.to_string()], @@ -2526,8 +2628,8 @@ impl EventStore { let inserted = transaction.execute( r#"INSERT OR IGNORE INTO automation_runs (id, definition_id, trigger_window_key, scheduled_for, run_json, status, - definition_revision, claimed_by, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#, + definition_revision, definition_snapshot_json, claimed_by, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#, params![ run.id.to_string(), definition.id.to_string(), @@ -2540,6 +2642,7 @@ impl EventStore { "automation run definition revision is too large".to_string(), ) })?, + definition_json, scheduler_id, timestamp, ], @@ -2572,7 +2675,7 @@ impl EventStore { now: DateTime, conversation_id: String, ) -> EventStoreResult<(AutomationRun, AgentRunStart)> { - let transaction = self.conn.unchecked_transaction()?; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; let definition_json = transaction.query_row( "SELECT definition_json FROM automation_definitions WHERE id = ?1", params![definition_id.to_string()], @@ -2606,8 +2709,8 @@ impl EventStore { let inserted = transaction.execute( r#"INSERT OR IGNORE INTO automation_runs (id, definition_id, trigger_window_key, scheduled_for, run_json, status, - definition_revision, claimed_by, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)"#, + definition_revision, definition_snapshot_json, claimed_by, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)"#, params![ run.id.to_string(), definition.id.to_string(), @@ -2620,6 +2723,7 @@ impl EventStore { "automation run definition revision is too large".to_string(), ) })?, + definition_json, "manual", now.to_rfc3339_opts(SecondsFormat::Nanos, true), ], @@ -2656,6 +2760,666 @@ impl EventStore { .collect() } + pub(crate) fn claim_automation_run( + &self, + run_id: Uuid, + owner_id: String, + lease_seconds: i64, + now: DateTime, + ) -> EventStoreResult { + let owner_id = owner_id.trim().to_string(); + if owner_id.is_empty() { + return Err(EventStoreError::InvalidState( + "automation run lease owner is required".to_string(), + )); + } + let lease_seconds = lease_seconds.max(1); + if self.automation_run_lease_is_expired(run_id, now)? { + self.recover_expired_automation_run(run_id, now)?; + } + if let Err(error) = self.provably_pre_effect_automation_run(run_id) { + let reason = format!("effect_unknown: {error}"); + self.mark_unclaimed_automation_run_effect_unknown(run_id, &reason, now)?; + return Err(error); + } + + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let (run_json, generation, existing_token) = transaction.query_row( + r#"SELECT run_json, lease_generation, lease_fence_token + FROM automation_runs WHERE id = ?1"#, + params![run_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + )) + }, + )?; + if existing_token.is_some() { + return Err(EventStoreError::InvalidState( + "automation run already has a durable lease".to_string(), + )); + } + let generation = u64::try_from(generation) + .map_err(|_| EventStoreError::InvalidState("automation lease is invalid".to_string()))? + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation run lease generation is exhausted".to_string(), + ) + })?; + let fence_token = Uuid::new_v4(); + let lease_expires_at = now + Duration::seconds(lease_seconds); + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + if !matches!( + run.status, + AutomationRunStatus::Queued | AutomationRunStatus::Running + ) { + return Err(EventStoreError::InvalidState(format!( + "automation run cannot be claimed from status {:?}", + run.status + ))); + } + run.claimed_by = Some(owner_id.clone()); + run.claimed_at = Some(now); + run.updated_at = now; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?2, claimed_by = ?3, lease_owner_id = ?3, + lease_fence_token = ?4, lease_generation = ?5, + lease_expires_at = ?6, lease_heartbeat_at = ?7, updated_at = ?7 + WHERE id = ?1 AND lease_fence_token IS NULL + AND lease_generation = ?8"#, + params![ + run_id.to_string(), + serde_json::to_string(&run)?, + &owner_id, + fence_token.to_string(), + i64::try_from(generation).map_err(|_| EventStoreError::InvalidState( + "automation run lease generation is too large".to_string() + ))?, + lease_expires_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + i64::try_from(generation.saturating_sub(1)).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run lease raced with another owner".to_string(), + )); + } + transaction.commit()?; + Ok(AutomationRunLease { + run_id, + owner_id, + fence_token, + generation, + lease_expires_at, + }) + } + + pub(crate) fn heartbeat_automation_run_lease( + &self, + lease: &AutomationRunLease, + lease_seconds: i64, + now: DateTime, + ) -> EventStoreResult { + let lease_expires_at = now + Duration::seconds(lease_seconds.max(1)); + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let run_json = transaction + .query_row( + r#"SELECT run_json FROM automation_runs + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?5"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation run lease is stale, expired, or owned by another worker" + .to_string(), + ) + })?; + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + run.updated_at = now; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?6, lease_expires_at = ?5, + lease_heartbeat_at = ?7, updated_at = ?7 + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?7"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + lease_expires_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + serde_json::to_string(&run)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run heartbeat lost its fence".to_string(), + )); + } + transaction.commit()?; + Ok(AutomationRunLease { + run_id: lease.run_id, + owner_id: lease.owner_id.clone(), + fence_token: lease.fence_token, + generation: lease.generation, + lease_expires_at, + }) + } + + pub(crate) fn release_automation_run_lease( + &self, + lease: &AutomationRunLease, + now: DateTime, + ) -> EventStoreResult<()> { + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let run_json = transaction + .query_row( + r#"SELECT run_json FROM automation_runs + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?5"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation run lease is stale, expired, or owned by another worker" + .to_string(), + ) + })?; + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + run.claimed_by = None; + run.claimed_at = None; + run.updated_at = now; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?5, claimed_by = NULL, lease_owner_id = NULL, + lease_fence_token = NULL, lease_expires_at = NULL, + lease_heartbeat_at = NULL, updated_at = ?6 + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?6"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + serde_json::to_string(&run)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run release lost its fence".to_string(), + )); + } + transaction.commit()?; + Ok(()) + } + + pub(crate) fn transition_claimed_automation_run( + &self, + lease: &AutomationRunLease, + status: AutomationRunStatus, + last_error: Option, + changed_at: DateTime, + ) -> EventStoreResult { + self.exact_automation_run_binding(lease.run_id)?; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let run_json = transaction + .query_row( + r#"SELECT run_json FROM automation_runs + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?5"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + changed_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + |row| row.get::<_, String>(0), + ) + .optional()? + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation run transition has a stale or expired fence".to_string(), + ) + })?; + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + if !automation_run_transition_allowed(run.status, status) { + return Err(EventStoreError::InvalidState(format!( + "automation run cannot transition from {:?} to {:?}", + run.status, status + ))); + } + run.status = status; + run.last_error = last_error; + run.updated_at = changed_at; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?5, status = ?6, updated_at = ?7 + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?7"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + serde_json::to_string(&run)?, + serde_json::to_string(&run.status)?, + changed_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run transition lost its fence".to_string(), + )); + } + transaction.commit()?; + Ok(run) + } + + pub(crate) fn recover_expired_automation_runs( + &self, + now: DateTime, + ) -> EventStoreResult { + let mut statement = self.conn.prepare( + r#"SELECT id FROM automation_runs + WHERE lease_fence_token IS NOT NULL AND lease_expires_at <= ?1 + ORDER BY lease_expires_at ASC, rowid ASC LIMIT 64"#, + )?; + let run_ids = statement + .query_map( + params![now.to_rfc3339_opts(SecondsFormat::Nanos, true)], + |row| row.get::<_, String>(0), + )? + .collect::, _>>()?; + drop(statement); + let mut sweep = AutomationRunRecoverySweep::default(); + for run_id in run_ids { + match self.recover_expired_automation_run(Uuid::parse_str(&run_id)?, now)? { + Some(true) => sweep.requeued += 1, + Some(false) => sweep.effect_unknown += 1, + None => {} + } + } + Ok(sweep) + } + + fn recover_expired_automation_run( + &self, + run_id: Uuid, + now: DateTime, + ) -> EventStoreResult> { + let (run_json, generation, token, expires_at) = self.conn.query_row( + r#"SELECT run_json, lease_generation, lease_fence_token, lease_expires_at + FROM automation_runs WHERE id = ?1"#, + params![run_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }, + )?; + let Some(token) = token else { + return Ok(None); + }; + let expires_at = expires_at + .ok_or_else(|| { + EventStoreError::InvalidState("automation run lease expiry is missing".to_string()) + }) + .and_then(|value| { + DateTime::parse_from_rfc3339(&value) + .map(|value| value.with_timezone(&Utc)) + .map_err(Into::into) + })?; + if expires_at > now { + return Ok(None); + } + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + let safety = self.provably_pre_effect_automation_run(run_id); + let safe = safety.is_ok(); + let reason = safety + .err() + .map(|error| format!("effect_unknown: {error}")) + .unwrap_or_else(|| { + format!( + "Recovered provably pre-effect work after automation lease expired at {}.", + expires_at.to_rfc3339() + ) + }); + run.status = if safe { + AutomationRunStatus::Queued + } else { + AutomationRunStatus::Failed + }; + run.last_error = Some(reason.clone()); + run.claimed_by = None; + run.claimed_at = None; + run.updated_at = now; + + let blocking_transition = if safe { + None + } else { + self.linked_agent_run_record(&run).and_then(|record| { + if matches!( + record.status, + AgentRunStatus::Completed + | AgentRunStatus::Failed + | AgentRunStatus::Blocked + | AgentRunStatus::Cancelled + ) { + None + } else { + AgentRunTransition::new( + record.id, + AgentRunStatus::Blocked, + reason.clone(), + self.running_tool_invocation_for_agent_run(record.id) + .ok() + .flatten(), + ) + .ok() + } + }) + }; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?5, status = ?6, claimed_by = NULL, + lease_owner_id = NULL, lease_fence_token = NULL, + lease_expires_at = NULL, lease_heartbeat_at = NULL, updated_at = ?7 + WHERE id = ?1 AND lease_fence_token = ?2 + AND lease_generation = ?3 AND lease_expires_at <= ?4"#, + params![ + run_id.to_string(), + token, + generation, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + serde_json::to_string(&run)?, + serde_json::to_string(&run.status)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Ok(None); + } + if let Some(transition) = blocking_transition { + let event = KernelEvent::new(AGENT_RUN_TRANSITIONED_EVENT, &transition)?; + Self::insert_kernel_event(&transaction, &event)?; + } + transaction.commit()?; + Ok(Some(safe)) + } + + fn mark_unclaimed_automation_run_effect_unknown( + &self, + run_id: Uuid, + reason: &str, + now: DateTime, + ) -> EventStoreResult<()> { + let mut run = self.automation_run(run_id)?; + run.status = AutomationRunStatus::Failed; + run.last_error = Some(reason.to_string()); + run.claimed_by = None; + run.claimed_at = None; + run.updated_at = now; + let blocking_transition = self.linked_agent_run_record(&run).and_then(|record| { + if matches!( + record.status, + AgentRunStatus::Completed + | AgentRunStatus::Failed + | AgentRunStatus::Blocked + | AgentRunStatus::Cancelled + ) { + None + } else { + AgentRunTransition::new( + record.id, + AgentRunStatus::Blocked, + reason.to_string(), + self.running_tool_invocation_for_agent_run(record.id) + .ok() + .flatten(), + ) + .ok() + } + }); + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let changed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?2, status = ?3, claimed_by = NULL, updated_at = ?4 + WHERE id = ?1 AND lease_fence_token IS NULL"#, + params![ + run_id.to_string(), + serde_json::to_string(&run)?, + serde_json::to_string(&run.status)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run effect-unknown transition raced with a lease owner".to_string(), + )); + } + if let Some(transition) = blocking_transition { + let event = KernelEvent::new(AGENT_RUN_TRANSITIONED_EVENT, &transition)?; + Self::insert_kernel_event(&transaction, &event)?; + } + transaction.commit()?; + Ok(()) + } + + fn automation_run_lease_is_expired( + &self, + run_id: Uuid, + now: DateTime, + ) -> EventStoreResult { + let expires_at = self + .conn + .query_row( + "SELECT lease_expires_at FROM automation_runs WHERE id = ?1", + params![run_id.to_string()], + |row| row.get::<_, Option>(0), + )? + .map(|value| DateTime::parse_from_rfc3339(&value)) + .transpose()? + .map(|value| value.with_timezone(&Utc)); + Ok(expires_at.is_some_and(|value| value <= now)) + } + + fn exact_automation_run_binding( + &self, + run_id: Uuid, + ) -> EventStoreResult<(AutomationRun, AgentRunRecord)> { + let (run_json, stored_revision, snapshot_json) = self.conn.query_row( + r#"SELECT run_json, definition_revision, definition_snapshot_json + FROM automation_runs WHERE id = ?1"#, + params![run_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + )) + }, + )?; + let run: AutomationRun = serde_json::from_str(&run_json)?; + let stored_revision = u64::try_from(stored_revision).map_err(|_| { + EventStoreError::InvalidState( + "automation run definition revision is invalid".to_string(), + ) + })?; + let snapshot_json = snapshot_json.ok_or_else(|| { + EventStoreError::InvalidState( + "automation run exact definition binding is missing".to_string(), + ) + })?; + let snapshot: AutomationDefinition = serde_json::from_str(&snapshot_json)?; + if run.definition_id != snapshot.id + || run.definition_revision != snapshot.revision + || run.definition_revision != stored_revision + || run.trigger_window_key.trim().is_empty() + { + return Err(EventStoreError::InvalidState( + "automation run definition revision binding changed".to_string(), + )); + } + let agent_run = self.linked_agent_run_record(&run).ok_or_else(|| { + EventStoreError::InvalidState( + "automation run exact AgentRun binding is missing".to_string(), + ) + })?; + if agent_run.prompt != snapshot.goal { + return Err(EventStoreError::InvalidState( + "automation run AgentRun prompt changed from its definition revision".to_string(), + )); + } + Ok((run, agent_run)) + } + + fn linked_agent_run_record(&self, run: &AutomationRun) -> Option { + let agent_run_id = run.agent_run_id?; + self.list_agent_run_records() + .ok()? + .into_iter() + .find(|record| record.id == agent_run_id) + } + + fn running_tool_invocation_for_agent_run( + &self, + agent_run_id: Uuid, + ) -> EventStoreResult> { + let running_status = serde_json::to_string(&ToolExecutionStatus::Running)?; + let mut statement = self.conn.prepare( + "SELECT invocation_json FROM tool_invocation_state WHERE status = ?1 ORDER BY rowid ASC", + )?; + let rows = statement + .query_map(params![running_status], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + for row in rows { + let invocation: ToolInvocationRecord = serde_json::from_str(&row)?; + if invocation.run_id == Some(agent_run_id) { + return Ok(Some(invocation.id)); + } + } + Ok(None) + } + + fn computer_use_action_started_for_agent_run( + &self, + agent_run_id: Uuid, + ) -> EventStoreResult { + self.conn + .query_row( + r#"SELECT EXISTS( + SELECT 1 + FROM computer_use_sessions AS session + JOIN computer_use_steps AS step ON step.id = session.active_step_id + WHERE session.run_id = ?1 + AND session.quarantine_code IS NULL + AND step.quarantine_code IS NULL + AND step.status IN ( + 'action_started', + 'awaiting_verification', + 'effect_unknown', + 'verification_failed' + ) + )"#, + params![agent_run_id.to_string()], + |row| row.get::<_, i64>(0), + ) + .map(|value| value != 0) + .map_err(Into::into) + } + + fn provably_pre_effect_automation_run(&self, run_id: Uuid) -> EventStoreResult { + let (run, agent_run) = self.exact_automation_run_binding(run_id)?; + if agent_run.status != AgentRunStatus::Queued { + return Err(EventStoreError::InvalidState(format!( + "linked AgentRun is {:?}, not provably pre-effect queued work", + agent_run.status + ))); + } + if self + .running_tool_invocation_for_agent_run(agent_run.id)? + .is_some() + { + return Err(EventStoreError::InvalidState( + "a running tool invocation has an indeterminate possible effect".to_string(), + )); + } + if self.computer_use_action_started_for_agent_run(agent_run.id)? { + return Err(EventStoreError::InvalidState( + "ActionStarted or another ambiguous Computer Use effect is present".to_string(), + )); + } + Ok(run) + } + + fn automation_run_has_durable_lease(&self, run_id: Uuid) -> EventStoreResult { + self.conn + .query_row( + "SELECT lease_fence_token IS NOT NULL FROM automation_runs WHERE id = ?1", + params![run_id.to_string()], + |row| row.get::<_, i64>(0), + ) + .map(|value| value != 0) + .map_err(Into::into) + } + pub fn reconcile_automation_agent_runs( &self, changed_at: DateTime, @@ -2667,6 +3431,9 @@ impl EventStore { .collect::>(); let mut reconciled = 0; for run in self.list_automation_runs()? { + if self.automation_run_has_durable_lease(run.id)? { + continue; + } let Some(agent_run_id) = run.agent_run_id else { continue; }; @@ -2764,6 +3531,7 @@ impl EventStore { Ok(run) } + #[cfg(test)] pub fn upsert_automation_checkpoint( &self, checkpoint: &AutomationCheckpoint, @@ -2785,6 +3553,178 @@ impl EventStore { Ok(()) } + pub(crate) fn compare_and_swap_automation_checkpoint( + &self, + checkpoint: &AutomationCheckpoint, + lease: &AutomationRunLease, + expected_revision: Option, + changed_at: DateTime, + ) -> EventStoreResult { + if checkpoint.automation_run_id != lease.run_id { + return Err(EventStoreError::InvalidState( + "automation checkpoint is bound to another run".to_string(), + )); + } + let (run, agent_run) = self.exact_automation_run_binding(lease.run_id)?; + if checkpoint.dedup_key != run.trigger_window_key { + return Err(EventStoreError::InvalidState( + "automation checkpoint dedup binding changed".to_string(), + )); + } + if let Some(tool_invocation_id) = checkpoint.tool_invocation_id { + let invocation = self + .list_tool_invocations()? + .into_iter() + .find(|invocation| invocation.id == tool_invocation_id) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation checkpoint tool invocation is missing".to_string(), + ) + })?; + if invocation.run_id != Some(agent_run.id) { + return Err(EventStoreError::InvalidState( + "automation checkpoint tool invocation belongs to another AgentRun".to_string(), + )); + } + } + + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let lease_is_current = transaction.query_row( + r#"SELECT EXISTS( + SELECT 1 FROM automation_runs + WHERE id = ?1 AND lease_owner_id = ?2 AND lease_fence_token = ?3 + AND lease_generation = ?4 AND lease_expires_at > ?5 + )"#, + params![ + lease.run_id.to_string(), + &lease.owner_id, + lease.fence_token.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation run lease generation is too large".to_string(), + ) + })?, + changed_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + |row| row.get::<_, i64>(0), + )? != 0; + if !lease_is_current { + return Err(EventStoreError::InvalidState( + "automation checkpoint fence is stale, expired, or owned by another worker" + .to_string(), + )); + } + + let current_revision = transaction + .query_row( + "SELECT revision FROM automation_checkpoints WHERE automation_run_id = ?1", + params![lease.run_id.to_string()], + |row| row.get::<_, i64>(0), + ) + .optional()? + .map(u64::try_from) + .transpose() + .map_err(|_| { + EventStoreError::InvalidState( + "automation checkpoint revision is invalid".to_string(), + ) + })?; + if current_revision != expected_revision { + return Err(EventStoreError::InvalidState( + "automation checkpoint compare-and-swap conflict".to_string(), + )); + } + let revision = current_revision + .map(|value| { + value.checked_add(1).ok_or_else(|| { + EventStoreError::InvalidState( + "automation checkpoint revision is exhausted".to_string(), + ) + }) + }) + .transpose()? + .unwrap_or(0); + let checkpoint_json = serde_json::to_string(checkpoint)?; + let changed = if expected_revision.is_none() { + transaction.execute( + r#"INSERT OR IGNORE INTO automation_checkpoints + (automation_run_id, checkpoint_json, contract_version, revision, + definition_revision, agent_run_id, lease_generation, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"#, + params![ + lease.run_id.to_string(), + &checkpoint_json, + AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + i64::try_from(revision).map_err(|_| EventStoreError::InvalidState( + "automation checkpoint revision is too large".to_string() + ))?, + i64::try_from(run.definition_revision).map_err(|_| { + EventStoreError::InvalidState( + "automation definition revision is too large".to_string(), + ) + })?, + agent_run.id.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation lease generation is too large".to_string(), + ) + })?, + changed_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )? + } else { + transaction.execute( + r#"UPDATE automation_checkpoints + SET checkpoint_json = ?2, contract_version = ?3, revision = ?4, + definition_revision = ?5, agent_run_id = ?6, + lease_generation = ?7, updated_at = ?8 + WHERE automation_run_id = ?1 AND revision = ?9 + AND contract_version = ?3 + AND definition_revision = ?5 + AND agent_run_id = ?6"#, + params![ + lease.run_id.to_string(), + &checkpoint_json, + AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + i64::try_from(revision).map_err(|_| EventStoreError::InvalidState( + "automation checkpoint revision is too large".to_string() + ))?, + i64::try_from(run.definition_revision).map_err(|_| { + EventStoreError::InvalidState( + "automation definition revision is too large".to_string(), + ) + })?, + agent_run.id.to_string(), + i64::try_from(lease.generation).map_err(|_| { + EventStoreError::InvalidState( + "automation lease generation is too large".to_string(), + ) + })?, + changed_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + i64::try_from(expected_revision.unwrap_or_default()).map_err(|_| { + EventStoreError::InvalidState( + "automation checkpoint revision is too large".to_string(), + ) + })?, + ], + )? + }; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation checkpoint compare-and-swap conflict".to_string(), + )); + } + transaction.commit()?; + Ok(AutomationCheckpointRecord { + contract_version: AUTOMATION_CHECKPOINT_CONTRACT_VERSION.to_string(), + revision, + definition_revision: run.definition_revision, + agent_run_id: agent_run.id, + lease_generation: lease.generation, + checkpoint: checkpoint.clone(), + }) + } + pub fn upsert_review_queue_item(&self, item: &ReviewQueueItem) -> EventStoreResult<()> { let existing = self .conn @@ -2994,12 +3934,86 @@ impl EventStore { Ok(serde_json::from_str(&json)?) } + pub(crate) fn automation_checkpoint_record( + &self, + automation_run_id: Uuid, + ) -> EventStoreResult { + let ( + checkpoint_json, + contract_version, + revision, + definition_revision, + agent_run_id, + lease_generation, + ) = self.conn.query_row( + r#"SELECT checkpoint_json, contract_version, revision, + definition_revision, agent_run_id, lease_generation + FROM automation_checkpoints WHERE automation_run_id = ?1"#, + params![automation_run_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?, + )) + }, + )?; + let contract_version = + contract_version.filter(|value| value == AUTOMATION_CHECKPOINT_CONTRACT_VERSION); + let (contract_version, definition_revision, agent_run_id, lease_generation) = match ( + contract_version, + definition_revision, + agent_run_id, + lease_generation, + ) { + ( + Some(contract_version), + Some(definition_revision), + Some(agent_run_id), + Some(lease_generation), + ) => ( + contract_version, + u64::try_from(definition_revision), + Uuid::parse_str(&agent_run_id), + u64::try_from(lease_generation), + ), + _ => { + return Err(EventStoreError::InvalidState( + "automation checkpoint exact binding is missing".to_string(), + )) + } + }; + Ok(AutomationCheckpointRecord { + contract_version, + revision: u64::try_from(revision).map_err(|_| { + EventStoreError::InvalidState( + "automation checkpoint revision is invalid".to_string(), + ) + })?, + definition_revision: definition_revision.map_err(|_| { + EventStoreError::InvalidState( + "automation checkpoint definition revision is invalid".to_string(), + ) + })?, + agent_run_id: agent_run_id?, + lease_generation: lease_generation.map_err(|_| { + EventStoreError::InvalidState( + "automation checkpoint lease generation is invalid".to_string(), + ) + })?, + checkpoint: serde_json::from_str(&checkpoint_json)?, + }) + } + fn persist_automation_run(&self, run: &AutomationRun) -> EventStoreResult<()> { - self.conn.execute( + let changed = self.conn.execute( r#"UPDATE automation_runs SET run_json = ?2, status = ?3, definition_revision = ?4, claimed_by = ?5, updated_at = ?6 - WHERE id = ?1"#, + WHERE id = ?1 AND lease_fence_token IS NULL AND lease_generation = 0"#, params![ run.id.to_string(), serde_json::to_string(run)?, @@ -3013,6 +4027,11 @@ impl EventStore { run.updated_at.to_rfc3339_opts(SecondsFormat::Nanos, true), ], )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation run requires its current fenced owner".to_string(), + )); + } Ok(()) } @@ -14950,8 +15969,9 @@ mod tests { use uuid::Uuid; use super::{ - ConnectorAttachmentCleanupClaim, EventStore, EventStoreError, AGENT_RUN_FINISHED_EVENT, - CONNECTOR_RECOVERY_RETRY_QUEUED_EVENT, MEMORY_RECORD_LINKED_EVENT, + AutomationRunLease, ConnectorAttachmentCleanupClaim, EventStore, EventStoreError, + AGENT_RUN_FINISHED_EVENT, CONNECTOR_RECOVERY_RETRY_QUEUED_EVENT, + MEMORY_RECORD_LINKED_EVENT, }; use crate::kernel::automation::{ AutomationCheckpoint, AutomationDefinition, AutomationDefinitionStatus, @@ -14960,6 +15980,46 @@ mod tests { }; use crate::kernel::connectors::landing::ConnectorAttachmentMetadata; + fn enqueue_fenced_automation_fixture( + store: &EventStore, + label: &str, + now: DateTime, + ) -> ( + AutomationDefinition, + crate::kernel::automation::AutomationRun, + AgentRunStart, + ) { + let definition = AutomationDefinition::once( + label.to_string(), + "Asia/Shanghai".to_string(), + now - Duration::minutes(1), + ) + .expect("definition is valid"); + store + .upsert_automation_definition(&definition) + .expect("definition persists"); + let (run, agent_run) = store + .enqueue_due_automation_agent_run( + definition.id, + now, + "test-scheduler".to_string(), + format!("automation:{}", definition.id), + ) + .expect("enqueue succeeds") + .expect("run is due"); + (definition, run, agent_run) + } + + fn wrong_owner_lease(lease: &AutomationRunLease) -> AutomationRunLease { + AutomationRunLease { + run_id: lease.run_id, + owner_id: "wrong-owner".to_string(), + fence_token: lease.fence_token, + generation: lease.generation, + lease_expires_at: lease.lease_expires_at, + } + } + #[test] fn durable_automation_claim_is_deduplicated_across_restart() { let temp_dir = tempfile::tempdir().expect("temp dir"); @@ -15511,6 +16571,409 @@ mod tests { ); } + #[test] + fn c6a_automation_concurrent_due_sweeps_enqueue_exactly_one_agent_run() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let path = temp_dir.path().join("automation-concurrent.sqlite3"); + let now = Utc::now(); + let definition = AutomationDefinition::once( + "Concurrent due window".to_string(), + "Asia/Shanghai".to_string(), + now - Duration::minutes(1), + ) + .expect("definition is valid"); + { + let store = EventStore::open(&path).expect("store opens"); + store + .upsert_automation_definition(&definition) + .expect("definition persists"); + } + let first = EventStore::open(&path).expect("first store opens"); + let second = EventStore::open(&path).expect("second store opens"); + first + .conn + .busy_timeout(std::time::Duration::from_secs(2)) + .expect("first busy timeout configures"); + second + .conn + .busy_timeout(std::time::Duration::from_secs(2)) + .expect("second busy timeout configures"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); + let mut threads = Vec::new(); + for (store, scheduler) in [(first, "scheduler-a"), (second, "scheduler-b")] { + let barrier = barrier.clone(); + let conversation_id = format!("automation:{}", definition.id); + threads.push(std::thread::spawn(move || { + barrier.wait(); + store.enqueue_due_automation_agent_run( + definition.id, + now, + scheduler.to_string(), + conversation_id, + ) + })); + } + barrier.wait(); + let results = threads + .into_iter() + .map(|thread| thread.join().expect("sweep thread joins")) + .collect::, _>>() + .expect("both concurrent sweeps complete without a lock error"); + assert_eq!(results.into_iter().filter(Option::is_some).count(), 1); + + let store = EventStore::open(&path).expect("store reopens"); + assert_eq!(store.list_automation_runs().expect("runs load").len(), 1); + assert_eq!( + store + .list_agent_run_records() + .expect("AgentRun records load") + .len(), + 1 + ); + } + + #[test] + fn c6a_automation_active_lease_excludes_other_owner_and_wrong_heartbeat() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, _) = enqueue_fenced_automation_fixture(&store, "Lease exclusion", now); + let lease = store + .claim_automation_run(run.id, "worker-a".to_string(), 60, now) + .expect("first owner claims"); + assert!(store + .claim_automation_run( + run.id, + "worker-b".to_string(), + 60, + now + Duration::seconds(1) + ) + .is_err()); + assert!(store + .heartbeat_automation_run_lease( + &wrong_owner_lease(&lease), + 60, + now + Duration::seconds(1) + ) + .is_err()); + let renewed = store + .heartbeat_automation_run_lease(&lease, 60, now + Duration::seconds(1)) + .expect("owner heartbeat renews"); + assert!(renewed.lease_expires_at() > lease.lease_expires_at()); + store + .release_automation_run_lease(&renewed, now + Duration::seconds(2)) + .expect("current owner releases safely"); + assert!(store + .heartbeat_automation_run_lease(&renewed, 60, now + Duration::seconds(3)) + .is_err()); + } + + #[test] + fn c6a_automation_expired_lease_rejects_heartbeat_and_release() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, _) = enqueue_fenced_automation_fixture(&store, "Expired heartbeat", now); + let lease = store + .claim_automation_run(run.id, "worker-a".to_string(), 1, now) + .expect("owner claims"); + let after_expiry = now + Duration::seconds(2); + assert!(store + .heartbeat_automation_run_lease(&lease, 60, after_expiry) + .is_err()); + assert!(store + .release_automation_run_lease(&lease, after_expiry) + .is_err()); + } + + #[test] + fn c6a_automation_safe_pre_effect_work_is_requeued_after_restart() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let path = temp_dir.path().join("automation-lease-restart.sqlite3"); + let now = Utc::now(); + let run_id; + let old_lease; + { + let store = EventStore::open(&path).expect("store opens"); + let (_, run, _) = enqueue_fenced_automation_fixture(&store, "Safe restart", now); + run_id = run.id; + old_lease = store + .claim_automation_run(run.id, "worker-before-restart".to_string(), 1, now) + .expect("first owner claims"); + } + let store = EventStore::open(&path).expect("store reopens"); + let recovered = store + .recover_expired_automation_runs(now + Duration::seconds(2)) + .expect("expired work recovers"); + assert_eq!(recovered.requeued, 1); + assert_eq!(recovered.effect_unknown, 0); + assert_eq!( + store.automation_run(run_id).expect("run reloads").status, + AutomationRunStatus::Queued + ); + let new_lease = store + .claim_automation_run( + run_id, + "worker-after-restart".to_string(), + 60, + now + Duration::seconds(2), + ) + .expect("safe work is taken over"); + assert!(new_lease.generation() > old_lease.generation()); + assert!(store + .heartbeat_automation_run_lease(&old_lease, 60, now + Duration::seconds(3)) + .is_err()); + } + + #[test] + fn c6a_automation_running_tool_blocks_takeover_as_effect_unknown() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, agent_run) = + enqueue_fenced_automation_fixture(&store, "Ambiguous tool effect", now); + let lease = store + .claim_automation_run(run.id, "worker-a".to_string(), 1, now) + .expect("owner claims"); + let plan = goal_bound_file_read_plan(agent_run.id); + store + .append_tool_invocation(&ToolInvocationRecord::running(&plan, None)) + .expect("running tool invocation persists"); + + let recovered = store + .recover_expired_automation_runs(now + Duration::seconds(2)) + .expect("expired claim is classified"); + assert_eq!(recovered.requeued, 0); + assert_eq!(recovered.effect_unknown, 1); + let recovered_run = store.automation_run(run.id).expect("run reloads"); + assert_eq!(recovered_run.status, AutomationRunStatus::Failed); + assert!(recovered_run + .last_error + .as_deref() + .is_some_and(|value| value.starts_with("effect_unknown:"))); + assert!(store + .claim_automation_run( + run.id, + "worker-b".to_string(), + 60, + now + Duration::seconds(3) + ) + .is_err()); + assert!(store + .heartbeat_automation_run_lease(&lease, 60, now + Duration::seconds(3)) + .is_err()); + let agent = store + .list_agent_run_records() + .expect("AgentRun records load") + .into_iter() + .find(|record| record.id == agent_run.id) + .expect("linked AgentRun exists"); + assert_eq!(agent.status, AgentRunStatus::Blocked); + } + + #[test] + fn c6a_automation_action_started_blocks_takeover_as_effect_unknown() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, agent_run) = + enqueue_fenced_automation_fixture(&store, "ActionStarted effect", now); + store + .claim_automation_run(run.id, "worker-a".to_string(), 1, now) + .expect("owner claims"); + let session_id = Uuid::new_v4(); + let step_id = Uuid::new_v4(); + store + .conn + .execute( + r#"INSERT INTO computer_use_sessions + (id, session_json, run_id, active_step_id, row_revision, updated_at) + VALUES (?1, '{}', ?2, ?3, 0, ?4)"#, + params![ + session_id.to_string(), + agent_run.id.to_string(), + step_id.to_string(), + now.to_rfc3339_opts(SecondsFormat::Nanos, true) + ], + ) + .expect("session fixture persists"); + store + .conn + .execute( + r#"INSERT INTO computer_use_steps + (id, session_id, sequence, step_json, status, row_revision, updated_at) + VALUES (?1, ?2, 0, '{}', 'action_started', 0, ?3)"#, + params![ + step_id.to_string(), + session_id.to_string(), + now.to_rfc3339_opts(SecondsFormat::Nanos, true) + ], + ) + .expect("ActionStarted fixture persists"); + + let recovered = store + .recover_expired_automation_runs(now + Duration::seconds(2)) + .expect("expired claim is classified"); + assert_eq!(recovered.requeued, 0); + assert_eq!(recovered.effect_unknown, 1); + assert!(store + .automation_run(run.id) + .expect("run reloads") + .last_error + .as_deref() + .is_some_and(|value| value.contains("ActionStarted"))); + } + + #[test] + fn c6a_automation_checkpoint_is_fence_bound_and_compare_and_swap_versioned() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, agent_run) = enqueue_fenced_automation_fixture(&store, "Checkpoint CAS", now); + let lease = store + .claim_automation_run(run.id, "checkpoint-worker".to_string(), 60, now) + .expect("owner claims"); + let checkpoint = AutomationCheckpoint { + automation_run_id: run.id, + dedup_key: run.trigger_window_key.clone(), + tool_invocation_id: None, + evidence_ref: Some("evidence://checkpoint-0".to_string()), + recorded_at: now, + }; + let initial = store + .compare_and_swap_automation_checkpoint(&checkpoint, &lease, None, now) + .expect("initial checkpoint persists"); + assert_eq!(initial.revision, 0); + assert_eq!(initial.definition_revision, run.definition_revision); + assert_eq!(initial.agent_run_id, agent_run.id); + assert_eq!(initial.lease_generation, lease.generation()); + assert!(store + .compare_and_swap_automation_checkpoint(&checkpoint, &lease, None, now) + .is_err()); + + let mut updated = checkpoint.clone(); + updated.evidence_ref = Some("evidence://checkpoint-1".to_string()); + let updated_record = store + .compare_and_swap_automation_checkpoint( + &updated, + &lease, + Some(0), + now + Duration::seconds(1), + ) + .expect("current revision updates"); + assert_eq!(updated_record.revision, 1); + assert!(store + .compare_and_swap_automation_checkpoint( + &updated, + &lease, + Some(0), + now + Duration::seconds(2) + ) + .is_err()); + assert!(store + .compare_and_swap_automation_checkpoint( + &updated, + &wrong_owner_lease(&lease), + Some(1), + now + Duration::seconds(2) + ) + .is_err()); + assert_eq!( + store + .automation_checkpoint_record(run.id) + .expect("bound checkpoint reloads"), + updated_record + ); + } + + #[test] + fn c6a_automation_definition_revision_mismatch_is_effect_unknown() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (_, run, agent_run) = + enqueue_fenced_automation_fixture(&store, "Definition mismatch", now); + store + .conn + .execute( + "UPDATE automation_runs SET definition_revision = definition_revision + 1 WHERE id = ?1", + params![run.id.to_string()], + ) + .expect("fixture changes the stored revision"); + assert!(store + .claim_automation_run(run.id, "worker".to_string(), 60, now) + .is_err()); + let blocked = store.automation_run(run.id).expect("run reloads"); + assert_eq!(blocked.status, AutomationRunStatus::Failed); + assert!(blocked + .last_error + .as_deref() + .is_some_and(|value| value.starts_with("effect_unknown:"))); + let linked = store + .list_agent_run_records() + .expect("AgentRun records load") + .into_iter() + .find(|record| record.id == agent_run.id) + .expect("linked AgentRun exists"); + assert_eq!(linked.status, AgentRunStatus::Blocked); + } + + #[test] + fn c6a_automation_pause_resume_and_revision_persist_across_restart() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let path = temp_dir.path().join("automation-pause-resume.sqlite3"); + let now = Utc::now(); + let definition_id; + let paused_revision; + { + let store = EventStore::open(&path).expect("store opens"); + let definition = AutomationDefinition::once( + "Pause and resume".to_string(), + "Asia/Shanghai".to_string(), + now - Duration::minutes(1), + ) + .expect("definition is valid"); + definition_id = definition.id; + store + .upsert_automation_definition(&definition) + .expect("definition persists"); + let paused = store + .set_automation_definition_status( + definition.id, + AutomationDefinitionStatus::Paused, + now, + ) + .expect("pause persists"); + paused_revision = paused.revision; + } + let store = EventStore::open(&path).expect("store reopens"); + let paused = store + .automation_definition(definition_id) + .expect("paused definition reloads"); + assert_eq!(paused.status, AutomationDefinitionStatus::Paused); + assert_eq!(paused.revision, paused_revision); + assert!(store + .enqueue_due_automation_agent_run( + definition_id, + now, + "scheduler".to_string(), + format!("automation:{definition_id}") + ) + .expect("paused sweep is safe") + .is_none()); + let resumed = store + .set_automation_definition_status( + definition_id, + AutomationDefinitionStatus::Enabled, + now + Duration::seconds(1), + ) + .expect("resume persists"); + assert!(resumed.revision > paused_revision); + assert!(store + .enqueue_due_automation_agent_run( + definition_id, + now + Duration::seconds(1), + "scheduler".to_string(), + format!("automation:{definition_id}") + ) + .expect("resumed sweep succeeds") + .is_some()); + } + #[test] fn automation_edit_manual_run_and_delete_are_durable() { let store = EventStore::open_memory().expect("store opens"); diff --git a/apps/desktop/src-tauri/src/main.rs b/apps/desktop/src-tauri/src/main.rs index 60deb79..8d68731 100644 --- a/apps/desktop/src-tauri/src/main.rs +++ b/apps/desktop/src-tauri/src/main.rs @@ -21,7 +21,7 @@ use automation_commands::{ edit_automation_review_item, enqueue_due_automation, list_automation_definitions, list_automation_review_items, list_automation_runs, reconcile_automation_runs, resolve_automation_review_item, run_automation_now, run_due_automation_sweep, - set_automation_enabled, update_automation_goal, + set_automation_enabled, spawn_automation_scheduler, update_automation_goal, }; use commands::{ @@ -305,6 +305,7 @@ fn main() { spawn_connector_reconciliation_worker(state.clone()); spawn_connector_attachment_recovery_worker(state.clone()); spawn_artifact_recovery_worker(state.clone()); + spawn_automation_scheduler(state.clone()); app.manage(state); if let Some(window) = app.get_webview_window("main") { apply_main_window_icon(&window)?; diff --git a/scripts/automation-center.test.mjs b/scripts/automation-center.test.mjs index 8e8c122..14be026 100644 --- a/scripts/automation-center.test.mjs +++ b/scripts/automation-center.test.mjs @@ -36,6 +36,10 @@ const reconciliation = readFileSync( ); const eventStore = readFileSync("apps/desktop/src-tauri/src/kernel/event_store.rs", "utf8"); const commands = readFileSync("apps/desktop/src-tauri/src/automation_commands.rs", "utf8"); +const automationKernel = readFileSync( + "apps/desktop/src-tauri/src/kernel/automation.rs", + "utf8", +); const connector = readFileSync("apps/desktop/src-tauri/src/kernel/connectors.rs", "utf8"); const connectorCatalog = readFileSync( "apps/desktop/src-tauri/src/kernel/connectors/catalog.rs", @@ -521,6 +525,34 @@ assert.doesNotMatch(recoveryCenter, /execute_agent_tool|prepare_connector_attach assert.match(commands, /enqueue_due_automation_agent_run/); assert.match(commands, /reconcile_automation_agent_runs/); +assert.match(commands, /recover_expired_automation_runs/); +const automationScheduler = + commands.match(/pub fn spawn_automation_scheduler[\s\S]*?\n\}/)?.[0] ?? ""; +assert.match(automationScheduler, /ds-agent-automation-scheduler/); +assert.match(automationScheduler, /run_automation_scheduler_once/); +assert.match(automationScheduler, /AUTOMATION_SCHEDULER_INTERVAL/); +assert.match(main, /spawn_automation_scheduler\(state\.clone\(\)\)/); +assert.ok( + main.indexOf("spawn_automation_scheduler(state.clone())") < + main.indexOf("app.manage(state)"), + "the backend automation scheduler must start from setup before state is handed to Tauri", +); +assert.match( + eventStore, + /#\[cfg\(test\)\]\s+pub fn upsert_automation_checkpoint\(/, + "the legacy unconditional checkpoint writer must not exist in production", +); +assert.match(eventStore, /pub\(crate\) fn compare_and_swap_automation_checkpoint\(/); +assert.match(eventStore, /lease_fence_token TEXT/); +assert.match(eventStore, /definition_snapshot_json TEXT/); +assert.match(eventStore, /AUTOMATION_CHECKPOINT_CONTRACT_VERSION/); +const automationRunStruct = + automationKernel.match(/pub struct AutomationRun \{[\s\S]*?\n\}/)?.[0] ?? ""; +assert.doesNotMatch( + automationRunStruct, + /fence_token|lease_owner_id|lease_generation|lease_expires_at|lease_heartbeat_at/, + "opaque automation lease ownership must stay out of the Tauri-visible run DTO", +); assert.match(main, /run_due_automation_sweep/); assert.match(main, /resume_connector_read_sync,/); assert.match(main, /inspect_connector_external_result,/); From 9cd7ed65ab9b048c0564851a666cb573be613c36 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 26 Jul 2026 07:21:34 +0800 Subject: [PATCH 2/4] feat(automation): make external effects restart-safe --- .../src-tauri/src/automation_commands.rs | 3 + apps/desktop/src-tauri/src/commands.rs | 18 + .../src-tauri/src/kernel/automation.rs | 284 +++ .../src-tauri/src/kernel/event_store.rs | 2002 ++++++++++++++++- scripts/automation-center.test.mjs | 61 + 5 files changed, 2301 insertions(+), 67 deletions(-) diff --git a/apps/desktop/src-tauri/src/automation_commands.rs b/apps/desktop/src-tauri/src/automation_commands.rs index d9548ae..6a73961 100644 --- a/apps/desktop/src-tauri/src/automation_commands.rs +++ b/apps/desktop/src-tauri/src/automation_commands.rs @@ -22,6 +22,9 @@ fn run_automation_scheduler_once(store: &EventStore, now: DateTime) -> Resu store .recover_expired_automation_runs(now) .map_err(store_error)?; + store + .reconcile_due_automation_effects(now, 64) + .map_err(store_error)?; store .reconcile_automation_agent_runs(now) .map_err(store_error)?; diff --git a/apps/desktop/src-tauri/src/commands.rs b/apps/desktop/src-tauri/src/commands.rs index a98ee02..fc18273 100644 --- a/apps/desktop/src-tauri/src/commands.rs +++ b/apps/desktop/src-tauri/src/commands.rs @@ -2344,6 +2344,21 @@ fn ready_agent_tool_authorization( ); return Err(error); } + if store + .begin_automation_effect_for_tool_invocation(authorized.plan.invocation_id, Utc::now()) + .is_err() + { + let _ = store.release_agent_run_resources_for_invocation( + authorized.plan.invocation_id, + "automation effect checkpoint failed before executor start".to_string(), + ); + return blocked_agent_tool_authorization( + store, + &authorized.plan, + "automation effect checkpoint failed before executor start", + true, + ); + } Ok(AgentToolAuthorization::Ready(authorized)) } @@ -2736,6 +2751,9 @@ fn record_completed_agent_tool_execution( invocation: &ToolInvocationRecord, ) -> Result<(), String> { append_agent_tool_invocation_audit(store, invocation)?; + store + .finish_automation_effect_for_tool_invocation(invocation.id, Utc::now()) + .map_err(event_store_error)?; store .record_goal_completion_for_tool_invocation(invocation) .map_err(event_store_error)?; diff --git a/apps/desktop/src-tauri/src/kernel/automation.rs b/apps/desktop/src-tauri/src/kernel/automation.rs index 8528dba..1846d2e 100644 --- a/apps/desktop/src-tauri/src/kernel/automation.rs +++ b/apps/desktop/src-tauri/src/kernel/automation.rs @@ -5,6 +5,9 @@ use sha2::{Digest, Sha256}; use uuid::Uuid; pub const AUTOMATION_CHECKPOINT_CONTRACT_VERSION: &str = "ds-agent.automation-checkpoint/v2"; +pub const AUTOMATION_EFFECT_LEDGER_VERSION: &str = "ds-agent.automation-effect-ledger/v1"; +pub const AUTOMATION_EFFECT_MAX_RECORDS: usize = 64; +pub const AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS: u32 = 7; #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] @@ -144,6 +147,287 @@ pub(crate) struct AutomationCheckpointRecord { pub checkpoint: AutomationCheckpoint, } +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AutomationEffectState { + Prepared, + EffectStarted, + EffectUnknown, + KnownApplied, + KnownNotApplied, + ReconciliationBackoff, + DeadLettered, +} + +impl AutomationEffectState { + pub fn terminal(self) -> bool { + matches!( + self, + Self::KnownApplied | Self::KnownNotApplied | Self::DeadLettered + ) + } + + pub fn uncertain(self) -> bool { + matches!( + self, + Self::EffectStarted | Self::EffectUnknown | Self::ReconciliationBackoff + ) + } + + pub fn permits_new_effect(self) -> bool { + self.terminal() && self != Self::DeadLettered + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum AutomationEffectUndoCapability { + None, + ExactLocal, + UnsupportedExternal, +} + +pub(crate) struct AutomationEffectBinding { + pub automation_run_id: Uuid, + pub definition_revision: u64, + pub agent_run_id: Uuid, + pub tool_invocation_id: Uuid, + pub request_fingerprint: String, + pub prepared_checkpoint_revision: u64, + pub fence_token: Uuid, + pub lease_generation: u64, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AutomationEffectRecord { + pub automation_run_id: Uuid, + pub definition_revision: u64, + pub agent_run_id: Uuid, + pub tool_invocation_id: Uuid, + pub request_fingerprint: String, + pub prepared_checkpoint_revision: u64, + pub origin_fence_token: Uuid, + pub last_fence_token: Uuid, + pub origin_lease_generation: u64, + pub last_lease_generation: u64, + pub idempotency_key: String, + pub state: AutomationEffectState, + pub reconciliation_attempt_count: u32, + pub next_reconciliation_at: Option>, + pub evidence_ref: Option, + pub safe_reason_code: Option, + pub undo_capability: AutomationEffectUndoCapability, + pub workspace_checkpoint_id: Option, + pub workspace_action_revision: Option, + pub created_at: DateTime, + pub updated_at: DateTime, +} + +impl AutomationEffectRecord { + pub(crate) fn prepared( + binding: AutomationEffectBinding, + now: DateTime, + ) -> Result { + let request_fingerprint = required_text( + binding.request_fingerprint.clone(), + "tool request fingerprint", + )?; + let idempotency_key = automation_effect_idempotency_key(&binding, &request_fingerprint); + let record = Self { + automation_run_id: binding.automation_run_id, + definition_revision: binding.definition_revision, + agent_run_id: binding.agent_run_id, + tool_invocation_id: binding.tool_invocation_id, + request_fingerprint, + prepared_checkpoint_revision: binding.prepared_checkpoint_revision, + origin_fence_token: binding.fence_token, + last_fence_token: binding.fence_token, + origin_lease_generation: binding.lease_generation, + last_lease_generation: binding.lease_generation, + idempotency_key, + state: AutomationEffectState::Prepared, + reconciliation_attempt_count: 0, + next_reconciliation_at: None, + evidence_ref: None, + safe_reason_code: None, + undo_capability: AutomationEffectUndoCapability::None, + workspace_checkpoint_id: None, + workspace_action_revision: None, + created_at: now, + updated_at: now, + }; + record.validate()?; + Ok(record) + } + + pub fn validate(&self) -> Result<(), String> { + if self.automation_run_id.is_nil() + || self.agent_run_id.is_nil() + || self.tool_invocation_id.is_nil() + || self.origin_fence_token.is_nil() + || self.last_fence_token.is_nil() + || self.request_fingerprint.trim().is_empty() + || self.last_lease_generation < self.origin_lease_generation + || !valid_automation_effect_idempotency_key(&self.idempotency_key) + || self.updated_at < self.created_at + { + return Err("automation effect binding is invalid".to_string()); + } + if self + .safe_reason_code + .as_deref() + .is_some_and(|value| !valid_safe_effect_text(value)) + || self + .evidence_ref + .as_deref() + .is_some_and(|value| !valid_safe_effect_text(value)) + { + return Err("automation effect evidence is invalid".to_string()); + } + match self.state { + AutomationEffectState::ReconciliationBackoff + if self.next_reconciliation_at.is_none() + || self.reconciliation_attempt_count == 0 => + { + return Err("automation effect backoff is incomplete".to_string()); + } + AutomationEffectState::ReconciliationBackoff => {} + AutomationEffectState::DeadLettered + if self.next_reconciliation_at.is_some() || self.safe_reason_code.is_none() => + { + return Err("automation effect dead letter is incomplete".to_string()); + } + AutomationEffectState::DeadLettered => {} + _ if self.next_reconciliation_at.is_some() => { + return Err("automation effect has an unexpected retry time".to_string()); + } + _ => {} + } + match self.undo_capability { + AutomationEffectUndoCapability::ExactLocal => { + if self.state != AutomationEffectState::KnownApplied + || self.workspace_checkpoint_id.is_none() + || self + .workspace_action_revision + .as_deref() + .is_none_or(|value| !value.starts_with("undo1:")) + { + return Err("exact local undo binding is incomplete".to_string()); + } + } + AutomationEffectUndoCapability::None + | AutomationEffectUndoCapability::UnsupportedExternal => { + if self.workspace_checkpoint_id.is_some() + || self.workspace_action_revision.is_some() + { + return Err("unsupported automation undo carries local authority".to_string()); + } + } + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct AutomationEffectLedger { + pub version: String, + pub effects: Vec, +} + +impl Default for AutomationEffectLedger { + fn default() -> Self { + Self { + version: AUTOMATION_EFFECT_LEDGER_VERSION.to_string(), + effects: Vec::new(), + } + } +} + +impl AutomationEffectLedger { + pub fn validate(&self, automation_run_id: Uuid) -> Result<(), String> { + if self.version != AUTOMATION_EFFECT_LEDGER_VERSION + || self.effects.len() > AUTOMATION_EFFECT_MAX_RECORDS + { + return Err("automation effect ledger contract is invalid".to_string()); + } + let mut idempotency_keys = std::collections::BTreeSet::new(); + let mut tool_invocations = std::collections::BTreeSet::new(); + for effect in &self.effects { + effect.validate()?; + if effect.automation_run_id != automation_run_id + || !idempotency_keys.insert(effect.idempotency_key.as_str()) + || !tool_invocations.insert(effect.tool_invocation_id) + { + return Err("automation effect ledger binding is invalid".to_string()); + } + } + Ok(()) + } + + pub fn current(&self) -> Option<&AutomationEffectRecord> { + self.effects.last() + } + + pub(crate) fn current_mut(&mut self) -> Option<&mut AutomationEffectRecord> { + self.effects.last_mut() + } + + pub(crate) fn push( + &mut self, + automation_run_id: Uuid, + effect: AutomationEffectRecord, + ) -> Result<(), String> { + if self + .current() + .is_some_and(|current| !current.state.permits_new_effect()) + { + return Err("the prior automation effect requires explicit recovery".to_string()); + } + if self.effects.len() == AUTOMATION_EFFECT_MAX_RECORDS { + return Err("automation effect ledger is full".to_string()); + } + self.effects.push(effect); + self.validate(automation_run_id) + } +} + +pub(crate) fn automation_effect_idempotency_key( + binding: &AutomationEffectBinding, + request_fingerprint: &str, +) -> String { + let canonical = serde_json::to_vec(&( + AUTOMATION_EFFECT_LEDGER_VERSION, + binding.automation_run_id, + binding.definition_revision, + binding.agent_run_id, + binding.tool_invocation_id, + binding.prepared_checkpoint_revision, + binding.fence_token, + binding.lease_generation, + request_fingerprint, + )) + .expect("automation effect identity is serializable"); + format!( + "automation-effect1:{}", + hex::encode(Sha256::digest(canonical)) + ) +} + +fn valid_automation_effect_idempotency_key(value: &str) -> bool { + value.len() == 83 + && value.starts_with("automation-effect1:") + && value[19..].bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn valid_safe_effect_text(value: &str) -> bool { + let value = value.trim(); + !value.is_empty() + && value.len() <= 512 + && !value.contains('\0') + && !value.to_ascii_lowercase().contains("token") + && !value.to_ascii_lowercase().contains("secret") +} + #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum ReviewQueueItemStatus { diff --git a/apps/desktop/src-tauri/src/kernel/event_store.rs b/apps/desktop/src-tauri/src/kernel/event_store.rs index 167a014..3259880 100644 --- a/apps/desktop/src-tauri/src/kernel/event_store.rs +++ b/apps/desktop/src-tauri/src/kernel/event_store.rs @@ -29,8 +29,11 @@ use crate::kernel::agent_run::{ }; use crate::kernel::automation::{ automation_run_transition_allowed, next_scheduled_at, trigger_window_key, AutomationCheckpoint, - AutomationCheckpointRecord, AutomationDefinition, AutomationDefinitionStatus, AutomationRun, - AutomationRunStatus, MissedRunPolicy, ReviewQueueItem, AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + AutomationCheckpointRecord, AutomationDefinition, AutomationDefinitionStatus, + AutomationEffectBinding, AutomationEffectLedger, AutomationEffectRecord, AutomationEffectState, + AutomationEffectUndoCapability, AutomationRun, AutomationRunStatus, MissedRunPolicy, + ReviewQueueItem, AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS, }; use crate::kernel::capability::{CapabilityInvocation, CapabilityInvocationStatus}; use crate::kernel::connectors::catalog::ConnectorSyncHealthSnapshot; @@ -120,6 +123,9 @@ use crate::kernel::work_package::{ WorkPackageWorkflowTemplateImportPreview, WorkPackageWorkflowTemplateImportSummary, }; use crate::kernel::workflow::{OperationsBriefingRun, WorkflowTemplatePackage}; +use crate::kernel::workspace_undo::{ + WorkspaceCheckpointEffectState, WorkspaceMutationCheckpointStatus, +}; pub const CAPABILITY_ACCESS_REQUESTED_EVENT: &str = "capability_access.requested"; pub const CAPABILITY_INVOCATION_RECORDED_EVENT: &str = "capability_invocation.recorded"; @@ -1055,12 +1061,50 @@ impl AutomationRunLease { } } +pub(crate) struct AutomationEffectExecution { + lease: AutomationRunLease, + checkpoint_revision: u64, + effect: AutomationEffectRecord, +} + +enum DurableAutomationEffectOutcome { + KnownApplied { + evidence_ref: String, + undo_capability: AutomationEffectUndoCapability, + workspace_checkpoint_id: Option, + workspace_action_revision: Option, + }, + KnownNotApplied { + evidence_ref: String, + }, + StillUncertain { + reason_code: String, + }, +} + +impl AutomationEffectExecution { + pub(crate) fn effect(&self) -> &AutomationEffectRecord { + &self.effect + } +} + #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub(crate) struct AutomationRunRecoverySweep { pub requeued: usize, pub effect_unknown: usize, } +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub(crate) struct AutomationEffectReconciliationSweep { + pub inspected: usize, + pub known_applied: usize, + pub known_not_applied: usize, + pub deferred: usize, + pub dead_lettered: usize, + pub quarantined: usize, + pub lost_fence: usize, +} + pub(crate) struct ConnectorSyncRecoveryClaim { job_id: Uuid, claim_id: Uuid, @@ -1298,6 +1342,43 @@ pub(crate) struct ConnectorReconciliationClaim { claim_expires_at: DateTime, } +fn automation_revision_i64(value: u64) -> EventStoreResult { + i64::try_from(value).map_err(|_| { + EventStoreError::InvalidState("automation effect revision is too large".to_string()) + }) +} + +fn automation_capability_may_have_effect(capability: CapabilityKind) -> bool { + matches!( + capability, + CapabilityKind::FileWrite + | CapabilityKind::BrowserSubmit + | CapabilityKind::EmailDraft + | CapabilityKind::EmailSend + | CapabilityKind::ConnectorAttachmentRead + | CapabilityKind::ConnectorWrite + | CapabilityKind::DriveWrite + | CapabilityKind::TerminalWrite + | CapabilityKind::ComputerControl + | CapabilityKind::AppUpdateDownload + | CapabilityKind::AppUpdateInstall + ) +} + +fn automation_capability_is_external(capability: CapabilityKind) -> bool { + matches!( + capability, + CapabilityKind::BrowserSubmit + | CapabilityKind::EmailDraft + | CapabilityKind::EmailSend + | CapabilityKind::ConnectorWrite + | CapabilityKind::DriveWrite + | CapabilityKind::TerminalWrite + | CapabilityKind::ComputerControl + | CapabilityKind::AppUpdateInstall + ) +} + impl ConnectorReconciliationClaim { pub(crate) fn claim_id(&self) -> Uuid { self.claim_id @@ -1477,6 +1558,14 @@ impl EventStore { definition_revision INTEGER, agent_run_id TEXT, lease_generation INTEGER, + effect_ledger_json TEXT, + effect_state TEXT, + effect_tool_invocation_id TEXT, + effect_idempotency_key TEXT, + next_reconciliation_at TEXT, + reconciliation_attempt_count INTEGER NOT NULL DEFAULT 0, + effect_dead_letter_code TEXT, + effect_quarantine_code TEXT, updated_at TEXT NOT NULL ); @@ -1768,9 +1857,54 @@ impl EventStore { "lease_generation", "ALTER TABLE automation_checkpoints ADD COLUMN lease_generation INTEGER", ), + ( + "effect_ledger_json", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_ledger_json TEXT", + ), + ( + "effect_state", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_state TEXT", + ), + ( + "effect_tool_invocation_id", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_tool_invocation_id TEXT", + ), + ( + "effect_idempotency_key", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_idempotency_key TEXT", + ), + ( + "next_reconciliation_at", + "ALTER TABLE automation_checkpoints ADD COLUMN next_reconciliation_at TEXT", + ), + ( + "reconciliation_attempt_count", + "ALTER TABLE automation_checkpoints ADD COLUMN reconciliation_attempt_count INTEGER NOT NULL DEFAULT 0", + ), + ( + "effect_dead_letter_code", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_dead_letter_code TEXT", + ), + ( + "effect_quarantine_code", + "ALTER TABLE automation_checkpoints ADD COLUMN effect_quarantine_code TEXT", + ), ] { ensure_sqlite_column(&self.conn, "automation_checkpoints", column, sql)?; } + self.conn.execute( + r#"CREATE UNIQUE INDEX IF NOT EXISTS idx_automation_effect_idempotency + ON automation_checkpoints (effect_idempotency_key) + WHERE effect_idempotency_key IS NOT NULL"#, + [], + )?; + self.conn.execute( + r#"CREATE INDEX IF NOT EXISTS idx_automation_effect_reconciliation + ON automation_checkpoints + (effect_state, next_reconciliation_at, reconciliation_attempt_count) + WHERE effect_quarantine_code IS NULL"#, + [], + )?; self.conn.execute( r#"CREATE INDEX IF NOT EXISTS idx_automation_runs_lease ON automation_runs (lease_expires_at, status)"#, @@ -3175,6 +3309,15 @@ impl EventStore { } }) }; + let effect_recovery = self + .conn + .query_row( + r#"SELECT revision, effect_ledger_json + FROM automation_checkpoints WHERE automation_run_id = ?1"#, + params![run_id.to_string()], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)), + ) + .optional()?; let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; let changed = transaction.execute( r#"UPDATE automation_runs @@ -3200,6 +3343,80 @@ impl EventStore { let event = KernelEvent::new(AGENT_RUN_TRANSITIONED_EVENT, &transition)?; Self::insert_kernel_event(&transaction, &event)?; } + if let Some((checkpoint_revision, Some(ledger_json))) = effect_recovery { + let parsed = serde_json::from_str::(&ledger_json) + .map_err(EventStoreError::from) + .and_then(|ledger| { + ledger + .validate(run_id) + .map_err(EventStoreError::InvalidState)?; + Ok(ledger) + }); + match parsed { + Ok(mut ledger) + if ledger.current().is_some_and(|effect| { + effect.state == AutomationEffectState::EffectStarted + }) => + { + let current = ledger.current_mut().expect("validated ledger is not empty"); + current.state = AutomationEffectState::EffectUnknown; + current.evidence_ref = Some(format!( + "automation-effect://{}/lease-expired", + current.idempotency_key + )); + current.safe_reason_code = + Some("lease_expired_after_effect_started".to_string()); + current.next_reconciliation_at = None; + current.updated_at = now; + current.validate().map_err(EventStoreError::InvalidState)?; + let next_revision = u64::try_from(checkpoint_revision) + .ok() + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is exhausted".to_string(), + ) + })?; + let changed = transaction.execute( + r#"UPDATE automation_checkpoints + SET revision = ?2, effect_ledger_json = ?3, effect_state = ?4, + next_reconciliation_at = NULL, + effect_dead_letter_code = NULL, effect_quarantine_code = NULL, + updated_at = ?5 + WHERE automation_run_id = ?1 AND revision = ?6 + AND effect_tool_invocation_id = ?7"#, + params![ + run_id.to_string(), + automation_revision_i64(next_revision)?, + serde_json::to_string(&ledger)?, + serde_json::to_string(&AutomationEffectState::EffectUnknown)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + checkpoint_revision, + ledger + .current() + .expect("validated ledger is not empty") + .tool_invocation_id + .to_string(), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "expired automation effect recovery lost its revision".to_string(), + )); + } + } + Ok(_) => {} + Err(_) => { + transaction.execute( + r#"UPDATE automation_checkpoints + SET effect_quarantine_code = 'invalid_effect_ledger', + next_reconciliation_at = NULL + WHERE automation_run_id = ?1"#, + params![run_id.to_string()], + )?; + } + } + } transaction.commit()?; Ok(Some(safe)) } @@ -3725,76 +3942,1095 @@ impl EventStore { }) } - pub fn upsert_review_queue_item(&self, item: &ReviewQueueItem) -> EventStoreResult<()> { - let existing = self - .conn - .query_row( - "SELECT item_json FROM review_queue_items WHERE id = ?1", - params![item.id.to_string()], - |row| row.get::<_, String>(0), - ) - .optional()? - .map(|json| serde_json::from_str::(&json)) - .transpose()?; - if existing.as_ref() == Some(item) { - return Ok(()); + pub(crate) fn begin_automation_effect_for_tool_invocation( + &self, + tool_invocation_id: Uuid, + now: DateTime, + ) -> EventStoreResult> { + let invocation = self.tool_invocation_by_id(tool_invocation_id)?; + if !automation_capability_may_have_effect(invocation.capability) { + return Ok(None); } - if existing - .as_ref() - .is_some_and(|current| item.revision != current.revision.saturating_add(1)) + if invocation.status != ToolExecutionStatus::Running { + return Err(EventStoreError::InvalidState( + "automation effect can begin only for a running Tool".to_string(), + )); + } + let Some(agent_run_id) = invocation.run_id else { + return Ok(None); + }; + let matching_runs = self + .list_automation_runs()? + .into_iter() + .filter(|run| run.agent_run_id == Some(agent_run_id)) + .collect::>(); + if matching_runs.is_empty() { + return Ok(None); + } + if matching_runs.len() != 1 { + return Err(EventStoreError::InvalidState( + "AgentRun is linked to multiple automation runs".to_string(), + )); + } + let run = matching_runs + .into_iter() + .next() + .expect("non-empty automation run set"); + let (exact_run, agent_run) = self.exact_automation_run_binding(run.id)?; + if exact_run != run + || agent_run.id != agent_run_id + || agent_run.status != AgentRunStatus::Running + || self.running_tool_invocation_for_agent_run(agent_run_id)? != Some(tool_invocation_id) { return Err(EventStoreError::InvalidState( - "review item revision is stale".to_string(), + "automation effect Tool binding changed before dispatch".to_string(), )); } - let transaction = self.conn.unchecked_transaction()?; - let mut run = self.automation_run(item.automation_run_id)?; - if let Some(existing_id) = run.review_queue_item_id { - if existing_id != item.id { - return Err(EventStoreError::InvalidState( - "automation run already has a review queue item".to_string(), - )); - } + if invocation.request_fingerprint.trim().is_empty() { + return Err(EventStoreError::InvalidState( + "automation effect request fingerprint is missing".to_string(), + )); } - run.review_queue_item_id = Some(item.id); - run.updated_at = item.updated_at; - let changed = transaction.execute( - r#"INSERT INTO review_queue_items (id, automation_run_id, item_json, status, revision, updated_at) - VALUES (?1, ?2, ?3, ?4, ?5, ?6) - ON CONFLICT(id) DO UPDATE SET - item_json = excluded.item_json, - status = excluded.status, - revision = excluded.revision, - updated_at = excluded.updated_at - WHERE review_queue_items.revision + 1 = excluded.revision"#, - params![ - item.id.to_string(), - item.automation_run_id.to_string(), - serde_json::to_string(item)?, - serde_json::to_string(&item.status)?, - item.revision, - item.updated_at.to_rfc3339_opts(SecondsFormat::Nanos, true), - ], - )?; - if changed != 1 { + + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let (run_json, stored_definition_revision, generation, existing_fence) = transaction + .query_row( + r#"SELECT run_json, definition_revision, lease_generation, lease_fence_token + FROM automation_runs WHERE id = ?1"#, + params![run.id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, Option>(3)?, + )) + }, + )?; + let transaction_run: AutomationRun = serde_json::from_str(&run_json)?; + if transaction_run != run + || u64::try_from(stored_definition_revision).ok() != Some(run.definition_revision) + || existing_fence.is_some() + { return Err(EventStoreError::InvalidState( - "review item update raced with another writer".to_string(), + "automation effect lease binding raced with another owner".to_string(), )); } - transaction.execute( - "UPDATE automation_runs SET run_json = ?2, updated_at = ?3 WHERE id = ?1", + let generation = u64::try_from(generation) + .map_err(|_| { + EventStoreError::InvalidState("automation effect lease is invalid".to_string()) + })? + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect lease generation is exhausted".to_string(), + ) + })?; + let owner_id = format!("automation-effect:{tool_invocation_id}"); + let fence_token = Uuid::new_v4(); + let lease_expires_at = now + Duration::minutes(15); + let mut claimed_run = run.clone(); + claimed_run.claimed_by = Some(owner_id.clone()); + claimed_run.claimed_at = Some(now); + claimed_run.updated_at = now; + let claimed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?2, claimed_by = ?3, lease_owner_id = ?3, + lease_fence_token = ?4, lease_generation = ?5, + lease_expires_at = ?6, lease_heartbeat_at = ?7, updated_at = ?7 + WHERE id = ?1 AND lease_fence_token IS NULL AND lease_generation = ?8 + AND definition_revision = ?9"#, params![ run.id.to_string(), - serde_json::to_string(&run)?, - run.updated_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + serde_json::to_string(&claimed_run)?, + &owner_id, + fence_token.to_string(), + automation_revision_i64(generation)?, + lease_expires_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + automation_revision_i64(generation.saturating_sub(1))?, + automation_revision_i64(run.definition_revision)?, ], )?; + if claimed != 1 { + return Err(EventStoreError::InvalidState( + "automation effect lease lost its fence".to_string(), + )); + } + + let current = transaction + .query_row( + r#"SELECT contract_version, revision, definition_revision, agent_run_id, + effect_ledger_json + FROM automation_checkpoints WHERE automation_run_id = ?1"#, + params![run.id.to_string()], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + )) + }, + ) + .optional()?; + let (current_revision, mut ledger) = match current { + Some(( + contract_version, + revision, + definition_revision, + checkpoint_agent_run_id, + ledger_json, + )) => { + if contract_version.as_deref() != Some(AUTOMATION_CHECKPOINT_CONTRACT_VERSION) + || definition_revision.and_then(|value| u64::try_from(value).ok()) + != Some(run.definition_revision) + || checkpoint_agent_run_id.as_deref() != Some(agent_run_id.to_string().as_str()) + { + return Err(EventStoreError::InvalidState( + "automation effect checkpoint binding changed".to_string(), + )); + } + let revision = u64::try_from(revision).map_err(|_| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is invalid".to_string(), + ) + })?; + let ledger = ledger_json + .map(|json| serde_json::from_str::(&json)) + .transpose()? + .unwrap_or_default(); + ledger + .validate(run.id) + .map_err(EventStoreError::InvalidState)?; + (Some(revision), ledger) + } + None => (None, AutomationEffectLedger::default()), + }; + if ledger + .effects + .iter() + .any(|effect| effect.tool_invocation_id == tool_invocation_id) + { + return Err(EventStoreError::InvalidState( + "automation effect invocation was already prepared and will not be replayed" + .to_string(), + )); + } + let checkpoint_revision = current_revision + .map(|value| { + value.checked_add(1).ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is exhausted".to_string(), + ) + }) + }) + .transpose()? + .unwrap_or(0); + let mut effect = AutomationEffectRecord::prepared( + AutomationEffectBinding { + automation_run_id: run.id, + definition_revision: run.definition_revision, + agent_run_id, + tool_invocation_id, + request_fingerprint: invocation.request_fingerprint.clone(), + prepared_checkpoint_revision: checkpoint_revision, + fence_token, + lease_generation: generation, + }, + now, + ) + .map_err(EventStoreError::InvalidState)?; + effect.state = AutomationEffectState::EffectStarted; + effect.safe_reason_code = Some("effect_started_before_dispatch".to_string()); + effect.validate().map_err(EventStoreError::InvalidState)?; + ledger + .push(run.id, effect.clone()) + .map_err(EventStoreError::InvalidState)?; + let checkpoint = AutomationCheckpoint { + automation_run_id: run.id, + dedup_key: run.trigger_window_key.clone(), + tool_invocation_id: Some(tool_invocation_id), + evidence_ref: Some(format!( + "automation-effect://{}/effect-started", + effect.idempotency_key + )), + recorded_at: now, + }; + let checkpoint_json = serde_json::to_string(&checkpoint)?; + let ledger_json = serde_json::to_string(&ledger)?; + let effect_state = serde_json::to_string(&effect.state)?; + let changed = if let Some(expected_revision) = current_revision { + transaction.execute( + r#"UPDATE automation_checkpoints + SET checkpoint_json = ?2, contract_version = ?3, revision = ?4, + definition_revision = ?5, agent_run_id = ?6, lease_generation = ?7, + effect_ledger_json = ?8, effect_state = ?9, + effect_tool_invocation_id = ?10, effect_idempotency_key = ?11, + next_reconciliation_at = NULL, reconciliation_attempt_count = 0, + effect_dead_letter_code = NULL, effect_quarantine_code = NULL, + updated_at = ?12 + WHERE automation_run_id = ?1 AND revision = ?13 + AND contract_version = ?3 AND definition_revision = ?5 + AND agent_run_id = ?6"#, + params![ + run.id.to_string(), + checkpoint_json, + AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + automation_revision_i64(checkpoint_revision)?, + automation_revision_i64(run.definition_revision)?, + agent_run_id.to_string(), + automation_revision_i64(generation)?, + ledger_json, + effect_state, + tool_invocation_id.to_string(), + &effect.idempotency_key, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + automation_revision_i64(expected_revision)?, + ], + )? + } else { + transaction.execute( + r#"INSERT INTO automation_checkpoints + (automation_run_id, checkpoint_json, contract_version, revision, + definition_revision, agent_run_id, lease_generation, effect_ledger_json, + effect_state, effect_tool_invocation_id, effect_idempotency_key, + next_reconciliation_at, reconciliation_attempt_count, + effect_dead_letter_code, effect_quarantine_code, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, + NULL, 0, NULL, NULL, ?12)"#, + params![ + run.id.to_string(), + checkpoint_json, + AUTOMATION_CHECKPOINT_CONTRACT_VERSION, + automation_revision_i64(checkpoint_revision)?, + automation_revision_i64(run.definition_revision)?, + agent_run_id.to_string(), + automation_revision_i64(generation)?, + ledger_json, + effect_state, + tool_invocation_id.to_string(), + &effect.idempotency_key, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )? + }; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation effect checkpoint compare-and-swap conflict".to_string(), + )); + } transaction.commit()?; - Ok(()) + Ok(Some(effect)) } - pub fn list_review_queue_items(&self) -> EventStoreResult> { - let mut statement = self.conn.prepare( + pub(crate) fn finish_automation_effect_for_tool_invocation( + &self, + tool_invocation_id: Uuid, + now: DateTime, + ) -> EventStoreResult> { + let Some(execution) = + self.automation_effect_execution_for_invocation(tool_invocation_id, now)? + else { + return Ok(None); + }; + let outcome = self.classify_automation_effect(&execution.effect)?; + let mut effect = execution.effect.clone(); + match outcome { + DurableAutomationEffectOutcome::KnownApplied { + evidence_ref, + undo_capability, + workspace_checkpoint_id, + workspace_action_revision, + } => { + effect.state = AutomationEffectState::KnownApplied; + effect.evidence_ref = Some(evidence_ref); + effect.safe_reason_code = Some("effect_known_applied".to_string()); + effect.undo_capability = undo_capability; + effect.workspace_checkpoint_id = workspace_checkpoint_id; + effect.workspace_action_revision = workspace_action_revision; + } + DurableAutomationEffectOutcome::KnownNotApplied { evidence_ref } => { + effect.state = AutomationEffectState::KnownNotApplied; + effect.evidence_ref = Some(evidence_ref); + effect.safe_reason_code = Some("effect_known_not_applied".to_string()); + } + DurableAutomationEffectOutcome::StillUncertain { reason_code } => { + effect.state = AutomationEffectState::EffectUnknown; + effect.evidence_ref = Some(format!( + "automation-effect://{}/requires-reconciliation", + effect.idempotency_key + )); + effect.safe_reason_code = Some(reason_code); + } + } + effect.next_reconciliation_at = None; + effect.updated_at = now; + effect.last_lease_generation = execution.lease.generation; + self.persist_automation_effect_transition(&execution, &effect, now)?; + self.release_automation_run_lease(&execution.lease, now)?; + Ok(Some(effect)) + } + + pub(crate) fn reconcile_due_automation_effects( + &self, + now: DateTime, + limit: usize, + ) -> EventStoreResult { + let effect_unknown = serde_json::to_string(&AutomationEffectState::EffectUnknown)?; + let reconciliation_backoff = + serde_json::to_string(&AutomationEffectState::ReconciliationBackoff)?; + let mut statement = self.conn.prepare( + r#"SELECT automation_run_id + FROM automation_checkpoints + WHERE effect_quarantine_code IS NULL + AND effect_state IN (?1, ?2) + AND (next_reconciliation_at IS NULL OR next_reconciliation_at <= ?3) + ORDER BY COALESCE(next_reconciliation_at, updated_at) ASC, rowid ASC + LIMIT ?4"#, + )?; + let run_ids = statement + .query_map( + params![ + effect_unknown, + reconciliation_backoff, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + i64::try_from(limit.min(64)).unwrap_or(64), + ], + |row| row.get::<_, String>(0), + )? + .collect::, _>>()?; + drop(statement); + + let mut sweep = AutomationEffectReconciliationSweep::default(); + for run_id in run_ids { + sweep.inspected += 1; + let run_id = match Uuid::parse_str(&run_id) { + Ok(run_id) => run_id, + Err(_) => { + self.conn.execute( + r#"UPDATE automation_checkpoints + SET effect_quarantine_code = 'invalid_effect_projection', + next_reconciliation_at = NULL + WHERE automation_run_id = ?1"#, + params![run_id], + )?; + sweep.quarantined += 1; + continue; + } + }; + if self.automation_effect_ledger(run_id).is_err() { + self.conn.execute( + r#"UPDATE automation_checkpoints + SET effect_quarantine_code = 'invalid_effect_ledger', + next_reconciliation_at = NULL + WHERE automation_run_id = ?1"#, + params![run_id.to_string()], + )?; + sweep.quarantined += 1; + continue; + } + let execution = match self.claim_automation_effect_reconciliation(run_id, now) { + Ok(execution) => execution, + Err(EventStoreError::InvalidState(_)) => { + sweep.lost_fence += 1; + continue; + } + Err(error) => return Err(error), + }; + let outcome = self.classify_automation_effect(&execution.effect); + let mut effect = execution.effect.clone(); + match outcome { + Ok(DurableAutomationEffectOutcome::KnownApplied { + evidence_ref, + undo_capability, + workspace_checkpoint_id, + workspace_action_revision, + }) => { + effect.state = AutomationEffectState::KnownApplied; + effect.evidence_ref = Some(evidence_ref); + effect.safe_reason_code = Some("effect_known_applied".to_string()); + effect.undo_capability = undo_capability; + effect.workspace_checkpoint_id = workspace_checkpoint_id; + effect.workspace_action_revision = workspace_action_revision; + effect.next_reconciliation_at = None; + sweep.known_applied += 1; + } + Ok(DurableAutomationEffectOutcome::KnownNotApplied { evidence_ref }) => { + effect.state = AutomationEffectState::KnownNotApplied; + effect.evidence_ref = Some(evidence_ref); + effect.safe_reason_code = Some("effect_known_not_applied".to_string()); + effect.next_reconciliation_at = None; + sweep.known_not_applied += 1; + } + Ok(DurableAutomationEffectOutcome::StillUncertain { reason_code }) => { + effect.reconciliation_attempt_count = effect + .reconciliation_attempt_count + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect reconciliation count is exhausted".to_string(), + ) + })?; + if effect.reconciliation_attempt_count + >= AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS + { + effect.state = AutomationEffectState::DeadLettered; + effect.safe_reason_code = + Some("effect_dead_letter_manual_recovery_required".to_string()); + effect.next_reconciliation_at = None; + sweep.dead_lettered += 1; + } else { + let exponent = effect.reconciliation_attempt_count.saturating_sub(1).min(6); + let seconds = 30_i64.checked_shl(exponent).unwrap_or(30_i64 << 6); + effect.state = AutomationEffectState::ReconciliationBackoff; + effect.safe_reason_code = Some(reason_code); + effect.next_reconciliation_at = Some(now + Duration::seconds(seconds)); + sweep.deferred += 1; + } + } + Err(_) => { + effect.reconciliation_attempt_count = effect + .reconciliation_attempt_count + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect reconciliation count is exhausted".to_string(), + ) + })?; + if effect.reconciliation_attempt_count + >= AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS + { + effect.state = AutomationEffectState::DeadLettered; + effect.safe_reason_code = + Some("effect_dead_letter_manual_recovery_required".to_string()); + effect.next_reconciliation_at = None; + sweep.dead_lettered += 1; + } else { + let exponent = effect.reconciliation_attempt_count.saturating_sub(1).min(6); + let seconds = 30_i64.checked_shl(exponent).unwrap_or(30_i64 << 6); + effect.state = AutomationEffectState::ReconciliationBackoff; + effect.safe_reason_code = Some("reconciliation_read_failed".to_string()); + effect.next_reconciliation_at = Some(now + Duration::seconds(seconds)); + sweep.deferred += 1; + } + } + } + effect.last_lease_generation = execution.lease.generation; + effect.updated_at = now; + self.persist_automation_effect_transition(&execution, &effect, now)?; + self.release_automation_run_lease(&execution.lease, now)?; + } + Ok(sweep) + } + + fn claim_automation_effect_reconciliation( + &self, + run_id: Uuid, + now: DateTime, + ) -> EventStoreResult { + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let (checkpoint_revision, ledger_json, checkpoint_definition_revision, agent_run_id) = + transaction.query_row( + r#"SELECT revision, effect_ledger_json, definition_revision, agent_run_id + FROM automation_checkpoints + WHERE automation_run_id = ?1 AND effect_quarantine_code IS NULL"#, + params![run_id.to_string()], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + }, + )?; + let checkpoint_revision = u64::try_from(checkpoint_revision).map_err(|_| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is invalid".to_string(), + ) + })?; + let mut ledger: AutomationEffectLedger = + serde_json::from_str(ledger_json.as_deref().ok_or_else(|| { + EventStoreError::InvalidState("automation effect ledger is missing".to_string()) + })?)?; + ledger + .validate(run_id) + .map_err(EventStoreError::InvalidState)?; + let effect = ledger.current().cloned().ok_or_else(|| { + EventStoreError::InvalidState("automation effect ledger is empty".to_string()) + })?; + if !effect.state.uncertain() + || effect.state == AutomationEffectState::EffectStarted + || effect + .next_reconciliation_at + .is_some_and(|retry_at| retry_at > now) + || checkpoint_definition_revision.and_then(|value| u64::try_from(value).ok()) + != Some(effect.definition_revision) + || agent_run_id.as_deref() != Some(effect.agent_run_id.to_string().as_str()) + { + return Err(EventStoreError::InvalidState( + "automation effect is not due for reconciliation".to_string(), + )); + } + + let (run_json, generation, existing_fence) = transaction.query_row( + r#"SELECT run_json, lease_generation, lease_fence_token + FROM automation_runs WHERE id = ?1"#, + params![run_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + )) + }, + )?; + if existing_fence.is_some() { + return Err(EventStoreError::InvalidState( + "automation effect reconciliation lease is already owned".to_string(), + )); + } + let generation = u64::try_from(generation) + .map_err(|_| { + EventStoreError::InvalidState( + "automation effect reconciliation lease is invalid".to_string(), + ) + })? + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect reconciliation lease generation is exhausted".to_string(), + ) + })?; + let owner_id = format!("automation-reconcile:{}", effect.tool_invocation_id); + let fence_token = Uuid::new_v4(); + let lease_expires_at = now + Duration::minutes(5); + let mut run: AutomationRun = serde_json::from_str(&run_json)?; + run.claimed_by = Some(owner_id.clone()); + run.claimed_at = Some(now); + run.updated_at = now; + let claimed = transaction.execute( + r#"UPDATE automation_runs + SET run_json = ?2, claimed_by = ?3, lease_owner_id = ?3, + lease_fence_token = ?4, lease_generation = ?5, + lease_expires_at = ?6, lease_heartbeat_at = ?7, updated_at = ?7 + WHERE id = ?1 AND lease_fence_token IS NULL AND lease_generation = ?8"#, + params![ + run_id.to_string(), + serde_json::to_string(&run)?, + &owner_id, + fence_token.to_string(), + automation_revision_i64(generation)?, + lease_expires_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + automation_revision_i64(generation.saturating_sub(1))?, + ], + )?; + if claimed != 1 { + return Err(EventStoreError::InvalidState( + "automation effect reconciliation lost its lease fence".to_string(), + )); + } + let mut claimed_effect = effect; + claimed_effect.last_fence_token = fence_token; + claimed_effect.last_lease_generation = generation; + claimed_effect.updated_at = now; + *ledger.current_mut().expect("validated ledger is not empty") = claimed_effect.clone(); + ledger + .validate(run_id) + .map_err(EventStoreError::InvalidState)?; + let claimed_revision = checkpoint_revision.checked_add(1).ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is exhausted".to_string(), + ) + })?; + let changed = transaction.execute( + r#"UPDATE automation_checkpoints + SET revision = ?2, lease_generation = ?3, effect_ledger_json = ?4, + updated_at = ?5 + WHERE automation_run_id = ?1 AND revision = ?6 + AND effect_tool_invocation_id = ?7 AND effect_idempotency_key = ?8"#, + params![ + run_id.to_string(), + automation_revision_i64(claimed_revision)?, + automation_revision_i64(generation)?, + serde_json::to_string(&ledger)?, + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + automation_revision_i64(checkpoint_revision)?, + claimed_effect.tool_invocation_id.to_string(), + &claimed_effect.idempotency_key, + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation effect reconciliation lost its checkpoint revision".to_string(), + )); + } + transaction.commit()?; + Ok(AutomationEffectExecution { + lease: AutomationRunLease { + run_id, + owner_id, + fence_token, + generation, + lease_expires_at, + }, + checkpoint_revision: claimed_revision, + effect: claimed_effect, + }) + } + + pub(crate) fn automation_effect_ledger( + &self, + automation_run_id: Uuid, + ) -> EventStoreResult> { + let ledger_json = self + .conn + .query_row( + "SELECT effect_ledger_json FROM automation_checkpoints WHERE automation_run_id = ?1", + params![automation_run_id.to_string()], + |row| row.get::<_, Option>(0), + ) + .optional()? + .flatten(); + let Some(ledger_json) = ledger_json else { + return Ok(None); + }; + let ledger: AutomationEffectLedger = serde_json::from_str(&ledger_json)?; + ledger + .validate(automation_run_id) + .map_err(EventStoreError::InvalidState)?; + Ok(Some(ledger)) + } + + fn automation_effect_execution_for_invocation( + &self, + tool_invocation_id: Uuid, + now: DateTime, + ) -> EventStoreResult> { + let row = self + .conn + .query_row( + r#"SELECT checkpoint.automation_run_id, checkpoint.revision, + checkpoint.effect_ledger_json, run.lease_owner_id, + run.lease_fence_token, run.lease_generation, run.lease_expires_at + FROM automation_checkpoints AS checkpoint + JOIN automation_runs AS run ON run.id = checkpoint.automation_run_id + WHERE checkpoint.effect_tool_invocation_id = ?1 + AND checkpoint.effect_quarantine_code IS NULL"#, + params![tool_invocation_id.to_string()], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, i64>(5)?, + row.get::<_, Option>(6)?, + )) + }, + ) + .optional()?; + let Some((run_id, revision, ledger_json, owner_id, fence_token, generation, expires_at)) = + row + else { + return Ok(None); + }; + let run_id = Uuid::parse_str(&run_id)?; + let ledger_json = ledger_json.ok_or_else(|| { + EventStoreError::InvalidState("automation effect ledger is missing".to_string()) + })?; + let ledger: AutomationEffectLedger = serde_json::from_str(&ledger_json)?; + ledger + .validate(run_id) + .map_err(EventStoreError::InvalidState)?; + let effect = ledger + .current() + .filter(|effect| effect.tool_invocation_id == tool_invocation_id) + .cloned() + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect projection does not match its ledger".to_string(), + ) + })?; + if effect.state != AutomationEffectState::EffectStarted { + return Err(EventStoreError::InvalidState( + "automation effect is not awaiting its first outcome".to_string(), + )); + } + let owner_id = owner_id.ok_or_else(|| { + EventStoreError::InvalidState("automation effect lease owner is missing".to_string()) + })?; + let fence_token = fence_token + .ok_or_else(|| { + EventStoreError::InvalidState("automation effect fence is missing".to_string()) + }) + .and_then(|value| Uuid::parse_str(&value).map_err(Into::into))?; + let lease_expires_at = expires_at + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect lease expiry is missing".to_string(), + ) + }) + .and_then(|value| { + DateTime::parse_from_rfc3339(&value) + .map(|value| value.with_timezone(&Utc)) + .map_err(Into::into) + })?; + if lease_expires_at <= now + || owner_id != format!("automation-effect:{tool_invocation_id}") + || fence_token != effect.last_fence_token + || u64::try_from(generation).ok() != Some(effect.last_lease_generation) + { + return Err(EventStoreError::InvalidState( + "automation effect fence is stale or expired".to_string(), + )); + } + Ok(Some(AutomationEffectExecution { + lease: AutomationRunLease { + run_id, + owner_id, + fence_token, + generation: effect.last_lease_generation, + lease_expires_at, + }, + checkpoint_revision: u64::try_from(revision).map_err(|_| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is invalid".to_string(), + ) + })?, + effect, + })) + } + + fn persist_automation_effect_transition( + &self, + execution: &AutomationEffectExecution, + effect: &AutomationEffectRecord, + now: DateTime, + ) -> EventStoreResult { + effect.validate().map_err(EventStoreError::InvalidState)?; + if effect.automation_run_id != execution.lease.run_id + || effect.tool_invocation_id != execution.effect.tool_invocation_id + || effect.idempotency_key != execution.effect.idempotency_key + || effect.prepared_checkpoint_revision != execution.effect.prepared_checkpoint_revision + || effect.origin_fence_token != execution.effect.origin_fence_token + || effect.last_fence_token != execution.lease.fence_token + { + return Err(EventStoreError::InvalidState( + "automation effect transition changed immutable authority".to_string(), + )); + } + let mut ledger = self + .automation_effect_ledger(effect.automation_run_id)? + .ok_or_else(|| { + EventStoreError::InvalidState("automation effect ledger is missing".to_string()) + })?; + let current = ledger.current_mut().ok_or_else(|| { + EventStoreError::InvalidState("automation effect ledger is empty".to_string()) + })?; + if current != &execution.effect { + return Err(EventStoreError::InvalidState( + "automation effect transition raced with another reconciler".to_string(), + )); + } + *current = effect.clone(); + ledger + .validate(effect.automation_run_id) + .map_err(EventStoreError::InvalidState)?; + let revision = execution + .checkpoint_revision + .checked_add(1) + .ok_or_else(|| { + EventStoreError::InvalidState( + "automation effect checkpoint revision is exhausted".to_string(), + ) + })?; + let checkpoint = AutomationCheckpoint { + automation_run_id: effect.automation_run_id, + dedup_key: self + .automation_run(effect.automation_run_id)? + .trigger_window_key, + tool_invocation_id: Some(effect.tool_invocation_id), + evidence_ref: effect.evidence_ref.clone(), + recorded_at: now, + }; + let transaction = Transaction::new_unchecked(&self.conn, TransactionBehavior::Immediate)?; + let changed = transaction.execute( + r#"UPDATE automation_checkpoints + SET checkpoint_json = ?2, revision = ?3, lease_generation = ?4, + effect_ledger_json = ?5, effect_state = ?6, + next_reconciliation_at = ?7, reconciliation_attempt_count = ?8, + effect_dead_letter_code = ?9, effect_quarantine_code = NULL, + updated_at = ?10 + WHERE automation_run_id = ?1 AND revision = ?11 + AND definition_revision = ?12 AND agent_run_id = ?13 + AND effect_tool_invocation_id = ?14 AND effect_idempotency_key = ?15 + AND EXISTS ( + SELECT 1 FROM automation_runs AS run + WHERE run.id = ?1 AND run.lease_owner_id = ?16 + AND run.lease_fence_token = ?17 AND run.lease_generation = ?4 + AND run.lease_expires_at > ?10 + )"#, + params![ + effect.automation_run_id.to_string(), + serde_json::to_string(&checkpoint)?, + automation_revision_i64(revision)?, + automation_revision_i64(execution.lease.generation)?, + serde_json::to_string(&ledger)?, + serde_json::to_string(&effect.state)?, + effect + .next_reconciliation_at + .map(|value| value.to_rfc3339_opts(SecondsFormat::Nanos, true)), + i64::from(effect.reconciliation_attempt_count), + (effect.state == AutomationEffectState::DeadLettered) + .then(|| effect.safe_reason_code.clone()) + .flatten(), + now.to_rfc3339_opts(SecondsFormat::Nanos, true), + automation_revision_i64(execution.checkpoint_revision)?, + automation_revision_i64(effect.definition_revision)?, + effect.agent_run_id.to_string(), + effect.tool_invocation_id.to_string(), + &effect.idempotency_key, + execution.lease.owner_id(), + execution.lease.fence_token.to_string(), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "automation effect transition lost its fence or revision".to_string(), + )); + } + transaction.commit()?; + Ok(revision) + } + + fn classify_automation_effect( + &self, + effect: &AutomationEffectRecord, + ) -> EventStoreResult { + match self.workspace_mutation_checkpoint_for_invocation(effect.tool_invocation_id) { + Ok(checkpoint) => { + if checkpoint.run_id != Some(effect.agent_run_id) { + return Ok(DurableAutomationEffectOutcome::StillUncertain { + reason_code: "workspace_checkpoint_run_mismatch".to_string(), + }); + } + return Ok(match (checkpoint.status, checkpoint.effect_state) { + ( + WorkspaceMutationCheckpointStatus::Ready, + WorkspaceCheckpointEffectState::KnownApplied, + ) if checkpoint.action_revision().is_some() => { + DurableAutomationEffectOutcome::KnownApplied { + evidence_ref: format!( + "workspace-checkpoint://{}/known-applied", + checkpoint.id + ), + undo_capability: AutomationEffectUndoCapability::ExactLocal, + workspace_checkpoint_id: Some(checkpoint.id), + workspace_action_revision: checkpoint.action_revision(), + } + } + ( + WorkspaceMutationCheckpointStatus::Ready + | WorkspaceMutationCheckpointStatus::NotUndoable + | WorkspaceMutationCheckpointStatus::Undone, + WorkspaceCheckpointEffectState::KnownApplied, + ) => DurableAutomationEffectOutcome::KnownApplied { + evidence_ref: format!( + "workspace-checkpoint://{}/known-applied", + checkpoint.id + ), + undo_capability: AutomationEffectUndoCapability::None, + workspace_checkpoint_id: None, + workspace_action_revision: None, + }, + ( + WorkspaceMutationCheckpointStatus::Failed, + WorkspaceCheckpointEffectState::NoEffect, + ) => DurableAutomationEffectOutcome::KnownNotApplied { + evidence_ref: format!( + "workspace-checkpoint://{}/known-not-applied", + checkpoint.id + ), + }, + _ => DurableAutomationEffectOutcome::StillUncertain { + reason_code: "workspace_effect_requires_repair".to_string(), + }, + }); + } + Err(EventStoreError::NotFound(_)) => {} + Err(error) => return Err(error), + } + + if let Some(connector) = self + .list_connector_invocations()? + .into_iter() + .find(|invocation| invocation.tool_invocation_id == Some(effect.tool_invocation_id)) + { + return Ok(match connector.status { + ConnectorInvocationStatus::Succeeded => { + DurableAutomationEffectOutcome::KnownApplied { + evidence_ref: format!("connector-effect://{}/known-applied", connector.id), + undo_capability: AutomationEffectUndoCapability::UnsupportedExternal, + workspace_checkpoint_id: None, + workspace_action_revision: None, + } + } + ConnectorInvocationStatus::Failed => { + DurableAutomationEffectOutcome::KnownNotApplied { + evidence_ref: format!( + "connector-effect://{}/known-not-applied", + connector.id + ), + } + } + ConnectorInvocationStatus::PendingApproval + | ConnectorInvocationStatus::Running + | ConnectorInvocationStatus::ReconciliationRequired => { + DurableAutomationEffectOutcome::StillUncertain { + reason_code: "connector_effect_requires_reconciliation".to_string(), + } + } + }); + } + + let computer_use_status = self + .conn + .query_row( + r#"SELECT step.status + FROM computer_use_sessions AS session + JOIN computer_use_steps AS step ON step.id = session.active_step_id + WHERE session.run_id = ?1 AND session.quarantine_code IS NULL + AND step.quarantine_code IS NULL + ORDER BY session.updated_at DESC, session.rowid DESC LIMIT 1"#, + params![effect.agent_run_id.to_string()], + |row| row.get::<_, String>(0), + ) + .optional()?; + if let Some(status) = computer_use_status { + return Ok(match status.as_str() { + "verified" => DurableAutomationEffectOutcome::KnownApplied { + evidence_ref: format!( + "computer-use-effect://{}/known-applied", + effect.tool_invocation_id + ), + undo_capability: AutomationEffectUndoCapability::UnsupportedExternal, + workspace_checkpoint_id: None, + workspace_action_revision: None, + }, + "observed" | "awaiting_approval" | "ready" | "needs_replan" | "cancelled" => { + DurableAutomationEffectOutcome::KnownNotApplied { + evidence_ref: format!( + "computer-use-effect://{}/known-not-applied", + effect.tool_invocation_id + ), + } + } + _ => DurableAutomationEffectOutcome::StillUncertain { + reason_code: "computer_use_effect_requires_reconciliation".to_string(), + }, + }); + } + + let invocation = self.tool_invocation_by_id(effect.tool_invocation_id)?; + if invocation.request_fingerprint != effect.request_fingerprint + || invocation.run_id != Some(effect.agent_run_id) + { + return Ok(DurableAutomationEffectOutcome::StillUncertain { + reason_code: "tool_effect_binding_changed".to_string(), + }); + } + if invocation.status == ToolExecutionStatus::Succeeded && invocation.verification.passed { + return Ok(DurableAutomationEffectOutcome::KnownApplied { + evidence_ref: format!("tool-effect://{}/verified", effect.tool_invocation_id), + undo_capability: if automation_capability_is_external(invocation.capability) { + AutomationEffectUndoCapability::UnsupportedExternal + } else { + AutomationEffectUndoCapability::None + }, + workspace_checkpoint_id: None, + workspace_action_revision: None, + }); + } + Ok(DurableAutomationEffectOutcome::StillUncertain { + reason_code: "tool_effect_requires_reconciliation".to_string(), + }) + } + + pub fn upsert_review_queue_item(&self, item: &ReviewQueueItem) -> EventStoreResult<()> { + let existing = self + .conn + .query_row( + "SELECT item_json FROM review_queue_items WHERE id = ?1", + params![item.id.to_string()], + |row| row.get::<_, String>(0), + ) + .optional()? + .map(|json| serde_json::from_str::(&json)) + .transpose()?; + if existing.as_ref() == Some(item) { + return Ok(()); + } + if existing + .as_ref() + .is_some_and(|current| item.revision != current.revision.saturating_add(1)) + { + return Err(EventStoreError::InvalidState( + "review item revision is stale".to_string(), + )); + } + let transaction = self.conn.unchecked_transaction()?; + let mut run = self.automation_run(item.automation_run_id)?; + if let Some(existing_id) = run.review_queue_item_id { + if existing_id != item.id { + return Err(EventStoreError::InvalidState( + "automation run already has a review queue item".to_string(), + )); + } + } + run.review_queue_item_id = Some(item.id); + run.updated_at = item.updated_at; + let changed = transaction.execute( + r#"INSERT INTO review_queue_items (id, automation_run_id, item_json, status, revision, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6) + ON CONFLICT(id) DO UPDATE SET + item_json = excluded.item_json, + status = excluded.status, + revision = excluded.revision, + updated_at = excluded.updated_at + WHERE review_queue_items.revision + 1 = excluded.revision"#, + params![ + item.id.to_string(), + item.automation_run_id.to_string(), + serde_json::to_string(item)?, + serde_json::to_string(&item.status)?, + item.revision, + item.updated_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + if changed != 1 { + return Err(EventStoreError::InvalidState( + "review item update raced with another writer".to_string(), + )); + } + transaction.execute( + "UPDATE automation_runs SET run_json = ?2, updated_at = ?3 WHERE id = ?1", + params![ + run.id.to_string(), + serde_json::to_string(&run)?, + run.updated_at.to_rfc3339_opts(SecondsFormat::Nanos, true), + ], + )?; + transaction.commit()?; + Ok(()) + } + + pub fn list_review_queue_items(&self) -> EventStoreResult> { + let mut statement = self.conn.prepare( "SELECT item_json FROM review_queue_items ORDER BY updated_at ASC, rowid ASC", )?; let rows = statement @@ -15975,10 +17211,14 @@ mod tests { }; use crate::kernel::automation::{ AutomationCheckpoint, AutomationDefinition, AutomationDefinitionStatus, - AutomationRunStatus, AutomationSchedule, MissedRunPolicy, ReviewQueueItem, - ReviewQueueItemStatus, + AutomationEffectState, AutomationEffectUndoCapability, AutomationRunStatus, + AutomationSchedule, MissedRunPolicy, ReviewQueueItem, ReviewQueueItemStatus, }; + use crate::kernel::capability::{FileSystemMutationOperation, LocalFileSystemMutationClient}; use crate::kernel::connectors::landing::ConnectorAttachmentMetadata; + use crate::kernel::workspace_undo::{ + execute_checkpointed_mutation, WorkspaceMutationCheckpoint, + }; fn enqueue_fenced_automation_fixture( store: &EventStore, @@ -16010,14 +17250,156 @@ mod tests { (definition, run, agent_run) } - fn wrong_owner_lease(lease: &AutomationRunLease) -> AutomationRunLease { - AutomationRunLease { - run_id: lease.run_id, - owner_id: "wrong-owner".to_string(), - fence_token: lease.fence_token, - generation: lease.generation, - lease_expires_at: lease.lease_expires_at, - } + fn wrong_owner_lease(lease: &AutomationRunLease) -> AutomationRunLease { + AutomationRunLease { + run_id: lease.run_id, + owner_id: "wrong-owner".to_string(), + fence_token: lease.fence_token, + generation: lease.generation, + lease_expires_at: lease.lease_expires_at, + } + } + + fn c6b_effect_plan( + store: &EventStore, + label: &str, + now: DateTime, + ) -> ( + crate::kernel::automation::AutomationRun, + AgentRunStart, + crate::kernel::tool_runtime::ToolExecutionPlan, + ) { + let (_, run, agent_run) = enqueue_fenced_automation_fixture(store, label, now); + let plan = prepare_tool_execution(&ToolExecutionRequest { + tool_id: FILE_WRITE_TOOL_ID.to_string(), + input: json!({ + "path": format!("c6b/{label}.txt"), + "summary": "Write one exact C6B fixture.", + "content": "C6B fixture" + }), + access_mode: AccessMode::FullAccess, + run_id: Some(agent_run.id), + }) + .expect("C6B mutation plan prepares"); + store + .append_tool_invocation(&ToolInvocationRecord::running(&plan, None)) + .expect("running mutation persists"); + store + .append_agent_run_transition( + &AgentRunTransition::new( + agent_run.id, + AgentRunStatus::Running, + "C6B fixture tool is running.".to_string(), + Some(plan.invocation_id), + ) + .expect("running transition is valid"), + ) + .expect("AgentRun becomes running"); + (run, agent_run, plan) + } + + fn c6b_connector_effect_plan( + store: &EventStore, + label: &str, + now: DateTime, + ) -> ( + crate::kernel::automation::AutomationRun, + AgentRunStart, + crate::kernel::tool_runtime::ToolExecutionPlan, + ) { + let (_, run, agent_run) = enqueue_fenced_automation_fixture(store, label, now); + let plan = prepare_tool_execution(&ToolExecutionRequest { + tool_id: CONNECTOR_MUTATE_TOOL_ID.to_string(), + input: json!({ + "provider_id": "fixture", + "account_id": Uuid::new_v4().to_string(), + "account_generation": 0, + "capability": "mail_send_draft", + "target_ref": "fixture-target", + "preview_hash": "sha256:fixture-preview", + "idempotency_key": format!("fixture:{label}"), + "automation_run_id": run.id.to_string() + }), + access_mode: AccessMode::FullAccess, + run_id: Some(agent_run.id), + }) + .expect("C6B connector plan prepares"); + store + .append_tool_invocation(&ToolInvocationRecord::running(&plan, Some(Uuid::new_v4()))) + .expect("running connector mutation persists"); + store + .append_agent_run_transition( + &AgentRunTransition::new( + agent_run.id, + AgentRunStatus::Running, + "C6B connector fixture is running.".to_string(), + Some(plan.invocation_id), + ) + .expect("running transition is valid"), + ) + .expect("AgentRun becomes running"); + (run, agent_run, plan) + } + + fn append_c6b_file_success( + store: &EventStore, + plan: &crate::kernel::tool_runtime::ToolExecutionPlan, + ) -> ToolInvocationRecord { + let approval_request_id = store + .tool_invocation_by_id(plan.invocation_id) + .expect("running invocation loads") + .approval_request_id; + let invocation = ToolInvocationRecord::succeeded( + plan, + json!({ + "path": "c6b/result.txt", + "bytes": 11, + "encoding": "utf-8" + }), + vec![ToolEvidence { + kind: "written_file".to_string(), + reference: format!("workspace-file://{}", plan.invocation_id), + summary: "Exact C6B file postcondition verified.".to_string(), + }], + ToolVerificationResult::passed("Exact C6B file postcondition verified."), + approval_request_id, + 1, + ) + .expect("successful C6B file record is valid"); + store + .append_tool_invocation(&invocation) + .expect("successful invocation persists"); + invocation + } + + fn append_c6b_connector_success( + store: &EventStore, + plan: &crate::kernel::tool_runtime::ToolExecutionPlan, + ) -> ToolInvocationRecord { + let approval_request_id = store + .tool_invocation_by_id(plan.invocation_id) + .expect("running connector invocation loads") + .approval_request_id; + let invocation = ToolInvocationRecord::succeeded( + plan, + json!({ + "remote_object_ref": "remote:fixture-object", + "outcome": "applied" + }), + vec![ToolEvidence { + kind: "connector_remote_state".to_string(), + reference: format!("connector-evidence://{}", plan.invocation_id), + summary: "Exact provider result verified.".to_string(), + }], + ToolVerificationResult::passed("Exact provider result verified."), + approval_request_id, + 1, + ) + .expect("successful connector record is valid"); + store + .append_tool_invocation(&invocation) + .expect("successful connector invocation persists"); + invocation } #[test] @@ -16912,6 +18294,492 @@ mod tests { assert_eq!(linked.status, AgentRunStatus::Blocked); } + #[test] + fn c6b_concurrent_restart_safe_idempotency_records_one_effect() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let path = temp_dir.path().join("c6b-concurrent.sqlite3"); + let now = Utc::now(); + let plan_id; + let run_id; + { + let store = EventStore::open(&path).expect("store opens"); + let (run, _, plan) = c6b_effect_plan(&store, "concurrent-idempotency", now); + plan_id = plan.invocation_id; + run_id = run.id; + } + let first = EventStore::open(&path).expect("first store opens"); + let second = EventStore::open(&path).expect("second store opens"); + first + .conn + .busy_timeout(std::time::Duration::from_secs(2)) + .expect("first timeout configures"); + second + .conn + .busy_timeout(std::time::Duration::from_secs(2)) + .expect("second timeout configures"); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(3)); + let mut threads = Vec::new(); + for store in [first, second] { + let barrier = barrier.clone(); + threads.push(std::thread::spawn(move || { + barrier.wait(); + store.begin_automation_effect_for_tool_invocation(plan_id, now) + })); + } + barrier.wait(); + let outcomes = threads + .into_iter() + .map(|thread| thread.join().expect("effect thread joins")) + .collect::>(); + assert_eq!( + outcomes + .iter() + .filter(|outcome| matches!(outcome, Ok(Some(_)))) + .count(), + 1 + ); + assert_eq!( + outcomes.iter().filter(|outcome| outcome.is_err()).count(), + 1 + ); + + let store = EventStore::open(&path).expect("store reopens"); + assert!(store + .begin_automation_effect_for_tool_invocation(plan_id, now + Duration::seconds(1)) + .is_err()); + let ledger = store + .automation_effect_ledger(run_id) + .expect("ledger loads") + .expect("ledger exists"); + assert_eq!(ledger.effects.len(), 1); + assert_eq!( + ledger.current().unwrap().state, + AutomationEffectState::EffectStarted + ); + } + + #[test] + fn c6b_pre_effect_binding_failure_never_creates_effect_authority() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "pre-effect-failure", now); + store + .conn + .execute( + "UPDATE automation_runs SET definition_revision = definition_revision + 1 WHERE id = ?1", + params![run.id.to_string()], + ) + .expect("fixture changes definition revision"); + assert!(store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .is_err()); + assert!(store + .automation_effect_ledger(run.id) + .expect("ledger lookup succeeds") + .is_none()); + let lease_present: i64 = store + .conn + .query_row( + "SELECT lease_fence_token IS NOT NULL FROM automation_runs WHERE id = ?1", + params![run.id.to_string()], + |row| row.get(0), + ) + .expect("lease projection loads"); + assert_eq!(lease_present, 0); + } + + #[test] + fn c6b_effect_started_restart_becomes_unknown_without_replay() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let path = temp_dir.path().join("c6b-effect-started.sqlite3"); + let now = Utc::now(); + let run_id; + let invocation_id; + { + let store = EventStore::open(&path).expect("store opens"); + let (run, _, plan) = c6b_effect_plan(&store, "restart-unknown", now); + run_id = run.id; + invocation_id = plan.invocation_id; + let effect = store + .begin_automation_effect_for_tool_invocation(invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + assert_eq!(effect.state, AutomationEffectState::EffectStarted); + store + .conn + .execute( + "UPDATE automation_runs SET lease_expires_at = ?2 WHERE id = ?1", + params![ + run_id.to_string(), + (now - Duration::seconds(1)).to_rfc3339_opts(SecondsFormat::Nanos, true) + ], + ) + .expect("fixture expires lease"); + } + let store = EventStore::open(&path).expect("store reopens"); + let recovered = store + .recover_expired_automation_runs(now) + .expect("expired effect recovers"); + assert_eq!(recovered.effect_unknown, 1); + let ledger = store + .automation_effect_ledger(run_id) + .expect("ledger loads") + .expect("ledger exists"); + assert_eq!( + ledger.current().unwrap().state, + AutomationEffectState::EffectUnknown + ); + let sweep = store + .reconcile_due_automation_effects(now + Duration::seconds(1), 64) + .expect("read-only reconciliation runs"); + assert_eq!(sweep.deferred, 1); + assert_eq!( + store + .list_tool_invocations() + .expect("Tool invocations load") + .into_iter() + .filter(|invocation| invocation.id == invocation_id) + .count(), + 1 + ); + } + + #[test] + fn c6b_known_applied_is_durable_and_distinct() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "known-applied", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + append_c6b_file_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("effect finishes") + .expect("effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!(effect.undo_capability, AutomationEffectUndoCapability::None); + assert_eq!( + store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .current() + .unwrap(), + &effect + ); + } + + #[test] + fn c6b_known_not_applied_is_durable_and_distinct() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "known-not-applied", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + let path = temp_dir.path().join("never-written.txt"); + let intent = WorkspaceMutationCheckpoint::intent( + &plan, + FileSystemMutationOperation::CreateFile, + path.to_str().expect("fixture path is UTF-8"), + None, + ) + .expect("workspace intent is valid"); + store + .insert_workspace_mutation_intent(&intent) + .expect("workspace intent persists"); + store + .fail_workspace_mutation_checkpoint(plan.invocation_id, "precheck_failed") + .expect("workspace pre-effect failure persists"); + let failed = ToolInvocationRecord::failed(&plan, "fixture precheck failed", None, 1); + store + .append_tool_invocation(&failed) + .expect("failed Tool persists"); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("effect finishes") + .expect("effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownNotApplied); + assert_ne!( + store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .current() + .unwrap() + .state, + AutomationEffectState::KnownApplied + ); + } + + #[test] + fn c6b_stale_fence_is_rejected_before_effect_outcome_changes() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "stale-fence", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + append_c6b_file_success(&store, &plan); + store + .conn + .execute( + "UPDATE automation_runs SET lease_fence_token = ?2 WHERE id = ?1", + params![run.id.to_string(), Uuid::new_v4().to_string()], + ) + .expect("fixture changes the fence"); + assert!(store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1) + ) + .is_err()); + assert_eq!( + store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .current() + .unwrap() + .state, + AutomationEffectState::EffectStarted + ); + } + + #[test] + fn c6b_bounded_backoff_dead_letters_without_replaying() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "bounded-backoff", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + let failed = ToolInvocationRecord::failed(&plan, "ambiguous fixture failure", None, 1); + store + .append_tool_invocation(&failed) + .expect("failed Tool persists"); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("effect is persisted unknown") + .expect("effect exists"); + assert_eq!(effect.state, AutomationEffectState::EffectUnknown); + for attempt in 1..=7 { + store + .reconcile_due_automation_effects(now + Duration::days(attempt), 64) + .expect("bounded reconciliation runs"); + } + let effect = store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .current() + .unwrap() + .clone(); + assert_eq!(effect.state, AutomationEffectState::DeadLettered); + assert_eq!(effect.reconciliation_attempt_count, 7); + assert!(effect.next_reconciliation_at.is_none()); + assert_eq!( + store + .list_tool_invocations() + .expect("Tool invocations load") + .into_iter() + .filter(|invocation| invocation.id == plan.invocation_id) + .count(), + 1 + ); + let replay_plan = prepare_tool_execution(&ToolExecutionRequest { + tool_id: FILE_WRITE_TOOL_ID.to_string(), + input: plan.request.input.clone(), + access_mode: plan.request.access_mode, + run_id: plan.request.run_id, + }) + .expect("replay fixture plan prepares"); + store + .append_tool_invocation(&ToolInvocationRecord::running(&replay_plan, None)) + .expect("replay fixture running Tool persists"); + assert!(store + .begin_automation_effect_for_tool_invocation( + replay_plan.invocation_id, + now + Duration::days(8) + ) + .is_err()); + } + + #[test] + fn c6b_invalid_ledger_is_quarantined_without_starving_valid_work() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (invalid_run, _, invalid_plan) = c6b_effect_plan(&store, "invalid-ledger", now); + store + .begin_automation_effect_for_tool_invocation(invalid_plan.invocation_id, now) + .expect("invalid fixture effect begins") + .expect("effect exists"); + let invalid_failed = + ToolInvocationRecord::failed(&invalid_plan, "ambiguous fixture failure", None, 1); + store + .append_tool_invocation(&invalid_failed) + .expect("invalid fixture Tool persists"); + store + .finish_automation_effect_for_tool_invocation( + invalid_plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("invalid fixture becomes unknown"); + store + .conn + .execute( + "UPDATE automation_checkpoints SET effect_ledger_json = '{bad-json' WHERE automation_run_id = ?1", + params![invalid_run.id.to_string()], + ) + .expect("fixture corrupts one ledger"); + + let (valid_run, _, valid_plan) = + c6b_effect_plan(&store, "valid-after-quarantine", now + Duration::seconds(2)); + store + .begin_automation_effect_for_tool_invocation( + valid_plan.invocation_id, + now + Duration::seconds(2), + ) + .expect("valid fixture effect begins") + .expect("effect exists"); + let valid_failed = + ToolInvocationRecord::failed(&valid_plan, "ambiguous fixture failure", None, 1); + store + .append_tool_invocation(&valid_failed) + .expect("valid fixture Tool persists"); + store + .finish_automation_effect_for_tool_invocation( + valid_plan.invocation_id, + now + Duration::seconds(3), + ) + .expect("valid fixture becomes unknown"); + + let sweep = store + .reconcile_due_automation_effects(now + Duration::minutes(1), 64) + .expect("quarantine sweep completes"); + assert_eq!(sweep.quarantined, 1); + assert_eq!(sweep.deferred, 1); + let quarantine_code: Option = store + .conn + .query_row( + "SELECT effect_quarantine_code FROM automation_checkpoints WHERE automation_run_id = ?1", + params![invalid_run.id.to_string()], + |row| row.get(0), + ) + .expect("quarantine code loads"); + assert_eq!(quarantine_code.as_deref(), Some("invalid_effect_ledger")); + assert_eq!( + store + .automation_effect_ledger(valid_run.id) + .expect("valid ledger loads") + .expect("valid ledger exists") + .current() + .unwrap() + .state, + AutomationEffectState::ReconciliationBackoff + ); + } + + #[test] + fn c6b_exact_local_undo_requires_exact_workspace_checkpoint() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_effect_plan(&store, "exact-local-undo", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + let path = temp_dir.path().join("created.txt"); + execute_checkpointed_mutation( + &store, + &plan, + &LocalFileSystemMutationClient, + FileSystemMutationOperation::CreateFile, + path.to_str().expect("fixture path is UTF-8"), + None, + Some("C6B fixture"), + ) + .expect("checkpointed mutation succeeds"); + append_c6b_file_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("effect finishes") + .expect("effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!( + effect.undo_capability, + AutomationEffectUndoCapability::ExactLocal + ); + assert!(effect.workspace_checkpoint_id.is_some()); + assert!(effect + .workspace_action_revision + .as_deref() + .is_some_and(|value| value.starts_with("undo1:"))); + assert_eq!( + store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .effects + .len(), + 1 + ); + } + + #[test] + fn c6b_external_effect_has_no_local_undo_authority() { + let store = EventStore::open_memory().expect("store opens"); + let now = Utc::now(); + let (run, _, plan) = c6b_connector_effect_plan(&store, "external-undo", now); + store + .begin_automation_effect_for_tool_invocation(plan.invocation_id, now) + .expect("effect begins") + .expect("automation effect exists"); + append_c6b_connector_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(1), + ) + .expect("effect finishes") + .expect("effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!( + effect.undo_capability, + AutomationEffectUndoCapability::UnsupportedExternal + ); + assert!(effect.workspace_checkpoint_id.is_none()); + assert!(effect.workspace_action_revision.is_none()); + assert_eq!( + store + .automation_effect_ledger(run.id) + .expect("ledger loads") + .expect("ledger exists") + .effects + .len(), + 1 + ); + } + #[test] fn c6a_automation_pause_resume_and_revision_persist_across_restart() { let temp_dir = tempfile::tempdir().expect("temp dir"); @@ -17243,7 +19111,7 @@ mod tests { use crate::kernel::tool_runtime::{ prepare_tool_execution, ToolEvidence, ToolExecutionRequest, ToolExecutionStatus, ToolInvocationRecord, ToolVerificationResult, APP_UPDATE_CHECK_TOOL_ID, - CONNECTOR_MUTATE_TOOL_ID, FILE_READ_TOOL_ID, + CONNECTOR_MUTATE_TOOL_ID, FILE_READ_TOOL_ID, FILE_WRITE_TOOL_ID, }; use crate::kernel::work_package::export_work_package; use crate::kernel::workflow::WorkflowTemplatePackage; diff --git a/scripts/automation-center.test.mjs b/scripts/automation-center.test.mjs index 14be026..6e582ff 100644 --- a/scripts/automation-center.test.mjs +++ b/scripts/automation-center.test.mjs @@ -526,6 +526,67 @@ assert.doesNotMatch(recoveryCenter, /execute_agent_tool|prepare_connector_attach assert.match(commands, /enqueue_due_automation_agent_run/); assert.match(commands, /reconcile_automation_agent_runs/); assert.match(commands, /recover_expired_automation_runs/); +assert.match(commands, /reconcile_due_automation_effects\(now, 64\)/); +assert.match( + commands, + /recover_expired_automation_runs\(now\)[\s\S]*reconcile_due_automation_effects\(now, 64\)[\s\S]*reconcile_automation_agent_runs\(now\)/, + "scheduler recovery must reconcile durable effects read-only before AgentRun projection", +); +assert.match(automationKernel, /ds-agent\.automation-effect-ledger\/v1/); +assert.match(automationKernel, /AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS: u32 = 7/); +assert.match(automationKernel, /EffectStarted[\s\S]*EffectUnknown[\s\S]*KnownApplied[\s\S]*KnownNotApplied[\s\S]*ReconciliationBackoff[\s\S]*DeadLettered/); +const automationEffectRecord = + automationKernel.match(/pub struct AutomationEffectRecord \{[\s\S]*?\n\}/)?.[0] ?? ""; +for (const authorityField of [ + "automation_run_id", + "definition_revision", + "agent_run_id", + "tool_invocation_id", + "prepared_checkpoint_revision", + "origin_fence_token", + "last_fence_token", + "origin_lease_generation", + "last_lease_generation", + "idempotency_key", +]) { + assert.match( + automationEffectRecord, + new RegExp(`${authorityField}:`), + `automation effects must persist exact ${authorityField} authority`, + ); +} +assert.match(eventStore, /effect_ledger_json TEXT/); +assert.match(eventStore, /idx_automation_effect_idempotency/); +assert.match(eventStore, /begin_automation_effect_for_tool_invocation/); +assert.match(eventStore, /finish_automation_effect_for_tool_invocation/); +const effectReconciliation = + eventStore.match( + /pub\(crate\) fn reconcile_due_automation_effects\([\s\S]*?\n \}\n\n fn claim_automation_effect_reconciliation/, + )?.[0] ?? ""; +assert.notEqual(effectReconciliation, ""); +assert.doesNotMatch( + effectReconciliation, + /apply_mutation|AgentToolExecutor|executor\.execute|execute_agent_tool|provider\(/, + "automation effect reconciliation must remain read-only and never replay a provider or Tool", +); +assert.match(effectReconciliation, /AUTOMATION_EFFECT_MAX_RECONCILIATION_ATTEMPTS/); +assert.match(effectReconciliation, /effect_dead_letter_manual_recovery_required/); +assert.match(eventStore, /WorkspaceMutationCheckpointStatus::Ready[\s\S]*AutomationEffectUndoCapability::ExactLocal/); +assert.match(eventStore, /AutomationEffectUndoCapability::UnsupportedExternal/); +const readyAuthorization = + appCommands.match(/fn ready_agent_tool_authorization\([\s\S]*?\n\}/)?.[0] ?? ""; +assert.match( + readyAuthorization, + /ToolInvocationRecord::running[\s\S]*AgentRunStatus::Running[\s\S]*begin_automation_effect_for_tool_invocation[\s\S]*AgentToolAuthorization::Ready/, + "mutating automation Tools must persist EffectStarted before executor dispatch", +); +const completedToolAudit = + appCommands.match(/fn record_completed_agent_tool_execution\([\s\S]*?\n\}/)?.[0] ?? ""; +assert.match( + completedToolAudit, + /append_agent_tool_invocation_audit[\s\S]*finish_automation_effect_for_tool_invocation/, + "durable Tool outcome must be audited before deterministic effect classification", +); const automationScheduler = commands.match(/pub fn spawn_automation_scheduler[\s\S]*?\n\}/)?.[0] ?? ""; assert.match(automationScheduler, /ds-agent-automation-scheduler/); From 8952beee1f75b6fb0ccc4952625adeaf3dd0ab0e Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 26 Jul 2026 08:17:47 +0800 Subject: [PATCH 3/4] feat(automation): surface restart-safe checkpoints --- .../src-tauri/src/kernel/event_store.rs | 552 ++++++++++++++++++ .../src-tauri/src/kernel/task_lifecycle.rs | 227 ++++++- apps/desktop/src/AutomationCenter.tsx | 36 +- apps/desktop/src/TaskLifecyclePanel.tsx | 90 ++- scripts/task-lifecycle.test.mjs | 19 +- 5 files changed, 904 insertions(+), 20 deletions(-) diff --git a/apps/desktop/src-tauri/src/kernel/event_store.rs b/apps/desktop/src-tauri/src/kernel/event_store.rs index 3259880..bb20210 100644 --- a/apps/desktop/src-tauri/src/kernel/event_store.rs +++ b/apps/desktop/src-tauri/src/kernel/event_store.rs @@ -17341,6 +17341,62 @@ mod tests { (run, agent_run, plan) } + fn c6c_effect_plan_for_existing_run( + store: &EventStore, + automation_run_id: Uuid, + agent_run_id: Uuid, + label: &str, + external: bool, + ) -> crate::kernel::tool_runtime::ToolExecutionPlan { + let request = if external { + ToolExecutionRequest { + tool_id: CONNECTOR_MUTATE_TOOL_ID.to_string(), + input: json!({ + "provider_id": "fixture", + "account_id": Uuid::new_v4().to_string(), + "account_generation": 0, + "capability": "mail_send_draft", + "target_ref": "fixture-target", + "preview_hash": "sha256:fixture-preview", + "idempotency_key": format!("c6c:{label}"), + "automation_run_id": automation_run_id.to_string() + }), + access_mode: AccessMode::FullAccess, + run_id: Some(agent_run_id), + } + } else { + ToolExecutionRequest { + tool_id: FILE_WRITE_TOOL_ID.to_string(), + input: json!({ + "path": format!("c6c/{label}.txt"), + "summary": "Write one exact C6C fault-matrix fixture.", + "content": "C6C fault-matrix fixture" + }), + access_mode: AccessMode::FullAccess, + run_id: Some(agent_run_id), + } + }; + let plan = prepare_tool_execution(&request).expect("C6C mutation plan prepares"); + store + .append_tool_invocation(&ToolInvocationRecord::running( + &plan, + external.then(Uuid::new_v4), + )) + .expect("C6C running mutation persists"); + store + .append_agent_run_transition( + &AgentRunTransition::new( + agent_run_id, + AgentRunStatus::Running, + "C6C fault-matrix Tool is running.".to_string(), + Some(plan.invocation_id), + ) + .expect("C6C running transition is valid"), + ) + .expect("C6C AgentRun becomes running"); + plan + } + fn append_c6b_file_success( store: &EventStore, plan: &crate::kernel::tool_runtime::ToolExecutionPlan, @@ -18780,6 +18836,502 @@ mod tests { ); } + #[test] + fn c6c_step6_fault_matrix_100_runs_proves_exit_contract() { + let base = DateTime::parse_from_rfc3339("2026-07-26T00:00:00Z") + .expect("fixed matrix timestamp parses") + .with_timezone(&Utc); + let mut scheduled = 0usize; + let mut pause_resume = 0usize; + let mut lease = 0usize; + let mut checkpoint = 0usize; + let mut idempotency = 0usize; + let mut automatic_pre_effect_recovery = 0usize; + let mut explicit_recovery_required = 0usize; + let mut known_applied = 0usize; + let mut known_not_applied = 0usize; + let mut reconciliation_dead_letter = 0usize; + let mut exact_local_undo = 0usize; + let mut unsupported_external_undo = 0usize; + + for case in 0..100 { + let temp_dir = tempfile::tempdir().expect("matrix case temp dir"); + let path = temp_dir.path().join(format!("c6c-case-{case:03}.sqlite3")); + let now = base + Duration::hours(i64::from(case)); + let label = format!("c6c-fault-{case:03}"); + let run_id; + let agent_run_id; + let initial_generation; + let explicit_invocation_id; + + { + let store = EventStore::open(&path).expect("matrix store opens"); + let definition = AutomationDefinition::once( + label.clone(), + "Asia/Shanghai".to_string(), + now - Duration::minutes(1), + ) + .expect("matrix definition is valid"); + store + .upsert_automation_definition(&definition) + .expect("matrix definition persists"); + let paused = store + .set_automation_definition_status( + definition.id, + AutomationDefinitionStatus::Paused, + now, + ) + .expect("matrix definition pauses"); + assert!(store + .enqueue_due_automation_agent_run( + definition.id, + now, + "c6c-paused-scheduler".to_string(), + format!("automation:{}", definition.id), + ) + .expect("paused matrix sweep is safe") + .is_none()); + let resumed = store + .set_automation_definition_status( + definition.id, + AutomationDefinitionStatus::Enabled, + now + Duration::milliseconds(10), + ) + .expect("matrix definition resumes"); + assert!(resumed.revision > paused.revision); + let (run, agent_run) = store + .enqueue_due_automation_agent_run( + definition.id, + now + Duration::milliseconds(20), + "c6c-scheduler".to_string(), + format!("automation:{}", definition.id), + ) + .expect("matrix run enqueues") + .expect("resumed matrix run is due"); + assert!(store + .enqueue_due_automation_agent_run( + definition.id, + now + Duration::milliseconds(20), + "c6c-duplicate-scheduler".to_string(), + format!("automation:{}", definition.id), + ) + .expect("duplicate matrix sweep is safe") + .is_none()); + run_id = run.id; + agent_run_id = agent_run.id; + + let initial_lease = store + .claim_automation_run( + run.id, + format!("c6c-worker-{case:03}"), + 1, + now + Duration::milliseconds(30), + ) + .expect("matrix worker claims"); + initial_generation = initial_lease.generation(); + let initial_checkpoint = AutomationCheckpoint { + automation_run_id: run.id, + dedup_key: run.trigger_window_key.clone(), + tool_invocation_id: None, + evidence_ref: Some(format!("evidence://c6c/{case:03}/pre-effect")), + recorded_at: now + Duration::milliseconds(40), + }; + let checkpoint_record = store + .compare_and_swap_automation_checkpoint( + &initial_checkpoint, + &initial_lease, + None, + now + Duration::milliseconds(40), + ) + .expect("matrix checkpoint persists"); + assert_eq!(checkpoint_record.revision, 0); + assert!(store + .compare_and_swap_automation_checkpoint( + &initial_checkpoint, + &initial_lease, + None, + now + Duration::milliseconds(50), + ) + .is_err()); + + scheduled += 1; + pause_resume += 1; + lease += 1; + checkpoint += 1; + idempotency += 1; + + if case < 95 { + explicit_invocation_id = None; + } else { + store + .release_automation_run_lease( + &initial_lease, + now + Duration::milliseconds(60), + ) + .expect("matrix pre-effect lease releases"); + let plan = c6c_effect_plan_for_existing_run( + &store, + run.id, + agent_run.id, + &label, + case % 2 == 0, + ); + let effect = store + .begin_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::milliseconds(70), + ) + .expect("matrix effect begins") + .expect("matrix automation effect exists"); + assert_eq!(effect.state, AutomationEffectState::EffectStarted); + assert!(store + .begin_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::milliseconds(80), + ) + .is_err()); + assert_eq!( + store + .automation_effect_ledger(run.id) + .expect("matrix ledger loads") + .expect("matrix ledger exists") + .effects + .len(), + 1 + ); + store + .conn + .execute( + "UPDATE automation_runs SET lease_expires_at = ?2 WHERE id = ?1", + params![ + run.id.to_string(), + (now + Duration::seconds(1)) + .to_rfc3339_opts(SecondsFormat::Nanos, true) + ], + ) + .expect("matrix effect lease expires"); + explicit_invocation_id = Some(plan.invocation_id); + } + } + + let store = EventStore::open(&path).expect("matrix store reopens"); + let recovered = store + .recover_expired_automation_runs(now + Duration::seconds(2)) + .expect("matrix restart recovery completes"); + if case < 95 { + assert_eq!(recovered.requeued, 1); + assert_eq!(recovered.effect_unknown, 0); + let recovered_run = store.automation_run(run_id).expect("matrix run reloads"); + assert_eq!(recovered_run.status, AutomationRunStatus::Queued); + assert!(recovered_run + .last_error + .as_deref() + .is_some_and(|value| value.starts_with( + "Recovered provably pre-effect work after automation lease expired" + ))); + let resumed_lease = store + .claim_automation_run( + run_id, + format!("c6c-restart-worker-{case:03}"), + 60, + now + Duration::seconds(3), + ) + .expect("matrix recovered work is claimed"); + assert!(resumed_lease.generation() > initial_generation); + let resumed_checkpoint = AutomationCheckpoint { + automation_run_id: run_id, + dedup_key: recovered_run.trigger_window_key.clone(), + tool_invocation_id: None, + evidence_ref: Some(format!("evidence://c6c/{case:03}/restart")), + recorded_at: now + Duration::seconds(3), + }; + let checkpoint_record = store + .compare_and_swap_automation_checkpoint( + &resumed_checkpoint, + &resumed_lease, + Some(0), + now + Duration::seconds(3), + ) + .expect("matrix restart checkpoint advances"); + assert_eq!(checkpoint_record.revision, 1); + assert!(store + .compare_and_swap_automation_checkpoint( + &resumed_checkpoint, + &resumed_lease, + Some(0), + now + Duration::seconds(3), + ) + .is_err()); + store + .release_automation_run_lease(&resumed_lease, now + Duration::seconds(4)) + .expect("matrix restart lease releases"); + + let outcome = case % 5; + let plan = c6c_effect_plan_for_existing_run( + &store, + run_id, + agent_run_id, + &label, + outcome == 4, + ); + let effect = store + .begin_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(5), + ) + .expect("matrix post-restart effect begins") + .expect("matrix post-restart automation effect exists"); + assert_eq!(effect.state, AutomationEffectState::EffectStarted); + assert!(store + .begin_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(5), + ) + .is_err()); + + match outcome { + 0 => { + append_c6b_file_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(6), + ) + .expect("matrix known-applied effect finishes") + .expect("matrix known-applied effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!(effect.undo_capability, AutomationEffectUndoCapability::None); + known_applied += 1; + } + 1 => { + let never_written = temp_dir.path().join("never-written.txt"); + let intent = WorkspaceMutationCheckpoint::intent( + &plan, + FileSystemMutationOperation::CreateFile, + never_written + .to_str() + .expect("matrix fixture path is UTF-8"), + None, + ) + .expect("matrix workspace intent is valid"); + store + .insert_workspace_mutation_intent(&intent) + .expect("matrix workspace intent persists"); + store + .fail_workspace_mutation_checkpoint( + plan.invocation_id, + "precheck_failed", + ) + .expect("matrix pre-effect failure persists"); + store + .append_tool_invocation(&ToolInvocationRecord::failed( + &plan, + "matrix precheck failed", + None, + 1, + )) + .expect("matrix failed Tool persists"); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(6), + ) + .expect("matrix known-not-applied effect finishes") + .expect("matrix known-not-applied effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownNotApplied); + known_not_applied += 1; + } + 2 => { + store + .append_tool_invocation(&ToolInvocationRecord::failed( + &plan, + "matrix ambiguous failure", + None, + 1, + )) + .expect("matrix ambiguous Tool result persists"); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(6), + ) + .expect("matrix uncertain effect persists") + .expect("matrix uncertain effect exists"); + assert_eq!(effect.state, AutomationEffectState::EffectUnknown); + for attempt in 1..=7 { + store + .reconcile_due_automation_effects(now + Duration::days(attempt), 64) + .expect("matrix bounded reconciliation runs"); + } + let effect = store + .automation_effect_ledger(run_id) + .expect("matrix dead-letter ledger loads") + .expect("matrix dead-letter ledger exists") + .current() + .expect("matrix dead-letter effect exists") + .clone(); + assert_eq!(effect.state, AutomationEffectState::DeadLettered); + assert_eq!(effect.reconciliation_attempt_count, 7); + assert!(effect.next_reconciliation_at.is_none()); + let replay_plan = prepare_tool_execution(&ToolExecutionRequest { + tool_id: FILE_WRITE_TOOL_ID.to_string(), + input: plan.request.input.clone(), + access_mode: plan.request.access_mode, + run_id: plan.request.run_id, + }) + .expect("matrix replay probe prepares"); + store + .append_tool_invocation(&ToolInvocationRecord::running( + &replay_plan, + None, + )) + .expect("matrix replay probe persists"); + assert!(store + .begin_automation_effect_for_tool_invocation( + replay_plan.invocation_id, + now + Duration::days(8), + ) + .is_err()); + reconciliation_dead_letter += 1; + } + 3 => { + let created = temp_dir.path().join("created.txt"); + execute_checkpointed_mutation( + &store, + &plan, + &LocalFileSystemMutationClient, + FileSystemMutationOperation::CreateFile, + created.to_str().expect("matrix fixture path is UTF-8"), + None, + Some("C6C matrix fixture"), + ) + .expect("matrix checkpointed mutation succeeds"); + append_c6b_file_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(6), + ) + .expect("matrix exact-local effect finishes") + .expect("matrix exact-local effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!( + effect.undo_capability, + AutomationEffectUndoCapability::ExactLocal + ); + assert!(effect.workspace_checkpoint_id.is_some()); + assert!(effect.workspace_action_revision.is_some()); + exact_local_undo += 1; + } + 4 => { + append_c6b_connector_success(&store, &plan); + let effect = store + .finish_automation_effect_for_tool_invocation( + plan.invocation_id, + now + Duration::seconds(6), + ) + .expect("matrix external effect finishes") + .expect("matrix external effect exists"); + assert_eq!(effect.state, AutomationEffectState::KnownApplied); + assert_eq!( + effect.undo_capability, + AutomationEffectUndoCapability::UnsupportedExternal + ); + assert!(effect.workspace_checkpoint_id.is_none()); + assert!(effect.workspace_action_revision.is_none()); + unsupported_external_undo += 1; + } + _ => unreachable!("matrix outcome is bounded"), + } + assert_eq!( + store + .automation_effect_ledger(run_id) + .expect("matrix final ledger loads") + .expect("matrix final ledger exists") + .effects + .len(), + 1 + ); + automatic_pre_effect_recovery += 1; + } else { + assert_eq!(recovered.requeued, 0); + assert_eq!(recovered.effect_unknown, 1); + let invocation_id = + explicit_invocation_id.expect("explicit-recovery invocation exists"); + assert_eq!( + store + .automation_effect_ledger(run_id) + .expect("matrix explicit ledger loads") + .expect("matrix explicit ledger exists") + .current() + .expect("matrix explicit effect exists") + .state, + AutomationEffectState::EffectUnknown + ); + assert!(store + .begin_automation_effect_for_tool_invocation( + invocation_id, + now + Duration::seconds(3), + ) + .is_err()); + let sweep = store + .reconcile_due_automation_effects(now + Duration::seconds(3), 64) + .expect("matrix explicit reconciliation defers safely"); + assert_eq!(sweep.deferred, 1); + assert_eq!( + store + .automation_effect_ledger(run_id) + .expect("matrix backoff ledger loads") + .expect("matrix backoff ledger exists") + .current() + .expect("matrix backoff effect exists") + .state, + AutomationEffectState::ReconciliationBackoff + ); + assert_eq!( + store + .list_tool_invocations() + .expect("matrix explicit Tool records load") + .into_iter() + .filter(|invocation| invocation.id == invocation_id) + .count(), + 1 + ); + assert_eq!( + store + .automation_effect_ledger(run_id) + .expect("matrix explicit final ledger loads") + .expect("matrix explicit final ledger exists") + .effects + .len(), + 1 + ); + explicit_recovery_required += 1; + } + } + + assert_eq!(scheduled, 100); + assert_eq!(pause_resume, 100); + assert_eq!(lease, 100); + assert_eq!(checkpoint, 100); + assert_eq!(idempotency, 100); + assert_eq!(automatic_pre_effect_recovery, 95); + assert_eq!(explicit_recovery_required, 5); + assert_eq!(known_applied, 19); + assert_eq!(known_not_applied, 19); + assert_eq!(reconciliation_dead_letter, 19); + assert_eq!(exact_local_undo, 19); + assert_eq!(unsupported_external_undo, 19); + + println!( + "C6C_STEP6_FAULT_MATRIX total=100 passed=100 automatic_pre_effect_recovery=95 \ + explicit_recovery_required=5 scheduled=100 pause_resume=100 lease=100 \ + checkpoint=100 idempotency=100 known_applied=19 known_not_applied=19 \ + reconciliation_dead_letter=19 exact_local_undo=19 unsupported_external_undo=19 \ + duplicate_effects=0 blind_replays=0" + ); + } + #[test] fn c6a_automation_pause_resume_and_revision_persist_across_restart() { let temp_dir = tempfile::tempdir().expect("temp dir"); diff --git a/apps/desktop/src-tauri/src/kernel/task_lifecycle.rs b/apps/desktop/src-tauri/src/kernel/task_lifecycle.rs index cd14ff1..4eda80e 100644 --- a/apps/desktop/src-tauri/src/kernel/task_lifecycle.rs +++ b/apps/desktop/src-tauri/src/kernel/task_lifecycle.rs @@ -6,7 +6,8 @@ use uuid::Uuid; use crate::kernel::agent_run::{AgentRunRecord, AgentRunStatus}; use crate::kernel::artifacts::{ArtifactDeliveryView, ArtifactPhase}; use crate::kernel::automation::{ - AutomationRun, AutomationRunStatus, ReviewQueueItem, ReviewQueueItemStatus, + AutomationEffectState, AutomationEffectUndoCapability, AutomationRun, AutomationRunStatus, + ReviewQueueItem, ReviewQueueItemStatus, }; use crate::kernel::computer_use_session::{ ComputerUseSession, ComputerUseStep, ComputerUseStepStatus, ComputerUseUndoCapability, @@ -15,7 +16,7 @@ use crate::kernel::connectors::{ ConnectorInvocation, ConnectorInvocationStatus, ConnectorRecoveryAction, ConnectorRecoveryExternalEffectState, ConnectorRecoveryItem, ConnectorRecoveryStatus, }; -use crate::kernel::event_store::{EventStore, EventStoreResult}; +use crate::kernel::event_store::{EventStore, EventStoreError, EventStoreResult}; use crate::kernel::expert_team::ExpertExternalEffectState; use crate::kernel::models::{TaskRecord, TaskRecordStatus}; use crate::kernel::policy::CapabilityKind; @@ -118,7 +119,11 @@ impl EventStore { .collect::>(); items.extend(self.list_task_records()?.iter().map(task_record_item)); items.extend(self.list_agent_run_records()?.iter().map(agent_run_item)); - items.extend(self.list_automation_runs()?.iter().map(automation_run_item)); + items.extend( + self.list_automation_runs()? + .iter() + .map(|run| automation_run_item(self, run)), + ); items.extend(self.list_review_queue_items()?.iter().map(review_item)); items.extend( self.list_tool_invocations()? @@ -229,8 +234,8 @@ fn agent_run_item(record: &AgentRunRecord) -> TaskLifecycleItem { ) } -fn automation_run_item(run: &AutomationRun) -> TaskLifecycleItem { - let phase = match run.status { +fn automation_run_item(store: &EventStore, run: &AutomationRun) -> TaskLifecycleItem { + let base_phase = match run.status { AutomationRunStatus::Queued => TaskLifecyclePhase::Queued, AutomationRunStatus::Running => TaskLifecyclePhase::Running, AutomationRunStatus::WaitingReview => TaskLifecyclePhase::WaitingReview, @@ -239,14 +244,152 @@ fn automation_run_item(run: &AutomationRun) -> TaskLifecycleItem { AutomationRunStatus::Failed => TaskLifecyclePhase::Failed, AutomationRunStatus::Cancelled => TaskLifecyclePhase::Cancelled, }; + let checkpoint = match store.automation_checkpoint_record(run.id) { + Ok(checkpoint) => Some(checkpoint), + Err(EventStoreError::Sqlite(rusqlite::Error::QueryReturnedNoRows)) => None, + Err(_) => return invalid_automation_checkpoint_item(run), + }; + if checkpoint.as_ref().is_some_and(|checkpoint| { + checkpoint.checkpoint.automation_run_id != run.id + || checkpoint.definition_revision != run.definition_revision + || Some(checkpoint.agent_run_id) != run.agent_run_id + }) { + return invalid_automation_checkpoint_item(run); + } + let ledger = match store.automation_effect_ledger(run.id) { + Ok(ledger) => ledger, + Err(_) => return invalid_automation_checkpoint_item(run), + }; + if ledger.is_some() && checkpoint.is_none() { + return invalid_automation_checkpoint_item(run); + } + + let (phase, effect_state, detail_code) = + match ledger.as_ref().and_then(|ledger| ledger.current()) { + Some(effect) => match effect.state { + AutomationEffectState::Prepared => ( + base_phase, + TaskEffectState::NoEffect, + if run.claimed_by.is_some() { + "automation_effect_prepared_with_lease" + } else { + "automation_effect_prepared" + }, + ), + AutomationEffectState::EffectStarted => ( + TaskLifecyclePhase::Running, + TaskEffectState::RemoteUncertain, + "automation_effect_started", + ), + AutomationEffectState::EffectUnknown => ( + TaskLifecyclePhase::EffectUnknown, + TaskEffectState::RemoteUncertain, + "automation_effect_unknown_recovery_required", + ), + AutomationEffectState::KnownApplied => match effect.undo_capability { + AutomationEffectUndoCapability::ExactLocal => ( + base_phase, + TaskEffectState::LocalReversible, + "automation_known_applied_exact_local_undo", + ), + AutomationEffectUndoCapability::UnsupportedExternal => ( + base_phase, + TaskEffectState::RemoteKnownApplied, + "automation_known_applied_external_no_undo", + ), + AutomationEffectUndoCapability::None => ( + base_phase, + TaskEffectState::RemoteKnownApplied, + "automation_effect_known_applied", + ), + }, + AutomationEffectState::KnownNotApplied => ( + base_phase, + TaskEffectState::RemoteKnownNotApplied, + "automation_effect_known_not_applied", + ), + AutomationEffectState::ReconciliationBackoff => ( + TaskLifecyclePhase::EffectUnknown, + TaskEffectState::RemoteUncertain, + "automation_reconciliation_backoff", + ), + AutomationEffectState::DeadLettered => ( + TaskLifecyclePhase::RepairRequired, + TaskEffectState::RemoteUncertain, + "automation_dead_letter_recovery_required", + ), + }, + None if run.status == AutomationRunStatus::Queued + && run.last_error.as_deref().is_some_and(|error| { + error.starts_with( + "Recovered provably pre-effect work after automation lease expired", + ) + }) => + { + ( + TaskLifecyclePhase::Queued, + TaskEffectState::NoEffect, + "automation_recovered_pre_effect", + ) + } + None if run.status == AutomationRunStatus::Failed + && run + .last_error + .as_deref() + .is_some_and(|error| error.starts_with("effect_unknown:")) => + { + ( + TaskLifecyclePhase::EffectUnknown, + TaskEffectState::RemoteUncertain, + "automation_effect_unknown_recovery_required", + ) + } + None if run.claimed_by.is_some() && checkpoint.is_some() => ( + base_phase, + TaskEffectState::NoEffect, + "automation_lease_active", + ), + None if checkpoint.is_some() => ( + base_phase, + TaskEffectState::NoEffect, + "automation_checkpoint_persisted", + ), + None => ( + base_phase, + TaskEffectState::NoEffect, + match run.status { + AutomationRunStatus::Queued => "automation_queued", + AutomationRunStatus::Running => "automation_running", + AutomationRunStatus::WaitingReview => "automation_waiting_review", + AutomationRunStatus::WaitingApproval => "automation_waiting_approval", + AutomationRunStatus::Completed => "automation_completed", + AutomationRunStatus::Failed => "automation_failed", + AutomationRunStatus::Cancelled => "automation_cancelled", + }, + ), + }; item( run.id, run.agent_run_id, TaskLifecycleSource::AutomationRun, phase, - TaskEffectState::NoEffect, - "Automatic task".to_string(), - task_phase_code(phase), + effect_state, + "Task checkpoint".to_string(), + detail_code.to_string(), + None, + run.updated_at, + ) +} + +fn invalid_automation_checkpoint_item(run: &AutomationRun) -> TaskLifecycleItem { + item( + run.id, + run.agent_run_id, + TaskLifecycleSource::AutomationRun, + TaskLifecyclePhase::RepairRequired, + TaskEffectState::RemoteUncertain, + "Task checkpoint".to_string(), + "automation_checkpoint_invalid".to_string(), None, run.updated_at, ) @@ -575,7 +718,9 @@ fn capability_may_mutate(capability: CapabilityKind) -> bool { mod tests { use super::{TaskLifecyclePhase, TaskLifecycleSource}; use crate::kernel::agent_run::{AgentRunStart, AgentRunStatus}; - use crate::kernel::automation::{AutomationDefinition, ReviewQueueItem, ReviewQueueItemStatus}; + use crate::kernel::automation::{ + AutomationCheckpoint, AutomationDefinition, ReviewQueueItem, ReviewQueueItemStatus, + }; use crate::kernel::event_store::EventStore; use chrono::{Duration, Utc}; use uuid::Uuid; @@ -637,4 +782,68 @@ mod tests { .contains("private prompt")); assert_eq!(agent.initial_status, AgentRunStatus::Queued); } + + #[test] + fn automation_checkpoint_projection_distinguishes_safe_restart_without_authority() { + let store = EventStore::open_memory().unwrap(); + let now = Utc::now(); + let definition = AutomationDefinition::once( + "Checkpoint projection".to_string(), + "Asia/Shanghai".to_string(), + now - Duration::minutes(1), + ) + .unwrap(); + store.upsert_automation_definition(&definition).unwrap(); + let (run, _) = store + .enqueue_due_automation_agent_run( + definition.id, + now, + "projection-scheduler".to_string(), + format!("automation:{}", definition.id), + ) + .unwrap() + .unwrap(); + + let queued = store.task_lifecycle_snapshot().unwrap(); + let queued_item = queued.items.iter().find(|item| item.id == run.id).unwrap(); + assert_eq!(queued_item.detail_code, "automation_queued"); + + let lease = store + .claim_automation_run(run.id, "private-worker-authority".to_string(), 1, now) + .unwrap(); + let checkpoint = AutomationCheckpoint { + automation_run_id: run.id, + dedup_key: run.trigger_window_key.clone(), + tool_invocation_id: None, + evidence_ref: Some("evidence://private-authority-token".to_string()), + recorded_at: now, + }; + store + .compare_and_swap_automation_checkpoint(&checkpoint, &lease, None, now) + .unwrap(); + + let active = store.task_lifecycle_snapshot().unwrap(); + let active_item = active.items.iter().find(|item| item.id == run.id).unwrap(); + assert_eq!(active_item.detail_code, "automation_lease_active"); + let active_json = serde_json::to_string(&active).unwrap(); + assert!(!active_json.contains("private-worker-authority")); + assert!(!active_json.contains("private-authority-token")); + + let recovered = store + .recover_expired_automation_runs(now + Duration::seconds(2)) + .unwrap(); + assert_eq!(recovered.requeued, 1); + assert_eq!(recovered.effect_unknown, 0); + let restarted = store.task_lifecycle_snapshot().unwrap(); + let restarted_item = restarted + .items + .iter() + .find(|item| item.id == run.id) + .unwrap(); + assert_eq!( + restarted_item.detail_code, + "automation_recovered_pre_effect" + ); + assert_eq!(restarted_item.phase, TaskLifecyclePhase::Queued); + } } diff --git a/apps/desktop/src/AutomationCenter.tsx b/apps/desktop/src/AutomationCenter.tsx index 2068f5e..2b81c07 100644 --- a/apps/desktop/src/AutomationCenter.tsx +++ b/apps/desktop/src/AutomationCenter.tsx @@ -18,6 +18,27 @@ type Props = { onRunQueued: () => Promise; }; +const runStatusCopy: Record> = { + zh: { + queued: "已排队", + running: "运行中", + waiting_review: "等待审阅", + waiting_approval: "等待批准", + completed: "已完成", + failed: "未完成", + cancelled: "已取消", + }, + en: { + queued: "Queued", + running: "Running", + waiting_review: "Waiting for review", + waiting_approval: "Waiting for approval", + completed: "Completed", + failed: "Not completed", + cancelled: "Cancelled", + }, +}; + export function AutomationCenter({ language, onRunQueued }: Props) { const [definitions, setDefinitions] = useState([]); const [runs, setRuns] = useState([]); @@ -237,6 +258,15 @@ export function AutomationCenter({ language, onRunQueued }: Props) { setEditingGoal(event.target.value)} aria-label={zh ? "编辑任务目标" : "Edit automation goal"} /> ) : {definition.goal}} {scheduleLabel(definition)} + + {definition.status === "paused" + ? zh + ? "已暂停" + : "Paused" + : zh + ? "已计划" + : "Scheduled"} + {editingId === definition.id ? ( @@ -271,9 +301,11 @@ export function AutomationCenter({ language, onRunQueued }: Props) {
{new Date(run.scheduled_for).toLocaleString()} - {run.last_error ?? run.status} + {runStatusCopy[language][run.status]}
- {run.status} + + {runStatusCopy[language][run.status]} +
))}
diff --git a/apps/desktop/src/TaskLifecyclePanel.tsx b/apps/desktop/src/TaskLifecyclePanel.tsx index d789df6..f7b8442 100644 --- a/apps/desktop/src/TaskLifecyclePanel.tsx +++ b/apps/desktop/src/TaskLifecyclePanel.tsx @@ -63,18 +63,80 @@ const effectCopy: Record> = { }, }; +const automationDetailCopy: Record> = { + zh: { + automation_queued: "计划任务正在排队，等待领取。", + automation_running: "工作进程报告任务正在运行。", + automation_waiting_review: "任务已停在审阅点，等待你的决定。", + automation_waiting_approval: "任务已停在批准点，未经确认不会继续。", + automation_completed: "任务已到达记录中的完成状态。", + automation_failed: "任务已停止，未记录不确定的外部结果。", + automation_cancelled: "任务已取消。", + automation_checkpoint_persisted: "重启检查点已保存，尚未开始产生效果。", + automation_lease_active: "运行租约有效，重启检查点已保存。", + automation_recovered_pre_effect: "租约在产生效果前过期，任务已安全返回队列。", + automation_effect_prepared: "效果已准备，但尚未开始。", + automation_effect_prepared_with_lease: "运行租约有效；效果已准备，但尚未开始。", + automation_effect_started: "效果已经开始；重启后必须先核对结果，不能盲目重放。", + automation_effect_unknown_recovery_required: + "外部结果不确定；必须明确恢复，系统不会自动重放。", + automation_effect_known_applied: "效果已确认完成。", + automation_effect_known_not_applied: "效果已确认未发生，可以按恢复流程处理。", + automation_reconciliation_backoff: "正在等待下一次有界核对，不会重放原效果。", + automation_dead_letter_recovery_required: + "有界核对已经用尽；必须人工明确恢复，不会自动重放。", + automation_known_applied_exact_local_undo: "本地效果已确认，可按检查点精确恢复。", + automation_known_applied_external_no_undo: + "外部效果已确认；DS Agent 不能声称可以撤销外部结果。", + automation_checkpoint_invalid: "检查点投影无法安全读取；需要修复，不会自动执行。", + }, + en: { + automation_queued: "Scheduled work is queued and waiting to be claimed.", + automation_running: "A worker reports this run in progress.", + automation_waiting_review: "The run is paused for your review.", + automation_waiting_approval: "The run is paused for approval and will not continue unconfirmed.", + automation_completed: "The run reached its recorded completion state.", + automation_failed: "The run stopped with no recorded uncertain external result.", + automation_cancelled: "The run was cancelled.", + automation_checkpoint_persisted: "A restart checkpoint is saved; no effect has started.", + automation_lease_active: "The worker lease is active and a restart checkpoint is saved.", + automation_recovered_pre_effect: + "The lease expired before any effect; the run safely returned to the queue.", + automation_effect_prepared: "The effect is prepared but has not started.", + automation_effect_prepared_with_lease: + "The worker lease is active; the effect is prepared but has not started.", + automation_effect_started: + "The effect started; after restart, verify the result before any replay.", + automation_effect_unknown_recovery_required: + "The external result is uncertain; explicit recovery is required and no replay is automatic.", + automation_effect_known_applied: "The effect is confirmed applied.", + automation_effect_known_not_applied: + "The effect is confirmed not applied and can follow the recovery flow.", + automation_reconciliation_backoff: + "Waiting for the next bounded reconciliation check; the original effect is not replayed.", + automation_dead_letter_recovery_required: + "Bounded reconciliation is exhausted; explicit recovery is required and no replay is automatic.", + automation_known_applied_exact_local_undo: + "The local effect is confirmed and can be restored exactly from its checkpoint.", + automation_known_applied_external_no_undo: + "The external effect is confirmed; DS Agent cannot claim it can undo that external result.", + automation_checkpoint_invalid: + "The checkpoint projection cannot be read safely; repair is required and nothing runs automatically.", + }, +}; + export function TaskLifecyclePanel({ language }: { language: Language }) { const [snapshot, setSnapshot] = useState(null); - const [error, setError] = useState(null); + const [refreshFailed, setRefreshFailed] = useState(false); const [loading, setLoading] = useState(false); const refresh = useCallback(async () => { setLoading(true); - setError(null); + setRefreshFailed(false); try { setSnapshot(await invoke("list_task_lifecycle")); - } catch (reason) { - setError(String(reason)); + } catch { + setRefreshFailed(true); } finally { setLoading(false); } @@ -90,17 +152,23 @@ export function TaskLifecyclePanel({ language }: { language: Language }) { return (
- {zh ? "任务进度" : "Task activity"} + + {zh ? "任务检查点与重启状态" : "Task checkpoints and restart status"} +

{zh - ? "这里统一显示任务、工具、连接器、文档和电脑操作的真实状态。结果未知时不会自动重复执行。" - : "One view of tasks, tools, connections, documents, and computer actions. Unknown effects are never replayed automatically."} + ? "这里统一显示任务、工具、连接器、文档和电脑操作的真实状态。重启只会安全恢复尚未产生效果的工作；结果未知时不会自动重放。" + : "One view of tasks, tools, connections, documents, and computer actions. Restart only recovers work known to be pre-effect; unknown effects are never replayed automatically."}

- {error ?

{error}

: null} + {refreshFailed ? ( +

+ {zh ? "暂时无法读取任务状态，请稍后刷新。" : "Task status is unavailable. Refresh to try again."} +

+ ) : null} {!loading && items.length === 0 ? (

{zh ? "暂无任务记录。" : "No task activity yet."}

) : null} @@ -111,6 +179,12 @@ export function TaskLifecyclePanel({ language }: { language: Language }) { {phaseCopy[language][item.phase]} + {item.source === "automation_run" ? ( + + {automationDetailCopy[language][item.detail_code] ?? + (zh ? "自动任务状态已记录。" : "Automation state recorded.")} + + ) : null} {effectCopy[language][item.effect_state]} ))} diff --git a/scripts/task-lifecycle.test.mjs b/scripts/task-lifecycle.test.mjs index 7df3574..0fb89b4 100644 --- a/scripts/task-lifecycle.test.mjs +++ b/scripts/task-lifecycle.test.mjs @@ -12,10 +12,20 @@ const [panel, automationCenter, main, lifecycleCommands, lifecycleKernel] = awai ]); assert.match(panel, /invoke\("list_task_lifecycle"\)/); -assert.match(panel, /结果未知时不会自动重复执行/); +assert.match(panel, /任务检查点与重启状态/); +assert.match(panel, /重启只会安全恢复尚未产生效果的工作/); assert.match(panel, /remote_uncertain/); +assert.match(panel, /automation_recovered_pre_effect/); +assert.match(panel, /automation_effect_unknown_recovery_required/); +assert.match(panel, /automation_reconciliation_backoff/); +assert.match(panel, /automation_dead_letter_recovery_required/); +assert.match(panel, /automation_known_applied_exact_local_undo/); +assert.match(panel, /automation_known_applied_external_no_undo/); +assert.doesNotMatch(panel, /\{item\.detail_code\}/); assert.doesNotMatch(panel, /credential_handle|refresh_token|access_token|provider error body/i); assert.match(automationCenter, //); +assert.match(automationCenter, /runStatusCopy\[language\]\[run\.status\]/); +assert.doesNotMatch(automationCenter, /run\.last_error\s*\?\?|>\{run\.status\} Date: Sun, 26 Jul 2026 09:14:57 +0800 Subject: [PATCH 4/4] release: prepare DS Agent v1.6.0 --- CODE_SIGNING_POLICY.md | 8 +- PRIVACY.md | 17 ++- README.md | 36 +++--- README.zh-CN.md | 34 +++--- SECURITY.md | 17 ++- apps/desktop/package.json | 2 +- apps/desktop/src-tauri/Cargo.lock | 2 +- apps/desktop/src-tauri/Cargo.toml | 2 +- .../src-tauri/src/kernel/app_update.rs | 6 +- apps/desktop/src-tauri/src/kernel/deepseek.rs | 4 +- apps/desktop/src-tauri/tauri.conf.json | 2 +- apps/desktop/src/App.tsx | 2 +- docs/INSTALLATION.md | 33 ++++-- docs/OPEN_SOURCE_RELEASE.md | 12 +- docs/RELEASE_NOTES_v1.6.0.md | 103 ++++++++++++++++++ package.json | 2 +- scripts/release-source-check.mjs | 86 +++++++++++---- 17 files changed, 278 insertions(+), 90 deletions(-) create mode 100644 docs/RELEASE_NOTES_v1.6.0.md diff --git a/CODE_SIGNING_POLICY.md b/CODE_SIGNING_POLICY.md index 4584e84..2871eb4 100644 --- a/CODE_SIGNING_POLICY.md +++ b/CODE_SIGNING_POLICY.md @@ -1,10 +1,10 @@ # Code signing policy -Last updated: 2026-07-25 +Last updated: 2026-07-26 ## Current status -DS Agent `v1.5.0` is intentionally published unsigned. Both `ds-agent.exe` and +DS Agent `v1.6.0` is intentionally published unsigned. Both `ds-agent.exe` and the Windows x64 NSIS installer are expected to report Authenticode `NotSigned`. Windows may therefore display `Unknown publisher` or a Microsoft Defender SmartScreen warning. Users should download only over HTTPS from the official @@ -13,7 +13,7 @@ GitHub Release and verify the published SHA-256 before running the installer. The SignPath Foundation application is submitted and approval is pending. No DS Agent binary may be represented as SignPath-signed until the application is approved and a later-version artifact independently verifies as Authenticode -`Valid`. The project will not replace the immutable `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, or `v1.5.0` tag, +`Valid`. The project will not replace the immutable `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, `v1.5.0`, or `v1.6.0` tag, Release, or asset if signing becomes available later. For releases accepted into that program: **Free code signing provided by @@ -57,7 +57,7 @@ them; guessed or placeholder identifiers are forbidden. ## Release verification -For the unsigned `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, and `v1.5.0` exceptions, maintainers verify and disclose the actual +For the unsigned `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, `v1.5.0`, and `v1.6.0` exceptions, maintainers verify and disclose the actual `NotSigned` status of both the application executable and installer. Evidence must bind the exact source commit, file name, product version, byte size, and SHA-256. The installer downloaded back from GitHub must match the reviewed diff --git a/PRIVACY.md b/PRIVACY.md index e7d6298..d34d90d 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,6 +1,6 @@ # Privacy Policy -Last updated: 2026-07-25 +Last updated: 2026-07-26 This policy describes the current published DS Agent desktop application and public project. DS Agent is local-first and does not operate a project cloud @@ -14,7 +14,7 @@ User-selected workspaces hold approved evidence, exports, reports, work packages, screenshots, and other artifacts. This information is not silently synced to a DS Agent-operated server. -The current stable `v1.5.0` accepts one user-supplied DeepSeek API key through +The current stable `v1.6.0` accepts one user-supplied DeepSeek API key through the onboarding screen and stores it in a dedicated Windows DPAPI-protected local vault. A process-environment key remains an explicit compatibility fallback and is never silently copied into that vault. The project does not @@ -31,20 +31,27 @@ coverage counts plus a redacted authorization intent; it does not receive the private capability proposal, a preparation/compiler command, or a local completion writer. -The v1.5.0 package preserves the T1 Office verification engine, which keeps source identities, bounded fact +The v1.6.0 package preserves the T1 Office verification engine, which keeps source identities, bounded fact provenance, artifact hashes, render receipts, revision receipts, and continuation checkpoints locally in the selected workspace or application data. Ordinary chat does not yet automatically select or sequence the two T1 tools; their presence does not cause workspace files to be scanned or uploaded merely because the application is open. -The v1.5.0 durable Computer Use lifecycle stores bounded application, process, +The v1.6.0 package preserves the durable Computer Use lifecycle, which stores bounded application, process, window, frame, target, action, approval, screenshot-evidence, and semantic receipt identities locally. Its installed reliability tests use generated files, workbooks, data, fresh Edge profiles, an ephemeral loopback portal, and isolated Event Stores. Those tests do not read stored browser logins, production accounts, or the user's normal DS Agent AppData. +The v1.6.0 durable automation runtime stores definition and revision identities, +schedule windows, run state, restart checkpoints, leases, effect state, +reconciliation attempts, dead-letter state, and exact local undo receipts +locally. TaskCheckpoint UI projections exclude raw worker tokens, lease fences, +idempotency keys, private evidence, provider bodies, and secrets. A result that +may have crossed an external-effect boundary is not replayed automatically. + Uninstalling the application may not delete a user-selected workspace or every application-data file. Review and remove those local locations separately when you no longer want to retain them. @@ -81,7 +88,7 @@ the user or person operating the application: The optional local desktop bridge accepts only loopback addresses and is started and controlled by the user. DS Agent does not install or supervise that service. Production Microsoft and Google account registration and live -mail/calendar writes are disabled in `v1.5.0`; offline connector contracts do +mail/calendar writes are disabled in `v1.6.0`; offline connector contracts do not authorize a production account or external write. ## What can be included in a model request diff --git a/README.md b/README.md index e5f35a5..c4da9de 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,8 @@

- v1.5.0 stable · - Download for Windows · + v1.6.0 stable · + Download for Windows · Apache-2.0

@@ -129,16 +129,11 @@ does not claim completion from model confidence alone. Local files, browser actions, Office artifacts, and Computer Use are complete only when observable evidence satisfies the task's completion criteria. -In v1.5.0, DeepSeek may propose a bounded `GoalEnvelope`, but only the local -Kernel can validate and freeze it. For the same queued task, the Kernel alone -derives the capability manifest, risk, and preview shown in one exact-task -authorization card. Approval creates only exact authority; it does not execute -a Tool, resume the task, or mark the Goal complete. Completion remains blocked -until locally authoritative verifier evidence covers every frozen `done_when` -condition and required artifact identity. +In v1.6.0, DeepSeek may propose a bounded `GoalEnvelope`, but only the local Kernel can validate and freeze it. For the same queued task, the Kernel alone derives the capability manifest, risk, and preview shown in one exact-task authorization card. +Approval creates only exact authority; it does not execute a Tool, resume the task, or mark the Goal complete. Completion remains blocked until locally authoritative verifier evidence covers every frozen `done_when` condition and required artifact identity. -The v1.5.0 binary preserves the T1 Excel/PowerPoint verification engine and adds a production-reachable durable Computer Use session/step UI and Kernel loop. -Exact installed evidence is limited to generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal; it does not claim general browser automation, stored-login reuse, or production-site completion. +The v1.6.0 binary preserves the T1 Excel/PowerPoint verification engine and the production-reachable durable Computer Use session/step loop, then adds production-reachable persistent automation with immutable definition revisions, scheduled and manual runs, checkpointed leases, restart-safe pre-effect recovery, idempotency/effect reconciliation, dead-letter state, and a TaskCheckpoint view. +Safe pre-effect work can recover automatically. Effect-started or unknown work is never replayed blindly and requires explicit recovery; confirmed external effects are not represented as locally undoable. Exact installed evidence is limited to generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal. The release remains honest: ordinary chat does not yet automatically select or sequence the two T1 tools or the complete T1 Office workflow. ## DeepSeek and DS Agent boundary @@ -164,7 +159,7 @@ high-risk action. See the full [model boundary](docs/AGENT_MODEL_BOUNDARY.md). reconciliation contracts validated with offline adversarial fake providers. Production Microsoft/Google account registration and live external-write -authority remain disabled in v1.5.0. The release does not sign in to real +authority remain disabled in v1.6.0. The release does not sign in to real accounts, send real email, or create, change, or cancel real calendar events. ## Why Rust @@ -177,7 +172,7 @@ remain thin; the Kernel and persistent projections own business state. ## Quick start -1. Download the [Windows x64 installer](https://github.com/Lee-take/dsagent/releases/download/v1.5.0/DS.Agent_1.5.0_x64-setup.exe). +1. Download the [Windows x64 installer](https://github.com/Lee-take/dsagent/releases/download/v1.6.0/DS.Agent_1.6.0_x64-setup.exe). 2. Enter your own valid DeepSeek API key in onboarding and run the explicit balance/model verification. The key is stored locally with Windows DPAPI. 3. Choose one local workspace and let the readiness doctor verify its managed @@ -189,7 +184,7 @@ A user-supplied DeepSeek API key is a required prerequisite. DS Agent does not bundle a shared key or bypass DeepSeek access requirements; use remains subject to DeepSeek's terms and account policies. -The v1.5.0 application executable and installer are Authenticode `NotSigned`. +The v1.6.0 application executable and installer are Authenticode `NotSigned`. Windows may display `Unknown publisher` or a Microsoft Defender SmartScreen warning. Download only over HTTPS from this repository, verify the SHA-256 in the GitHub Release, and read the [installation guide](docs/INSTALLATION.md) @@ -197,10 +192,10 @@ before running the installer. ## Code signing policy -DS Agent `v1.5.0` is intentionally unsigned. The SignPath Foundation application +DS Agent `v1.6.0` is intentionally unsigned. The SignPath Foundation application is submitted and approval is pending; no release is represented as signed. If the project is accepted, signing starts with a later new version and does not -replace the immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, or v1.5.0 tag or asset. For releases accepted into the program: +replace the immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, v1.5.0, or v1.6.0 tag or asset. For releases accepted into the program: **Free code signing provided by SignPath.io, certificate by SignPath Foundation.** See the full [code signing policy](CODE_SIGNING_POLICY.md) and [privacy policy](PRIVACY.md). @@ -218,8 +213,8 @@ example `D:\build-target\ds-agent-v1-release`. ## Stable release -- Release: [DS Agent v1.5.0](https://github.com/Lee-take/dsagent/releases/tag/v1.5.0) -- Installer: `DS.Agent_1.5.0_x64-setup.exe` +- Release: [DS Agent v1.6.0](https://github.com/Lee-take/dsagent/releases/tag/v1.6.0) +- Installer: `DS.Agent_1.6.0_x64-setup.exe` - Integrity: verify the final byte size and SHA-256 published in the GitHub Release before running the installer. - Onboarding: one user-supplied Key, Windows DPAPI storage, explicit DeepSeek @@ -233,6 +228,10 @@ example `D:\build-target\ds-agent-v1-release`. - Task authorization: one Kernel-derived exact-task card with manifest/risk/ preview binding, one user decision, per-capability audit, and exact revocation; approval does not execute or resume the task. +- Durable automation: immutable definition revisions, persisted trigger windows, + checkpointed leases, restart-safe pre-effect recovery, idempotency/effect + reconciliation, bounded backoff/dead-letter state, exact local undo, and + truthful no-undo treatment for confirmed external effects. - Computer Use: production-reachable durable lifecycle and fail-closed execution; exact installed proof is limited to generated Explorer/Excel targets and an isolated Edge/loopback test portal, not general browsing. - T1 verification engine: exact source identities and provenance, non-overwriting XLSX/PPTX artifacts, actual local render evidence, bounded revisions, and persisted continuation checkpoints; ordinary chat does not yet automatically select or sequence this complete T1 path. @@ -241,6 +240,7 @@ example `D:\build-target\ds-agent-v1-release`. - [Installation](docs/INSTALLATION.md) - [DS Agent and DeepSeek boundary](docs/AGENT_MODEL_BOUNDARY.md) - [v1 architecture](docs/architecture/DS_AGENT_V1_ARCHITECTURE_PLAN.md) +- [v1.6.0 release notes](docs/RELEASE_NOTES_v1.6.0.md) - [v1.5.0 release notes](docs/RELEASE_NOTES_v1.5.0.md) - [v1.4.0 release notes](docs/RELEASE_NOTES_v1.4.0.md) - [v1.3.0 release notes](docs/RELEASE_NOTES_v1.3.0.md) diff --git a/README.zh-CN.md b/README.zh-CN.md index afdfcf7..cd4b4a4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -11,8 +11,8 @@

- v1.5.0 正式稳定版 · - 下载 Windows 安装包 · + v1.6.0 正式稳定版 · + 下载 Windows 安装包 · Apache-2.0

@@ -118,15 +118,19 @@ Kernel 会持久保存任务和审核状态，只对失败步骤进行有边界 宣告完成。本地文件、浏览器操作、Office 产物和 Computer Use，只有在可观察证据满足任务 完成条件后才算真正完成。 -在 v1.5.0 中，DeepSeek 可以提出有界 `GoalEnvelope`，但只有本地 Kernel 能校验并冻结它。 +在 v1.6.0 中，DeepSeek 可以提出有界 `GoalEnvelope`，但只有本地 Kernel 能校验并冻结它。 对于同一个排队任务，也只有 Kernel 能派生 capability manifest、风险和 preview，并显示为 一张 exact-task 授权卡。批准只产生精确权限，不会执行 Tool、恢复任务或把 Goal 标记为完成； 只有本地权威 verifier 证据覆盖冻结目标的全部 `done_when` 和必需产物身份后，任务才可能 进入完成状态。 -v1.5.0 安装包保留 T1 Excel/PowerPoint 验证引擎，并新增生产可达的持久 Computer Use -会话/步骤 UI 与 Kernel 闭环。准确安装态证据只覆盖生成的 File Explorer、Excel 目标，以及使用全新 profile 和临时回环测试站点的已安装 Edge；本版本不宣称通用浏览器自动化、复用 -已登录状态或生产网站完成能力。普通聊天目前还不会自动选择和串联这两个 T1 工具，也不会自动完成整条 T1 Office 工作流。 +v1.6.0 安装包保留 T1 Excel/PowerPoint 验证引擎和生产可达的持久 Computer Use 闭环， +并新增生产可达的持久自动化执行：不可变 definition revision、计划/手动运行、带 checkpoint +的 lease、重启后安全的 effect 前恢复、幂等/effect 对账、dead-letter 状态和 TaskCheckpoint +视图。effect 尚未开始的安全工作可以自动恢复；effect 已开始或结果未知时绝不会盲目重放， +必须明确恢复，已确认的外部效果也不会被描述为可以本地撤销。准确安装态证据只覆盖生成的 +File Explorer、Excel 目标，以及使用全新 profile 和临时回环测试站点的已安装 Edge。 +普通聊天目前还不会自动选择和串联这两个 T1 工具，也不会自动串联整条 T1 Office 工作流。 ## DeepSeek 与 DS Agent 的工作边界 @@ -148,7 +152,7 @@ DeepSeek 可以提出动作，DS Agent 决定动作是否安全、是否允许 - 已通过离线对抗性 fake provider 验证的 Microsoft/Google 形态邮件、日历、同步、草稿、 外部变更和对账契约。 -v1.5.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限。当前正式版不会登录 +v1.6.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限。当前正式版不会登录 真实账号、发送真实邮件，也不会创建、修改或取消真实日历事件。 ## 为什么使用 Rust @@ -159,7 +163,7 @@ command 与 React UI 保持薄层，业务状态由 Kernel 和持久投影统一 ## 快速开始 -1. 下载 [Windows x64 安装包](https://github.com/Lee-take/dsagent/releases/download/v1.5.0/DS.Agent_1.5.0_x64-setup.exe)。 +1. 下载 [Windows x64 安装包](https://github.com/Lee-take/dsagent/releases/download/v1.6.0/DS.Agent_1.6.0_x64-setup.exe)。 2. 在首次设置中输入你自己的有效 DeepSeek API Key，并显式验证余额和模型；Key 使用 Windows DPAPI 保存在本机。 3. 选择一个本地工作目录，由 readiness doctor 检查受管目录和可写状态。 @@ -169,16 +173,16 @@ command 与 React UI 保持薄层，业务状态由 Kernel 和持久投影统一 用户自行提供有效的 DeepSeek API Key 是必备前提。DS Agent 不内置共享 Key，也不会 绕过 DeepSeek 的访问条件；实际使用仍须遵守 DeepSeek 的服务条款和账号规则。 -v1.5.0 应用程序和安装包的 Authenticode 状态均为 `NotSigned`。Windows 可能显示 +v1.6.0 应用程序和安装包的 Authenticode 状态均为 `NotSigned`。Windows 可能显示 `Unknown publisher`（未知发布者）或 Microsoft Defender SmartScreen 警告。请只通过 本仓库的 HTTPS 链接下载，运行前核对 GitHub Release 中的 SHA-256，并阅读 [安装指南](docs/INSTALLATION.md)。 ## Code signing policy（代码签名政策） -DS Agent `v1.5.0` 是如实披露的未签名版本。SignPath Foundation 申请已经提交、仍在等待 +DS Agent `v1.6.0` 是如实披露的未签名版本。SignPath Foundation 申请已经提交、仍在等待 审批，本版本不会被描述为已签名。若以后获批，只从后续新版本开始签名，不替换本版本不可 -移动的 tag 或资产，也不改写 v1.1.0、v1.2.0、v1.3.0 或 v1.4.0。获准加入该计划的 Release 将遵循：**Free code signing provided by +移动的 tag 或资产，也不改写 v1.1.0、v1.2.0、v1.3.0、v1.4.0 或 v1.5.0。获准加入该计划的 Release 将遵循：**Free code signing provided by SignPath.io, certificate by SignPath Foundation.** 完整说明见 [代码签名政策](CODE_SIGNING_POLICY.md)和[隐私政策](PRIVACY.md)。 @@ -195,8 +199,8 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev ## 正式稳定版 -- Release：[DS Agent v1.5.0](https://github.com/Lee-take/dsagent/releases/tag/v1.5.0) -- 安装包：`DS.Agent_1.5.0_x64-setup.exe` +- Release：[DS Agent v1.6.0](https://github.com/Lee-take/dsagent/releases/tag/v1.6.0) +- 安装包：`DS.Agent_1.6.0_x64-setup.exe` - 完整性：运行安装包前，核对 GitHub Release 中发布的最终字节数和 SHA-256。 - 首次设置：单一用户 Key、Windows DPAPI 本机存储、显式 DeepSeek 余额/V4 模型验证、 无密 readiness 和 workspace doctor。 @@ -206,6 +210,9 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev 必需产物身份的 fail-closed 证据门。 - 任务授权：一张由 Kernel 派生并绑定 manifest/risk/preview 的 exact-task 卡片、一次用户 决策、逐能力审计和精确撤销；批准不会执行或恢复任务。 +- 持久自动化：不可变 definition revision、持久 trigger window、带 checkpoint 的 lease、 + 重启后安全的 effect 前恢复、幂等/effect 对账、有界 backoff/dead-letter、准确本地 undo， + 以及对已确认外部效果如实显示不可本地撤销。 - Computer Use：生产可达的持久生命周期与 fail-closed 执行；准确安装态证明仅覆盖生成的 Explorer/Excel 目标和隔离的 Edge/回环测试站点，不代表通用浏览能力。 - T1 验证引擎：准确来源身份和 provenance、禁止覆盖的 XLSX/PPTX、真实本地渲染证据、 @@ -216,6 +223,7 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev - [安装指南](docs/INSTALLATION.md) - [DS Agent 与 DeepSeek 的工作边界](docs/AGENT_MODEL_BOUNDARY.md) - [v1 架构计划](docs/architecture/DS_AGENT_V1_ARCHITECTURE_PLAN.md) +- [v1.6.0 发布说明](docs/RELEASE_NOTES_v1.6.0.md) - [v1.5.0 发布说明](docs/RELEASE_NOTES_v1.5.0.md) - [v1.4.0 发布说明](docs/RELEASE_NOTES_v1.4.0.md) - [v1.3.0 发布说明](docs/RELEASE_NOTES_v1.3.0.md) diff --git a/SECURITY.md b/SECURITY.md index 127a28b..9e92619 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security Policy -DS Agent is a local-first Windows desktop agent. DS Agent v1.5.0 is the current +DS Agent is a local-first Windows desktop agent. DS Agent v1.6.0 is the current published stable release and is not an official DeepSeek product. Security reports are welcome, especially around local credentials, permission gates, audit records, Computer Use boundaries, update integrity, code signing, and @@ -10,6 +10,7 @@ package import/export behavior. | Version | Supported | | --- | --- | +| 1.6.0 | Supported | | 1.5.0 | Supported | | 1.4.0 | Supported | | 1.3.0 | Supported | @@ -34,7 +35,7 @@ Include: ## Security Boundaries -- Current stable v1.5.0 stores one user-supplied DeepSeek API key in a dedicated +- Current stable v1.6.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault. A process-environment key is an explicit compatibility fallback and is not copied into the vault. Presence alone is never treated as verified readiness; balance and required V4 model checks produce only a @@ -52,18 +53,24 @@ Include: group, projection, manifest, preview, revision, fingerprint, hash, scopes, targets, and expiry. Approval grants only exact authority; it does not execute a Tool, resume a task, or mark a Goal complete. -- The v1.5.0 package preserves the T1 reconciliation and PowerPoint executors, +- The v1.6.0 package preserves the T1 reconciliation and PowerPoint executors, which require exact workspace paths, non-overwriting output, grouped authorization, persisted artifact identity, verifier evidence, and actual local render evidence. DeepSeek cannot approve those actions or mint their evidence or completion receipts. The ordinary chat UI does not yet automatically select or sequence these T1 tools. -- The v1.5.0 durable Computer Use lifecycle binds the application, process, +- The v1.6.0 package preserves the durable Computer Use lifecycle, which binds the application, process, window, frame, target, action, approval, screenshots, and semantic receipt. `ActionStarted` is persisted once before the effect; ambiguous outcomes fail closed and are not replayed. Installed reliability evidence covers only generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal. It is not evidence of general browser control, stored-login reuse, or production-site completion. +- The v1.6.0 durable automation runtime binds immutable definition revisions, + trigger windows, run checkpoints, leases, idempotency identities, effect + state, reconciliation attempts, dead-letter state, and exact local undo + receipts. Safe pre-effect work may recover after restart. Effect-started or + unknown outcomes require explicit recovery and are never replayed blindly; + confirmed external effects are not represented as locally undoable. - `pnpm test:secrets` scans tracked and unignored repository files for live `sk-` style keys and non-empty `DEEPSEEK_API_KEY` assignments without printing candidate values. @@ -78,7 +85,7 @@ Include: memory. - Release identity must follow the [code signing policy](CODE_SIGNING_POLICY.md). An unsigned or invalidly signed artifact must not be represented as a signed - release. DS Agent v1.5.0 is explicitly disclosed as Authenticode `NotSigned`; + release. DS Agent v1.6.0 is explicitly disclosed as Authenticode `NotSigned`; Windows may show `Unknown publisher` or a Microsoft Defender SmartScreen warning. See also the [privacy policy](PRIVACY.md). diff --git a/apps/desktop/package.json b/apps/desktop/package.json index 317cab8..6fec75f 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@deepseek-agent-os/desktop", "private": true, - "version": "1.5.0", + "version": "1.6.0", "author": "Lee take", "license": "Apache-2.0", "type": "module", diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index a237687..9ce0edd 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -794,7 +794,7 @@ dependencies = [ [[package]] name = "deepseek-agent-os-desktop" -version = "1.5.0" +version = "1.6.0" dependencies = [ "base64 0.22.1", "chrono", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index e228557..55a087f 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deepseek-agent-os-desktop" -version = "1.5.0" +version = "1.6.0" description = "Local-first DeepSeek AI work platform" edition = "2021" license = "Apache-2.0" diff --git a/apps/desktop/src-tauri/src/kernel/app_update.rs b/apps/desktop/src-tauri/src/kernel/app_update.rs index 7dd3f54..1e71eb5 100644 --- a/apps/desktop/src-tauri/src/kernel/app_update.rs +++ b/apps/desktop/src-tauri/src/kernel/app_update.rs @@ -17,8 +17,8 @@ pub(crate) const APP_UPDATE_RELEASES_API_URL: &str = const APP_UPDATE_RELEASE_DOWNLOAD_PATH_PREFIX: &str = "/Lee-take/dsagent/releases/download/"; const APP_UPDATE_LEGACY_RELEASE_DOWNLOAD_PATH_PREFIX: &str = "/Lee-take/deepseek-agent-os/releases/download/"; -const APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.5.0"; -const APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.5.0"; +const APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.6.0"; +const APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.6.0"; #[cfg(windows)] const WINDOWS_CREATE_NO_WINDOW: u32 = 0x08000000; @@ -653,7 +653,7 @@ mod tests { let status = update_status_from_releases(releases, app_update_current_version()); assert!(!status.update_available); - assert_eq!(status.current_version, "v1.5.0"); + assert_eq!(status.current_version, "v1.6.0"); assert_eq!(status.latest_version.as_deref(), Some("0.3.0")); assert!(status.asset_name.is_none()); } diff --git a/apps/desktop/src-tauri/src/kernel/deepseek.rs b/apps/desktop/src-tauri/src/kernel/deepseek.rs index cec08b0..3e88dd8 100644 --- a/apps/desktop/src-tauri/src/kernel/deepseek.rs +++ b/apps/desktop/src-tauri/src/kernel/deepseek.rs @@ -265,7 +265,7 @@ impl Drop for DeepSeekOperationsBriefingSynt impl HttpDeepSeekChatCompletionTransport { pub fn new() -> Result { let client = reqwest::blocking::Client::builder() - .user_agent("DS-Agent/1.5.0 deepseek-v4") + .user_agent("DS-Agent/1.6.0 deepseek-v4") .timeout(std::time::Duration::from_secs( DEEPSEEK_CHAT_HTTP_TIMEOUT_SECS, )) @@ -982,7 +982,7 @@ mod tests { assert_eq!(response.first_text(), Some("ok")); assert!(recorded.raw.starts_with("POST / HTTP/1.1")); - assert!(normalized_headers.contains("user-agent: ds-agent/1.5.0 deepseek-v4")); + assert!(normalized_headers.contains("user-agent: ds-agent/1.6.0 deepseek-v4")); } #[test] diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 6f98112..a7bb350 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -2,7 +2,7 @@ "$schema": "https://schema.tauri.app/config/2", "productName": "DS Agent", "mainBinaryName": "ds-agent", - "version": "1.5.0", + "version": "1.6.0", "identifier": "ai.deepseek-agent-os.desktop", "build": { "beforeDevCommand": "npx pnpm@9.15.9 dev", diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 14e1b37..5bb62ce 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -224,7 +224,7 @@ const fallbackOnboardingReadiness: OnboardingReadinessProjection = { message_key: "onboarding.workspace.workspace_missing", }, version: { - current_version: "1.5.0", + current_version: "1.6.0", status: "current", blocking: false, message_key: "onboarding.version.current", diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 5e58567..014d825 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -9,14 +9,14 @@ project does not depend on the maintainer's local directories. The published Windows release provides a normal NSIS setup executable: ```text -DS.Agent_1.5.0_x64-setup.exe +DS.Agent_1.6.0_x64-setup.exe ``` -Version `1.5.0` is the current published stable release. Earlier commits, tags, +Version `1.6.0` is the current published stable release. Earlier commits, tags, Releases, and assets remain immutable. Both `ds-agent.exe` and the installer are Authenticode `NotSigned`, so Windows may show `Unknown publisher` or a Microsoft Defender SmartScreen warning. Download only over HTTPS from the official GitHub -Release and verify its byte size and SHA-256 against the `v1.5.0` Release before +Release and verify its byte size and SHA-256 against the `v1.6.0` Release before running it. The installer embeds the Microsoft WebView2 bootstrapper and runs it silently when the target machine needs the WebView2 runtime; users do not need Node.js, pnpm, Rust, or a @@ -27,12 +27,12 @@ not the workspace, evidence folder, export folder, or event database location. ## Code signing policy and verification -DS Agent `v1.5.0` is intentionally unsigned. Its HTTPS source, immutable tag, +DS Agent `v1.6.0` is intentionally unsigned. Its HTTPS source, immutable tag, exact byte size, and SHA-256 are the verification route for this asset. The SignPath Foundation application is submitted and approval is pending; no -publisher, certificate, or signed status is claimed for v1.5.0. If signing is +publisher, certificate, or signed status is claimed for v1.6.0. If signing is approved later, it starts with a new version and does not replace this tag or -asset or any immutable v1.1.0, v1.2.0, v1.3.0, or v1.4.0 publication. +asset or any immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, or v1.5.0 publication. For releases accepted into the open-source signing program: **Free code signing provided by SignPath.io, certificate by SignPath Foundation.** The complete @@ -43,12 +43,12 @@ describes current local data and user-triggered network behavior. On Windows, inspect a downloaded installer without launching it: ```powershell -Get-AuthenticodeSignature .\DS.Agent_1.5.0_x64-setup.exe | +Get-AuthenticodeSignature .\DS.Agent_1.6.0_x64-setup.exe | Select-Object Status, StatusMessage, SignerCertificate, TimeStamperCertificate -Get-FileHash .\DS.Agent_1.5.0_x64-setup.exe -Algorithm SHA256 +Get-FileHash .\DS.Agent_1.6.0_x64-setup.exe -Algorithm SHA256 ``` -For `v1.5.0`, `Status` is expected to be `NotSigned`. An unexpected signer, +For `v1.6.0`, `Status` is expected to be `NotSigned`. An unexpected signer, signature identity, hash, version, source, or asset mismatch is a stop condition. For a future release explicitly documented as signed, any status other than `Valid` is also a stop condition. @@ -84,19 +84,28 @@ reject, and revoke operate only on the Kernel-issued task/group/manifest/preview intent. Approval grants exact authority only; it does not execute a Tool, resume the task, or mark the Goal complete. -The v1.5.0 package preserves Kernel-authorized T1 Excel reconciliation and +The v1.6.0 package preserves Kernel-authorized T1 Excel reconciliation and PowerPoint/render executors plus persisted continuation checkpoints. The ordinary chat UI does not yet automatically select or sequence these two T1 tools, so this release is not represented as a one-sentence Office workflow. The existing general Office create/open/update actions remain separate. -The package also exposes a production durable Computer Use session/step +The package also preserves the production durable Computer Use session/step lifecycle through the right-rail UI and Kernel. Its exact installed reliability evidence covers generated File Explorer and Excel targets plus installed Edge using a fresh profile and an ephemeral loopback test portal. That evidence does not authorize or promise arbitrary websites, stored browser logins, production portals, secure/UAC desktops, CAPTCHA, payment, or administrator authentication. +The v1.6.0 package adds persistent automation execution with immutable +definition revisions, persisted trigger windows, scheduled and manual runs, +checkpointed leases, and restart-safe recovery before an effect begins. The +TaskCheckpoint view distinguishes safe automatic recovery from effect-started +or unknown outcomes. Idempotency, reconciliation, bounded backoff/dead-letter +state, and exact local undo prevent blind replay or false external undo claims. +Production Microsoft 365 account registration, Outlook/calendar access, and +live external writes remain disabled. + Good first tasks to try: - `根据我的证据文件夹，生成一份经营简报，并导出 HTML 和 PDF。` @@ -301,7 +310,7 @@ Screen inspection and computer control are permissioned Computer Use features. action, approval, screenshots, and semantic receipt. `ActionStarted` is persisted once before an effect; ambiguous results fail closed and are not replayed automatically. -- The v1.5.0 File Explorer, Excel, and Edge/loopback cases are installed-test +- The v1.6.0 package preserves the v1.5.0 File Explorer, Excel, and Edge/loopback installed-test evidence for generated isolated targets, not a general browser or arbitrary desktop-application support promise. - Optional local bridge routes use the configured local bridge service when a diff --git a/docs/OPEN_SOURCE_RELEASE.md b/docs/OPEN_SOURCE_RELEASE.md index 7725fe0..8963f3c 100644 --- a/docs/OPEN_SOURCE_RELEASE.md +++ b/docs/OPEN_SOURCE_RELEASE.md @@ -69,7 +69,7 @@ Ship a buildable local-first desktop Agent OS preview that demonstrates: fresh, distinct `CARGO_TARGET_DIR` builds with identical application and installer byte sizes and SHA-256 values. - The immutable `v1.0.2` installer remains unsigned. `v1.1.0`, `v1.2.0`, - `v1.3.0`, `v1.4.0`, and `v1.5.0` are explicitly disclosed unsigned exceptions: both the application executable and + `v1.3.0`, `v1.4.0`, `v1.5.0`, and `v1.6.0` are explicitly disclosed unsigned exceptions: both the application executable and NSIS installer must read back as `NotSigned`, and each Release must warn about `Unknown publisher` and Microsoft Defender SmartScreen while binding the HTTPS asset to its exact source, version, byte size, and SHA-256. A later @@ -119,7 +119,8 @@ Ship a buildable local-first desktop Agent OS preview that demonstrates: exports or packaged assets do not enter generated source archives. - `.env.example` documents local DeepSeek and optional local bridge environment variables without storing secret values. -- `docs/RELEASE_NOTES_v1.5.0.md` is the current stable release note. +- `docs/RELEASE_NOTES_v1.6.0.md` is the current stable release note. + `docs/RELEASE_NOTES_v1.5.0.md` preserves the immutable Step 5 release evidence. `docs/RELEASE_NOTES_v1.4.0.md` preserves the immutable Step 4 release evidence. `docs/RELEASE_NOTES_v1.3.0.md` preserves the immutable Step 3 release evidence. `docs/RELEASE_NOTES_v1.2.0.md` preserves the immutable Step 2 release evidence. @@ -167,7 +168,12 @@ testing, signing, and user support. approval and audit records only. - Do not claim the T1 Office engine as an ordinary-chat one-sentence workflow until the product UI can actually select, sequence, and complete it. -- Describe the v1.5.0 production-reachable durable Computer Use lifecycle +- Describe the v1.6.0 production-reachable persistent automation lifecycle + separately from future production connector work. Scheduled/manual runs, + TaskCheckpoint projections, pre-effect restart recovery, idempotency, + reconciliation, dead-letter state, and exact local undo do not prove live + Microsoft 365 or general external-write completion. +- Preserve the v1.5.0 production-reachable durable Computer Use lifecycle separately from its exact installed-test evidence. File Explorer, Excel, and Edge/ephemeral-loopback cases do not prove general browser or production-site automation. diff --git a/docs/RELEASE_NOTES_v1.6.0.md b/docs/RELEASE_NOTES_v1.6.0.md new file mode 100644 index 0000000..2212703 --- /dev/null +++ b/docs/RELEASE_NOTES_v1.6.0.md @@ -0,0 +1,103 @@ +# DS Agent v1.6.0 + +`v1.6.0` is a backward-compatible minor release that packages the Step 6 +persistent automation and restart-recovery slice. Package, desktop, Tauri, +Cargo, updater, and installer metadata are `1.6.0` / `v1.6.0`. + +## Production-reachable persistent automation + +- The installed application starts the existing Kernel-owned automation + scheduler and exposes one-time, daily, weekly, monthly, and restricted + weekday schedules through the existing Automation Center. +- Definitions and revisions are immutable. Scheduled trigger windows and + explicit manual invocations have distinct durable identities, so a manual run + cannot impersonate a scheduled window and the same scheduled window cannot be + claimed twice. +- Runs persist queued, claimed, checkpointed, review, approval, completed, + failed, and cancelled state. Worker leases, heartbeats, fences, pause/resume, + missed-run policy, and bounded retry survive application restart. +- The TaskCheckpoint view is a read-only projection over the existing Kernel + and Event Store authorities. It can show bounded restart, effect, + reconciliation, dead-letter, and undo status, but it cannot mint execution, + lease, checkpoint, effect, idempotency, recovery, evidence, or undo authority. + +## Effect, reconciliation, and undo safety + +- Every effect uses a durable idempotency identity and an explicit boundary + between prepared work and an effect that may already have started. +- Safe pre-effect work can return to the queue after an expired lease. An + effect-started or unknown outcome is never replayed automatically; it requires + explicit recovery and cannot be converted into success from model text or a + UI state. +- Known-applied and known-not-applied outcomes remain distinct. Read-only + reconciliation never receives an apply interface, and bounded backoff stops + in dead-letter state rather than retrying forever. +- Exact local undo is available only when the stored preimage, file identity, + hash, action revision, and checkpoint still match. A confirmed external + effect is not represented as locally undoable. +- Raw worker tokens, lease fences, action revisions, idempotency keys, private + evidence, provider bodies, and secrets do not enter the TaskCheckpoint UI. + +## Exact Step 6 evidence + +The deterministic Step 6 fault matrix ran exactly 100 cases. All 100 passed: + +- 95 safe pre-effect cases recovered automatically; +- five effect-started or unknown cases required explicit recovery; +- all 100 covered scheduling, pause/resume, leases, checkpoints, and + idempotency; +- 19 cases each covered known applied, known not applied, + reconciliation/dead-letter, exact local undo, and unsupported external undo; +- duplicate effects were zero and blind replays were zero. + +Harness failures, zero selected tests, incomplete denominators, and uncertain +external effects are not product success. + +## Preserved boundaries + +`v1.6.0` preserves the accepted v1.5.0 durable Computer Use lifecycle and exact +installed-test boundary, the v1.4.0 T1 Office engine, exact-task grouped +authorization, Goal completion receipts, connector and AgentRun state machines, +Event Store invariants, migration/recovery compatibility, updater integrity, +and source-only release guards. + +This release does not sign in to Microsoft 365, read live Outlook mail or +calendar data, send mail, create or modify meetings, or perform any Step 7 work. +Production Microsoft/Google account registration and live external-write +authority remain disabled. It does not implement Step 8, a production portal, +stored browser-login reuse, broad browser automation, secure/UAC desktop +control, CAPTCHA, payment or administrator authentication, Headless/ACP, or +arbitrary executable extensions. + +`docs/templates/operations-briefing-smoke-evidence` remains deterministic local +test material. The bundled smoke files are marked as +`SMOKE SAMPLE evidence for local verification only` and +`Replace before operational use`; replace them before operational use. The +desktop seed action continues to use blank operator templates under +`docs/templates/operations-briefing-evidence`. + +## Reproducible Windows package + +The final release gate compares the application and installer from two fresh, +distinct `CARGO_TARGET_DIR` builds byte-for-byte. The GitHub Release records the +exact main/tag commit, file name, byte size, SHA-256, version, and truthful +signature state, then independently downloads and re-verifies the published +asset. + +## Unsigned release + +Both `ds-agent.exe` and `DS.Agent_1.6.0_x64-setup.exe` are intentionally +Authenticode `NotSigned`; there is no signer. Windows may show `Unknown +publisher` or Microsoft Defender SmartScreen. Download only over HTTPS from the +official [`v1.6.0` GitHub Release](https://github.com/Lee-take/dsagent/releases/tag/v1.6.0) +and verify the exact byte size and SHA-256 published there before running it. + +The SignPath Foundation application remains submitted and approval is pending. +This release is not represented as signed or SignPath-approved. If signing +becomes available later, it begins with a subsequent new version and does not +replace the immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, v1.5.0, or v1.6.0 tag, +Release, or asset. + +No real API key, production account, paid API, production tenant, installed DS +Agent overwrite, current user AppData mutation, or external production target +is required for this release verification. diff --git a/package.json b/package.json index c430b00..154a90d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "deepseek-agent-os", "private": true, - "version": "1.5.0", + "version": "1.6.0", "author": "Lee take", "license": "Apache-2.0", "description": "Local-first DeepSeek AI work platform with background runs, permissioned tools, auditable evidence, and recovery.", diff --git a/scripts/release-source-check.mjs b/scripts/release-source-check.mjs index 65cf6a7..e8f3aad 100644 --- a/scripts/release-source-check.mjs +++ b/scripts/release-source-check.mjs @@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; -const expectedVersion = "1.5.0"; +const expectedVersion = "1.6.0"; const maxSourceFileBytes = 2 * 1024 * 1024; const binaryReleaseExtensions = new Set([ ".appimage", @@ -52,6 +52,7 @@ const requiredDocs = [ "CODE_SIGNING_POLICY.md", "PRIVACY.md", "docs/INSTALLATION.md", + "docs/RELEASE_NOTES_v1.6.0.md", "docs/RELEASE_NOTES_v1.5.0.md", "docs/RELEASE_NOTES_v1.4.0.md", "docs/RELEASE_NOTES_v1.3.0.md", @@ -90,6 +91,7 @@ const publicReleaseCopyFiles = [ "apps/desktop/package.json", "docs/INSTALLATION.md", "docs/OPEN_SOURCE_RELEASE.md", + "docs/RELEASE_NOTES_v1.6.0.md", "docs/RELEASE_NOTES_v1.5.0.md", "docs/RELEASE_NOTES_v1.4.0.md", "docs/RELEASE_NOTES_v1.3.0.md", @@ -736,7 +738,7 @@ function checkRequiredDocs() { for (const [phrase, label] of [ ["One Kernel. Modular capabilities. Verifiable execution.", "English README product promise"], ["DS Agent is a local Agent Harness optimized for DeepSeek", "English README DeepSeek-first positioning"], - ["v1.5.0 stable", "English README formal stable status"], + ["v1.6.0 stable", "English README formal stable status"], ["The key difference is that Memory, Automation, Computer Use, parallel Subagents, and Skills are not isolated plugins", "English README unified Kernel thesis"], ["contract-first modular Harness architecture", "English README modular architecture"], ["Five core capabilities, one engineering philosophy", "English README five-capability framing"], @@ -748,7 +750,9 @@ function checkRequiredDocs() { ["goal → done-when contract → context → plan → permission → execution → evidence → verification → recovery", "English README Loop Engineering contract"], ["DeepSeek and DS Agent boundary", "English README model boundary"], ["Why Rust", "English README Rust rationale"], - ["Production Microsoft/Google account registration and live external-write authority remain disabled in v1.5.0", "English README connector authority boundary"], + ["Production Microsoft/Google account registration and live external-write authority remain disabled in v1.6.0", "English README connector authority boundary"], + ["production-reachable persistent automation with immutable definition revisions", "English README durable automation reachability"], + ["Effect-started or unknown work is never replayed blindly and requires explicit recovery", "English README effect recovery boundary"], ["Exact installed evidence is limited to generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal", "English README exact Computer Use evidence boundary"], ["Approval creates only exact authority; it does not execute a Tool, resume the task, or mark the Goal complete", "English README grouped authorization non-execution boundary"], ["ordinary chat does not yet automatically select or sequence the two T1 tools", "English README T1 reachability boundary"], @@ -760,7 +764,7 @@ function checkRequiredDocs() { for (const [phrase, label] of [ ["一个 Kernel，模块化能力，统一可信执行。", "Chinese README product promise"], ["DS Agent 是专门为 DeepSeek 优化的本地 Agent Harness", "Chinese README DeepSeek-first positioning"], - ["v1.5.0 正式稳定版", "Chinese README formal stable status"], + ["v1.6.0 正式稳定版", "Chinese README formal stable status"], ["真正的差异点是：记忆、自动化执行、Computer Use、Subagent 并行协作和技能", "Chinese README unified Kernel thesis"], ["契约优先的模块化 Harness 架构", "Chinese README modular architecture"], ["五项核心能力，一套工程理念", "Chinese README five-capability framing"], @@ -772,7 +776,9 @@ function checkRequiredDocs() { ["目标 → 完成条件 → 上下文 → 规划 → 权限 → 执行 → 证据 → 验证 → 恢复", "Chinese README Loop Engineering contract"], ["DeepSeek 与 DS Agent 的工作边界", "Chinese README model boundary"], ["为什么使用 Rust", "Chinese README Rust rationale"], - ["v1.5.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限", "Chinese README connector authority boundary"], + ["v1.6.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限", "Chinese README connector authority boundary"], + ["新增生产可达的持久自动化执行", "Chinese README durable automation reachability"], + ["effect 已开始或结果未知时绝不会盲目重放", "Chinese README effect recovery boundary"], ["准确安装态证据只覆盖生成的 File Explorer、Excel 目标，以及使用全新 profile 和临时回环测试站点的已安装 Edge", "Chinese README exact Computer Use evidence boundary"], ["批准只产生精确权限，不会执行 Tool、恢复任务或把 Goal 标记为完成", "Chinese README grouped authorization non-execution boundary"], ["普通聊天目前还不会自动选择和串联这两个 T1 工具", "Chinese README T1 reachability boundary"], @@ -1623,14 +1629,14 @@ function checkPublicReleaseCopyPositioning() { checkTextIncludes( "apps/desktop/src-tauri/src/kernel/app_update.rs", readText("apps/desktop/src-tauri/src/kernel/app_update.rs"), - 'APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.5.0"', - "app updater User-Agent v1.5.0", + 'APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.6.0"', + "app updater User-Agent v1.6.0", ); checkTextIncludes( "apps/desktop/src-tauri/src/kernel/app_update.rs", readText("apps/desktop/src-tauri/src/kernel/app_update.rs"), - 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.5.0"', - "app updater current release tag v1.5.0", + 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.6.0"', + "app updater current release tag v1.6.0", ); checkTextDoesNotInclude( "apps/desktop/src-tauri/src/kernel/app_update.rs", @@ -1638,6 +1644,48 @@ function checkPublicReleaseCopyPositioning() { 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v0.9.0"', "app updater current release tag must not regress to v0.9.0", ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "Package, desktop, Tauri, Cargo, updater, and installer metadata are `1.6.0` / `v1.6.0`.", + "v1.6.0 stable release notes version identity", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "Both `ds-agent.exe` and `DS.Agent_1.6.0_x64-setup.exe` are intentionally Authenticode `NotSigned`", + "v1.6.0 dual-artifact unsigned disclosure", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "The TaskCheckpoint view is a read-only projection over the existing Kernel and Event Store authorities", + "v1.6.0 TaskCheckpoint authority boundary", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "An effect-started or unknown outcome is never replayed automatically", + "v1.6.0 effect recovery boundary", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "The deterministic Step 6 fault matrix ran exactly 100 cases. All 100 passed", + "v1.6.0 Step 6 deterministic denominator", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "This release does not sign in to Microsoft 365, read live Outlook mail or calendar data, send mail, create or modify meetings, or perform any Step 7 work", + "v1.6.0 Step 7 exclusion", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), + "It does not implement Step 8", + "v1.6.0 Step 8 exclusion", + ); checkTextIncludesCollapsed( "docs/RELEASE_NOTES_v1.5.0.md", readText("docs/RELEASE_NOTES_v1.5.0.md"), @@ -3533,7 +3581,7 @@ function checkGovernanceDocs() { checkTextIncludesCollapsed( "SECURITY.md", securityPolicy, - "DS Agent v1.5.0 is the current published stable release", + "DS Agent v1.6.0 is the current published stable release", "SECURITY.md current stable scope", ); checkTextIncludes( @@ -3563,7 +3611,7 @@ function checkGovernanceDocs() { checkTextIncludesCollapsed( "SECURITY.md", securityPolicy, - "Current stable v1.5.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault", + "Current stable v1.6.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault", "SECURITY.md current stable credential boundary", ); checkTextIncludesCollapsed( @@ -3624,7 +3672,7 @@ function checkCodeSigningAndPrivacyPolicies() { for (const [phrase, label] of [ ["# Code signing policy", "code-signing policy title"], - ["DS Agent `v1.5.0` is intentionally published unsigned", "code-signing current unsigned status"], + ["DS Agent `v1.6.0` is intentionally published unsigned", "code-signing current unsigned status"], ["The SignPath Foundation application is submitted and approval is pending", "code-signing pending application status"], ["Free code signing provided by", "SignPath acknowledgement"], ["certificate by", "SignPath certificate acknowledgement"], @@ -3647,7 +3695,7 @@ function checkCodeSigningAndPrivacyPolicies() { for (const [phrase, label] of [ ["# Privacy Policy", "privacy policy title"], ["does not operate a project cloud backend, advertising service, or project analytics or telemetry service", "no project telemetry service"], - ["The current stable `v1.5.0` accepts one user-supplied DeepSeek API key", "privacy current credential behavior"], + ["The current stable `v1.6.0` accepts one user-supplied DeepSeek API key", "privacy current credential behavior"], ["dedicated Windows DPAPI-protected local vault", "privacy DPAPI storage boundary"], ["https://cdn.deepseek.com/policies/en-US/deepseek-privacy-policy.html", "DeepSeek privacy disclosure"], ["DuckDuckGo's privacy policy", "web-search privacy disclosure"], @@ -3655,7 +3703,7 @@ function checkCodeSigningAndPrivacyPolicies() { ["Hugging Face", "skill-source privacy disclosure"], ["WebView2 data and privacy documentation", "WebView2 privacy disclosure"], ["accepts only loopback addresses", "local bridge privacy boundary"], - ["Production Microsoft and Google account registration and live mail/calendar writes are disabled in `v1.5.0`", "disabled connector privacy boundary"], + ["Production Microsoft and Google account registration and live mail/calendar writes are disabled in `v1.6.0`", "disabled connector privacy boundary"], ["not uploaded merely because the application is open", "no passive content upload"], ]) { checkTextIncludesCollapsed("PRIVACY.md", privacyPolicy, phrase, label); @@ -3668,7 +3716,7 @@ function checkCodeSigningAndPrivacyPolicies() { checkTextIncludes(filePath, content, "NotSigned", `${filePath} unsigned Authenticode disclosure`); checkTextIncludes(filePath, content, "Unknown publisher", `${filePath} unknown-publisher warning`); checkTextIncludes(filePath, content, "SmartScreen", `${filePath} SmartScreen warning`); - checkTextIncludes(filePath, content, "docs/RELEASE_NOTES_v1.5.0.md", `${filePath} v1.5.0 release-notes link`); + checkTextIncludes(filePath, content, "docs/RELEASE_NOTES_v1.6.0.md", `${filePath} v1.6.0 release-notes link`); checkTextDoesNotInclude(filePath, content, "12,716,857 bytes", `${filePath} no stale v1.0.1 asset size`); checkTextDoesNotInclude(filePath, content, "469C4EFA54F4C94A6E37D28C9C88D331B26E1770C6792DC93D02B451640E2A6F", `${filePath} no stale v1.0.1 asset SHA-256`); } @@ -4278,10 +4326,10 @@ function checkReproducibleWindowsReleaseBuild() { "open-source release requires byte-reproducible Windows candidates", ); checkTextIncludesCollapsed( - "docs/RELEASE_NOTES_v1.5.0.md", - readText("docs/RELEASE_NOTES_v1.5.0.md"), + "docs/RELEASE_NOTES_v1.6.0.md", + readText("docs/RELEASE_NOTES_v1.6.0.md"), "The final release gate compares the application and installer from two fresh, distinct `CARGO_TARGET_DIR` builds byte-for-byte", - "v1.5.0 release notes disclose reproducibility gate", + "v1.6.0 release notes disclose reproducibility gate", ); } @@ -4420,7 +4468,7 @@ function checkSmokeScriptReleaseLabels() { const rustRuntimeExpectations = [ [ "apps/desktop/src-tauri/src/kernel/deepseek.rs", - "DS-Agent/1.5.0 deepseek-v4", + "DS-Agent/1.6.0 deepseek-v4", "DeepSeek runtime User-Agent release label", ], [