From 8e43ef4cd51eeb1236fce64e36fe6fe01a8d37f6 Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 24 Jul 2026 21:11:07 +0800 Subject: [PATCH 1/5] Implement C5A computer use action contract --- apps/desktop/src-tauri/src/commands.rs | 59 +-- .../src/kernel/computer_use_runtime.rs | 430 ++++++++++++++++-- .../src/kernel/computer_use_session.rs | 422 ++++++++++++++--- .../src/kernel/event_store/computer_use.rs | 80 +++- 4 files changed, 850 insertions(+), 141 deletions(-) diff --git a/apps/desktop/src-tauri/src/commands.rs b/apps/desktop/src-tauri/src/commands.rs index 5d49623..a98ee02 100644 --- a/apps/desktop/src-tauri/src/commands.rs +++ b/apps/desktop/src-tauri/src/commands.rs @@ -58,15 +58,15 @@ use crate::kernel::computer_use::{ computer_use_backend_status_for_strategy, ComputerUseBackendStatus, }; use crate::kernel::computer_use_runtime::{ - accessibility_value_semantic_fingerprint, execute_ready_computer_use_step, + accessibility_value_semantic_fingerprint, approve_computer_use_step, bind_computer_use_action, + execute_ready_computer_use_step, persist_observed_computer_use_session, take_over_computer_use_step, ComputerUseAccessibilityClient, ComputerUseExecutionPermit, ComputerUseSessionView, ComputerUseStepView, LocalComputerUseAccessibilityClient, RedactedComputerUseState, }; use crate::kernel::computer_use_session::{ - ComputerUseActionBinding, ComputerUseObservation, ComputerUseObservationPhase, - ComputerUsePostcondition, ComputerUseSession, ComputerUseStep, ComputerUseStepStatus, - ComputerUseUndoCapability, + ComputerUseApprovalActor, ComputerUseObservation, ComputerUseObservationPhase, + ComputerUsePostcondition, ComputerUseStep, ComputerUseStepStatus, ComputerUseUndoCapability, }; use crate::kernel::connectors::reconciliation::ConnectorReconcilerRegistry; use crate::kernel::connectors::runtime_registry::{ @@ -15020,8 +15020,11 @@ fn computer_use_observation_from_redacted_state( state.validate()?; ComputerUseObservation::new( phase, + state.application_fingerprint, + state.process_fingerprint, state.window_fingerprint, state.window_title_fingerprint, + state.frame_fingerprint, Some(state.target_fingerprint), state.semantic_fingerprint, screenshot_evidence_ref, @@ -15096,19 +15099,16 @@ pub fn start_durable_computer_use_session( evidence_ref, screenshot_captured_at, )?; - let now = observation.captured_at; - let mut session = ComputerUseSession::new(run_id, safe_goal_summary, now)?; - let step = ComputerUseStep::new_observed(session.id, 1, observation, undo_capability, now)?; - { + let (session, step) = { let store = state.event_store.lock().map_err(|_| lock_error())?; - store - .insert_computer_use_session(&session) - .map_err(event_store_error)?; - store - .insert_computer_use_step(&step) - .map_err(event_store_error)?; - } - session.activate_step(step.id, now)?; + persist_observed_computer_use_session( + &store, + run_id, + safe_goal_summary, + undo_capability, + observation, + )? + }; Ok(ComputerUseSessionStartResult { session: ComputerUseSessionView::from(&session), step: ComputerUseStepView::from(&step), @@ -15170,16 +15170,7 @@ pub fn bind_durable_computer_use_action( let action = parse_computer_control_action(&action_contract)?; let postcondition = computer_use_postcondition_from_request(postcondition)?; let store = state.event_store.lock().map_err(|_| lock_error())?; - let mut step = store - .get_computer_use_step(step_id) - .map_err(event_store_error)?; - let expected_revision = step.revision; - let binding = - ComputerUseActionBinding::new(&step.pre_observation, action, safe_summary, postcondition)?; - step.bind_action(binding, Utc::now())?; - store - .update_computer_use_step(&step, expected_revision) - .map_err(event_store_error)?; + let step = bind_computer_use_action(&store, step_id, action, safe_summary, postcondition)?; Ok(ComputerUseStepView::from(&step)) } @@ -15231,18 +15222,13 @@ pub fn run_durable_computer_use_step( .ok_or_else(|| "computer use step has no exact action".to_string())?; { let store = state.event_store.lock().map_err(|_| lock_error())?; - let mut current = store - .get_computer_use_step(step_id) - .map_err(event_store_error)?; - let expected_revision = current.revision; - current.approve( + approve_computer_use_step( + &store, + step_id, approval_request_id, &approved_action_fingerprint, - Utc::now(), + ComputerUseApprovalActor::User, )?; - store - .update_computer_use_step(¤t, expected_revision) - .map_err(event_store_error)?; } let app_data_dir = app.resolved_app_data_dir()?; @@ -20334,8 +20320,11 @@ mod tests { let now = Utc::now(); let pre = ComputerUseObservation::new( ComputerUseObservationPhase::PreAction, + "f".repeat(64), + "d".repeat(64), "a".repeat(64), "e".repeat(64), + "9".repeat(64), Some("b".repeat(64)), Some("c".repeat(64)), "computer-screenshots/pre.png".to_string(), diff --git a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs index ac6710f..d7c943d 100644 --- a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs +++ b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs @@ -4,25 +4,28 @@ use sha2::{Digest, Sha256}; use std::sync::Mutex; use uuid::Uuid; -#[cfg(test)] -use crate::kernel::capability::ComputerControlAction; -use crate::kernel::capability::{ComputerControlClient, ComputerScreenshotClient}; -#[cfg(test)] -use crate::kernel::computer_use_session::ComputerUseActionBinding; +use crate::kernel::capability::{ + ComputerControlAction, ComputerControlClient, ComputerScreenshotClient, +}; use crate::kernel::computer_use_session::{ - ComputerUseObservation, ComputerUseObservationPhase, ComputerUsePostcondition, - ComputerUseSession, ComputerUseStep, ComputerUseStepStatus, ComputerUseUndoCapability, - ComputerUseVerificationOutcome, ComputerUseVerificationReceipt, + ComputerUseActionBinding, ComputerUseApprovalActor, ComputerUseObservation, + ComputerUseObservationPhase, ComputerUsePostcondition, ComputerUseSession, ComputerUseStep, + ComputerUseStepStatus, ComputerUseUndoCapability, ComputerUseVerificationOutcome, + ComputerUseVerificationReceipt, }; use crate::kernel::event_store::EventStore; const MAX_REDACTED_SUMMARY_CHARS: usize = 1_000; const MAX_SEMANTIC_VALUE_CHARS: usize = 32_768; +pub const COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS: usize = 2; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct RedactedComputerUseState { + pub application_fingerprint: String, + pub process_fingerprint: String, pub window_fingerprint: String, pub window_title_fingerprint: String, + pub frame_fingerprint: String, pub target_fingerprint: String, pub semantic_fingerprint: Option, pub safe_summary: String, @@ -30,8 +33,11 @@ pub struct RedactedComputerUseState { impl RedactedComputerUseState { pub fn validate(&self) -> Result<(), String> { + require_fingerprint(&self.application_fingerprint, "application fingerprint")?; + require_fingerprint(&self.process_fingerprint, "process fingerprint")?; require_fingerprint(&self.window_fingerprint, "window fingerprint")?; require_fingerprint(&self.window_title_fingerprint, "window title fingerprint")?; + require_fingerprint(&self.frame_fingerprint, "frame fingerprint")?; require_fingerprint(&self.target_fingerprint, "target fingerprint")?; if let Some(value) = self.semantic_fingerprint.as_deref() { require_fingerprint(value, "semantic fingerprint")?; @@ -126,7 +132,10 @@ pub struct ComputerUseStepView { pub status: ComputerUseStepStatus, pub revision: u64, pub pre_observation_fingerprint: String, + pub application_fingerprint: String, + pub process_fingerprint: String, pub window_fingerprint: String, + pub frame_fingerprint: String, pub target_fingerprint: Option, pub pre_semantic_fingerprint: Option, pub pre_screenshot_evidence_ref: String, @@ -135,6 +144,8 @@ pub struct ComputerUseStepView { pub action_safe_summary: Option, pub action_fingerprint: Option, pub approval_request_id: Option, + pub approval_actor: Option, + pub observation_valid_until: DateTime, pub post_observation_fingerprint: Option, pub post_semantic_fingerprint: Option, pub post_screenshot_evidence_ref: Option, @@ -172,7 +183,10 @@ impl From<&ComputerUseStep> for ComputerUseStepView { status: step.status, revision: step.revision, pre_observation_fingerprint: step.pre_observation.fingerprint.clone(), + application_fingerprint: step.pre_observation.application_fingerprint.clone(), + process_fingerprint: step.pre_observation.process_fingerprint.clone(), window_fingerprint: step.pre_observation.window_fingerprint.clone(), + frame_fingerprint: step.pre_observation.frame_fingerprint.clone(), target_fingerprint: step.pre_observation.target_fingerprint.clone(), pre_semantic_fingerprint: step.pre_observation.semantic_fingerprint.clone(), pre_screenshot_evidence_ref: step.pre_observation.screenshot_evidence_ref.clone(), @@ -181,6 +195,8 @@ impl From<&ComputerUseStep> for ComputerUseStepView { action_safe_summary: action.map(|value| value.safe_summary.clone()), action_fingerprint: action.map(|value| value.action_fingerprint.clone()), approval_request_id: step.approval_request_id, + approval_actor: step.approval_actor, + observation_valid_until: step.pre_observation.valid_until, post_observation_fingerprint: post.map(|value| value.fingerprint.clone()), post_semantic_fingerprint: post.and_then(|value| value.semantic_fingerprint.clone()), post_screenshot_evidence_ref: post.map(|value| value.screenshot_evidence_ref.clone()), @@ -194,20 +210,18 @@ impl From<&ComputerUseStep> for ComputerUseStepView { } } -#[cfg(test)] -pub fn create_observed_computer_use_session( +pub fn persist_observed_computer_use_session( store: &EventStore, run_id: Option, safe_goal_summary: String, undo_capability: ComputerUseUndoCapability, - screenshot_client: &impl ComputerScreenshotClient, - accessibility_client: &impl ComputerUseAccessibilityClient, + observation: ComputerUseObservation, ) -> Result<(ComputerUseSession, ComputerUseStep), String> { - let observation = capture_computer_use_observation( - ComputerUseObservationPhase::PreAction, - screenshot_client, - accessibility_client, - )?; + observation.validate()?; + observation.require_fresh_at(Utc::now())?; + if observation.phase != ComputerUseObservationPhase::PreAction { + return Err("computer use session requires a pre-action observation".to_string()); + } let now = observation.captured_at; let mut session = ComputerUseSession::new(run_id, safe_goal_summary, now)?; let step = ComputerUseStep::new_observed(session.id, 1, observation, undo_capability, now)?; @@ -221,7 +235,6 @@ pub fn create_observed_computer_use_session( Ok((session, step)) } -#[cfg(test)] pub fn bind_computer_use_action( store: &EventStore, step_id: Uuid, @@ -242,18 +255,33 @@ pub fn bind_computer_use_action( Ok(step) } -#[cfg(test)] pub fn approve_computer_use_step( store: &EventStore, step_id: Uuid, approval_request_id: Uuid, approved_action_fingerprint: &str, + actor: ComputerUseApprovalActor, ) -> Result { let mut step = store .get_computer_use_step(step_id) .map_err(|error| error.to_string())?; let expected_revision = step.revision; - step.approve(approval_request_id, approved_action_fingerprint, Utc::now())?; + let now = Utc::now(); + if step.pre_observation.require_fresh_at(now).is_err() { + step.require_replan( + "The approved observation expired before authority could be bound; re-observation and a new local-user approval are required." + .to_string(), + now, + )?; + store + .update_computer_use_step(&step, expected_revision) + .map_err(|error| error.to_string())?; + return Err( + "computer use observation expired before approval; the step now requires re-observation" + .to_string(), + ); + } + step.approve(approval_request_id, approved_action_fingerprint, actor, now)?; store .update_computer_use_step(&step, expected_revision) .map_err(|error| error.to_string())?; @@ -283,6 +311,26 @@ pub fn execute_ready_computer_use_step( screenshot_client: &impl ComputerScreenshotClient, accessibility_client: &impl ComputerUseAccessibilityClient, control_client: &impl ComputerControlClient, +) -> Result { + execute_ready_computer_use_step_at( + store, + step_id, + permit, + screenshot_client, + accessibility_client, + control_client, + Utc::now(), + ) +} + +fn execute_ready_computer_use_step_at( + store: &impl ComputerUseStepPersistence, + step_id: Uuid, + permit: ComputerUseExecutionPermit, + screenshot_client: &impl ComputerScreenshotClient, + accessibility_client: &impl ComputerUseAccessibilityClient, + control_client: &impl ComputerControlClient, + now: DateTime, ) -> Result { if !permit.local_unlock_confirmed { return Err("computer use execution requires an active local unlock".to_string()); @@ -292,20 +340,41 @@ pub fn execute_ready_computer_use_step( } let mut step = store.load_step(step_id)?; + step.validate()?; if step.status != ComputerUseStepStatus::Ready { return Err(format!( "computer use step in {:?} is not ready for execution", step.status )); } + if step.pre_observation.require_fresh_at(now).is_err() { + let expected_revision = step.revision; + step.require_replan( + "The approved desktop observation expired before execution; re-observation and a new local-user approval are required." + .to_string(), + now, + )?; + store.persist_step(&step, expected_revision)?; + return Ok(ComputerUseExecutionResult { + step, + execution_summary: None, + safe_error: Some( + "Desktop observation expired before execution; no input action was sent." + .to_string(), + ), + }); + } let action = step .action .clone() .ok_or_else(|| "computer use ready step has no exact action".to_string())?; let current = accessibility_client.capture_redacted_state()?; current.validate()?; - if current.window_fingerprint != action.window_fingerprint + if current.application_fingerprint != action.application_fingerprint + || current.process_fingerprint != action.process_fingerprint + || current.window_fingerprint != action.window_fingerprint || current.window_title_fingerprint != action.pre_window_title_fingerprint + || current.frame_fingerprint != action.frame_fingerprint || current.target_fingerprint != action.target_fingerprint || current.semantic_fingerprint != step.pre_observation.semantic_fingerprint { @@ -328,10 +397,13 @@ pub fn execute_ready_computer_use_step( let expected_revision = step.revision; step.mark_action_started( permit.approval_request_id, + ¤t.application_fingerprint, + ¤t.process_fingerprint, ¤t.window_fingerprint, ¤t.window_title_fingerprint, + ¤t.frame_fingerprint, ¤t.target_fingerprint, - Utc::now(), + now, )?; store.persist_step(&step, expected_revision)?; @@ -428,14 +500,36 @@ pub fn capture_computer_use_observation( phase: ComputerUseObservationPhase, screenshot_client: &impl ComputerScreenshotClient, accessibility_client: &impl ComputerUseAccessibilityClient, +) -> Result { + let mut last_error = None; + for _ in 0..COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS { + match capture_computer_use_observation_once(phase, screenshot_client, accessibility_client) + { + Ok(observation) => return Ok(observation), + Err(error) => last_error = Some(error), + } + } + Err(format!( + "computer use observation failed after {COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS} bounded attempts: {}", + last_error.unwrap_or_else(|| "no observation receipt was returned".to_string()) + )) +} + +fn capture_computer_use_observation_once( + phase: ComputerUseObservationPhase, + screenshot_client: &impl ComputerScreenshotClient, + accessibility_client: &impl ComputerUseAccessibilityClient, ) -> Result { let screenshot = screenshot_client.capture_screenshot()?; let state = accessibility_client.capture_redacted_state()?; state.validate()?; ComputerUseObservation::new( phase, + state.application_fingerprint, + state.process_fingerprint, state.window_fingerprint, state.window_title_fingerprint, + state.frame_fingerprint, Some(state.target_fingerprint), state.semantic_fingerprint, screenshot.evidence_ref, @@ -635,6 +729,7 @@ fn capture_windows_redacted_state() -> Result let handle_identity = format!("{:p}", hwnd.0); let process_text = process_id.to_string(); let thread_text = thread_id.to_string(); + let process_fingerprint = fingerprint_parts(&["windows-process/v1", process_text.as_str()]); let window_fingerprint = fingerprint_parts(&[ "windows-foreground-window/v1", &handle_identity, @@ -699,6 +794,19 @@ fn capture_windows_redacted_state() -> Result "not_keyboard_focusable" }, ]); + let application_fingerprint = fingerprint_parts(&[ + "windows-application/v1", + &fingerprint_parts(&[&window_class]), + &fingerprint_parts(&[&target_class]), + ]); + let frame_fingerprint = fingerprint_parts(&[ + "windows-accessibility-frame/v1", + &process_fingerprint, + &window_fingerprint, + &control_type_text, + &fingerprint_parts(&[&automation_id]), + &fingerprint_parts(&[&target_class]), + ]); let semantic_fingerprint = if is_password { None @@ -739,8 +847,11 @@ fn capture_windows_redacted_state() -> Result "semantic state unavailable" }; Ok(RedactedComputerUseState { + application_fingerprint, + process_fingerprint, window_fingerprint, window_title_fingerprint, + frame_fingerprint, target_fingerprint, semantic_fingerprint, safe_summary: format!( @@ -779,6 +890,12 @@ mod tests { states: Mutex::new(states.into_iter().map(Ok).collect()), } } + + fn with_results(states: Vec>) -> Self { + Self { + states: Mutex::new(states.into()), + } + } } impl ComputerUseAccessibilityClient for FakeAccessibilityClient { @@ -829,6 +946,41 @@ mod tests { fail: bool, } + struct InMemoryStepPersistence { + step: Mutex, + } + + impl InMemoryStepPersistence { + fn new(step: ComputerUseStep) -> Self { + Self { + step: Mutex::new(step), + } + } + } + + impl ComputerUseStepPersistence for InMemoryStepPersistence { + fn load_step(&self, step_id: Uuid) -> Result { + let step = self.step.lock().unwrap().clone(); + if step.id != step_id { + return Err("in-memory computer use step does not exist".to_string()); + } + Ok(step) + } + + fn persist_step( + &self, + step: &ComputerUseStep, + expected_revision: u64, + ) -> Result<(), String> { + let mut current = self.step.lock().unwrap(); + if current.id != step.id || current.revision != expected_revision { + return Err("in-memory computer use step changed concurrently".to_string()); + } + *current = step.clone(); + Ok(()) + } + } + impl FakeControlClient { fn succeeding() -> Self { Self { @@ -875,10 +1027,34 @@ mod tests { title: &str, target: &str, semantic: Option<&str>, + ) -> RedactedComputerUseState { + redacted_state_with_identity( + "application", + "process", + window, + title, + "frame", + target, + semantic, + ) + } + + #[allow(clippy::too_many_arguments)] + fn redacted_state_with_identity( + application: &str, + process: &str, + window: &str, + title: &str, + frame: &str, + target: &str, + semantic: Option<&str>, ) -> RedactedComputerUseState { RedactedComputerUseState { + application_fingerprint: fingerprint_parts(&[application]), + process_fingerprint: fingerprint_parts(&[process]), window_fingerprint: fingerprint_parts(&[window]), window_title_fingerprint: fingerprint_parts(&[title]), + frame_fingerprint: fingerprint_parts(&[frame]), target_fingerprint: fingerprint_parts(&[target]), semantic_fingerprint: semantic.map(|value| fingerprint_parts(&[value])), safe_summary: "Isolated Notepad-like editor is foreground and focused.".to_string(), @@ -891,13 +1067,18 @@ mod tests { accessibility: &FakeAccessibilityClient, expected_semantic: &str, ) -> (ComputerUseStep, Uuid) { - let (_, observed) = create_observed_computer_use_session( + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + screenshots, + accessibility, + ) + .unwrap(); + let (_, observed) = persist_observed_computer_use_session( store, None, "Update an isolated Notepad-like editor.".to_string(), ComputerUseUndoCapability::None, - screenshots, - accessibility, + observation, ) .unwrap(); let bound = bind_computer_use_action( @@ -918,11 +1099,199 @@ mod tests { bound.id, approval_id, &bound.action.as_ref().unwrap().action_fingerprint, + ComputerUseApprovalActor::User, ) .unwrap(); (ready, approval_id) } + #[test] + fn observation_retry_is_bounded_and_never_retries_an_action() { + let screenshots = FakeScreenshotClient::new(COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS); + let accessibility = FakeAccessibilityClient::with_results(vec![ + Err("transient accessibility failure".to_string()), + Ok(redacted_state("window", "target", Some("before"))), + ]); + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshots, + &accessibility, + ) + .expect("the second observation attempt succeeds"); + assert_eq!( + observation.screenshot_evidence_ref, + "computer-screenshots/fake-2.png" + ); + + let screenshots = FakeScreenshotClient::new(COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS); + let accessibility = FakeAccessibilityClient::with_results(vec![ + Err("first failure".to_string()), + Err("second failure".to_string()), + ]); + let error = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshots, + &accessibility, + ) + .expect_err("observation stops at the fixed retry bound"); + assert!(error.contains("failed after 2 bounded attempts")); + } + + #[test] + fn wrong_application_process_window_or_frame_stops_before_input() { + let cases = [ + ( + "application", + redacted_state_with_identity( + "changed-application", + "process", + "window", + "stable-title", + "frame", + "target", + Some("before"), + ), + ), + ( + "process", + redacted_state_with_identity( + "application", + "changed-process", + "window", + "stable-title", + "frame", + "target", + Some("before"), + ), + ), + ( + "window", + redacted_state_with_identity( + "application", + "process", + "changed-window", + "stable-title", + "frame", + "target", + Some("before"), + ), + ), + ( + "frame", + redacted_state_with_identity( + "application", + "process", + "window", + "stable-title", + "changed-frame", + "target", + Some("before"), + ), + ), + ]; + + for (label, drifted) in cases { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state("window", "target", Some("before")), + drifted, + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!( + result.step.status, + ComputerUseStepStatus::NeedsReplan, + "{label} drift must require replanning" + ); + assert!(result.step.approval_request_id.is_none()); + assert!(result.step.approval_actor.is_none()); + assert_eq!( + control.calls.load(Ordering::SeqCst), + 0, + "{label} drift must stop before input" + ); + } + } + + #[test] + fn stale_approved_observation_requires_replan_before_revalidation() { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = + FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = setup_ready_step(&store, &screenshots, &accessibility, "after"); + let expired_at = ready.pre_observation.valid_until + chrono::Duration::milliseconds(1); + + let result = execute_ready_computer_use_step_at( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + expired_at, + ) + .unwrap(); + + assert_eq!(result.step.status, ComputerUseStepStatus::NeedsReplan); + assert!(result.step.approval_request_id.is_none()); + assert!(result.step.approval_actor.is_none()); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn action_mutation_after_approval_is_rejected_before_input() { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = + FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); + let (mut ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + ready.action.as_mut().unwrap().action = ComputerControlAction::PressKey { + key: "ENTER".to_string(), + }; + let persistence = InMemoryStepPersistence::new(ready.clone()); + let no_observation = FakeAccessibilityClient::new(Vec::new()); + let no_screenshot = FakeScreenshotClient::new(0); + let control = FakeControlClient::succeeding(); + + let error = execute_ready_computer_use_step( + &persistence, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &no_screenshot, + &no_observation, + &control, + ) + .expect_err("mutated approved action must fail domain validation"); + + assert!(error.contains("action fingerprint is inconsistent")); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + #[test] fn durable_vertical_step_observes_acts_once_and_verifies() { let directory = tempdir().unwrap(); @@ -1369,13 +1738,17 @@ $form.Add_Shown({ [void]$editor.Focus() }) let store = EventStore::open(directory.path().join("notepad-like-smoke.db")) .map_err(|error| error.to_string())?; let screenshot = LocalComputerScreenshotClient::new(directory.path().to_path_buf()); - let (_, observed) = create_observed_computer_use_session( + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshot, + &accessibility, + )?; + let (_, observed) = persist_observed_computer_use_session( &store, None, "Verify one isolated Notepad-like editor action.".to_string(), ComputerUseUndoCapability::None, - &screenshot, - &accessibility, + observation, )?; if observed.pre_observation.semantic_fingerprint.is_none() { return Err( @@ -1401,6 +1774,7 @@ $form.Add_Shown({ [void]$editor.Focus() }) .as_ref() .ok_or_else(|| "smoke action is missing".to_string())? .action_fingerprint, + ComputerUseApprovalActor::User, )?; let control = LocalComputerControlClient::new(); let result = execute_ready_computer_use_step( diff --git a/apps/desktop/src-tauri/src/kernel/computer_use_session.rs b/apps/desktop/src-tauri/src/kernel/computer_use_session.rs index 8a03f43..96a897e 100644 --- a/apps/desktop/src-tauri/src/kernel/computer_use_session.rs +++ b/apps/desktop/src-tauri/src/kernel/computer_use_session.rs @@ -1,11 +1,12 @@ -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Duration, SecondsFormat, Utc}; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use uuid::Uuid; use crate::kernel::capability::ComputerControlAction; -pub const COMPUTER_USE_STEP_CONTRACT_VERSION: &str = "computer-use-step/v1"; +pub const COMPUTER_USE_STEP_CONTRACT_VERSION: &str = "computer-use-step/v2"; +pub const COMPUTER_USE_OBSERVATION_FRESHNESS_SECS: i64 = 120; const MAX_SAFE_SUMMARY_CHARS: usize = 1_000; #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -53,6 +54,15 @@ pub enum ComputerUseVerificationOutcome { Failed, } +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ComputerUseApprovalActor { + User, + KernelLifecycle, + DeepSeekModel, + FrontendPayload, +} + #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] pub struct ComputerUseRecoverySweep { pub needs_replan: usize, @@ -78,13 +88,17 @@ pub struct ComputerUseObservation { pub id: Uuid, pub phase: ComputerUseObservationPhase, pub fingerprint: String, + pub application_fingerprint: String, + pub process_fingerprint: String, pub window_fingerprint: String, pub window_title_fingerprint: String, + pub frame_fingerprint: String, pub target_fingerprint: Option, pub semantic_fingerprint: Option, pub screenshot_evidence_ref: String, pub safe_summary: String, pub captured_at: DateTime, + pub valid_until: DateTime, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -92,8 +106,11 @@ pub struct ComputerUseActionBinding { pub action: ComputerControlAction, pub safe_summary: String, pub pre_observation_fingerprint: String, + pub application_fingerprint: String, + pub process_fingerprint: String, pub window_fingerprint: String, pub pre_window_title_fingerprint: String, + pub frame_fingerprint: String, pub target_fingerprint: String, pub pre_semantic_fingerprint: Option, pub postcondition: ComputerUsePostcondition, @@ -129,6 +146,7 @@ pub struct ComputerUseStep { pub pre_observation: ComputerUseObservation, pub action: Option, pub approval_request_id: Option, + pub approval_actor: Option, pub action_started_at: Option>, pub action_start_count: u32, pub post_observation: Option, @@ -186,16 +204,22 @@ impl ComputerUseObservation { #[allow(clippy::too_many_arguments)] pub fn new( phase: ComputerUseObservationPhase, + application_fingerprint: String, + process_fingerprint: String, window_fingerprint: String, window_title_fingerprint: String, + frame_fingerprint: String, target_fingerprint: Option, semantic_fingerprint: Option, screenshot_evidence_ref: String, safe_summary: String, captured_at: DateTime, ) -> Result { + require_fingerprint(&application_fingerprint, "application fingerprint")?; + require_fingerprint(&process_fingerprint, "process fingerprint")?; require_fingerprint(&window_fingerprint, "window fingerprint")?; require_fingerprint(&window_title_fingerprint, "window title fingerprint")?; + require_fingerprint(&frame_fingerprint, "frame fingerprint")?; if let Some(value) = target_fingerprint.as_deref() { require_fingerprint(value, "target fingerprint")?; } @@ -204,28 +228,58 @@ impl ComputerUseObservation { } let screenshot_evidence_ref = safe_evidence_ref(screenshot_evidence_ref)?; let safe_summary = safe_text(safe_summary, "observation summary")?; + let id = Uuid::new_v4(); + let valid_until = captured_at + .checked_add_signed(Duration::seconds(COMPUTER_USE_OBSERVATION_FRESHNESS_SECS)) + .ok_or_else(|| "computer use observation freshness window overflowed".to_string())?; + let id_text = id.to_string(); + let captured_at_text = timestamp_text(captured_at); + let valid_until_text = timestamp_text(valid_until); let fingerprint = hash_parts(&[ COMPUTER_USE_STEP_CONTRACT_VERSION, + &id_text, observation_phase_name(phase), + &application_fingerprint, + &process_fingerprint, &window_fingerprint, &window_title_fingerprint, + &frame_fingerprint, target_fingerprint.as_deref().unwrap_or("none"), semantic_fingerprint.as_deref().unwrap_or("none"), &screenshot_evidence_ref, + &captured_at_text, + &valid_until_text, ]); Ok(Self { - id: Uuid::new_v4(), + id, phase, fingerprint, + application_fingerprint, + process_fingerprint, window_fingerprint, window_title_fingerprint, + frame_fingerprint, target_fingerprint, semantic_fingerprint, screenshot_evidence_ref, safe_summary, captured_at, + valid_until, }) } + + pub fn require_fresh_at(&self, now: DateTime) -> Result<(), String> { + if now < self.captured_at { + return Err("computer use observation timestamp is in the future".to_string()); + } + if now > self.valid_until { + return Err( + "computer use observation is stale; re-observation and a new approval are required" + .to_string(), + ); + } + Ok(()) + } } impl ComputerUseActionBinding { @@ -251,8 +305,11 @@ impl ComputerUseActionBinding { let action_fingerprint = hash_parts(&[ COMPUTER_USE_STEP_CONTRACT_VERSION, &pre_observation.fingerprint, + &pre_observation.application_fingerprint, + &pre_observation.process_fingerprint, &pre_observation.window_fingerprint, &pre_observation.window_title_fingerprint, + &pre_observation.frame_fingerprint, &target_fingerprint, &action_json, &postcondition_json, @@ -261,8 +318,11 @@ impl ComputerUseActionBinding { action, safe_summary, pre_observation_fingerprint: pre_observation.fingerprint.clone(), + application_fingerprint: pre_observation.application_fingerprint.clone(), + process_fingerprint: pre_observation.process_fingerprint.clone(), window_fingerprint: pre_observation.window_fingerprint.clone(), pre_window_title_fingerprint: pre_observation.window_title_fingerprint.clone(), + frame_fingerprint: pre_observation.frame_fingerprint.clone(), target_fingerprint, pre_semantic_fingerprint: pre_observation.semantic_fingerprint.clone(), postcondition, @@ -295,6 +355,7 @@ impl ComputerUseStep { pre_observation, action: None, approval_request_id: None, + approval_actor: None, action_started_at: None, action_start_count: 0, post_observation: None, @@ -317,9 +378,13 @@ impl ComputerUseStep { now: DateTime, ) -> Result<(), String> { self.require_status(ComputerUseStepStatus::Observed, "bind an action")?; + self.pre_observation.require_fresh_at(now)?; if action.pre_observation_fingerprint != self.pre_observation.fingerprint + || action.application_fingerprint != self.pre_observation.application_fingerprint + || action.process_fingerprint != self.pre_observation.process_fingerprint || action.window_fingerprint != self.pre_observation.window_fingerprint || action.pre_window_title_fingerprint != self.pre_observation.window_title_fingerprint + || action.frame_fingerprint != self.pre_observation.frame_fingerprint || self.pre_observation.target_fingerprint.as_deref() != Some(action.target_fingerprint.as_str()) || action.pre_semantic_fingerprint != self.pre_observation.semantic_fingerprint @@ -353,9 +418,17 @@ impl ComputerUseStep { &mut self, approval_request_id: Uuid, approved_action_fingerprint: &str, + actor: ComputerUseApprovalActor, now: DateTime, ) -> Result<(), String> { self.require_status(ComputerUseStepStatus::AwaitingApproval, "bind an approval")?; + if actor != ComputerUseApprovalActor::User { + return Err( + "computer use approval authority can only come from a local user decision" + .to_string(), + ); + } + self.pre_observation.require_fresh_at(now)?; if approval_request_id.is_nil() { return Err("computer use approval request id is invalid".to_string()); } @@ -370,6 +443,7 @@ impl ComputerUseStep { return Err("computer use approval is stale or bound to another action".to_string()); } self.approval_request_id = Some(approval_request_id); + self.approval_actor = Some(actor); self.transition( ComputerUseStepStatus::Ready, "Exact desktop action is approved and ready for revalidation.".to_string(), @@ -378,11 +452,15 @@ impl ComputerUseStep { Ok(()) } + #[allow(clippy::too_many_arguments)] pub fn mark_action_started( &mut self, approval_request_id: Uuid, + current_application_fingerprint: &str, + current_process_fingerprint: &str, current_window_fingerprint: &str, current_window_title_fingerprint: &str, + current_frame_fingerprint: &str, current_target_fingerprint: &str, now: DateTime, ) -> Result<(), String> { @@ -398,16 +476,36 @@ impl ComputerUseStep { if self.approval_request_id != Some(approval_request_id) { return Err("computer use approval does not match the ready step".to_string()); } + if self.approval_actor != Some(ComputerUseApprovalActor::User) { + return Err("computer use step lacks local-user approval authority".to_string()); + } + require_fingerprint( + current_application_fingerprint, + "current application fingerprint", + )?; + require_fingerprint(current_process_fingerprint, "current process fingerprint")?; require_fingerprint(current_window_fingerprint, "current window fingerprint")?; require_fingerprint( current_window_title_fingerprint, "current window title fingerprint", )?; + require_fingerprint(current_frame_fingerprint, "current frame fingerprint")?; require_fingerprint(current_target_fingerprint, "current target fingerprint")?; let action = self .action .as_ref() .ok_or_else(|| "computer use step has no exact action".to_string())?; + if action.application_fingerprint != current_application_fingerprint { + return Err( + "foreground application changed after approval; re-observation is required" + .to_string(), + ); + } + if action.process_fingerprint != current_process_fingerprint { + return Err( + "foreground process changed after approval; re-observation is required".to_string(), + ); + } if action.window_fingerprint != current_window_fingerprint { return Err( "foreground window changed after approval; re-observation is required".to_string(), @@ -419,6 +517,11 @@ impl ComputerUseStep { .to_string(), ); } + if action.frame_fingerprint != current_frame_fingerprint { + return Err( + "foreground frame changed after approval; re-observation is required".to_string(), + ); + } if action.target_fingerprint != current_target_fingerprint { return Err( "computer use target changed after approval; re-observation is required" @@ -452,9 +555,14 @@ impl ComputerUseStep { .action .as_ref() .ok_or_else(|| "computer use step has no exact action".to_string())?; - if observation.window_fingerprint != action.window_fingerprint { + if observation.application_fingerprint != action.application_fingerprint + || observation.process_fingerprint != action.process_fingerprint + || observation.window_fingerprint != action.window_fingerprint + || observation.frame_fingerprint != action.frame_fingerprint + { return Err( - "post-action observation is bound to a different foreground window".to_string(), + "post-action observation is bound to a different application, process, window, or frame" + .to_string(), ); } if observation.target_fingerprint.as_deref() != Some(action.target_fingerprint.as_str()) { @@ -574,6 +682,7 @@ impl ComputerUseStep { )); } self.approval_request_id = None; + self.approval_actor = None; self.transition( ComputerUseStepStatus::NeedsReplan, safe_text(reason, "replan reason")?, @@ -606,6 +715,7 @@ impl ComputerUseStep { | ComputerUseStepStatus::AwaitingApproval | ComputerUseStepStatus::Ready => { self.approval_request_id = None; + self.approval_actor = None; self.transition( ComputerUseStepStatus::NeedsReplan, "Desktop state may have changed after restart; re-observation is required." @@ -679,9 +789,13 @@ impl ComputerUseStep { Some(action) => { action.validate()?; if action.pre_observation_fingerprint != self.pre_observation.fingerprint + || action.application_fingerprint + != self.pre_observation.application_fingerprint + || action.process_fingerprint != self.pre_observation.process_fingerprint || action.window_fingerprint != self.pre_observation.window_fingerprint || action.pre_window_title_fingerprint != self.pre_observation.window_title_fingerprint + || action.frame_fingerprint != self.pre_observation.frame_fingerprint || self.pre_observation.target_fingerprint.as_deref() != Some(action.target_fingerprint.as_str()) || action.pre_semantic_fingerprint != self.pre_observation.semantic_fingerprint @@ -704,13 +818,15 @@ impl ComputerUseStep { let action = self.action.as_ref().ok_or_else(|| { "computer use post-observation has no exact action binding".to_string() })?; - if observation.window_fingerprint != action.window_fingerprint + if observation.application_fingerprint != action.application_fingerprint + || observation.process_fingerprint != action.process_fingerprint + || observation.window_fingerprint != action.window_fingerprint + || observation.frame_fingerprint != action.frame_fingerprint || observation.target_fingerprint.as_deref() != Some(action.target_fingerprint.as_str()) { return Err( - "computer use post-observation is bound to another window or target" - .to_string(), + "computer use post-observation is bound to another application, process, window, frame, or target".to_string(), ); } if self @@ -802,8 +918,28 @@ impl ComputerUseStep { | ComputerUseStepStatus::EffectUnknown | ComputerUseStepStatus::VerificationFailed ); - if approval_required && self.approval_request_id.is_none() { - return Err("computer use step status requires exact approval".to_string()); + if self.approval_request_id.is_some() != self.approval_actor.is_some() { + return Err("computer use approval identity and actor are inconsistent".to_string()); + } + if self + .approval_request_id + .is_some_and(|approval_request_id| approval_request_id.is_nil()) + { + return Err("computer use approval request id is invalid".to_string()); + } + if self + .approval_actor + .is_some_and(|actor| actor != ComputerUseApprovalActor::User) + { + return Err( + "computer use approval evidence can only name a local user actor".to_string(), + ); + } + if approval_required + && (self.approval_request_id.is_none() + || self.approval_actor != Some(ComputerUseApprovalActor::User)) + { + return Err("computer use step status requires exact local-user approval".to_string()); } let started_required = matches!( self.status, @@ -864,8 +1000,11 @@ impl ComputerUseObservation { return Err("computer use observation identity is invalid".to_string()); } require_fingerprint(&self.fingerprint, "observation fingerprint")?; + require_fingerprint(&self.application_fingerprint, "application fingerprint")?; + require_fingerprint(&self.process_fingerprint, "process fingerprint")?; require_fingerprint(&self.window_fingerprint, "window fingerprint")?; require_fingerprint(&self.window_title_fingerprint, "window title fingerprint")?; + require_fingerprint(&self.frame_fingerprint, "frame fingerprint")?; if let Some(value) = self.target_fingerprint.as_deref() { require_fingerprint(value, "target fingerprint")?; } @@ -874,14 +1013,30 @@ impl ComputerUseObservation { } safe_evidence_ref(self.screenshot_evidence_ref.clone())?; safe_text(self.safe_summary.clone(), "observation summary")?; + let expected_valid_until = self + .captured_at + .checked_add_signed(Duration::seconds(COMPUTER_USE_OBSERVATION_FRESHNESS_SECS)) + .ok_or_else(|| "computer use observation freshness window overflowed".to_string())?; + if self.valid_until != expected_valid_until { + return Err("computer use observation freshness window is inconsistent".to_string()); + } + let id_text = self.id.to_string(); + let captured_at_text = timestamp_text(self.captured_at); + let valid_until_text = timestamp_text(self.valid_until); let expected = hash_parts(&[ COMPUTER_USE_STEP_CONTRACT_VERSION, + &id_text, observation_phase_name(self.phase), + &self.application_fingerprint, + &self.process_fingerprint, &self.window_fingerprint, &self.window_title_fingerprint, + &self.frame_fingerprint, self.target_fingerprint.as_deref().unwrap_or("none"), self.semantic_fingerprint.as_deref().unwrap_or("none"), &self.screenshot_evidence_ref, + &captured_at_text, + &valid_until_text, ]); if expected != self.fingerprint { return Err("computer use observation fingerprint is inconsistent".to_string()); @@ -896,11 +1051,14 @@ impl ComputerUseActionBinding { &self.pre_observation_fingerprint, "pre-observation fingerprint", )?; + require_fingerprint(&self.application_fingerprint, "application fingerprint")?; + require_fingerprint(&self.process_fingerprint, "process fingerprint")?; require_fingerprint(&self.window_fingerprint, "window fingerprint")?; require_fingerprint( &self.pre_window_title_fingerprint, "pre-action window title fingerprint", )?; + require_fingerprint(&self.frame_fingerprint, "frame fingerprint")?; require_fingerprint(&self.target_fingerprint, "target fingerprint")?; if let Some(value) = self.pre_semantic_fingerprint.as_deref() { require_fingerprint(value, "pre-action semantic fingerprint")?; @@ -916,8 +1074,11 @@ impl ComputerUseActionBinding { let expected = hash_parts(&[ COMPUTER_USE_STEP_CONTRACT_VERSION, &self.pre_observation_fingerprint, + &self.application_fingerprint, + &self.process_fingerprint, &self.window_fingerprint, &self.pre_window_title_fingerprint, + &self.frame_fingerprint, &self.target_fingerprint, &action_json, &postcondition_json, @@ -980,6 +1141,10 @@ fn hash_parts(parts: &[&str]) -> String { hex::encode(hasher.finalize()) } +fn timestamp_text(value: DateTime) -> String { + value.to_rfc3339_opts(SecondsFormat::Nanos, true) +} + fn observation_phase_name(phase: ComputerUseObservationPhase) -> &'static str { match phase { ComputerUseObservationPhase::PreAction => "pre_action", @@ -1003,8 +1168,11 @@ mod tests { ) -> ComputerUseObservation { ComputerUseObservation::new( phase, + fingerprint("notepad-application"), + fingerprint("notepad-process"), fingerprint("notepad-window"), fingerprint("notepad-title"), + fingerprint("notepad-frame"), Some(fingerprint("notepad-editor")), semantic.map(fingerprint), format!( @@ -1034,6 +1202,43 @@ mod tests { .expect("action is valid") } + fn approve_as_user( + step: &mut ComputerUseStep, + approval_id: Uuid, + action_fingerprint: &str, + now: DateTime, + ) -> Result<(), String> { + step.approve( + approval_id, + action_fingerprint, + ComputerUseApprovalActor::User, + now, + ) + } + + fn mark_started( + step: &mut ComputerUseStep, + approval_id: Uuid, + window: &str, + window_title: &str, + target: &str, + now: DateTime, + ) -> Result<(), String> { + let application = step.pre_observation.application_fingerprint.clone(); + let process = step.pre_observation.process_fingerprint.clone(); + let frame = step.pre_observation.frame_fingerprint.clone(); + step.mark_action_started( + approval_id, + &application, + &process, + window, + window_title, + &frame, + target, + now, + ) + } + #[test] fn verified_step_binds_observation_action_approval_and_postcondition() { let now = Utc::now(); @@ -1048,8 +1253,7 @@ mod tests { .expect("step is valid"); step.bind_action(binding, now).expect("action binds"); - step.approve(approval_id, &action_fingerprint, now) - .expect("approval binds"); + approve_as_user(&mut step, approval_id, &action_fingerprint, now).expect("approval binds"); let window = step.pre_observation.window_fingerprint.clone(); let window_title = step.pre_observation.window_title_fingerprint.clone(); let target = step @@ -1057,7 +1261,7 @@ mod tests { .target_fingerprint .clone() .expect("target exists"); - step.mark_action_started(approval_id, &window, &window_title, &target, now) + mark_started(&mut step, approval_id, &window, &window_title, &target, now) .expect("action starts once"); let post = observation( ComputerUseObservationPhase::PostAction, @@ -1097,8 +1301,11 @@ mod tests { let mut false_verified = step.clone(); let altered_post = ComputerUseObservation::new( ComputerUseObservationPhase::PostAction, + fingerprint("notepad-application"), + fingerprint("notepad-process"), fingerprint("notepad-window"), fingerprint("notepad-title"), + fingerprint("notepad-frame"), Some(fingerprint("notepad-editor")), Some(fingerprint("wrong-result")), "computer-screenshots/altered-notepad.png".to_string(), @@ -1131,38 +1338,102 @@ mod tests { ) .expect("step is valid"); step.bind_action(binding, now).expect("action binds"); - assert!(step - .approve(Uuid::new_v4(), &fingerprint("stale-action"), now) - .is_err()); - step.approve(approval_id, &action_fingerprint, now) + assert!( + approve_as_user(&mut step, Uuid::new_v4(), &fingerprint("stale-action"), now,).is_err() + ); + approve_as_user(&mut step, approval_id, &action_fingerprint, now) .expect("exact approval binds"); let window = step.pre_observation.window_fingerprint.clone(); let window_title = step.pre_observation.window_title_fingerprint.clone(); let target = step.pre_observation.target_fingerprint.clone().unwrap(); - assert!(step - .mark_action_started( - approval_id, - &window, - &window_title, - &fingerprint("changed-target"), - now, - ) - .is_err()); - assert!(step - .mark_action_started( - approval_id, - &window, - &fingerprint("changed-title"), - &target, - now, - ) - .is_err()); - step.mark_action_started(approval_id, &window, &window_title, &target, now) + assert!(mark_started( + &mut step, + approval_id, + &window, + &window_title, + &fingerprint("changed-target"), + now, + ) + .is_err()); + assert!(mark_started( + &mut step, + approval_id, + &window, + &fingerprint("changed-title"), + &target, + now, + ) + .is_err()); + mark_started(&mut step, approval_id, &window, &window_title, &target, now) .expect("exact action starts"); + assert!( + mark_started(&mut step, approval_id, &window, &window_title, &target, now,).is_err() + ); + assert_eq!(step.action_start_count, 1); + } + + #[test] + fn stale_observation_and_non_user_approval_fail_closed() { + let now = Utc::now(); + let stale_captured_at = + now - Duration::seconds(COMPUTER_USE_OBSERVATION_FRESHNESS_SECS + 1); + let stale_pre = observation( + ComputerUseObservationPhase::PreAction, + Some("empty"), + stale_captured_at, + ); + let stale_binding = action(&stale_pre, "verified text"); + let mut stale_step = ComputerUseStep::new_observed( + Uuid::new_v4(), + 1, + stale_pre, + ComputerUseUndoCapability::None, + stale_captured_at, + ) + .unwrap(); + assert!(stale_step.bind_action(stale_binding, now).is_err()); + assert_eq!(stale_step.status, ComputerUseStepStatus::Observed); + + let pre = observation(ComputerUseObservationPhase::PreAction, Some("empty"), now); + let binding = action(&pre, "verified text"); + let action_fingerprint = binding.action_fingerprint.clone(); + let mut step = ComputerUseStep::new_observed( + Uuid::new_v4(), + 1, + pre, + ComputerUseUndoCapability::None, + now, + ) + .unwrap(); + step.bind_action(binding, now).unwrap(); + for actor in [ + ComputerUseApprovalActor::DeepSeekModel, + ComputerUseApprovalActor::FrontendPayload, + ComputerUseApprovalActor::KernelLifecycle, + ] { + assert!(step + .approve(Uuid::new_v4(), &action_fingerprint, actor, now) + .is_err()); + assert_eq!(step.status, ComputerUseStepStatus::AwaitingApproval); + assert!(step.approval_request_id.is_none()); + assert!(step.approval_actor.is_none()); + } + let mut forged = step.clone(); + forged.approval_request_id = Some(Uuid::new_v4()); + forged.approval_actor = Some(ComputerUseApprovalActor::DeepSeekModel); + assert!(forged.validate().is_err()); + let stale_approval_time = step.pre_observation.valid_until + Duration::milliseconds(1); assert!(step - .mark_action_started(approval_id, &window, &window_title, &target, now) + .approve( + Uuid::new_v4(), + &action_fingerprint, + ComputerUseApprovalActor::User, + stale_approval_time, + ) .is_err()); - assert_eq!(step.action_start_count, 1); + assert_eq!(step.status, ComputerUseStepStatus::AwaitingApproval); + assert!(step.approval_request_id.is_none()); + assert!(step.approval_actor.is_none()); } #[test] @@ -1182,9 +1453,7 @@ mod tests { ) .unwrap(); stale.bind_action(binding.clone(), now).unwrap(); - stale - .approve(approval_id, &action_fingerprint, now) - .unwrap(); + approve_as_user(&mut stale, approval_id, &action_fingerprint, now).unwrap(); stale .require_replan( "Foreground target changed; a new observation is required.".to_string(), @@ -1203,9 +1472,7 @@ mod tests { ) .unwrap(); uncertain.bind_action(binding, now).unwrap(); - uncertain - .approve(approval_id, &action_fingerprint, now) - .unwrap(); + approve_as_user(&mut uncertain, approval_id, &action_fingerprint, now).unwrap(); let window = uncertain.pre_observation.window_fingerprint.clone(); let window_title = uncertain.pre_observation.window_title_fingerprint.clone(); let target = uncertain @@ -1213,9 +1480,15 @@ mod tests { .target_fingerprint .clone() .unwrap(); - uncertain - .mark_action_started(approval_id, &window, &window_title, &target, now) - .unwrap(); + mark_started( + &mut uncertain, + approval_id, + &window, + &window_title, + &target, + now, + ) + .unwrap(); assert!(uncertain .cancel("Do not hide a possible external effect.".to_string(), now) .is_err()); @@ -1245,22 +1518,20 @@ mod tests { ) .unwrap(); ready.bind_action(binding.clone(), now).unwrap(); - ready - .approve(approval_id, &action_fingerprint, now) - .unwrap(); + approve_as_user(&mut ready, approval_id, &action_fingerprint, now).unwrap(); ready .take_over("User moved the mouse.".to_string(), now) .expect("takeover records"); assert_eq!(ready.status, ComputerUseStepStatus::UserTakenOver); - assert!(ready - .mark_action_started( - approval_id, - &pre.window_fingerprint, - &pre.window_title_fingerprint, - pre.target_fingerprint.as_deref().unwrap(), - now, - ) - .is_err()); + assert!(mark_started( + &mut ready, + approval_id, + &pre.window_fingerprint, + &pre.window_title_fingerprint, + pre.target_fingerprint.as_deref().unwrap(), + now, + ) + .is_err()); let mut started = ComputerUseStep::new_observed( Uuid::new_v4(), @@ -1271,21 +1542,31 @@ mod tests { ) .unwrap(); started.bind_action(binding, now).unwrap(); - started - .approve(approval_id, &action_fingerprint, now) - .unwrap(); + approve_as_user(&mut started, approval_id, &action_fingerprint, now).unwrap(); let window = started.pre_observation.window_fingerprint.clone(); let window_title = started.pre_observation.window_title_fingerprint.clone(); let target = started.pre_observation.target_fingerprint.clone().unwrap(); - started - .mark_action_started(approval_id, &window, &window_title, &target, now) - .unwrap(); + mark_started( + &mut started, + approval_id, + &window, + &window_title, + &target, + now, + ) + .unwrap(); started.recover_after_restart(now).unwrap(); assert_eq!(started.status, ComputerUseStepStatus::EffectUnknown); assert_eq!(started.action_start_count, 1); - assert!(started - .mark_action_started(approval_id, &window, &window_title, &target, now) - .is_err()); + assert!(mark_started( + &mut started, + approval_id, + &window, + &window_title, + &target, + now, + ) + .is_err()); } #[test] @@ -1304,12 +1585,11 @@ mod tests { ) .unwrap(); step.bind_action(binding, now).unwrap(); - step.approve(approval_id, &action_fingerprint, now).unwrap(); + approve_as_user(&mut step, approval_id, &action_fingerprint, now).unwrap(); let window = step.pre_observation.window_fingerprint.clone(); let window_title = step.pre_observation.window_title_fingerprint.clone(); let target = step.pre_observation.target_fingerprint.clone().unwrap(); - step.mark_action_started(approval_id, &window, &window_title, &target, now) - .unwrap(); + mark_started(&mut step, approval_id, &window, &window_title, &target, now).unwrap(); let post = observation(ComputerUseObservationPhase::PostAction, None, now); let post_fingerprint = post.fingerprint.clone(); step.record_post_observation(post, now).unwrap(); diff --git a/apps/desktop/src-tauri/src/kernel/event_store/computer_use.rs b/apps/desktop/src-tauri/src/kernel/event_store/computer_use.rs index afd87d1..8653da3 100644 --- a/apps/desktop/src-tauri/src/kernel/event_store/computer_use.rs +++ b/apps/desktop/src-tauri/src/kernel/event_store/computer_use.rs @@ -459,8 +459,8 @@ mod tests { use super::*; use crate::kernel::capability::ComputerControlAction; use crate::kernel::computer_use_session::{ - ComputerUseActionBinding, ComputerUseObservation, ComputerUseObservationPhase, - ComputerUsePostcondition, ComputerUseUndoCapability, + ComputerUseActionBinding, ComputerUseApprovalActor, ComputerUseObservation, + ComputerUseObservationPhase, ComputerUsePostcondition, ComputerUseUndoCapability, }; use sha2::{Digest, Sha256}; @@ -480,8 +480,11 @@ mod tests { .unwrap(); let observation = ComputerUseObservation::new( ComputerUseObservationPhase::PreAction, + fingerprint("application"), + fingerprint("process"), fingerprint("window"), fingerprint("window-title"), + fingerprint("frame"), Some(fingerprint("target")), Some(fingerprint("before")), "computer-screenshots/before-notepad.png".to_string(), @@ -527,14 +530,32 @@ mod tests { step.bind_action(action, now).unwrap(); store.update_computer_use_step(&step, expected).unwrap(); let expected = step.revision; - step.approve(approval_id, &action_fingerprint, now).unwrap(); + step.approve( + approval_id, + &action_fingerprint, + ComputerUseApprovalActor::User, + now, + ) + .unwrap(); store.update_computer_use_step(&step, expected).unwrap(); let expected = step.revision; + let application = step.pre_observation.application_fingerprint.clone(); + let process = step.pre_observation.process_fingerprint.clone(); let window = step.pre_observation.window_fingerprint.clone(); let window_title = step.pre_observation.window_title_fingerprint.clone(); + let frame = step.pre_observation.frame_fingerprint.clone(); let target = step.pre_observation.target_fingerprint.clone().unwrap(); - step.mark_action_started(approval_id, &window, &window_title, &target, now) - .unwrap(); + step.mark_action_started( + approval_id, + &application, + &process, + &window, + &window_title, + &frame, + &target, + now, + ) + .unwrap(); store.update_computer_use_step(&step, expected).unwrap(); } @@ -564,8 +585,13 @@ mod tests { step.bind_action(action, now).unwrap(); store.update_computer_use_step(&step, expected).unwrap(); let expected = step.revision; - step.approve(Uuid::new_v4(), &action_fingerprint, now) - .unwrap(); + step.approve( + Uuid::new_v4(), + &action_fingerprint, + ComputerUseApprovalActor::User, + now, + ) + .unwrap(); store.update_computer_use_step(&step, expected).unwrap(); let sweep = store.recover_computer_use_steps_after_restart(now).unwrap(); @@ -603,4 +629,44 @@ mod tests { .unwrap(); assert_eq!(quarantined, 1); } + + #[test] + fn obsolete_contract_version_is_quarantined_without_starving_healthy_recovery() { + let store = EventStore::open_memory().unwrap(); + let now = Utc::now(); + let (session, step, _) = observed_records(now); + store.insert_computer_use_session(&session).unwrap(); + store.insert_computer_use_step(&step).unwrap(); + + let mut obsolete = step.clone(); + obsolete.id = Uuid::new_v4(); + obsolete.sequence = 99; + obsolete.contract_version = "computer-use-step/v1".to_string(); + store + .conn + .execute( + "INSERT INTO computer_use_steps (id, session_id, sequence, step_json, status, row_revision, updated_at) VALUES (?1, ?2, ?3, ?4, 'observed', 0, ?5)", + params![ + obsolete.id.to_string(), + session.id.to_string(), + i64::from(obsolete.sequence), + serde_json::to_string(&obsolete).unwrap(), + timestamp(now) + ], + ) + .unwrap(); + + let sweep = store.recover_computer_use_steps_after_restart(now).unwrap(); + assert_eq!(sweep.quarantined, 1); + assert_eq!(sweep.needs_replan, 1); + let quarantined: i64 = store + .conn + .query_row( + "SELECT COUNT(*) FROM computer_use_steps WHERE quarantine_code='invalid_record'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(quarantined, 1); + } } From 7e06035a8c79464ca867f3c622dfb3a6f125877c Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 00:37:50 +0800 Subject: [PATCH 2/5] feat: add verified Explorer and Excel computer use --- .../src-tauri/src/kernel/capability.rs | 1778 ++++++++++++++- .../src/kernel/computer_use_runtime.rs | 1985 ++++++++++++++++- apps/desktop/src/App.tsx | 28 +- scripts/durable-computer-use-ui.test.mjs | 10 + 4 files changed, 3688 insertions(+), 113 deletions(-) diff --git a/apps/desktop/src-tauri/src/kernel/capability.rs b/apps/desktop/src-tauri/src/kernel/capability.rs index 20fac24..a194283 100644 --- a/apps/desktop/src-tauri/src/kernel/capability.rs +++ b/apps/desktop/src-tauri/src/kernel/capability.rs @@ -367,6 +367,10 @@ pub enum ComputerControlAction { TypeText { text: String, }, + SetAccessibilityValue { + value: String, + }, + SelectAccessibilityTarget, PressKey { key: String, }, @@ -389,6 +393,15 @@ impl ComputerControlAction { ComputerControlAction::TypeText { text } => { format!("type text ({} chars)", text.chars().count()) } + ComputerControlAction::SetAccessibilityValue { value } => { + format!( + "set focused accessibility value ({} chars)", + value.chars().count() + ) + } + ComputerControlAction::SelectAccessibilityTarget => { + "select focused accessibility target".to_string() + } ComputerControlAction::PressKey { key } => format!("press key {key}"), ComputerControlAction::Hotkey { keys } => format!("press hotkey {}", keys.join("+")), ComputerControlAction::Scroll { delta, axis } => { @@ -418,6 +431,10 @@ pub trait LocalComputerControlInputBackend { fn type_text(&mut self, text: &str) -> Result<(), String>; + fn set_accessibility_value(&mut self, value: &str) -> Result<(), String>; + + fn select_accessibility_target(&mut self) -> Result<(), String>; + fn key_down(&mut self, key: &str) -> Result<(), String>; fn key_up(&mut self, key: &str) -> Result<(), String>; @@ -1032,6 +1049,341 @@ impl ComputerScreenshotClient for LocalComputerScreenshotClient { } } +#[cfg(windows)] +#[allow(dead_code)] +pub struct WindowsBoundComputerScreenshotClient { + local: LocalComputerScreenshotClient, + window_handle: isize, + process_id: u32, +} + +#[cfg(windows)] +#[allow(dead_code)] +impl WindowsBoundComputerScreenshotClient { + pub fn new( + evidence_base_dir: PathBuf, + window_handle: isize, + process_id: u32, + ) -> Result { + if window_handle == 0 || process_id == 0 { + return Err("bound Windows screenshot requires an exact HWND and process".to_string()); + } + Ok(Self { + local: LocalComputerScreenshotClient::new(evidence_base_dir), + window_handle, + process_id, + }) + } +} + +#[cfg(windows)] +impl ComputerScreenshotClient for WindowsBoundComputerScreenshotClient { + fn capture_screenshot(&self) -> Result { + self.local + .capture_with_backend(&WindowsBoundScreenshotCaptureBackend { + window_handle: self.window_handle, + process_id: self.process_id, + }) + } +} + +#[cfg(windows)] +#[allow(dead_code)] +struct WindowsBoundScreenshotCaptureBackend { + window_handle: isize, + process_id: u32, +} + +#[cfg(windows)] +impl LocalScreenshotCaptureBackend for WindowsBoundScreenshotCaptureBackend { + fn capture_primary_display(&self) -> Result { + capture_bound_windows_window_image(self.window_handle, self.process_id) + } +} + +#[cfg(windows)] +#[allow(dead_code)] +fn require_bound_windows_process_identity( + actual_process_id: u32, + expected_process_id: u32, +) -> Result<(), String> { + if actual_process_id == 0 { + return Err("bound Windows screenshot HWND has no live process identity".to_string()); + } + if actual_process_id != expected_process_id { + return Err( + "bound Windows screenshot HWND no longer belongs to the expected process".to_string(), + ); + } + Ok(()) +} + +#[cfg(windows)] +#[allow(clippy::too_many_arguments, dead_code)] +fn require_bound_excel_editor_identity( + main_window_handle: isize, + expected_process_id: u32, + expected_thread_id: u32, + editor_window_handle: isize, + editor_process_id: u32, + editor_thread_id: u32, + editor_is_exact_or_child: bool, + editor_window_class: &str, +) -> Result<(), String> { + if main_window_handle == 0 + || expected_process_id == 0 + || expected_thread_id == 0 + || editor_window_handle == 0 + { + return Err("bound Excel editor identity is incomplete".to_string()); + } + if editor_process_id != expected_process_id || editor_thread_id != expected_thread_id { + return Err( + "bound Excel editor no longer belongs to the exact process and window thread" + .to_string(), + ); + } + if !editor_is_exact_or_child { + return Err("bound Excel editor is outside the exact Excel HWND".to_string()); + } + if !editor_window_class.eq_ignore_ascii_case("EXCEL6") { + return Err("bound Excel editor has the wrong native window class".to_string()); + } + Ok(()) +} + +#[cfg(windows)] +#[allow(clippy::too_many_arguments, dead_code)] +fn require_bound_excel_grid_identity( + main_window_handle: isize, + expected_process_id: u32, + expected_thread_id: u32, + grid_window_handle: isize, + grid_process_id: u32, + grid_thread_id: u32, + grid_is_exact_or_child: bool, + grid_window_class: &str, +) -> Result<(), String> { + if main_window_handle == 0 + || expected_process_id == 0 + || expected_thread_id == 0 + || grid_window_handle == 0 + { + return Err("bound Excel grid identity is incomplete".to_string()); + } + if grid_process_id != expected_process_id || grid_thread_id != expected_thread_id { + return Err( + "bound Excel grid no longer belongs to the exact process and window thread".to_string(), + ); + } + if !grid_is_exact_or_child { + return Err("bound Excel grid is outside the exact Excel HWND".to_string()); + } + if !grid_window_class.eq_ignore_ascii_case("EXCEL7") { + return Err("bound Excel grid has the wrong native window class".to_string()); + } + Ok(()) +} + +#[cfg(windows)] +#[allow(dead_code)] +fn excel_grid_automation_id(row: i32, column: i32) -> Result { + if row <= 0 || column <= 0 { + return Err("bound Excel grid identity requires one-based row and column".to_string()); + } + let mut remaining = + u32::try_from(column).map_err(|_| "bound Excel grid column is invalid".to_string())?; + let mut letters = Vec::new(); + while remaining > 0 { + remaining -= 1; + letters.push( + char::from_u32(u32::from(b'A') + (remaining % 26)) + .ok_or_else(|| "bound Excel grid column is invalid".to_string())?, + ); + remaining /= 26; + } + letters.reverse(); + Ok(format!("{}{row}", letters.into_iter().collect::())) +} + +#[cfg(windows)] +#[allow(dead_code)] +fn revalidate_bound_windows_screenshot_identity( + window_handle: isize, + expected_process_id: u32, +) -> Result { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; + + let hwnd = HWND(window_handle as _); + let mut actual_process_id = 0u32; + let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut actual_process_id)) }; + if thread_id == 0 { + return Err("bound Windows screenshot HWND is no longer available".to_string()); + } + require_bound_windows_process_identity(actual_process_id, expected_process_id)?; + Ok(hwnd) +} + +#[cfg(windows)] +#[allow(dead_code)] +fn capture_bound_windows_window_image( + window_handle: isize, + expected_process_id: u32, +) -> Result { + use std::ffi::c_void; + use std::mem; + + use windows::Win32::Graphics::Gdi::{ + CreateCompatibleBitmap, CreateCompatibleDC, DeleteDC, DeleteObject, GetDIBits, GetWindowDC, + ReleaseDC, SelectObject, BITMAPINFO, BITMAPINFOHEADER, DIB_RGB_COLORS, + }; + use windows::Win32::UI::WindowsAndMessaging::GetWindowRect; + + #[link(name = "user32")] + unsafe extern "system" { + fn PrintWindow(window: isize, target_dc: isize, flags: u32) -> i32; + } + + let hwnd = revalidate_bound_windows_screenshot_identity(window_handle, expected_process_id)?; + let mut rect = windows::Win32::Foundation::RECT::default(); + unsafe { GetWindowRect(hwnd, &mut rect) }.map_err(|error| { + format!("bound Windows screenshot could not read the window bounds: {error}") + })?; + let width_i32 = rect.right.saturating_sub(rect.left); + let height_i32 = rect.bottom.saturating_sub(rect.top); + if width_i32 <= 0 || height_i32 <= 0 { + return Err(format!( + "bound Windows screenshot returned invalid dimensions: {width_i32}x{height_i32}" + )); + } + let width = u32::try_from(width_i32) + .map_err(|_| "bound Windows screenshot width exceeds PNG limits".to_string())?; + let height = u32::try_from(height_i32) + .map_err(|_| "bound Windows screenshot height exceeds PNG limits".to_string())?; + let buffer_len = usize::try_from(width) + .ok() + .and_then(|width| { + usize::try_from(height) + .ok() + .and_then(|height| width.checked_mul(height)) + }) + .and_then(|pixels| pixels.checked_mul(4)) + .ok_or_else(|| "bound Windows screenshot pixel buffer size overflowed".to_string())?; + let buffer_len_u32 = u32::try_from(buffer_len) + .map_err(|_| "bound Windows screenshot pixel buffer exceeds GDI limits".to_string())?; + + let pixels = unsafe { + let window_dc = GetWindowDC(Some(hwnd)); + if window_dc.is_invalid() { + return Err( + "bound Windows screenshot GetWindowDC returned an invalid device context" + .to_string(), + ); + } + let memory_dc = CreateCompatibleDC(Some(window_dc)); + if memory_dc.is_invalid() { + ReleaseDC(Some(hwnd), window_dc); + return Err( + "bound Windows screenshot CreateCompatibleDC returned an invalid device context" + .to_string(), + ); + } + let bitmap = CreateCompatibleBitmap(window_dc, width_i32, height_i32); + if bitmap.is_invalid() { + let _ = DeleteDC(memory_dc); + ReleaseDC(Some(hwnd), window_dc); + return Err( + "bound Windows screenshot CreateCompatibleBitmap returned an invalid bitmap" + .to_string(), + ); + } + let previous_object = SelectObject(memory_dc, bitmap.into()); + if previous_object.is_invalid() { + let _ = DeleteObject(bitmap.into()); + let _ = DeleteDC(memory_dc); + ReleaseDC(Some(hwnd), window_dc); + return Err( + "bound Windows screenshot SelectObject could not bind the capture bitmap" + .to_string(), + ); + } + + let print_result = if PrintWindow(hwnd.0 as isize, memory_dc.0 as isize, 2) == 0 { + Err(format!( + "bound Windows screenshot PrintWindow failed: {}", + std::io::Error::last_os_error() + )) + } else { + Ok(()) + }; + SelectObject(memory_dc, previous_object); + let result = print_result.and_then(|()| { + let mut bitmap_info = BITMAPINFO { + bmiHeader: BITMAPINFOHEADER { + biSize: mem::size_of::() as u32, + biWidth: width_i32, + biHeight: -height_i32, + biPlanes: 1, + biBitCount: 32, + biCompression: 0, + biSizeImage: buffer_len_u32, + ..Default::default() + }, + ..Default::default() + }; + let mut pixels = vec![0_u8; buffer_len]; + let copied_lines = GetDIBits( + memory_dc, + bitmap, + 0, + height, + Some(pixels.as_mut_ptr().cast::()), + &mut bitmap_info, + DIB_RGB_COLORS, + ); + if copied_lines != height_i32 { + return Err(format!( + "bound Windows screenshot GetDIBits copied {copied_lines} of {height_i32} window lines" + )); + } + Ok(pixels) + }); + + let _ = DeleteObject(bitmap.into()); + let _ = DeleteDC(memory_dc); + ReleaseDC(Some(hwnd), window_dc); + result? + }; + revalidate_bound_windows_screenshot_identity(window_handle, expected_process_id)?; + + let mut pixels = pixels; + for pixel in pixels.chunks_exact_mut(4) { + pixel.swap(0, 2); + pixel[3] = 255; + } + if pixels + .chunks_exact(4) + .all(|pixel| pixel[0] == 0 && pixel[1] == 0 && pixel[2] == 0) + { + return Err("bound Windows screenshot PrintWindow returned only black pixels".to_string()); + } + let image = xcap::image::RgbaImage::from_raw(width, height, pixels).ok_or_else(|| { + "bound Windows screenshot pixels did not match window dimensions".to_string() + })?; + let mut buffer = Cursor::new(Vec::new()); + xcap::image::DynamicImage::ImageRgba8(image) + .write_to(&mut buffer, xcap::image::ImageFormat::Png) + .map_err(|error| format!("bound Windows screenshot PNG encoding failed: {error}"))?; + + Ok(CapturedScreenshotImage { + display_label: "Exact bound Windows window".to_string(), + width, + height, + png_bytes: buffer.into_inner(), + }) +} + struct XcapLocalScreenshotCaptureBackend; impl LocalScreenshotCaptureBackend for XcapLocalScreenshotCaptureBackend { @@ -1445,6 +1797,12 @@ impl LocalComputerControlClient { ComputerControlAction::TypeText { text } => { backend.type_text(text)?; } + ComputerControlAction::SetAccessibilityValue { value } => { + backend.set_accessibility_value(value)?; + } + ComputerControlAction::SelectAccessibilityTarget => { + backend.select_accessibility_target()?; + } ComputerControlAction::PressKey { key } => { backend.key_click(key)?; } @@ -1485,6 +1843,149 @@ impl ComputerControlClient for LocalComputerControlClient { } } +#[cfg(windows)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub struct WindowsBoundComputerControlClient { + window_handle: isize, + process_id: u32, + target: WindowsBoundComputerControlTarget, +} + +#[cfg(windows)] +#[allow(dead_code)] +const BOUND_WINDOWS_ACCESSIBILITY_ACTION_TIMEOUT: Duration = Duration::from_secs(6); + +// Bound-13 reached fresh post-commit B3 observation at 5.381 seconds. Nine seconds +// dominates the two-second semantic-convergence ceiling plus exact B4-to-B3 +// restoration, while still bounding any non-returning Excel provider call. +#[cfg(windows)] +#[allow(dead_code)] +const BOUND_EXCEL_ACCESSIBILITY_ACTION_TIMEOUT: Duration = Duration::from_secs(9); + +#[cfg(windows)] +#[allow(dead_code)] +fn bound_windows_accessibility_action_timeout( + target: &WindowsBoundComputerControlTarget, +) -> Duration { + if matches!(target, WindowsBoundComputerControlTarget::Excel { .. }) { + BOUND_EXCEL_ACCESSIBILITY_ACTION_TIMEOUT + } else { + BOUND_WINDOWS_ACCESSIBILITY_ACTION_TIMEOUT + } +} + +#[cfg(windows)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum WindowsBoundComputerControlTarget { + Any, + FileExplorer { + target_name: String, + }, + Excel { + worksheet_automation_id: String, + cell_automation_id: String, + row: i32, + column: i32, + }, +} + +#[cfg(windows)] +#[allow(dead_code)] +impl WindowsBoundComputerControlClient { + pub fn new(window_handle: isize, process_id: u32) -> Result { + if window_handle == 0 || process_id == 0 { + return Err( + "bound Windows computer control requires an exact HWND and process".to_string(), + ); + } + Ok(Self { + window_handle, + process_id, + target: WindowsBoundComputerControlTarget::Any, + }) + } + + pub fn new_file_explorer( + window_handle: isize, + process_id: u32, + target_name: String, + ) -> Result { + let mut client = Self::new(window_handle, process_id)?; + if target_name.trim().is_empty() { + return Err("bound File Explorer control requires an exact target name".to_string()); + } + client.target = WindowsBoundComputerControlTarget::FileExplorer { target_name }; + Ok(client) + } + + pub fn new_excel( + window_handle: isize, + process_id: u32, + worksheet_automation_id: String, + cell_automation_id: String, + row: i32, + column: i32, + ) -> Result { + let mut client = Self::new(window_handle, process_id)?; + if worksheet_automation_id.trim().is_empty() + || cell_automation_id.trim().is_empty() + || row < 0 + || column < 0 + { + return Err( + "bound Excel control requires an exact worksheet, cell, row, and column" + .to_string(), + ); + } + client.target = WindowsBoundComputerControlTarget::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + }; + Ok(client) + } +} + +#[cfg(windows)] +impl ComputerControlClient for WindowsBoundComputerControlClient { + fn execute_control( + &self, + _target: &str, + action: &ComputerControlAction, + ) -> Result { + use std::sync::mpsc; + + let action = action.clone(); + let summary = action.audit_summary(); + let window_handle = self.window_handle; + let process_id = self.process_id; + let target = self.target.clone(); + let action_timeout = bound_windows_accessibility_action_timeout(&target); + let (sender, receiver) = mpsc::sync_channel(1); + std::thread::spawn(move || { + let result = execute_bound_windows_accessibility_action( + window_handle, + process_id, + &target, + &action, + ); + let _ = sender.send(result); + }); + receiver + .recv_timeout(action_timeout) + .map_err(|error| { + format!( + "bound Windows accessibility action timed out after {} seconds; the effect is unknown and automatic replay is forbidden: {error}", + action_timeout.as_secs(), + ) + })??; + Ok(ComputerControlExecution { summary }) + } +} + pub struct CodexBridgeComputerControlClient { runtime: CodexBridgeClientRuntime, } @@ -1574,6 +2075,14 @@ impl LocalComputerControlInputBackend for EnigoLocalComputerControlInputBackend .map_err(|error| format!("computer control text input failed: {error}")) } + fn set_accessibility_value(&mut self, value: &str) -> Result<(), String> { + execute_local_accessibility_set_value(value) + } + + fn select_accessibility_target(&mut self) -> Result<(), String> { + execute_local_accessibility_select() + } + fn key_down(&mut self, key: &str) -> Result<(), String> { let key = enigo_key(key)?; enigo::Keyboard::key(&mut self.enigo, key, enigo::Direction::Press) @@ -1598,6 +2107,1057 @@ impl LocalComputerControlInputBackend for EnigoLocalComputerControlInputBackend } } +fn execute_local_accessibility_set_value(value: &str) -> Result<(), String> { + #[cfg(windows)] + { + let value = value.to_string(); + std::thread::spawn(move || set_windows_focused_accessibility_value(&value)) + .join() + .map_err(|_| "Windows accessibility value action thread failed".to_string())? + } + #[cfg(not(windows))] + { + let _ = value; + Err("focused accessibility value actions are Windows-first in Step 5".to_string()) + } +} + +fn execute_local_accessibility_select() -> Result<(), String> { + #[cfg(windows)] + { + std::thread::spawn(select_windows_focused_accessibility_target) + .join() + .map_err(|_| "Windows accessibility selection action thread failed".to_string())? + } + #[cfg(not(windows))] + { + Err("focused accessibility selection actions are Windows-first in Step 5".to_string()) + } +} + +#[cfg(windows)] +fn set_windows_focused_accessibility_value(value: &str) -> Result<(), String> { + use windows::core::BSTR; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationValuePattern, UIA_LegacyIAccessiblePatternId, UIA_ValuePatternId, + }; + use windows::Win32::UI::WindowsAndMessaging::{GetForegroundWindow, GetWindowThreadProcessId}; + + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|error| format!("Windows accessibility COM initialization failed: {error}"))?; + let _guard = ComGuard; + let hwnd = unsafe { GetForegroundWindow() }; + if hwnd.0.is_null() { + return Err("Windows accessibility found no foreground window".to_string()); + } + let mut foreground_process_id = 0u32; + let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut foreground_process_id)) }; + if thread_id == 0 || foreground_process_id == 0 { + return Err("Windows accessibility could not identify the foreground process".to_string()); + } + let automation: IUIAutomation = unsafe { + CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|error| format!("Windows UI Automation client creation failed: {error}"))? + }; + let focused = unsafe { + automation + .GetFocusedElement() + .map_err(|error| format!("Windows UI Automation found no focused target: {error}"))? + }; + let target_process_id = unsafe { focused.CurrentProcessId() } + .map_err(|error| format!("Windows UI Automation target process is unavailable: {error}"))?; + if target_process_id <= 0 || target_process_id as u32 != foreground_process_id { + return Err( + "Windows UI Automation focus does not belong to the foreground window".to_string(), + ); + } + let value = BSTR::from(value); + if let Ok(pattern) = + unsafe { focused.GetCurrentPatternAs::(UIA_ValuePatternId) } + { + return unsafe { pattern.SetValue(&value) } + .map_err(|error| format!("Windows accessibility value action failed: {error}")); + } + if let Ok(pattern) = unsafe { + focused.GetCurrentPatternAs::( + UIA_LegacyIAccessiblePatternId, + ) + } { + return unsafe { pattern.SetValue(&value) } + .map_err(|error| format!("Windows legacy accessibility value action failed: {error}")); + } + Err("focused Windows accessibility target does not support an exact value action".to_string()) +} + +#[cfg(windows)] +fn select_windows_focused_accessibility_target() -> Result<(), String> { + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationSelectionItemPattern, + IUIAutomationTreeWalker, UIA_SelectionItemPatternId, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + GetClassNameW, GetForegroundWindow, GetWindowThreadProcessId, + }; + + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|error| format!("Windows accessibility COM initialization failed: {error}"))?; + let _guard = ComGuard; + let hwnd = unsafe { GetForegroundWindow() }; + if hwnd.0.is_null() { + return Err("Windows accessibility found no foreground window".to_string()); + } + let mut foreground_process_id = 0u32; + let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut foreground_process_id)) }; + if thread_id == 0 || foreground_process_id == 0 { + return Err("Windows accessibility could not identify the foreground process".to_string()); + } + let automation: IUIAutomation = unsafe { + CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|error| format!("Windows UI Automation client creation failed: {error}"))? + }; + let focused = unsafe { + automation + .GetFocusedElement() + .map_err(|error| format!("Windows UI Automation found no focused target: {error}"))? + }; + let focused_process_id = unsafe { focused.CurrentProcessId() } + .map_err(|error| format!("Windows UI Automation target process is unavailable: {error}"))?; + if focused_process_id <= 0 || focused_process_id as u32 != foreground_process_id { + return Err( + "Windows UI Automation focus does not belong to the foreground window".to_string(), + ); + } + let mut class_buffer = [0u16; 256]; + let class_len = unsafe { GetClassNameW(hwnd, &mut class_buffer) }.max(0) as usize; + let window_class = String::from_utf16_lossy(&class_buffer[..class_len]); + let target = if window_class.eq_ignore_ascii_case("CabinetWClass") + || window_class.eq_ignore_ascii_case("ExploreWClass") + { + let root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!("Windows UI Automation could not inspect File Explorer: {error}") + })? + }; + let walker = unsafe { + automation.ControlViewWalker().map_err(|error| { + format!("Windows UI Automation control walker is unavailable: {error}") + })? + }; + let find_selected = |root: &IUIAutomationElement, + walker: &IUIAutomationTreeWalker| + -> Option { + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + if visited > 4_096 { + return None; + } + let process_matches = unsafe { element.CurrentProcessId() } + .map(|process_id| process_id == focused_process_id) + .unwrap_or(false); + if process_matches { + let is_selected = unsafe { + element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .ok() + .and_then(|pattern| pattern.CurrentIsSelected().ok()) + .map(|selected| selected.as_bool()) + .unwrap_or(false) + }; + if is_selected { + return Some(element); + } + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } + } + None + }; + find_selected(&root, &walker).ok_or_else(|| { + "Windows UI Automation found no selected File Explorer target".to_string() + })? + } else { + focused + }; + let target_process_id = unsafe { target.CurrentProcessId() } + .map_err(|error| format!("Windows UI Automation target process is unavailable: {error}"))?; + if target_process_id <= 0 || target_process_id as u32 != foreground_process_id { + return Err( + "Windows UI Automation target does not belong to the foreground window".to_string(), + ); + } + let pattern = unsafe { + target + .GetCurrentPatternAs::(UIA_SelectionItemPatternId) + .map_err(|error| { + format!( + "focused Windows accessibility target does not support exact selection: {error}" + ) + })? + }; + unsafe { pattern.Select() } + .map_err(|error| format!("Windows accessibility selection action failed: {error}")) +} + +#[cfg(windows)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum WindowsBoundAccessibilityProfile { + FileExplorer, + Excel, +} + +#[cfg(windows)] +#[allow(dead_code)] +fn execute_bound_windows_accessibility_action( + window_handle: isize, + expected_process_id: u32, + expected_target: &WindowsBoundComputerControlTarget, + action: &ComputerControlAction, +) -> Result<(), String> { + use windows::core::BSTR; + use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationElement, IUIAutomationGridItemPattern, + IUIAutomationInvokePattern, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationSelectionItemPattern, IUIAutomationTextPattern, IUIAutomationTreeWalker, + IUIAutomationValuePattern, UIA_DataItemControlTypeId, UIA_EditControlTypeId, + UIA_GridItemPatternId, UIA_InvokePatternId, UIA_LegacyIAccessiblePatternId, + UIA_ListItemControlTypeId, UIA_SelectionItemPatternId, UIA_TextPatternId, + UIA_ValuePatternId, + }; + use windows::Win32::UI::WindowsAndMessaging::{ + GetClassNameW, GetGUIThreadInfo, GetWindowThreadProcessId, IsChild, PostMessageW, + GUITHREADINFO, WM_CHAR, WM_KEYDOWN, WM_KEYUP, + }; + + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + struct ExcelEditCancelGuard { + editor_window: HWND, + active: bool, + } + impl Drop for ExcelEditCancelGuard { + fn drop(&mut self) { + if !self.active { + return; + } + let key_down = LPARAM((1u32 | (0x01u32 << 16)) as isize); + let key_up = LPARAM((1u32 | (0x01u32 << 16) | (1u32 << 30) | (1u32 << 31)) as isize); + let _ = unsafe { + PostMessageW(Some(self.editor_window), WM_KEYDOWN, WPARAM(0x1b), key_down) + }; + let _ = + unsafe { PostMessageW(Some(self.editor_window), WM_KEYUP, WPARAM(0x1b), key_up) }; + } + } + + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|error| format!("Windows accessibility COM initialization failed: {error}"))?; + let _guard = ComGuard; + let hwnd = HWND(window_handle as _); + let mut actual_process_id = 0u32; + let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut actual_process_id)) }; + if thread_id == 0 || actual_process_id == 0 { + return Err("bound Windows accessibility HWND is no longer available".to_string()); + } + if actual_process_id != expected_process_id { + return Err( + "bound Windows accessibility HWND no longer belongs to the expected process" + .to_string(), + ); + } + let mut class_buffer = [0u16; 256]; + let class_len = unsafe { GetClassNameW(hwnd, &mut class_buffer) }.max(0) as usize; + let window_class = String::from_utf16_lossy(&class_buffer[..class_len]); + let profile = match action { + ComputerControlAction::SelectAccessibilityTarget + if window_class.eq_ignore_ascii_case("CabinetWClass") + || window_class.eq_ignore_ascii_case("ExploreWClass") => + { + WindowsBoundAccessibilityProfile::FileExplorer + } + ComputerControlAction::SetAccessibilityValue { .. } + if window_class.eq_ignore_ascii_case("XLMAIN") => + { + WindowsBoundAccessibilityProfile::Excel + } + ComputerControlAction::SelectAccessibilityTarget => { + return Err( + "bound accessibility selection requires the exact File Explorer HWND".to_string(), + ); + } + ComputerControlAction::SetAccessibilityValue { .. } => { + return Err( + "bound accessibility value action requires the exact Excel HWND".to_string(), + ); + } + _ => { + return Err( + "bound Windows computer control accepts only semantic File Explorer or Excel actions" + .to_string(), + ); + } + }; + + let automation: IUIAutomation = unsafe { + CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|error| format!("Windows UI Automation client creation failed: {error}"))? + }; + let root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!("Windows UI Automation could not inspect the exact bound window: {error}") + })? + }; + let walker = unsafe { + automation + .ControlViewWalker() + .map_err(|error| format!("Windows UI Automation control walker failed: {error}"))? + }; + let find_target = |root: &IUIAutomationElement, + walker: &IUIAutomationTreeWalker, + require_selected: bool| + -> Option { + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + let max_visited = if matches!(expected_target, WindowsBoundComputerControlTarget::Any) { + 4_096 + } else { + 1_024 + }; + if visited > max_visited { + return None; + } + let process_matches = unsafe { element.CurrentProcessId() } + .map(|process_id| process_id == expected_process_id as i32) + .unwrap_or(false); + if process_matches { + let is_selected = unsafe { + element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .ok() + .and_then(|pattern| pattern.CurrentIsSelected().ok()) + .map(|selected| selected.as_bool()) + .unwrap_or(false) + }; + let exact_target_kind = match profile { + WindowsBoundAccessibilityProfile::FileExplorer => unsafe { + element + .CurrentControlType() + .map(|control_type| control_type == UIA_ListItemControlTypeId) + .unwrap_or(false) + && match expected_target { + WindowsBoundComputerControlTarget::FileExplorer { target_name } => { + element + .CurrentName() + .map(|name| name == target_name.as_str()) + .unwrap_or(false) + } + WindowsBoundComputerControlTarget::Any => true, + WindowsBoundComputerControlTarget::Excel { .. } => false, + } + }, + WindowsBoundAccessibilityProfile::Excel => unsafe { + let is_data_item = element + .CurrentControlType() + .map(|control_type| control_type == UIA_DataItemControlTypeId) + .unwrap_or(false); + let supports_value = element + .GetCurrentPatternAs::(UIA_ValuePatternId) + .is_ok() + || element + .GetCurrentPatternAs::( + UIA_LegacyIAccessiblePatternId, + ) + .is_ok(); + let exact_excel_target = match expected_target { + WindowsBoundComputerControlTarget::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + } => { + let address_matches = element + .CurrentAutomationId() + .map(|value| value == cell_automation_id.as_str()) + .unwrap_or(false); + let grid_matches = element + .GetCurrentPatternAs::( + UIA_GridItemPatternId, + ) + .ok() + .is_some_and(|grid| { + grid.CurrentRow().ok() == Some(*row) + && grid.CurrentColumn().ok() == Some(*column) + }); + let worksheet_matches = element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .ok() + .and_then(|selection| { + selection.CurrentSelectionContainer().ok() + }) + .and_then(|container| container.CurrentAutomationId().ok()) + .map(|value| value == worksheet_automation_id.as_str()) + .unwrap_or(false); + address_matches && grid_matches && worksheet_matches + } + WindowsBoundComputerControlTarget::Any => true, + WindowsBoundComputerControlTarget::FileExplorer { .. } => false, + }; + is_data_item && supports_value && exact_excel_target + }, + }; + if (!require_selected || is_selected) && exact_target_kind { + return Some(element); + } + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } + } + None + }; + let target = find_target(&root, &walker, true).ok_or_else(|| { + "Windows UI Automation found no selected target in the exact bound window".to_string() + })?; + let target_process_id = unsafe { target.CurrentProcessId() } + .map_err(|error| format!("Windows UI Automation target process is unavailable: {error}"))?; + if target_process_id <= 0 || target_process_id as u32 != expected_process_id { + return Err( + "Windows UI Automation target does not belong to the exact bound process".to_string(), + ); + } + + match action { + ComputerControlAction::SelectAccessibilityTarget => { + let pattern = unsafe { + target + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .map_err(|error| { + format!("bound File Explorer target does not support selection: {error}") + })? + }; + unsafe { pattern.Select() } + .map_err(|error| format!("bound File Explorer selection failed: {error}")) + } + ComputerControlAction::SetAccessibilityValue { value } => { + if profile == WindowsBoundAccessibilityProfile::Excel { + let excel_action_started = std::time::Instant::now(); + let excel_phase = |phase: &str, attempt: Option| { + eprintln!( + "C5B bound Excel action phase={phase} attempt={} elapsed_ms={}", + attempt + .map(|value| value.to_string()) + .unwrap_or_else(|| "none".to_string()), + excel_action_started.elapsed().as_millis(), + ); + }; + excel_phase("validate-exact-target", None); + let ( + expected_worksheet_automation_id, + expected_cell_automation_id, + expected_row, + expected_column, + ) = + match expected_target { + WindowsBoundComputerControlTarget::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + } => (worksheet_automation_id, cell_automation_id, *row, *column), + _ => return Err( + "bound Excel value action requires an exact worksheet and cell target" + .to_string(), + ), + }; + if excel_grid_automation_id(expected_row, expected_column)? + != *expected_cell_automation_id + { + return Err( + "bound Excel cell automation id disagrees with its exact grid identity" + .to_string(), + ); + } + if value.chars().any(char::is_control) { + return Err( + "bound Excel exact cell value contains unsupported control characters" + .to_string(), + ); + } + + excel_phase("set-target-focus-start", None); + unsafe { target.SetFocus() } + .map_err(|error| format!("bound Excel target focus failed: {error}"))?; + excel_phase("set-target-focus-complete", None); + let invoke = unsafe { + target + .GetCurrentPatternAs::(UIA_InvokePatternId) + .map_err(|error| { + format!("bound Excel target does not support semantic editing: {error}") + })? + }; + excel_phase("invoke-cell-edit-start", None); + unsafe { invoke.Invoke() } + .map_err(|error| format!("bound Excel semantic edit failed: {error}"))?; + excel_phase("invoke-cell-edit-complete", None); + std::thread::sleep(Duration::from_millis(350)); + + excel_phase("discover-cell-edit-start", None); + let mut editor = None; + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(&root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + if visited > 1_024 { + break; + } + let exact_editor = unsafe { + element + .CurrentProcessId() + .map(|process_id| process_id == expected_process_id as i32) + .unwrap_or(false) + && element + .CurrentControlType() + .map(|control_type| control_type == UIA_EditControlTypeId) + .unwrap_or(false) + && element + .CurrentAutomationId() + .map(|automation_id| automation_id == "CellEdit") + .unwrap_or(false) + && element + .CurrentHasKeyboardFocus() + .map(|focused| focused.as_bool()) + .unwrap_or(false) + && element + .GetCurrentPatternAs::(UIA_TextPatternId) + .is_ok() + }; + if exact_editor { + if editor.is_some() { + return Err("bound Excel exposed multiple focused exact cell editors" + .to_string()); + } + editor = Some(element.clone()); + } + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } + } + let editor = editor.ok_or_else(|| { + "bound Excel did not expose one focused exact CellEdit through UI Automation" + .to_string() + })?; + excel_phase("discover-cell-edit-complete", None); + excel_phase("get-editor-text-pattern-start", None); + let editor_text = unsafe { + editor + .GetCurrentPatternAs::(UIA_TextPatternId) + .map_err(|error| { + format!("bound Excel exact CellEdit lost TextPattern: {error}") + })? + }; + excel_phase("get-editor-text-pattern-complete", None); + excel_phase("get-editor-document-range-start", None); + let editor_range = unsafe { editor_text.DocumentRange() }.map_err(|error| { + format!("bound Excel exact CellEdit document range failed: {error}") + })?; + excel_phase("get-editor-document-range-complete", None); + excel_phase("select-editor-document-range-start", None); + unsafe { editor_range.Select() }.map_err(|error| { + format!("bound Excel exact CellEdit selection failed: {error}") + })?; + excel_phase("select-editor-document-range-complete", None); + + excel_phase("validate-native-editor-start", None); + let mut gui = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + unsafe { GetGUIThreadInfo(thread_id, &mut gui) }.map_err(|error| { + format!("bound Excel editor thread identity is unavailable: {error}") + })?; + let editor_window = gui.hwndFocus; + let mut editor_process_id = 0u32; + let editor_thread_id = unsafe { + GetWindowThreadProcessId(editor_window, Some(&mut editor_process_id)) + }; + let editor_is_exact_or_child = + editor_window == hwnd || unsafe { IsChild(hwnd, editor_window).as_bool() }; + let mut editor_class_buffer = [0u16; 64]; + let editor_class_len = + unsafe { GetClassNameW(editor_window, &mut editor_class_buffer) }.max(0) + as usize; + let editor_window_class = + String::from_utf16_lossy(&editor_class_buffer[..editor_class_len]); + require_bound_excel_editor_identity( + window_handle, + expected_process_id, + thread_id, + editor_window.0 as isize, + editor_process_id, + editor_thread_id, + editor_is_exact_or_child, + &editor_window_class, + )?; + excel_phase("validate-native-editor-complete", None); + let mut cancel_guard = ExcelEditCancelGuard { + editor_window, + active: true, + }; + + excel_phase("post-cell-characters-start", None); + for unit in value.encode_utf16() { + unsafe { + PostMessageW( + Some(editor_window), + WM_CHAR, + WPARAM(unit as usize), + LPARAM(1), + ) + } + .map_err(|error| { + format!("bound Excel exact CellEdit character input failed: {error}") + })?; + } + excel_phase("post-cell-characters-complete", None); + let mut semantic_converged = false; + let mut last_observed_chars = None; + let mut last_read_error = None; + for attempt in 1..=20 { + excel_phase("semantic-convergence-sleep", Some(attempt)); + std::thread::sleep(Duration::from_millis(100)); + + excel_phase("semantic-convergence-main-identity-start", Some(attempt)); + let mut current_process_id = 0u32; + let current_thread_id = + unsafe { GetWindowThreadProcessId(hwnd, Some(&mut current_process_id)) }; + if current_thread_id != thread_id || current_process_id != expected_process_id { + return Err( + "bound Excel HWND identity changed during exact CellEdit semantic convergence" + .to_string(), + ); + } + excel_phase("semantic-convergence-main-identity-complete", Some(attempt)); + excel_phase("semantic-convergence-editor-identity-start", Some(attempt)); + let mut current_editor_process_id = 0u32; + let current_editor_thread_id = unsafe { + GetWindowThreadProcessId( + editor_window, + Some(&mut current_editor_process_id), + ) + }; + if current_editor_thread_id != thread_id + || current_editor_process_id != expected_process_id + { + return Err( + "bound Excel editor identity changed during semantic convergence" + .to_string(), + ); + } + excel_phase( + "semantic-convergence-editor-identity-complete", + Some(attempt), + ); + excel_phase("semantic-convergence-native-focus-start", Some(attempt)); + let mut current_gui = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + unsafe { GetGUIThreadInfo(thread_id, &mut current_gui) }.map_err(|error| { + format!( + "bound Excel editor focus is unavailable during semantic convergence: {error}" + ) + })?; + if current_gui.hwndFocus != editor_window { + return Err( + "bound Excel editor focus changed before semantic convergence" + .to_string(), + ); + } + excel_phase("semantic-convergence-native-focus-complete", Some(attempt)); + excel_phase("semantic-convergence-uia-focus-start", Some(attempt)); + let editor_has_focus = unsafe { editor.CurrentHasKeyboardFocus() } + .map(|focused| focused.as_bool()) + .unwrap_or(false); + excel_phase("semantic-convergence-uia-focus-complete", Some(attempt)); + if !editor_has_focus { + return Err( + "bound Excel exact CellEdit lost UI Automation focus before semantic convergence" + .to_string(), + ); + } + + excel_phase("semantic-convergence-document-range-start", Some(attempt)); + let current_range = unsafe { editor_text.DocumentRange() }; + excel_phase( + "semantic-convergence-document-range-complete", + Some(attempt), + ); + let observed_text = match current_range { + Ok(range) => { + excel_phase("semantic-convergence-get-text-start", Some(attempt)); + let result = unsafe { range.GetText(-1) }; + excel_phase("semantic-convergence-get-text-complete", Some(attempt)); + result + } + Err(error) => Err(error), + }; + match observed_text { + Ok(observed) => { + let observed = observed.to_string(); + last_observed_chars = Some(observed.chars().count()); + last_read_error = None; + if observed == *value { + semantic_converged = true; + excel_phase("semantic-convergence-exact", Some(attempt)); + break; + } + excel_phase("semantic-convergence-mismatch", Some(attempt)); + } + Err(error) => { + last_read_error = Some(error.to_string()); + excel_phase("semantic-convergence-read-error", Some(attempt)); + } + } + } + if !semantic_converged { + return Err(format!( + "bound Excel exact CellEdit semantic readback did not converge within 2 seconds (expected_chars={}, observed_chars={}, read_error={})", + value.chars().count(), + last_observed_chars + .map(|count| count.to_string()) + .unwrap_or_else(|| "unavailable".to_string()), + last_read_error.as_deref().unwrap_or("none"), + )); + } + + excel_phase("commit-cell-edit-start", None); + let enter_down = LPARAM((1u32 | (0x1cu32 << 16)) as isize); + let enter_up = + LPARAM((1u32 | (0x1cu32 << 16) | (1u32 << 30) | (1u32 << 31)) as isize); + unsafe { PostMessageW(Some(editor_window), WM_KEYDOWN, WPARAM(0x0d), enter_down) } + .map_err(|error| { + format!("bound Excel exact CellEdit commit keydown failed: {error}") + })?; + unsafe { PostMessageW(Some(editor_window), WM_KEYUP, WPARAM(0x0d), enter_up) } + .map_err(|error| { + format!("bound Excel exact CellEdit commit keyup failed: {error}") + })?; + cancel_guard.active = false; + excel_phase("commit-cell-edit-complete", None); + std::thread::sleep(Duration::from_millis(350)); + + excel_phase("revalidate-post-commit-hwnd-start", None); + let mut post_process_id = 0u32; + let post_thread_id = + unsafe { GetWindowThreadProcessId(hwnd, Some(&mut post_process_id)) }; + if post_thread_id != thread_id || post_process_id != expected_process_id { + return Err( + "bound Excel HWND identity changed after the exact cell commit".to_string(), + ); + } + excel_phase("revalidate-post-commit-hwnd-complete", None); + excel_phase("reobserve-post-commit-cell-start", None); + let post_root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!( + "Windows UI Automation could not re-observe the exact Excel window after commit: {error}" + ) + })? + }; + let _post_target = find_target(&post_root, &walker, false).ok_or_else(|| { + "bound Excel could not re-observe the exact worksheet and cell after commit" + .to_string() + })?; + excel_phase("reobserve-post-commit-cell-complete", None); + + excel_phase("observe-post-commit-selection-start", None); + let mut selected_cell = None; + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(&post_root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + if visited > 1_024 { + return Err( + "bound Excel post-commit selection traversal exceeded its exact bound" + .to_string(), + ); + } + let process_matches = unsafe { element.CurrentProcessId() } + .map(|process_id| process_id == expected_process_id as i32) + .unwrap_or(false); + let is_data_item = unsafe { element.CurrentControlType() } + .map(|control_type| control_type == UIA_DataItemControlTypeId) + .unwrap_or(false); + if process_matches && is_data_item { + if let Ok(selection) = unsafe { + element.GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + } { + let is_selected = unsafe { selection.CurrentIsSelected() } + .map(|selected| selected.as_bool()) + .unwrap_or(false); + if is_selected { + let grid = unsafe { + element + .GetCurrentPatternAs::( + UIA_GridItemPatternId, + ) + .map_err(|error| { + format!( + "bound Excel selected post-commit cell lost its grid identity: {error}" + ) + })? + }; + let worksheet_automation_id = unsafe { + selection + .CurrentSelectionContainer() + .and_then(|container| container.CurrentAutomationId()) + .map_err(|error| { + format!( + "bound Excel selected post-commit cell lost its worksheet identity: {error}" + ) + })? + .to_string() + }; + let automation_id = unsafe { element.CurrentAutomationId() } + .map_err(|error| { + format!( + "bound Excel selected post-commit cell lost its address: {error}" + ) + })? + .to_string(); + let row = unsafe { grid.CurrentRow() }.map_err(|error| { + format!( + "bound Excel selected post-commit cell lost its row: {error}" + ) + })?; + let column = + unsafe { grid.CurrentColumn() }.map_err(|error| { + format!( + "bound Excel selected post-commit cell lost its column: {error}" + ) + })?; + if selected_cell.is_some() { + return Err( + "bound Excel exposed multiple selected post-commit cells" + .to_string(), + ); + } + selected_cell = + Some((automation_id, worksheet_automation_id, row, column)); + } + } + } + if process_matches { + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } + } + let ( + selected_automation_id, + selected_worksheet_automation_id, + selected_row, + selected_column, + ) = selected_cell.ok_or_else(|| { + "bound Excel exposed no selected post-commit cell".to_string() + })?; + excel_phase("observe-post-commit-selection-complete", None); + + let target_is_selected = selected_automation_id == *expected_cell_automation_id + && selected_worksheet_automation_id == *expected_worksheet_automation_id + && selected_row == expected_row + && selected_column == expected_column; + if !target_is_selected { + let next_row = expected_row.checked_add(1).ok_or_else(|| { + "bound Excel post-commit row identity overflowed".to_string() + })?; + let expected_next_automation_id = + excel_grid_automation_id(next_row, expected_column)?; + if selected_automation_id != expected_next_automation_id + || selected_worksheet_automation_id != *expected_worksheet_automation_id + || selected_row != next_row + || selected_column != expected_column + { + return Err( + "bound Excel post-commit selection changed to an unexpected workbook cell" + .to_string(), + ); + } + + excel_phase("validate-post-commit-grid-start", None); + let mut grid_gui = GUITHREADINFO { + cbSize: std::mem::size_of::() as u32, + ..Default::default() + }; + unsafe { GetGUIThreadInfo(thread_id, &mut grid_gui) }.map_err(|error| { + format!("bound Excel post-commit grid focus is unavailable: {error}") + })?; + let grid_window = grid_gui.hwndFocus; + let mut grid_process_id = 0u32; + let grid_thread_id = unsafe { + GetWindowThreadProcessId(grid_window, Some(&mut grid_process_id)) + }; + let grid_is_exact_or_child = + grid_window == hwnd || unsafe { IsChild(hwnd, grid_window).as_bool() }; + let mut grid_class_buffer = [0u16; 64]; + let grid_class_len = + unsafe { GetClassNameW(grid_window, &mut grid_class_buffer) }.max(0) + as usize; + let grid_window_class = + String::from_utf16_lossy(&grid_class_buffer[..grid_class_len]); + require_bound_excel_grid_identity( + window_handle, + expected_process_id, + thread_id, + grid_window.0 as isize, + grid_process_id, + grid_thread_id, + grid_is_exact_or_child, + &grid_window_class, + )?; + excel_phase("validate-post-commit-grid-complete", None); + + excel_phase("restore-target-selection-start", None); + let up_down = LPARAM((1u32 | (0x48u32 << 16)) as isize); + let up_up = + LPARAM((1u32 | (0x48u32 << 16) | (1u32 << 30) | (1u32 << 31)) as isize); + unsafe { PostMessageW(Some(grid_window), WM_KEYDOWN, WPARAM(0x26), up_down) } + .map_err(|error| { + format!("bound Excel exact grid Up keydown failed: {error}") + })?; + unsafe { PostMessageW(Some(grid_window), WM_KEYUP, WPARAM(0x26), up_up) } + .map_err(|error| { + format!("bound Excel exact grid Up keyup failed: {error}") + })?; + excel_phase("restore-target-selection-complete", None); + std::thread::sleep(Duration::from_millis(300)); + } + + excel_phase("verify-post-commit-target-selection-start", None); + let final_root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!( + "Windows UI Automation could not verify the exact Excel target selection after commit: {error}" + ) + })? + }; + find_target(&final_root, &walker, true).ok_or_else(|| { + "bound Excel could not verify the exact worksheet and cell selection after commit" + .to_string() + })?; + excel_phase("verify-post-commit-target-selection-complete", None); + + excel_phase("final-hwnd-identity-start", None); + let mut final_process_id = 0u32; + let final_thread_id = + unsafe { GetWindowThreadProcessId(hwnd, Some(&mut final_process_id)) }; + if final_thread_id != thread_id || final_process_id != expected_process_id { + return Err( + "bound Excel HWND identity changed after exact target selection restoration" + .to_string(), + ); + } + excel_phase("final-hwnd-identity-complete", None); + return Ok(()); + } + let value = BSTR::from(value.as_str()); + if let Ok(pattern) = unsafe { + target.GetCurrentPatternAs::(UIA_ValuePatternId) + } { + return unsafe { pattern.SetValue(&value) } + .map_err(|error| format!("bound Excel value action failed: {error}")); + } + let pattern = unsafe { + target + .GetCurrentPatternAs::( + UIA_LegacyIAccessiblePatternId, + ) + .map_err(|error| { + format!("bound Excel target does not support an exact value: {error}") + })? + }; + unsafe { pattern.SetValue(&value) } + .map_err(|error| format!("bound Excel legacy value action failed: {error}")) + } + _ => Err("bound Windows semantic action changed unexpectedly".to_string()), + } +} + fn enigo_button(button: ComputerControlMouseButton) -> enigo::Button { match button { ComputerControlMouseButton::Left => enigo::Button::Left, @@ -3288,7 +4848,7 @@ pub fn parse_computer_control_action(value: &str) -> Result Result { + if payload.contains('\0') { + return Err("accessibility value action cannot contain null bytes".to_string()); + } + if payload.chars().count() > 2_000 { + return Err("accessibility value action exceeds 2000 characters".to_string()); + } + Ok(ComputerControlAction::SetAccessibilityValue { + value: payload.to_string(), + }) + } + "select" if payload.eq_ignore_ascii_case("focused") => { + Ok(ComputerControlAction::SelectAccessibilityTarget) + } + "select" => Err("select action must be select:focused".to_string()), "press" => Ok(ComputerControlAction::PressKey { key: normalize_control_key(payload)?, }), @@ -3369,7 +4944,7 @@ pub fn parse_computer_control_action(value: &str) -> Result Err(format!( - "unsupported computer control action '{verb}': use click, move, type, press, hotkey, or scroll" + "unsupported computer control action '{verb}': use click, move, type, set_value, select, press, hotkey, or scroll" )), } } @@ -3381,6 +4956,8 @@ pub fn computer_control_action_contract_string(action: &ComputerControlAction) - } ComputerControlAction::Move { x, y } => format!("move:{x},{y}"), ComputerControlAction::TypeText { text } => format!("type:{text}"), + ComputerControlAction::SetAccessibilityValue { value } => format!("set_value:{value}"), + ComputerControlAction::SelectAccessibilityTarget => "select:focused".to_string(), ComputerControlAction::PressKey { key } => format!("press:{key}"), ComputerControlAction::Hotkey { keys } => format!("hotkey:{}", keys.join("+")), ComputerControlAction::Scroll { delta, axis } => { @@ -4253,24 +5830,32 @@ mod tests { use base64::{engine::general_purpose, Engine as _}; use chrono::{TimeZone, Utc}; + #[cfg(windows)] + use crate::kernel::capability::{ + bound_windows_accessibility_action_timeout, excel_grid_automation_id, + require_bound_excel_editor_identity, require_bound_excel_grid_identity, + require_bound_windows_process_identity, WindowsBoundComputerControlTarget, + WindowsBoundComputerScreenshotClient, + }; use crate::kernel::capability::{ - parse_computer_control_action, run_browser_browse, run_browser_submit_boundary, - run_computer_control_boundary, run_computer_screenshot, run_drive_read_boundary, - run_drive_write_boundary, run_email_draft_boundary, run_email_read_boundary, - run_email_send_boundary, run_evidence_folder_ingest, run_file_read, - run_file_write_boundary, run_filesystem_mutation_boundary, run_network_search_boundary, - run_terminal_read, run_terminal_write_boundary, BrowserBrowseRequest, BrowserPage, - BrowserPageClient, BrowserSubmitRequest, CapabilityInvocation, CapabilityInvocationStatus, - CapturedScreenshotImage, CodexBridgeComputerControlClient, - CodexBridgeComputerScreenshotClient, CodexBridgeNetworkSearchClient, ComputerControlAction, - ComputerControlClient, ComputerControlExecution, ComputerControlMouseButton, - ComputerControlRequest, ComputerScreenshot, ComputerScreenshotClient, - ComputerScreenshotRequest, DriveFolderEntry, DriveReadRequest, DriveWriteExportFile, - DriveWriteRequest, EmailDraftRequest, EmailReadRequest, EmailSendRequest, - EvidenceFolderClient, EvidenceFolderFile, EvidenceFolderRequest, FileContent, - FileContentClient, FileReadRequest, FileSystemMutationClient, FileSystemMutationOperation, - FileSystemMutationRequest, FileSystemMutationResult, FileWriteRequest, FileWriteResult, - HttpBrowserPageClient, LocalComputerControlClient, LocalComputerControlInputBackend, + computer_control_action_contract_string, parse_computer_control_action, run_browser_browse, + run_browser_submit_boundary, run_computer_control_boundary, run_computer_screenshot, + run_drive_read_boundary, run_drive_write_boundary, run_email_draft_boundary, + run_email_read_boundary, run_email_send_boundary, run_evidence_folder_ingest, + run_file_read, run_file_write_boundary, run_filesystem_mutation_boundary, + run_network_search_boundary, run_terminal_read, run_terminal_write_boundary, + BrowserBrowseRequest, BrowserPage, BrowserPageClient, BrowserSubmitRequest, + CapabilityInvocation, CapabilityInvocationStatus, CapturedScreenshotImage, + CodexBridgeComputerControlClient, CodexBridgeComputerScreenshotClient, + CodexBridgeNetworkSearchClient, ComputerControlAction, ComputerControlClient, + ComputerControlExecution, ComputerControlMouseButton, ComputerControlRequest, + ComputerScreenshot, ComputerScreenshotClient, ComputerScreenshotRequest, DriveFolderEntry, + DriveReadRequest, DriveWriteExportFile, DriveWriteRequest, EmailDraftRequest, + EmailReadRequest, EmailSendRequest, EvidenceFolderClient, EvidenceFolderFile, + EvidenceFolderRequest, FileContent, FileContentClient, FileReadRequest, + FileSystemMutationClient, FileSystemMutationOperation, FileSystemMutationRequest, + FileSystemMutationResult, FileWriteRequest, FileWriteResult, HttpBrowserPageClient, + LocalComputerControlClient, LocalComputerControlInputBackend, LocalComputerScreenshotClient, LocalDriveFolderClient, LocalFileSystemMutationClient, LocalScreenshotCaptureBackend, LocalWorkspaceFileWriteClient, NetworkSearchClient, NetworkSearchRequest, NetworkSearchResult, NetworkSearchResultItem, TerminalCommandOutput, @@ -4702,6 +6287,17 @@ mod tests { Ok(()) } + fn set_accessibility_value(&mut self, value: &str) -> Result<(), String> { + self.operations + .push(format!("set_value:{} chars", value.chars().count())); + Ok(()) + } + + fn select_accessibility_target(&mut self) -> Result<(), String> { + self.operations.push("select:focused".to_string()); + Ok(()) + } + fn key_down(&mut self, key: &str) -> Result<(), String> { self.operations.push(format!("down:{key}")); Ok(()) @@ -5515,6 +7111,65 @@ mod tests { assert!(!temp_dir.path().join("computer-screenshots").exists()); } + #[cfg(windows)] + #[test] + fn bound_windows_screenshot_rejects_invalid_hwnd_and_process_identity() { + let temp_dir = tempfile::tempdir().expect("temp dir"); + assert!( + WindowsBoundComputerScreenshotClient::new(temp_dir.path().to_path_buf(), 0, 1,) + .is_err() + ); + assert!( + WindowsBoundComputerScreenshotClient::new(temp_dir.path().to_path_buf(), 1, 0,) + .is_err() + ); + assert!(require_bound_windows_process_identity(41, 42).is_err()); + + let client = WindowsBoundComputerScreenshotClient::new(temp_dir.path().to_path_buf(), 1, 1) + .expect("nonzero identity binds provisionally"); + assert!(client.capture_screenshot().is_err()); + assert!(!temp_dir.path().join("computer-screenshots").exists()); + } + + #[cfg(windows)] + #[test] + fn bound_excel_editor_identity_rejects_wrong_handle_process_thread_and_class() { + assert!( + require_bound_excel_editor_identity(100, 41, 7, 200, 41, 7, true, "EXCEL6").is_ok() + ); + assert!(require_bound_excel_editor_identity(0, 41, 7, 200, 41, 7, true, "EXCEL6").is_err()); + assert!(require_bound_excel_editor_identity(100, 41, 7, 0, 41, 7, true, "EXCEL6").is_err()); + assert!( + require_bound_excel_editor_identity(100, 41, 7, 200, 42, 7, true, "EXCEL6").is_err() + ); + assert!( + require_bound_excel_editor_identity(100, 41, 7, 200, 41, 8, true, "EXCEL6").is_err() + ); + assert!( + require_bound_excel_editor_identity(100, 41, 7, 200, 41, 7, false, "EXCEL6").is_err() + ); + assert!( + require_bound_excel_editor_identity(100, 41, 7, 200, 41, 7, true, "EXCEL7").is_err() + ); + } + + #[cfg(windows)] + #[test] + fn bound_excel_grid_identity_and_one_based_address_fail_closed() { + assert_eq!(excel_grid_automation_id(3, 2).unwrap(), "B3"); + assert_eq!(excel_grid_automation_id(10, 27).unwrap(), "AA10"); + assert!(excel_grid_automation_id(0, 2).is_err()); + assert!(excel_grid_automation_id(3, 0).is_err()); + + assert!(require_bound_excel_grid_identity(100, 41, 7, 200, 41, 7, true, "EXCEL7").is_ok()); + assert!(require_bound_excel_grid_identity(100, 41, 7, 200, 42, 7, true, "EXCEL7").is_err()); + assert!(require_bound_excel_grid_identity(100, 41, 7, 200, 41, 8, true, "EXCEL7").is_err()); + assert!( + require_bound_excel_grid_identity(100, 41, 7, 200, 41, 7, false, "EXCEL7").is_err() + ); + assert!(require_bound_excel_grid_identity(100, 41, 7, 200, 41, 7, true, "EXCEL6").is_err()); + } + #[cfg(windows)] #[test] #[ignore = "requires a visible interactive Windows desktop"] @@ -5555,6 +7210,32 @@ mod tests { ); } + #[cfg(windows)] + #[test] + fn bound_excel_deadline_dominates_measured_path_without_widening_other_actions() { + assert_eq!( + bound_windows_accessibility_action_timeout(&WindowsBoundComputerControlTarget::Excel { + worksheet_automation_id: "C5B_Target".to_string(), + cell_automation_id: "B3".to_string(), + row: 3, + column: 2, + }), + Duration::from_secs(9), + ); + assert_eq!( + bound_windows_accessibility_action_timeout( + &WindowsBoundComputerControlTarget::FileExplorer { + target_name: "c5b-target.txt".to_string(), + } + ), + Duration::from_secs(6), + ); + assert_eq!( + bound_windows_accessibility_action_timeout(&WindowsBoundComputerControlTarget::Any), + Duration::from_secs(6), + ); + } + #[test] fn computer_control_action_parser_accepts_click_and_hotkey_actions() { assert_eq!( @@ -5573,6 +7254,37 @@ mod tests { ); } + #[test] + fn computer_control_action_parser_accepts_bounded_accessibility_actions() { + assert_eq!( + parse_computer_control_action("set_value:DS Agent C5B") + .expect("accessibility value action parses"), + ComputerControlAction::SetAccessibilityValue { + value: "DS Agent C5B".to_string(), + } + ); + assert_eq!( + parse_computer_control_action("select:focused") + .expect("accessibility selection action parses"), + ComputerControlAction::SelectAccessibilityTarget + ); + assert_eq!( + computer_control_action_contract_string( + &ComputerControlAction::SetAccessibilityValue { + value: "DS Agent C5B".to_string(), + }, + ), + "set_value:DS Agent C5B" + ); + assert_eq!( + computer_control_action_contract_string( + &ComputerControlAction::SelectAccessibilityTarget, + ), + "select:focused" + ); + assert!(parse_computer_control_action("select:other").is_err()); + } + #[test] fn computer_control_action_parser_rejects_unstructured_natural_language() { let error = parse_computer_control_action("Click the Submit button") @@ -5783,6 +7495,34 @@ mod tests { ); } + #[test] + fn local_computer_control_client_routes_semantic_accessibility_actions_once() { + let client = LocalComputerControlClient::new(); + let mut backend = FakeLocalControlInputBackend::default(); + + let value_execution = client + .execute_with_backend( + &ComputerControlAction::SetAccessibilityValue { + value: "verified".to_string(), + }, + &mut backend, + ) + .expect("accessibility value action translates"); + let select_execution = client + .execute_with_backend( + &ComputerControlAction::SelectAccessibilityTarget, + &mut backend, + ) + .expect("accessibility selection action translates"); + + assert_eq!( + backend.operations, + vec!["set_value:8 chars", "select:focused"] + ); + assert!(value_execution.summary.contains("8 chars")); + assert!(select_execution.summary.contains("select focused")); + } + #[test] fn computer_control_boundary_rejects_missing_fields() { let client = FakeComputerControlClient::new(); diff --git a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs index d7c943d..bbf8aad 100644 --- a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs +++ b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs @@ -648,15 +648,57 @@ pub fn accessibility_value_semantic_fingerprint(value: &str) -> Result Self { + if window_class.eq_ignore_ascii_case("CabinetWClass") + || window_class.eq_ignore_ascii_case("ExploreWClass") + { + Self::FileExplorer + } else if window_class.eq_ignore_ascii_case("XLMAIN") { + Self::Excel + } else { + Self::Generic + } + } + + fn contract_name(self) -> &'static str { + match self { + Self::FileExplorer => "file-explorer", + Self::Excel => "excel", + Self::Generic => "generic", + } + } + + fn safe_label(self) -> &'static str { + match self { + Self::FileExplorer => "File Explorer", + Self::Excel => "Excel", + Self::Generic => "Windows", + } + } +} + #[cfg(windows)] fn current_windows_bounded_semantic_value( element: &windows::Win32::UI::Accessibility::IUIAutomationElement, + profile: WindowsComputerUseTargetProfile, ) -> Option { use windows::Win32::UI::Accessibility::{ - IUIAutomationTextPattern, IUIAutomationValuePattern, UIA_TextPatternId, UIA_ValuePatternId, + IUIAutomationLegacyIAccessiblePattern, IUIAutomationSelectionItemPattern, + IUIAutomationTextPattern, IUIAutomationValuePattern, UIA_LegacyIAccessiblePatternId, + UIA_SelectionItemPatternId, UIA_TextPatternId, UIA_ValuePatternId, }; - unsafe { + let value = || unsafe { element .GetCurrentPatternAs::(UIA_ValuePatternId) .ok() @@ -670,24 +712,396 @@ fn current_windows_bounded_semantic_value( .and_then(|range| range.GetText(MAX_SEMANTIC_VALUE_CHARS as i32).ok()) .map(|value| value.to_string()) }) + .or_else(|| { + element + .GetCurrentPatternAs::( + UIA_LegacyIAccessiblePatternId, + ) + .ok() + .and_then(|pattern| pattern.CurrentValue().ok()) + .map(|value| value.to_string()) + }) + }; + let selection = || unsafe { + element + .GetCurrentPatternAs::(UIA_SelectionItemPatternId) + .ok() + .and_then(|pattern| pattern.CurrentIsSelected().ok()) + .map(|selected| { + if selected.as_bool() { + "selection:selected".to_string() + } else { + "selection:not_selected".to_string() + } + }) + }; + match profile { + WindowsComputerUseTargetProfile::FileExplorer => selection().or_else(value), + WindowsComputerUseTargetProfile::Excel | WindowsComputerUseTargetProfile::Generic => { + value().or_else(selection) + } + } +} + +#[cfg(windows)] +fn current_windows_accessibility_ancestor_fingerprint( + focused: &windows::Win32::UI::Accessibility::IUIAutomationElement, + walker: &windows::Win32::UI::Accessibility::IUIAutomationTreeWalker, + target_process_id: i32, + include_volatile_labels: bool, +) -> String { + let mut ancestor_fingerprints = vec!["windows-accessibility-ancestor-frame/v1".to_string()]; + let mut current = focused.clone(); + for _ in 0..16 { + let Ok(parent) = (unsafe { walker.GetParentElement(¤t) }) else { + break; + }; + let Ok(parent_process_id) = (unsafe { parent.CurrentProcessId() }) else { + break; + }; + if parent_process_id != target_process_id { + break; + } + let control_type = unsafe { parent.CurrentControlType() } + .map(|value| value.0.to_string()) + .unwrap_or_default(); + let automation_id = unsafe { parent.CurrentAutomationId() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let class_name = unsafe { parent.CurrentClassName() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let framework_id = unsafe { parent.CurrentFrameworkId() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let name = unsafe { parent.CurrentName() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let item_status = unsafe { parent.CurrentItemStatus() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let help_text = unsafe { parent.CurrentHelpText() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let stable_fingerprint = fingerprint_parts(&[ + &control_type, + &fingerprint_parts(&[&automation_id]), + &fingerprint_parts(&[&class_name]), + &fingerprint_parts(&[&framework_id]), + ]); + ancestor_fingerprints.push(if include_volatile_labels { + fingerprint_parts(&[ + &stable_fingerprint, + &fingerprint_parts(&[&name]), + &fingerprint_parts(&[&item_status]), + &fingerprint_parts(&[&help_text]), + ]) + } else { + stable_fingerprint + }); + current = parent; + } + let parts = ancestor_fingerprints + .iter() + .map(String::as_str) + .collect::>(); + fingerprint_parts(&parts) +} + +#[cfg(windows)] +fn current_windows_selected_accessibility_descendant( + root: &windows::Win32::UI::Accessibility::IUIAutomationElement, + walker: &windows::Win32::UI::Accessibility::IUIAutomationTreeWalker, + target_process_id: i32, + profile: WindowsComputerUseTargetProfile, + expected_target: Option<&WindowsBoundAccessibilityTarget>, +) -> Option { + use windows::Win32::UI::Accessibility::{ + IUIAutomationGridItemPattern, IUIAutomationLegacyIAccessiblePattern, + IUIAutomationSelectionItemPattern, IUIAutomationValuePattern, UIA_DataItemControlTypeId, + UIA_GridItemPatternId, UIA_LegacyIAccessiblePatternId, UIA_ListItemControlTypeId, + UIA_SelectionItemPatternId, UIA_ValuePatternId, + }; + + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + let max_visited = if expected_target.is_some() { + 1_024 + } else { + 4_096 + }; + if visited > max_visited { + return None; + } + let process_matches = unsafe { element.CurrentProcessId() } + .map(|process_id| process_id == target_process_id) + .unwrap_or(false); + if process_matches { + let is_selected = unsafe { + element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .ok() + .and_then(|pattern| pattern.CurrentIsSelected().ok()) + .map(|selected| selected.as_bool()) + .unwrap_or(false) + }; + let exact_target_kind = match profile { + WindowsComputerUseTargetProfile::FileExplorer => unsafe { + element + .CurrentControlType() + .map(|control_type| control_type == UIA_ListItemControlTypeId) + .unwrap_or(false) + && match expected_target { + Some(WindowsBoundAccessibilityTarget::FileExplorer { target_name }) => { + element + .CurrentName() + .map(|name| name == target_name.as_str()) + .unwrap_or(false) + } + None | Some(WindowsBoundAccessibilityTarget::Any) => true, + Some(_) => false, + } + }, + WindowsComputerUseTargetProfile::Excel => unsafe { + let is_data_item = element + .CurrentControlType() + .map(|control_type| control_type == UIA_DataItemControlTypeId) + .unwrap_or(false); + let supports_value = element + .GetCurrentPatternAs::(UIA_ValuePatternId) + .is_ok() + || element + .GetCurrentPatternAs::( + UIA_LegacyIAccessiblePatternId, + ) + .is_ok(); + let exact_excel_target = match expected_target { + Some(WindowsBoundAccessibilityTarget::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + }) => { + let address_matches = element + .CurrentAutomationId() + .map(|value| value == cell_automation_id.as_str()) + .unwrap_or(false); + let grid_matches = element + .GetCurrentPatternAs::( + UIA_GridItemPatternId, + ) + .ok() + .is_some_and(|grid| { + grid.CurrentRow().ok() == Some(*row) + && grid.CurrentColumn().ok() == Some(*column) + }); + let worksheet_matches = element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .ok() + .and_then(|selection| selection.CurrentSelectionContainer().ok()) + .and_then(|container| container.CurrentAutomationId().ok()) + .map(|value| value == worksheet_automation_id.as_str()) + .unwrap_or(false); + address_matches && grid_matches && worksheet_matches + } + None | Some(WindowsBoundAccessibilityTarget::Any) => true, + Some(_) => false, + }; + is_data_item && supports_value && exact_excel_target + }, + WindowsComputerUseTargetProfile::Generic => false, + }; + if is_selected && exact_target_kind { + return Some(element); + } + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } } + None } #[cfg(windows)] #[derive(Clone, Copy, Debug, Default)] pub struct WindowsComputerUseAccessibilityClient; +#[cfg(windows)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum WindowsComputerUseWindowBinding { + Foreground, + Exact { + window_handle: isize, + process_id: u32, + }, +} + +#[cfg(windows)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +pub struct WindowsBoundComputerUseAccessibilityClient { + window_handle: isize, + process_id: u32, + target: WindowsBoundAccessibilityTarget, +} + +#[cfg(windows)] +#[derive(Clone, Debug, Eq, PartialEq)] +#[allow(dead_code)] +enum WindowsBoundAccessibilityTarget { + Any, + FileExplorer { + target_name: String, + }, + Excel { + worksheet_automation_id: String, + cell_automation_id: String, + row: i32, + column: i32, + }, +} + +#[cfg(windows)] +impl WindowsBoundAccessibilityTarget { + fn contract_fingerprint(&self) -> String { + match self { + Self::Any => fingerprint_parts(&["windows-bound-target/v1", "any"]), + Self::FileExplorer { target_name } => fingerprint_parts(&[ + "windows-bound-target/v1", + "file-explorer", + &fingerprint_parts(&[target_name]), + ]), + Self::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + } => fingerprint_parts(&[ + "windows-bound-target/v1", + "excel", + &fingerprint_parts(&[worksheet_automation_id]), + &fingerprint_parts(&[cell_automation_id]), + &row.to_string(), + &column.to_string(), + ]), + } + } +} + +#[cfg(windows)] +#[allow(dead_code)] +impl WindowsBoundComputerUseAccessibilityClient { + pub fn new(window_handle: isize, process_id: u32) -> Result { + if window_handle == 0 || process_id == 0 { + return Err( + "bound Windows accessibility observation requires an exact HWND and process" + .to_string(), + ); + } + Ok(Self { + window_handle, + process_id, + target: WindowsBoundAccessibilityTarget::Any, + }) + } + + pub fn new_file_explorer( + window_handle: isize, + process_id: u32, + target_name: String, + ) -> Result { + let mut client = Self::new(window_handle, process_id)?; + if target_name.trim().is_empty() { + return Err( + "bound File Explorer observation requires an exact target name".to_string(), + ); + } + client.target = WindowsBoundAccessibilityTarget::FileExplorer { target_name }; + Ok(client) + } + + pub fn new_excel( + window_handle: isize, + process_id: u32, + worksheet_automation_id: String, + cell_automation_id: String, + row: i32, + column: i32, + ) -> Result { + let mut client = Self::new(window_handle, process_id)?; + if worksheet_automation_id.trim().is_empty() + || cell_automation_id.trim().is_empty() + || row < 0 + || column < 0 + { + return Err( + "bound Excel observation requires an exact worksheet, cell, row, and column" + .to_string(), + ); + } + client.target = WindowsBoundAccessibilityTarget::Excel { + worksheet_automation_id, + cell_automation_id, + row, + column, + }; + Ok(client) + } +} + #[cfg(windows)] impl ComputerUseAccessibilityClient for WindowsComputerUseAccessibilityClient { fn capture_redacted_state(&self) -> Result { - std::thread::spawn(capture_windows_redacted_state) - .join() - .map_err(|_| "Windows accessibility observation thread failed".to_string())? + std::thread::spawn(|| { + capture_windows_redacted_state(WindowsComputerUseWindowBinding::Foreground, None) + }) + .join() + .map_err(|_| "Windows accessibility observation thread failed".to_string())? + } +} + +#[cfg(windows)] +impl ComputerUseAccessibilityClient for WindowsBoundComputerUseAccessibilityClient { + fn capture_redacted_state(&self) -> Result { + use std::sync::mpsc; + use std::time::Duration; + + let binding = WindowsComputerUseWindowBinding::Exact { + window_handle: self.window_handle, + process_id: self.process_id, + }; + let target = self.target.clone(); + let (sender, receiver) = mpsc::sync_channel(1); + std::thread::spawn(move || { + let _ = sender.send(capture_windows_redacted_state(binding, Some(&target))); + }); + receiver.recv_timeout(Duration::from_secs(3)).map_err(|error| { + format!( + "bound Windows accessibility observation timed out after 3 seconds during exact target discovery: {error}" + ) + })? } } #[cfg(windows)] -fn capture_windows_redacted_state() -> Result { +fn capture_windows_redacted_state( + binding: WindowsComputerUseWindowBinding, + expected_target: Option<&WindowsBoundAccessibilityTarget>, +) -> Result { + use windows::Win32::Foundation::HWND; use windows::Win32::System::Com::{ CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, COINIT_MULTITHREADED, @@ -710,18 +1124,41 @@ fn capture_windows_redacted_state() -> Result .map_err(|error| format!("Windows accessibility COM initialization failed: {error}"))?; let _guard = ComGuard; - let hwnd = unsafe { GetForegroundWindow() }; - if hwnd.0.is_null() { - return Err("Windows accessibility found no foreground window".to_string()); - } + let (hwnd, expected_process_id, window_binding_name, safe_window_label) = match binding { + WindowsComputerUseWindowBinding::Foreground => { + let hwnd = unsafe { GetForegroundWindow() }; + if hwnd.0.is_null() { + return Err("Windows accessibility found no foreground window".to_string()); + } + (hwnd, None, "windows-foreground-window/v1", "Foreground") + } + WindowsComputerUseWindowBinding::Exact { + window_handle, + process_id, + } => ( + HWND(window_handle as _), + Some(process_id), + "windows-exact-window/v1", + "Bound", + ), + }; let mut process_id = 0u32; let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut process_id)) }; if thread_id == 0 || process_id == 0 { - return Err("Windows accessibility could not identify the foreground process".to_string()); + return Err( + "Windows accessibility could not identify the bound window process".to_string(), + ); + } + if expected_process_id.is_some_and(|expected| expected != process_id) { + return Err( + "Windows accessibility bound HWND no longer belongs to the expected process" + .to_string(), + ); } let mut class_buffer = [0u16; 256]; let class_len = unsafe { GetClassNameW(hwnd, &mut class_buffer) }.max(0) as usize; let window_class = String::from_utf16_lossy(&class_buffer[..class_len]); + let target_profile = WindowsComputerUseTargetProfile::from_window_class(&window_class); let mut title_buffer = [0u16; 1_024]; let title_len = unsafe { GetWindowTextW(hwnd, &mut title_buffer) }.max(0) as usize; let window_title_fingerprint = @@ -731,7 +1168,7 @@ fn capture_windows_redacted_state() -> Result let thread_text = thread_id.to_string(); let process_fingerprint = fingerprint_parts(&["windows-process/v1", process_text.as_str()]); let window_fingerprint = fingerprint_parts(&[ - "windows-foreground-window/v1", + window_binding_name, &handle_identity, &process_text, &thread_text, @@ -742,16 +1179,81 @@ fn capture_windows_redacted_state() -> Result CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) .map_err(|error| format!("Windows UI Automation client creation failed: {error}"))? }; - let focused = unsafe { - automation - .GetFocusedElement() - .map_err(|error| format!("Windows UI Automation found no focused target: {error}"))? + let control_walker = unsafe { + automation.ControlViewWalker().map_err(|error| { + format!("Windows UI Automation control walker is unavailable: {error}") + })? + }; + let focused = match binding { + WindowsComputerUseWindowBinding::Exact { .. } => { + if !matches!( + target_profile, + WindowsComputerUseTargetProfile::FileExplorer + | WindowsComputerUseTargetProfile::Excel + ) { + return Err( + "bound Windows accessibility observation supports only File Explorer or Excel" + .to_string(), + ); + } + let root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!("Windows UI Automation could not inspect the bound window: {error}") + })? + }; + current_windows_selected_accessibility_descendant( + &root, + &control_walker, + process_id as i32, + target_profile, + expected_target, + ) + .ok_or_else(|| { + "Windows UI Automation found no selected target in the exact bound window" + .to_string() + })? + } + WindowsComputerUseWindowBinding::Foreground => { + let focused = unsafe { + automation.GetFocusedElement().map_err(|error| { + format!("Windows UI Automation found no focused target: {error}") + })? + }; + let focused_process_id = unsafe { focused.CurrentProcessId() }.map_err(|error| { + format!("Windows UI Automation target process is unavailable: {error}") + })?; + if focused_process_id <= 0 || focused_process_id as u32 != process_id { + return Err( + "Windows UI Automation focus does not belong to the foreground window" + .to_string(), + ); + } + if target_profile == WindowsComputerUseTargetProfile::FileExplorer { + let root = unsafe { + automation.ElementFromHandle(hwnd).map_err(|error| { + format!("Windows UI Automation could not inspect File Explorer: {error}") + })? + }; + current_windows_selected_accessibility_descendant( + &root, + &control_walker, + focused_process_id, + target_profile, + None, + ) + .ok_or_else(|| { + "Windows UI Automation found no selected File Explorer target".to_string() + })? + } else { + focused + } + } }; let target_process_id = unsafe { focused.CurrentProcessId() } .map_err(|error| format!("Windows UI Automation target process is unavailable: {error}"))?; if target_process_id <= 0 || target_process_id as u32 != process_id { return Err( - "Windows UI Automation focus does not belong to the foreground window".to_string(), + "Windows UI Automation target does not belong to the foreground window".to_string(), ); } let control_type = unsafe { focused.CurrentControlType() } @@ -766,6 +1268,18 @@ fn capture_windows_redacted_state() -> Result let target_name = unsafe { focused.CurrentName() } .map(|value| value.to_string()) .unwrap_or_default(); + let target_item_status = unsafe { focused.CurrentItemStatus() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let target_help_text = unsafe { focused.CurrentHelpText() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let target_localized_control_type = unsafe { focused.CurrentLocalizedControlType() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let target_framework_id = unsafe { focused.CurrentFrameworkId() } + .map(|value| value.to_string()) + .unwrap_or_default(); let is_password = unsafe { focused.CurrentIsPassword() } .map(|value| value.as_bool()) .unwrap_or(true); @@ -775,53 +1289,109 @@ fn capture_windows_redacted_state() -> Result let is_keyboard_focusable = unsafe { focused.CurrentIsKeyboardFocusable() } .map(|value| value.as_bool()) .unwrap_or(false); - let target_fingerprint = fingerprint_parts(&[ - "windows-accessibility-target/v1", - &process_text, - &control_type_text, - &fingerprint_parts(&[&automation_id]), - &fingerprint_parts(&[&target_class]), - &fingerprint_parts(&[&target_name]), - if is_password { - "password" - } else { - "not_password" - }, - if is_enabled { "enabled" } else { "disabled" }, - if is_keyboard_focusable { - "keyboard_focusable" - } else { - "not_keyboard_focusable" - }, - ]); + let expected_target_fingerprint = expected_target + .map(WindowsBoundAccessibilityTarget::contract_fingerprint) + .unwrap_or_else(|| fingerprint_parts(&["windows-bound-target/v1", "unspecified"])); + let stable_target_fingerprint = if binding == WindowsComputerUseWindowBinding::Foreground { + fingerprint_parts(&[ + "windows-accessibility-target/v1", + target_profile.contract_name(), + &process_text, + &control_type_text, + &fingerprint_parts(&[&automation_id]), + &fingerprint_parts(&[&target_class]), + &fingerprint_parts(&[&target_name]), + &fingerprint_parts(&[&target_framework_id]), + if is_password { + "password" + } else { + "not_password" + }, + if is_enabled { "enabled" } else { "disabled" }, + ]) + } else { + fingerprint_parts(&[ + "windows-accessibility-bound-target/v1", + target_profile.contract_name(), + &process_text, + &control_type_text, + &fingerprint_parts(&[&target_class]), + &fingerprint_parts(&[&target_name]), + &fingerprint_parts(&[&target_framework_id]), + &expected_target_fingerprint, + if is_password { + "password" + } else { + "not_password" + }, + if is_enabled { "enabled" } else { "disabled" }, + ]) + }; + let target_fingerprint = if binding == WindowsComputerUseWindowBinding::Foreground { + fingerprint_parts(&[ + &stable_target_fingerprint, + &fingerprint_parts(&[&target_item_status]), + &fingerprint_parts(&[&target_help_text]), + &fingerprint_parts(&[&target_localized_control_type]), + if is_keyboard_focusable { + "keyboard_focusable" + } else { + "not_keyboard_focusable" + }, + ]) + } else { + stable_target_fingerprint + }; let application_fingerprint = fingerprint_parts(&[ "windows-application/v1", + target_profile.contract_name(), &fingerprint_parts(&[&window_class]), &fingerprint_parts(&[&target_class]), ]); - let frame_fingerprint = fingerprint_parts(&[ - "windows-accessibility-frame/v1", - &process_fingerprint, - &window_fingerprint, - &control_type_text, - &fingerprint_parts(&[&automation_id]), - &fingerprint_parts(&[&target_class]), - ]); + let walker = unsafe { + automation + .RawViewWalker() + .map_err(|error| format!("Windows UI Automation tree walker is unavailable: {error}"))? + }; + let ancestor_fingerprint = current_windows_accessibility_ancestor_fingerprint( + &focused, + &walker, + target_process_id, + binding == WindowsComputerUseWindowBinding::Foreground, + ); + let frame_fingerprint = if binding == WindowsComputerUseWindowBinding::Foreground { + fingerprint_parts(&[ + "windows-accessibility-frame/v1", + target_profile.contract_name(), + &process_fingerprint, + &window_fingerprint, + &control_type_text, + &fingerprint_parts(&[&automation_id]), + &fingerprint_parts(&[&target_class]), + &ancestor_fingerprint, + ]) + } else { + fingerprint_parts(&[ + "windows-accessibility-bound-frame/v1", + target_profile.contract_name(), + &process_fingerprint, + &window_fingerprint, + &control_type_text, + &fingerprint_parts(&[&target_class]), + &expected_target_fingerprint, + ]) + }; let semantic_fingerprint = if is_password { None } else { - let walker = unsafe { automation.RawViewWalker().ok() }; let mut semantic_element = focused.clone(); let mut value = None; for _ in 0..=16 { - value = current_windows_bounded_semantic_value(&semantic_element); + value = current_windows_bounded_semantic_value(&semantic_element, target_profile); if value.is_some() { break; } - let Some(walker) = walker.as_ref() else { - break; - }; let Ok(parent) = (unsafe { walker.GetParentElement(&semantic_element) }) else { break; }; @@ -855,7 +1425,8 @@ fn capture_windows_redacted_state() -> Result target_fingerprint, semantic_fingerprint, safe_summary: format!( - "Foreground Windows accessibility target type {} is {} and {}; {}.", + "{safe_window_label} {} accessibility target type {} is {} and {}; {}.", + target_profile.safe_label(), control_type.0, if is_enabled { "enabled" } else { "disabled" }, if is_keyboard_focusable { @@ -880,6 +1451,26 @@ mod tests { use super::*; use crate::kernel::capability::{ComputerControlExecution, ComputerScreenshot}; + #[cfg(windows)] + #[test] + fn bound_windows_clients_require_exact_nonzero_hwnd_and_process() { + use crate::kernel::capability::{ComputerControlAction, WindowsBoundComputerControlClient}; + + assert!(WindowsBoundComputerUseAccessibilityClient::new(0, 1).is_err()); + assert!(WindowsBoundComputerUseAccessibilityClient::new(1, 0).is_err()); + assert!(WindowsBoundComputerControlClient::new(0, 1).is_err()); + assert!(WindowsBoundComputerControlClient::new(1, 0).is_err()); + let observation = WindowsBoundComputerUseAccessibilityClient::new(1, 1).unwrap(); + let control = WindowsBoundComputerControlClient::new(1, 1).unwrap(); + assert!(observation.capture_redacted_state().is_err()); + assert!(control + .execute_control( + "invalid-bound-window", + &ComputerControlAction::SelectAccessibilityTarget, + ) + .is_err()); + } + struct FakeAccessibilityClient { states: Mutex>>, } @@ -1066,6 +1657,24 @@ mod tests { screenshots: &FakeScreenshotClient, accessibility: &FakeAccessibilityClient, expected_semantic: &str, + ) -> (ComputerUseStep, Uuid) { + setup_ready_step_with_action( + store, + screenshots, + accessibility, + expected_semantic, + ComputerControlAction::TypeText { + text: "verified text".to_string(), + }, + ) + } + + fn setup_ready_step_with_action( + store: &EventStore, + screenshots: &FakeScreenshotClient, + accessibility: &FakeAccessibilityClient, + expected_semantic: &str, + action: ComputerControlAction, ) -> (ComputerUseStep, Uuid) { let observation = capture_computer_use_observation( ComputerUseObservationPhase::PreAction, @@ -1084,9 +1693,7 @@ mod tests { let bound = bind_computer_use_action( store, observed.id, - ComputerControlAction::TypeText { - text: "verified text".to_string(), - }, + action, "Type the exact approved text into the focused editor.".to_string(), ComputerUsePostcondition::TargetSemanticFingerprintEquals { expected: fingerprint_parts(&[expected_semantic]), @@ -1230,43 +1837,303 @@ mod tests { } #[test] - fn stale_approved_observation_requires_replan_before_revalidation() { - let store = EventStore::open_memory().unwrap(); - let screenshots = FakeScreenshotClient::new(1); - let accessibility = - FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); - let control = FakeControlClient::succeeding(); - let (ready, approval_id) = setup_ready_step(&store, &screenshots, &accessibility, "after"); - let expired_at = ready.pre_observation.valid_until + chrono::Duration::milliseconds(1); - - let result = execute_ready_computer_use_step_at( - &store, - ready.id, - ComputerUseExecutionPermit { - approval_request_id: approval_id, - local_unlock_confirmed: true, - }, - &screenshots, - &accessibility, - &control, - expired_at, - ) - .unwrap(); + fn file_explorer_folder_or_file_drift_stops_before_semantic_selection() { + let cases = [ + ( + "folder", + redacted_state_with_identity( + "file-explorer", + "explorer-process", + "explorer-window", + "isolated-folder", + "other-folder-frame", + "generated-file", + Some("selection:not_selected"), + ), + ), + ( + "file", + redacted_state_with_identity( + "file-explorer", + "explorer-process", + "explorer-window", + "isolated-folder", + "isolated-folder-frame", + "other-file", + Some("selection:not_selected"), + ), + ), + ]; - assert_eq!(result.step.status, ComputerUseStepStatus::NeedsReplan); - assert!(result.step.approval_request_id.is_none()); - assert!(result.step.approval_actor.is_none()); - assert_eq!(control.calls.load(Ordering::SeqCst), 0); - } + for (label, drifted) in cases { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state_with_identity( + "file-explorer", + "explorer-process", + "explorer-window", + "isolated-folder", + "isolated-folder-frame", + "generated-file", + Some("selection:not_selected"), + ), + drifted, + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = setup_ready_step_with_action( + &store, + &screenshots, + &accessibility, + "selection:selected", + ComputerControlAction::SelectAccessibilityTarget, + ); - #[test] - fn action_mutation_after_approval_is_rejected_before_input() { - let store = EventStore::open_memory().unwrap(); - let screenshots = FakeScreenshotClient::new(1); - let accessibility = - FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); - let (mut ready, approval_id) = - setup_ready_step(&store, &screenshots, &accessibility, "after"); + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!( + result.step.status, + ComputerUseStepStatus::NeedsReplan, + "{label} drift must require replanning" + ); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + } + + #[test] + fn excel_workbook_sheet_or_cell_drift_stops_before_semantic_write() { + let cases = [ + ( + "workbook", + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "other-workbook", + "sheet-one-frame", + "cell-b3", + Some("before"), + ), + ), + ( + "sheet", + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "other-sheet-frame", + "cell-b3", + Some("before"), + ), + ), + ( + "cell", + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "sheet-one-frame", + "cell-c4", + Some("before"), + ), + ), + ]; + + for (label, drifted) in cases { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "sheet-one-frame", + "cell-b3", + Some("before"), + ), + drifted, + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = setup_ready_step_with_action( + &store, + &screenshots, + &accessibility, + "after", + ComputerControlAction::SetAccessibilityValue { + value: "after".to_string(), + }, + ); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!( + result.step.status, + ComputerUseStepStatus::NeedsReplan, + "{label} drift must require replanning" + ); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + } + + #[cfg(windows)] + #[test] + fn excel_object_model_corroboration_blocks_uia_false_completion_and_wrong_targets() { + let directory = tempfile::tempdir().expect("temp dir"); + let workbook = directory.path().join("generated.xlsx"); + std::fs::write(&workbook, b"isolated workbook placeholder").expect("workbook placeholder"); + let workbook_text = workbook.to_string_lossy(); + let result = |value: &str, other_b3: &str, sheet: &str, cell: &str| { + format!( + "{value}\ntarget-sentinel\n{other_b3}\nother-sentinel\n{workbook_text}\n1\n{sheet}\n{cell}\n" + ) + }; + + let missing_write = validate_excel_object_model_result( + &result("before", "other-before", "C5B_Target", "B3"), + "after", + &workbook, + "C5B_Target", + "B3", + ) + .expect_err("UIA-only value change must not verify"); + assert!(missing_write.contains("exact target write is missing")); + + let wrong_sheet = validate_excel_object_model_result( + &result("after", "other-before", "C5B_Other", "B3"), + "after", + &workbook, + "C5B_Target", + "B3", + ) + .expect_err("wrong sheet must fail"); + assert!(wrong_sheet.contains("wrong workbook/sheet/cell binding")); + + let wrong_cell = validate_excel_object_model_result( + &result("after", "other-before", "C5B_Target", "C4"), + "after", + &workbook, + "C5B_Target", + "B3", + ) + .expect_err("wrong cell must fail"); + assert!(wrong_cell.contains("wrong workbook/sheet/cell binding")); + + let wrong_target = validate_excel_object_model_result( + &result("after", "changed-wrong-cell", "C5B_Target", "B3"), + "after", + &workbook, + "C5B_Target", + "B3", + ) + .expect_err("wrong-target write must fail"); + assert!(wrong_target.contains("wrong-target write")); + + validate_excel_object_model_result( + &result("after", "other-before", "C5B_Target", "B3"), + "after", + &workbook, + "C5B_Target", + "B3", + ) + .expect("UIA and object-model outcome corroborate"); + } + + #[cfg(windows)] + #[test] + fn explorer_shell_corroboration_rejects_multiselected_decoy() { + let directory = tempfile::tempdir().expect("temp dir"); + let target = directory.path().join("target.txt"); + let decoy = directory.path().join("decoy.txt"); + std::fs::write(&target, b"target").expect("target"); + std::fs::write(&decoy, b"decoy").expect("decoy"); + + let error = validate_exact_explorer_selection_paths(&[target.clone(), decoy], &target) + .expect_err("multiple selected paths must fail closed"); + assert!(error.contains("selection count was 2, expected 1")); + validate_exact_explorer_selection_paths(&[target.clone()], &target) + .expect("one exact selected path corroborates"); + } + + #[cfg(windows)] + #[test] + fn excel_smoke_deadline_fails_closed_with_exact_phase() { + validate_excel_smoke_deadline( + std::time::Duration::from_secs(45), + "discover-exact-uia-cell", + ) + .expect("deadline boundary remains allowed"); + let error = validate_excel_smoke_deadline( + std::time::Duration::from_millis(45_001), + "legacy-action-host", + ) + .expect_err("elapsed smoke phase must fail closed"); + assert!(error.contains("45 second internal deadline")); + assert!(error.contains("legacy-action-host")); + } + + #[test] + fn stale_approved_observation_requires_replan_before_revalidation() { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = + FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = setup_ready_step(&store, &screenshots, &accessibility, "after"); + let expired_at = ready.pre_observation.valid_until + chrono::Duration::milliseconds(1); + + let result = execute_ready_computer_use_step_at( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + expired_at, + ) + .unwrap(); + + assert_eq!(result.step.status, ComputerUseStepStatus::NeedsReplan); + assert!(result.step.approval_request_id.is_none()); + assert!(result.step.approval_actor.is_none()); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn action_mutation_after_approval_is_rejected_before_input() { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = + FakeAccessibilityClient::new(vec![redacted_state("window", "target", Some("before"))]); + let (mut ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); ready.action.as_mut().unwrap().action = ComputerControlAction::PressKey { key: "ENTER".to_string(), }; @@ -1507,6 +2374,70 @@ mod tests { assert_eq!(control.calls.load(Ordering::SeqCst), 1); } + #[test] + fn excel_semantic_write_without_a_cell_receipt_cannot_complete() { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(2); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "sheet-one-frame", + "cell-b3", + Some("before"), + ), + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "sheet-one-frame", + "cell-b3", + Some("before"), + ), + redacted_state_with_identity( + "excel", + "excel-process", + "excel-window", + "generated-workbook", + "sheet-one-frame", + "cell-b3", + None, + ), + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = setup_ready_step_with_action( + &store, + &screenshots, + &accessibility, + "after", + ComputerControlAction::SetAccessibilityValue { + value: "after".to_string(), + }, + ); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!( + result.step.status, + ComputerUseStepStatus::AwaitingVerification + ); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + } + #[test] fn deterministic_postcondition_failure_is_distinct_from_action_failure() { let directory = tempdir().unwrap(); @@ -1648,6 +2579,884 @@ mod tests { ); } + #[cfg(windows)] + fn powershell_literal(value: &std::path::Path) -> String { + format!("'{}'", value.to_string_lossy().replace('\'', "''")) + } + + #[cfg(windows)] + fn wait_for_file(path: &std::path::Path, attempts: usize) -> bool { + use std::thread; + use std::time::Duration; + + (0..attempts).any(|_| { + if path.is_file() { + true + } else { + thread::sleep(Duration::from_millis(250)); + false + } + }) + } + + #[cfg(windows)] + fn c5b_installed_smoke_directory(name: &str) -> Result { + const ROOT_ENV: &str = "DEEPSEEK_AGENT_OS_C5B_SMOKE_ROOT"; + + let root = std::env::var_os(ROOT_ENV) + .filter(|value| !value.is_empty()) + .map(std::path::PathBuf::from) + .ok_or_else(|| { + format!( + "{ROOT_ENV} must name a fresh absolute authorized isolation root for installed C5B smokes" + ) + })?; + if !root.is_absolute() { + return Err(format!( + "{ROOT_ENV} must be an absolute authorized isolation root" + )); + } + let directory = root.join(name); + if directory.exists() + && std::fs::read_dir(&directory) + .map_err(|error| format!("C5B smoke directory is unreadable: {error}"))? + .next() + .is_some() + { + return Err(format!( + "C5B smoke directory must be fresh and empty: {}", + directory.display() + )); + } + std::fs::create_dir_all(&directory) + .map_err(|error| format!("C5B smoke directory creation failed: {error}"))?; + Ok(directory) + } + + #[cfg(windows)] + fn select_file_in_exact_explorer_window( + directory: &std::path::Path, + file_name: &str, + ) -> Result { + use std::process::Command; + + let directory = powershell_literal(directory); + let file_name = format!("'{}'", file_name.replace('\'', "''")); + let script = format!( + r#" +$targetDirectory = [IO.Path]::GetFullPath({directory}).TrimEnd('\') +$shell = New-Object -ComObject Shell.Application +$window = @($shell.Windows()) | Where-Object {{ + try {{ + $_.FullName -like '*explorer.exe' -and + [IO.Path]::GetFullPath(([Uri]$_.LocationURL).LocalPath).TrimEnd('\') -eq $targetDirectory + }} catch {{ + $false + }} +}} | Select-Object -First 1 +if ($null -eq $window) {{ throw 'isolated File Explorer window was not found' }} +$item = $window.Document.Folder.ParseName({file_name}) +if ($null -eq $item) {{ throw 'isolated File Explorer target file was not found' }} +$window.Document.SelectItem($item, 29) +[Console]::Out.Write([string]$window.HWND) +"#, + ); + let output = Command::new("powershell.exe") + .args(["-NoProfile", "-STA", "-Command", &script]) + .output() + .map_err(|error| format!("File Explorer target setup failed: {error}"))?; + if output.status.success() { + String::from_utf8_lossy(&output.stdout) + .trim() + .parse::() + .map_err(|error| format!("File Explorer did not return its exact HWND: {error}")) + } else { + Err(format!( + "File Explorer target setup failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )) + } + } + + #[cfg(windows)] + fn windows_process_id_for_handle(window_handle: isize) -> Result { + use windows::Win32::Foundation::HWND; + use windows::Win32::UI::WindowsAndMessaging::GetWindowThreadProcessId; + + if window_handle == 0 { + return Err("exact Windows HWND is empty".to_string()); + } + let mut process_id = 0u32; + let thread_id = + unsafe { GetWindowThreadProcessId(HWND(window_handle as _), Some(&mut process_id)) }; + if thread_id == 0 || process_id == 0 { + return Err("exact Windows HWND has no live process identity".to_string()); + } + Ok(process_id) + } + + #[cfg(windows)] + fn close_exact_explorer_window(directory: &std::path::Path) { + use std::process::Command; + + let directory = powershell_literal(directory); + let script = format!( + r#" +$targetDirectory = [IO.Path]::GetFullPath({directory}).TrimEnd('\') +$shell = New-Object -ComObject Shell.Application +@($shell.Windows()) | Where-Object {{ + try {{ + $_.FullName -like '*explorer.exe' -and + [IO.Path]::GetFullPath(([Uri]$_.LocationURL).LocalPath).TrimEnd('\') -eq $targetDirectory + }} catch {{ + $false + }} +}} | ForEach-Object {{ $_.Quit() }} +"#, + ); + let _ = Command::new("powershell.exe") + .args(["-NoProfile", "-STA", "-Command", &script]) + .status(); + } + + #[cfg(windows)] + fn validate_exact_explorer_selection_paths( + selected_paths: &[std::path::PathBuf], + expected_path: &std::path::Path, + ) -> Result<(), String> { + if selected_paths.len() != 1 { + return Err(format!( + "exact File Explorer selection count was {}, expected 1", + selected_paths.len() + )); + } + let actual = selected_paths[0] + .canonicalize() + .map_err(|error| format!("selected File Explorer path is invalid: {error}"))?; + let expected = expected_path + .canonicalize() + .map_err(|error| format!("expected File Explorer path is invalid: {error}"))?; + if actual != expected { + return Err(format!( + "selected File Explorer path mismatch: {}", + actual.display() + )); + } + Ok(()) + } + + #[cfg(windows)] + fn corroborate_exact_explorer_selection( + window_handle: isize, + directory: &std::path::Path, + file_name: &str, + ) -> Result<(), String> { + use std::process::Command; + + let expected_path = directory.join(file_name); + let directory = powershell_literal(directory); + let script = format!( + r#" +$targetDirectory = [IO.Path]::GetFullPath({directory}).TrimEnd('\') +$shell = New-Object -ComObject Shell.Application +$window = @($shell.Windows()) | Where-Object {{ + try {{ + [Int64]$_.HWND -eq {window_handle} -and + $_.FullName -like '*explorer.exe' -and + [IO.Path]::GetFullPath(([Uri]$_.LocationURL).LocalPath).TrimEnd('\') -eq $targetDirectory + }} catch {{ + $false + }} +}} | Select-Object -First 1 +if ($null -eq $window) {{ throw 'exact File Explorer HWND and LocationURL did not corroborate' }} +@($window.Document.SelectedItems()) | ForEach-Object {{ + [Console]::Out.WriteLine([IO.Path]::GetFullPath([string]$_.Path)) +}} +"#, + ); + let output = Command::new("powershell.exe") + .args(["-NoProfile", "-STA", "-Command", &script]) + .output() + .map_err(|error| format!("File Explorer selection corroboration failed: {error}"))?; + if output.status.success() { + let selected_paths = String::from_utf8_lossy(&output.stdout) + .lines() + .filter(|line| !line.trim().is_empty()) + .map(|line| std::path::PathBuf::from(line.trim())) + .collect::>(); + validate_exact_explorer_selection_paths(&selected_paths, &expected_path) + } else { + Err(format!( + "File Explorer UIA target lacked Shell LocationURL/SelectedItems corroboration: {}", + String::from_utf8_lossy(&output.stderr).trim() + )) + } + } + + #[cfg(windows)] + struct CorroboratedExplorerAccessibilityClient { + inner: WindowsBoundComputerUseAccessibilityClient, + window_handle: isize, + directory: std::path::PathBuf, + target_name: String, + } + + #[cfg(windows)] + impl ComputerUseAccessibilityClient for CorroboratedExplorerAccessibilityClient { + fn capture_redacted_state(&self) -> Result { + let state = self.inner.capture_redacted_state()?; + corroborate_exact_explorer_selection( + self.window_handle, + &self.directory, + &self.target_name, + )?; + Ok(state) + } + } + + #[cfg(windows)] + fn validate_excel_smoke_deadline( + elapsed: std::time::Duration, + phase: &str, + ) -> Result<(), String> { + if elapsed > std::time::Duration::from_secs(45) { + Err(format!( + "Excel smoke exceeded its 45 second internal deadline in phase {phase} after {} ms", + elapsed.as_millis() + )) + } else { + Ok(()) + } + } + + #[cfg(windows)] + fn request_excel_object_model_result( + verify: &std::path::Path, + result_file: &std::path::Path, + ) -> Result { + use std::thread; + use std::time::Duration; + + if verify.exists() { + std::fs::remove_file(verify).map_err(|error| { + format!("stale Excel verifier request could not be removed: {error}") + })?; + } + if result_file.exists() { + std::fs::remove_file(result_file).map_err(|error| { + format!("stale Excel verifier result could not be removed: {error}") + })?; + } + std::fs::write(verify, b"verify") + .map_err(|error| format!("Excel verifier request could not be written: {error}"))?; + for _ in 0..24 { + if result_file.is_file() && !verify.exists() { + return std::fs::read_to_string(result_file) + .map_err(|error| format!("Excel verifier result is unreadable: {error}")); + } + thread::sleep(Duration::from_millis(250)); + } + Err( + "Excel object-model verifier timed out after 6 seconds without a bounded result" + .to_string(), + ) + } + + #[cfg(windows)] + fn validate_excel_object_model_result( + actual: &str, + expected_value: &str, + workbook: &std::path::Path, + expected_sheet: &str, + expected_cell: &str, + ) -> Result<(), String> { + let fields = actual.lines().collect::>(); + if fields.len() != 8 { + return Err(format!( + "Excel object-model verifier returned {} fields instead of 8", + fields.len() + )); + } + if fields[1] != "target-sentinel" + || fields[2] != "other-before" + || fields[3] != "other-sentinel" + { + return Err(format!( + "Excel object-model verifier found a wrong-target write: target A1={:?}, other B3={:?}, other A1={:?}", + fields[1], fields[2], fields[3] + )); + } + let actual_workbook = std::path::PathBuf::from(fields[4]) + .canonicalize() + .map_err(|error| format!("Excel reported workbook path is invalid: {error}"))?; + let expected_workbook = workbook + .canonicalize() + .map_err(|error| format!("generated workbook path is invalid: {error}"))?; + if actual_workbook != expected_workbook + || fields[5] != "1" + || fields[6] != expected_sheet + || fields[7] != expected_cell + { + return Err(format!( + "Excel object-model verifier found a wrong workbook/sheet/cell binding: workbook={:?}, count={:?}, sheet={:?}, cell={:?}", + fields[4], fields[5], fields[6], fields[7] + )); + } + if fields[0] != expected_value { + return Err(format!( + "Excel exact target write is missing: {expected_sheet}!{expected_cell} expected {expected_value:?}, actual {:?}; no wrong-target sentinel changed", + fields[0] + )); + } + Ok(()) + } + + #[cfg(windows)] + struct ExcelObjectModelCorroboratingControlClient { + inner: crate::kernel::capability::WindowsBoundComputerControlClient, + verify: std::path::PathBuf, + result_file: std::path::PathBuf, + workbook: std::path::PathBuf, + expected_value: String, + expected_sheet: String, + expected_cell: String, + } + + #[cfg(windows)] + impl ComputerControlClient for ExcelObjectModelCorroboratingControlClient { + fn execute_control( + &self, + target: &str, + action: &ComputerControlAction, + ) -> Result { + let execution = self.inner.execute_control(target, action)?; + let actual = request_excel_object_model_result(&self.verify, &self.result_file)?; + validate_excel_object_model_result( + &actual, + &self.expected_value, + &self.workbook, + &self.expected_sheet, + &self.expected_cell, + )?; + Ok(execution) + } + } + + #[cfg(windows)] + #[test] + #[ignore = "requires a visible installed File Explorer session over isolated generated files"] + fn windows_file_explorer_isolated_selection_smoke_verifies_exact_file() { + use std::process::Command; + use std::thread; + use std::time::Duration; + + use crate::kernel::capability::{ + WindowsBoundComputerControlClient, WindowsBoundComputerScreenshotClient, + }; + + let directory = c5b_installed_smoke_directory("file-explorer") + .expect("isolated File Explorer directory"); + let target_name = "c5b-target.txt"; + let decoy_name = "c5b-decoy.txt"; + let target_path = directory.join(target_name); + let decoy_path = directory.join(decoy_name); + std::fs::write(&target_path, b"exact isolated C5B file") + .expect("isolated target file is generated"); + std::fs::write(&decoy_path, b"decoy").expect("isolated decoy file is generated"); + let expected_bytes = std::fs::read(&target_path).expect("target file is readable"); + let expected_decoy_bytes = std::fs::read(&decoy_path).expect("decoy file is readable"); + let _ = Command::new("explorer.exe") + .arg(&directory) + .spawn() + .expect("File Explorer starts") + .wait(); + let result = (|| -> Result<(), String> { + let selected_semantic = accessibility_value_semantic_fingerprint("selection:selected")?; + let not_selected_semantic = + accessibility_value_semantic_fingerprint("selection:not_selected")?; + let mut last_diagnostic = "setup not attempted".to_string(); + let target_binding = (0..40).find_map(|_| { + let window_handle = + match select_file_in_exact_explorer_window(&directory, target_name) { + Ok(window_handle) => window_handle, + Err(error) => { + last_diagnostic = error; + thread::sleep(Duration::from_millis(250)); + return None; + } + }; + let process_id = match windows_process_id_for_handle(window_handle) { + Ok(process_id) => process_id, + Err(error) => { + last_diagnostic = error; + thread::sleep(Duration::from_millis(250)); + return None; + } + }; + let accessibility = + match WindowsBoundComputerUseAccessibilityClient::new_file_explorer( + window_handle, + process_id, + target_name.to_string(), + ) { + Ok(accessibility) => CorroboratedExplorerAccessibilityClient { + inner: accessibility, + window_handle, + directory: directory.clone(), + target_name: target_name.to_string(), + }, + Err(error) => { + last_diagnostic = error; + return None; + } + }; + thread::sleep(Duration::from_millis(250)); + match accessibility.capture_redacted_state() { + Ok(state) + if state.safe_summary.contains("File Explorer") + && state.semantic_fingerprint.as_deref() + == Some(selected_semantic.as_str()) => + { + Some((window_handle, process_id, accessibility, state)) + } + Ok(state) => { + let semantic = if state.semantic_fingerprint.as_deref() + == Some(not_selected_semantic.as_str()) + { + "not-selected" + } else if state.semantic_fingerprint.is_some() { + "other" + } else { + "unavailable" + }; + last_diagnostic = format!("{}; semantic {semantic}", state.safe_summary); + None + } + Err(error) => { + last_diagnostic = error; + None + } + } + }); + let (window_handle, process_id, accessibility, target_state) = + target_binding.ok_or_else(|| { + format!( + "File Explorer did not expose the selected isolated file in its exact HWND through UI Automation: {last_diagnostic}" + ) + })?; + let decoy_window = select_file_in_exact_explorer_window(&directory, decoy_name)?; + if decoy_window != window_handle { + return Err( + "File Explorer exact HWND changed while selecting the decoy".to_string() + ); + } + thread::sleep(Duration::from_millis(300)); + let decoy_accessibility = CorroboratedExplorerAccessibilityClient { + inner: WindowsBoundComputerUseAccessibilityClient::new_file_explorer( + window_handle, + process_id, + decoy_name.to_string(), + )?, + window_handle, + directory: directory.clone(), + target_name: decoy_name.to_string(), + }; + let decoy_state = decoy_accessibility.capture_redacted_state()?; + if decoy_state.target_fingerprint == target_state.target_fingerprint { + return Err( + "File Explorer target fingerprint did not distinguish the isolated files" + .to_string(), + ); + } + let restored_window = select_file_in_exact_explorer_window(&directory, target_name)?; + if restored_window != window_handle { + return Err( + "File Explorer exact HWND changed while restoring the target".to_string(), + ); + } + thread::sleep(Duration::from_millis(300)); + let restored_state = accessibility.capture_redacted_state()?; + if restored_state.frame_fingerprint != target_state.frame_fingerprint + || restored_state.target_fingerprint != target_state.target_fingerprint + || restored_state.semantic_fingerprint != target_state.semantic_fingerprint + { + let component = |label: &str, before: &str, after: &str| { + format!( + "{label}={} (pre={before}, restored={after})", + if before == after { "match" } else { "mismatch" } + ) + }; + return Err(format!( + "File Explorer could not restore the exact isolated folder/file identity: {}; {}; {}; semantic={} (pre={:?}, restored={:?})", + component( + "frame/ancestor", + &target_state.frame_fingerprint, + &restored_state.frame_fingerprint, + ), + component( + "target", + &target_state.target_fingerprint, + &restored_state.target_fingerprint, + ), + component( + "window", + &target_state.window_fingerprint, + &restored_state.window_fingerprint, + ), + if target_state.semantic_fingerprint == restored_state.semantic_fingerprint { + "match" + } else { + "mismatch" + }, + target_state.semantic_fingerprint, + restored_state.semantic_fingerprint, + )); + } + + let store = EventStore::open(directory.join("file-explorer-smoke.db")) + .map_err(|error| error.to_string())?; + let screenshot = WindowsBoundComputerScreenshotClient::new( + directory.join("screenshots"), + window_handle, + process_id, + )?; + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshot, + &accessibility, + )?; + let (_, observed) = persist_observed_computer_use_session( + &store, + None, + "Select one exact generated file in an isolated File Explorer window.".to_string(), + ComputerUseUndoCapability::None, + observation, + )?; + let bound = bind_computer_use_action( + &store, + observed.id, + ComputerControlAction::SelectAccessibilityTarget, + "Select the exact focused generated file through UI Automation.".to_string(), + ComputerUsePostcondition::TargetSemanticFingerprintEquals { + expected: selected_semantic, + }, + )?; + let approval_id = Uuid::new_v4(); + approve_computer_use_step( + &store, + bound.id, + approval_id, + &bound + .action + .as_ref() + .ok_or_else(|| "File Explorer smoke action is missing".to_string())? + .action_fingerprint, + ComputerUseApprovalActor::User, + )?; + let control = WindowsBoundComputerControlClient::new_file_explorer( + window_handle, + process_id, + target_name.to_string(), + )?; + let run = execute_ready_computer_use_step( + &store, + bound.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshot, + &accessibility, + &control, + )?; + if run.step.status != ComputerUseStepStatus::Verified + || run.step.action_start_count != 1 + { + return Err(format!( + "File Explorer smoke ended in {:?} with {} action starts", + run.step.status, run.step.action_start_count + )); + } + if std::fs::read(&target_path).map_err(|error| error.to_string())? != expected_bytes { + return Err("File Explorer selection changed the generated file".to_string()); + } + if std::fs::read(&decoy_path).map_err(|error| error.to_string())? + != expected_decoy_bytes + { + return Err("File Explorer selection changed the generated decoy file".to_string()); + } + Ok(()) + })(); + close_exact_explorer_window(&directory); + result.expect("isolated File Explorer selection verifies"); + } + + #[cfg(windows)] + #[test] + #[ignore = "requires visible installed Microsoft Excel over an isolated generated workbook"] + fn windows_excel_isolated_cell_value_smoke_verifies_exact_outcome() { + use std::process::Command; + use std::thread; + use std::time::Duration; + + use crate::kernel::capability::{ + WindowsBoundComputerControlClient, WindowsBoundComputerScreenshotClient, + }; + use wait_timeout::ChildExt; + + let directory = c5b_installed_smoke_directory("excel").expect("isolated Excel directory"); + let workbook = directory.join("c5b-generated.xlsx"); + let ready = directory.join("excel-ready.txt"); + let stop = directory.join("excel-stop.txt"); + let verify = directory.join("excel-verify.txt"); + let result_file = directory.join("excel-result.txt"); + let workbook_literal = powershell_literal(&workbook); + let ready_literal = powershell_literal(&ready); + let stop_literal = powershell_literal(&stop); + let verify_literal = powershell_literal(&verify); + let result_literal = powershell_literal(&result_file); + let script = format!( + r#" +$excel = $null +$workbook = $null +try {{ + $excel = New-Object -ComObject Excel.Application + $excel.Visible = $true + $excel.DisplayAlerts = $false + $workbook = $excel.Workbooks.Add() + $target = $workbook.Worksheets.Item(1) + $target.Name = 'C5B_Target' + $other = $workbook.Worksheets.Add() + $other.Name = 'C5B_Other' + $target.Activate() + $target.Range('A1').Value2 = 'target-sentinel' + $target.Range('B3').Value2 = 'before' + $other.Range('A1').Value2 = 'other-sentinel' + $other.Range('B3').Value2 = 'other-before' + $target.Activate() + $target.Range('B3').Select() + $workbook.SaveAs({workbook_literal}, 51) + [IO.File]::WriteAllText({ready_literal}, [string]$excel.Hwnd) + while (-not (Test-Path -LiteralPath {stop_literal})) {{ + if (Test-Path -LiteralPath {verify_literal}) {{ + [IO.File]::WriteAllLines({result_literal}, [string[]]@( + [string]$target.Range('B3').Value2, + [string]$target.Range('A1').Value2, + [string]$other.Range('B3').Value2, + [string]$other.Range('A1').Value2, + [string]$workbook.FullName, + [string]$excel.Workbooks.Count, + [string]$excel.ActiveSheet.Name, + [string]$excel.ActiveCell.Address($false, $false) + )) + Remove-Item -LiteralPath {verify_literal} -Force + }} + Start-Sleep -Milliseconds 100 + }} +}} finally {{ + if ($null -ne $workbook) {{ $workbook.Close($false) }} + if ($null -ne $excel) {{ $excel.Quit() }} +}} +"#, + ); + let mut excel_host = Command::new("powershell.exe") + .args(["-NoProfile", "-STA", "-Command", &script]) + .spawn() + .expect("isolated Excel host starts"); + let run_result = (|| -> Result<(), String> { + let smoke_started = std::time::Instant::now(); + let phase = |name: &str| -> Result<(), String> { + let elapsed = smoke_started.elapsed(); + eprintln!( + "C5B Excel smoke phase={name} elapsed_ms={}", + elapsed.as_millis() + ); + validate_excel_smoke_deadline(elapsed, name) + }; + phase("wait-for-workbook")?; + if !wait_for_file(&ready, 32) { + return Err( + "Excel timed out after 8 seconds opening the isolated generated workbook" + .to_string(), + ); + } + phase("bind-exact-hwnd")?; + let hwnd = std::fs::read_to_string(&ready) + .map_err(|error| error.to_string())? + .trim() + .to_string(); + if hwnd.is_empty() { + return Err("Excel did not report its exact window handle".to_string()); + } + let window_handle = hwnd + .parse::() + .map_err(|error| format!("Excel reported an invalid HWND: {error}"))?; + let process_id = windows_process_id_for_handle(window_handle)?; + let accessibility = WindowsBoundComputerUseAccessibilityClient::new_excel( + window_handle, + process_id, + "C5B_Target".to_string(), + "B3".to_string(), + 3, + 2, + )?; + let before_semantic = accessibility_value_semantic_fingerprint("before")?; + phase("discover-exact-uia-cell")?; + let mut last_target_diagnostic = "exact target discovery was not attempted".to_string(); + let mut pre_state = None; + for attempt in 1..=4 { + thread::sleep(Duration::from_millis(250)); + match accessibility.capture_redacted_state() { + Ok(state) + if state.safe_summary.contains("Excel") + && state.semantic_fingerprint.as_deref() + == Some(before_semantic.as_str()) => + { + pre_state = Some(state); + break; + } + Ok(state) => { + last_target_diagnostic = format!( + "attempt {attempt}: {}; semantic={:?}", + state.safe_summary, state.semantic_fingerprint + ); + } + Err(error) => { + last_target_diagnostic = format!("attempt {attempt}: {error}"); + } + } + } + let pre_state = pre_state.ok_or_else(|| { + format!( + "Excel did not expose C5B_Target!B3 (provider GridItem row 3 column 2) through UI Automation within 4 bounded attempts after {} ms: {last_target_diagnostic}", + smoke_started.elapsed().as_millis() + ) + })?; + if pre_state.application_fingerprint.is_empty() + || pre_state.frame_fingerprint.is_empty() + || pre_state.target_fingerprint.is_empty() + { + return Err("Excel target identity was incomplete".to_string()); + } + phase("corroborate-pre-object-model")?; + let pre_object_model = request_excel_object_model_result(&verify, &result_file)?; + validate_excel_object_model_result( + &pre_object_model, + "before", + &workbook, + "C5B_Target", + "B3", + )?; + + phase("capture-pre-observation")?; + let store = EventStore::open(directory.join("excel-smoke.db")) + .map_err(|error| error.to_string())?; + let screenshot = WindowsBoundComputerScreenshotClient::new( + directory.join("screenshots"), + window_handle, + process_id, + )?; + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshot, + &accessibility, + )?; + let (_, observed) = persist_observed_computer_use_session( + &store, + None, + "Set one exact cell in an isolated generated Excel workbook.".to_string(), + ComputerUseUndoCapability::None, + observation, + )?; + let expected_value = "DS Agent C5B exact cell"; + let bound = bind_computer_use_action( + &store, + observed.id, + ComputerControlAction::SetAccessibilityValue { + value: expected_value.to_string(), + }, + "Set the exact focused generated-workbook cell through UI Automation.".to_string(), + ComputerUsePostcondition::TargetSemanticFingerprintEquals { + expected: accessibility_value_semantic_fingerprint(expected_value)?, + }, + )?; + let approval_id = Uuid::new_v4(); + approve_computer_use_step( + &store, + bound.id, + approval_id, + &bound + .action + .as_ref() + .ok_or_else(|| "Excel smoke action is missing".to_string())? + .action_fingerprint, + ComputerUseApprovalActor::User, + )?; + let control = ExcelObjectModelCorroboratingControlClient { + inner: WindowsBoundComputerControlClient::new_excel( + window_handle, + process_id, + "C5B_Target".to_string(), + "B3".to_string(), + 3, + 2, + )?, + verify: verify.clone(), + result_file: result_file.clone(), + workbook: workbook.clone(), + expected_value: expected_value.to_string(), + expected_sheet: "C5B_Target".to_string(), + expected_cell: "B3".to_string(), + }; + phase("execute-exact-uia-edit-and-object-model-corroboration")?; + let run = execute_ready_computer_use_step( + &store, + bound.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshot, + &accessibility, + &control, + )?; + if run.step.status != ComputerUseStepStatus::Verified + || run.step.action_start_count != 1 + { + return Err(format!( + "Excel smoke ended in {:?} with {} action starts: {}", + run.step.status, + run.step.action_start_count, + run.safe_error + .as_deref() + .unwrap_or("no corroborated effect receipt was returned") + )); + } + phase("verified")?; + let actual = + std::fs::read_to_string(&result_file).map_err(|error| error.to_string())?; + validate_excel_object_model_result( + &actual, + expected_value, + &workbook, + "C5B_Target", + "B3", + )?; + Ok(()) + })(); + let _ = std::fs::write(&stop, b"stop"); + if excel_host + .wait_timeout(Duration::from_secs(5)) + .ok() + .flatten() + .is_none() + { + let _ = excel_host.kill(); + let _ = excel_host.wait(); + } + run_result.expect("isolated Excel cell action verifies"); + } + #[cfg(windows)] #[test] #[ignore = "requires a visible isolated Windows Notepad-like editor session"] diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 337a694..135b467 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -1053,6 +1053,20 @@ function capabilityFamilyIcon(family: CapabilityFamily) { } } +function durableComputerUsePostcondition(actionContract: string) { + const normalized = actionContract.trim(); + const separator = normalized.indexOf(":"); + const verb = separator >= 0 ? normalized.slice(0, separator).trim().toLowerCase() : ""; + const payload = separator >= 0 ? normalized.slice(separator + 1).trim() : ""; + if (verb === "set_value" && payload) { + return { kind: "semantic_equals_text", expectedText: payload } as const; + } + if (verb === "select" && payload.toLowerCase() === "focused") { + return { kind: "semantic_equals_text", expectedText: "selection:selected" } as const; + } + return { kind: "semantic_changed" } as const; +} + export function App() { const [state, setState] = useState(fallbackState); const [onboardingReadiness, setOnboardingReadiness] = @@ -1170,7 +1184,9 @@ export function App() { const [computerControlTarget, setComputerControlTarget] = useState(""); const [computerControlAction, setComputerControlAction] = useState(""); const [computerControlUnlockToken, setComputerControlUnlockToken] = useState(""); - const [computerUseActionDraft, setComputerUseActionDraft] = useState("type:DS Agent verified"); + const [computerUseActionDraft, setComputerUseActionDraft] = useState( + "set_value:DS Agent verified", + ); const [lastComputerScreenshotInvocationId, setLastComputerScreenshotInvocationId] = useState(null); const [setupWorkspaceName, setSetupWorkspaceName] = useState(""); @@ -3827,8 +3843,8 @@ export function App() { runId: null, safeGoalSummary: language === "zh" - ? "在隔离的记事本类应用中执行一个可验证动作。" - : "Execute one verifiable action in an isolated Notepad-like app.", + ? "在隔离的 Windows 前台目标中执行一个可验证动作。" + : "Execute one verifiable action on an isolated foreground Windows target.", undoCapability: "none", }, ); @@ -3856,9 +3872,9 @@ export function App() { actionContract: computerUseActionDraft.trim(), safeSummary: language === "zh" - ? "在隔离的前台编辑器中执行这一项已显示的结构化动作。" - : "Execute this displayed structured action in the isolated foreground editor.", - postcondition: { kind: "semantic_changed" }, + ? "在隔离的前台可访问性目标中执行这一项已显示的结构化动作。" + : "Execute this displayed structured action on the isolated foreground accessibility target.", + postcondition: durableComputerUsePostcondition(computerUseActionDraft), }); await refreshDurableComputerUseState(); setComputerUseStepNotice(step.status_reason ?? computerUseStepCopy.awaiting_approval); diff --git a/scripts/durable-computer-use-ui.test.mjs b/scripts/durable-computer-use-ui.test.mjs index 5af7d17..785092d 100644 --- a/scripts/durable-computer-use-ui.test.mjs +++ b/scripts/durable-computer-use-ui.test.mjs @@ -16,6 +16,16 @@ assert.match(app, /"bind_durable_computer_use_action"/); assert.match(app, /"run_durable_computer_use_step"/); assert.match(app, /"take_over_durable_computer_use_step"/); assert.match(app, /"reobserve_durable_computer_use_session"/); +assert.match(app, /function durableComputerUsePostcondition/); +assert.match(app, /"set_value:DS Agent verified"/); +assert.match(app, /expectedText: "selection:selected"/); +assert.match(app, /postcondition: durableComputerUsePostcondition\(computerUseActionDraft\)/); +assert.match(app, /isolated foreground Windows target/); +assert.doesNotMatch( + app, + /Execute one verifiable action in an isolated Notepad-like app/, + "the reachable C5B surface must not remain falsely limited to Notepad", +); assert.match(app, /computerUseStepPending \|\| !computerControlUnlockStatus\.unlocked/); assert.match( app, From 42a617af87f91388f4ef52125da16bb3619dba5f Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 02:18:54 +0800 Subject: [PATCH 3/5] feat: add verified Edge portal computer use --- .../src-tauri/src/kernel/capability.rs | 657 ++++++++- .../src/kernel/computer_use_runtime.rs | 1289 ++++++++++++++++- 2 files changed, 1933 insertions(+), 13 deletions(-) diff --git a/apps/desktop/src-tauri/src/kernel/capability.rs b/apps/desktop/src-tauri/src/kernel/capability.rs index a194283..2db13e3 100644 --- a/apps/desktop/src-tauri/src/kernel/capability.rs +++ b/apps/desktop/src-tauri/src/kernel/capability.rs @@ -1863,6 +1863,623 @@ const BOUND_WINDOWS_ACCESSIBILITY_ACTION_TIMEOUT: Duration = Duration::from_secs #[allow(dead_code)] const BOUND_EXCEL_ACCESSIBILITY_ACTION_TIMEOUT: Duration = Duration::from_secs(9); +#[cfg(all(windows, test))] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WindowsEdgePortalDomQuery { + pub devtools_port: u16, + pub url: String, + pub document_title: String, + pub document_token: String, + pub target_element_id: String, + pub target_name: String, + pub decoy_element_id: String, + pub decoy_name: String, + pub receipt_element_id: String, + pub receipt_prefix: String, +} + +#[cfg(all(windows, test))] +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct WindowsEdgePortalDomSnapshot { + pub target_id: String, + pub frame_id: String, + pub browser_window_id: i64, + pub url: String, + pub origin: String, + pub document_title: String, + pub document_token: String, + pub target_name: String, + pub target_value: String, + pub semantic_receipt: String, + pub decoy_value: String, +} + +#[cfg(all(windows, test))] +struct EdgeDevToolsSocket { + stream: std::net::TcpStream, + next_command_id: u64, +} + +#[cfg(all(windows, test))] +impl EdgeDevToolsSocket { + fn connect(websocket_url: &str, expected_port: u16) -> Result { + use base64::Engine; + use std::io::{Read, Write}; + use std::time::Duration; + + let parsed = reqwest::Url::parse(websocket_url) + .map_err(|error| format!("Edge DevTools WebSocket URL is invalid: {error}"))?; + if parsed.scheme() != "ws" + || !parsed + .host_str() + .is_some_and(|host| host == "127.0.0.1" || host.eq_ignore_ascii_case("localhost")) + || parsed.port() != Some(expected_port) + || !parsed.username().is_empty() + || parsed.password().is_some() + { + return Err( + "Edge DevTools WebSocket must remain on the exact loopback port".to_string(), + ); + } + let host = parsed + .host_str() + .ok_or_else(|| "Edge DevTools WebSocket host is missing".to_string())?; + let mut path = parsed.path().to_string(); + if let Some(query) = parsed.query() { + path.push('?'); + path.push_str(query); + } + let mut stream = std::net::TcpStream::connect((host, expected_port)) + .map_err(|error| format!("Edge DevTools WebSocket connection failed: {error}"))?; + stream + .set_read_timeout(Some(Duration::from_secs(3))) + .map_err(|error| format!("Edge DevTools read timeout setup failed: {error}"))?; + stream + .set_write_timeout(Some(Duration::from_secs(3))) + .map_err(|error| format!("Edge DevTools write timeout setup failed: {error}"))?; + + let key = base64::engine::general_purpose::STANDARD.encode(Uuid::new_v4().as_bytes()); + let request = format!( + "GET {path} HTTP/1.1\r\nHost: {host}:{expected_port}\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Key: {key}\r\nSec-WebSocket-Version: 13\r\n\r\n" + ); + stream + .write_all(request.as_bytes()) + .map_err(|error| format!("Edge DevTools handshake write failed: {error}"))?; + let mut response = Vec::new(); + let mut byte = [0u8; 1]; + while !response.ends_with(b"\r\n\r\n") { + if response.len() >= 16_384 { + return Err("Edge DevTools handshake headers exceeded their bound".to_string()); + } + stream + .read_exact(&mut byte) + .map_err(|error| format!("Edge DevTools handshake read failed: {error}"))?; + response.push(byte[0]); + } + let response = String::from_utf8(response) + .map_err(|_| "Edge DevTools handshake was not valid HTTP".to_string())?; + let mut lines = response.split("\r\n"); + let status = lines.next().unwrap_or_default(); + if !status.starts_with("HTTP/1.1 101 ") { + return Err(format!( + "Edge DevTools WebSocket upgrade was refused: {status}" + )); + } + let mut upgrade = false; + let mut connection = false; + let mut accept = None; + for line in lines { + let Some((name, value)) = line.split_once(':') else { + continue; + }; + let name = name.trim(); + let value = value.trim(); + if name.eq_ignore_ascii_case("upgrade") && value.eq_ignore_ascii_case("websocket") { + upgrade = true; + } else if name.eq_ignore_ascii_case("connection") + && value + .split(',') + .any(|part| part.trim().eq_ignore_ascii_case("upgrade")) + { + connection = true; + } else if name.eq_ignore_ascii_case("sec-websocket-accept") { + accept = Some(value.to_string()); + } + } + let expected_accept = base64::engine::general_purpose::STANDARD.encode(edge_sha1( + format!("{key}258EAFA5-E914-47DA-95CA-C5AB0DC85B11").as_bytes(), + )); + if !upgrade || !connection || accept.as_deref() != Some(expected_accept.as_str()) { + return Err("Edge DevTools WebSocket handshake identity did not verify".to_string()); + } + Ok(Self { + stream, + next_command_id: 1, + }) + } + + fn command( + &mut self, + method: &str, + params: serde_json::Value, + ) -> Result { + let command_id = self.next_command_id; + self.next_command_id = self.next_command_id.saturating_add(1); + let payload = serde_json::to_vec(&serde_json::json!({ + "id": command_id, + "method": method, + "params": params, + })) + .map_err(|error| format!("Edge DevTools command serialization failed: {error}"))?; + self.write_frame(0x1, &payload)?; + loop { + let message = self.read_text_message()?; + let message: serde_json::Value = serde_json::from_slice(&message) + .map_err(|error| format!("Edge DevTools response was invalid JSON: {error}"))?; + if message.get("id").and_then(serde_json::Value::as_u64) != Some(command_id) { + continue; + } + if let Some(error) = message.get("error") { + return Err(format!( + "Edge DevTools command {method} failed: {}", + edge_safe_json_summary(error) + )); + } + return message + .get("result") + .cloned() + .ok_or_else(|| format!("Edge DevTools command {method} returned no result")); + } + } + + fn write_frame(&mut self, opcode: u8, payload: &[u8]) -> Result<(), String> { + use std::io::Write; + + if payload.len() > 1_048_576 { + return Err("Edge DevTools outbound frame exceeded its bound".to_string()); + } + let mut frame = Vec::with_capacity(payload.len().saturating_add(14)); + frame.push(0x80 | (opcode & 0x0f)); + if payload.len() < 126 { + frame.push(0x80 | payload.len() as u8); + } else if payload.len() <= u16::MAX as usize { + frame.push(0x80 | 126); + frame.extend_from_slice(&(payload.len() as u16).to_be_bytes()); + } else { + frame.push(0x80 | 127); + frame.extend_from_slice(&(payload.len() as u64).to_be_bytes()); + } + let mask_seed = Uuid::new_v4(); + let mask = &mask_seed.as_bytes()[..4]; + frame.extend_from_slice(mask); + for (index, byte) in payload.iter().enumerate() { + frame.push(*byte ^ mask[index % mask.len()]); + } + self.stream + .write_all(&frame) + .map_err(|error| format!("Edge DevTools frame write failed: {error}")) + } + + fn read_text_message(&mut self) -> Result, String> { + use std::io::Read; + + let mut message = Vec::new(); + let mut started = false; + loop { + let mut header = [0u8; 2]; + self.stream + .read_exact(&mut header) + .map_err(|error| format!("Edge DevTools frame header read failed: {error}"))?; + let finished = header[0] & 0x80 != 0; + let opcode = header[0] & 0x0f; + if header[1] & 0x80 != 0 { + return Err("Edge DevTools server unexpectedly masked a frame".to_string()); + } + let mut length = u64::from(header[1] & 0x7f); + if length == 126 { + let mut bytes = [0u8; 2]; + self.stream + .read_exact(&mut bytes) + .map_err(|error| format!("Edge DevTools frame length read failed: {error}"))?; + length = u64::from(u16::from_be_bytes(bytes)); + } else if length == 127 { + let mut bytes = [0u8; 8]; + self.stream + .read_exact(&mut bytes) + .map_err(|error| format!("Edge DevTools frame length read failed: {error}"))?; + length = u64::from_be_bytes(bytes); + } + if length > 1_048_576 || message.len().saturating_add(length as usize) > 1_048_576 { + return Err("Edge DevTools inbound message exceeded its bound".to_string()); + } + let mut payload = vec![0u8; length as usize]; + self.stream + .read_exact(&mut payload) + .map_err(|error| format!("Edge DevTools frame payload read failed: {error}"))?; + match opcode { + 0x0 if started => message.extend_from_slice(&payload), + 0x1 if !started => { + started = true; + message.extend_from_slice(&payload); + } + 0x8 => return Err("Edge DevTools WebSocket closed unexpectedly".to_string()), + 0x9 => { + self.write_frame(0xA, &payload)?; + continue; + } + 0xA => continue, + _ => { + return Err(format!( + "Edge DevTools returned an unexpected WebSocket opcode: {opcode}" + )) + } + } + if finished { + return Ok(message); + } + } + } +} + +#[cfg(all(windows, test))] +fn edge_sha1(input: &[u8]) -> [u8; 20] { + let bit_length = (input.len() as u64).wrapping_mul(8); + let mut message = input.to_vec(); + message.push(0x80); + while message.len() % 64 != 56 { + message.push(0); + } + message.extend_from_slice(&bit_length.to_be_bytes()); + let mut state = [ + 0x6745_2301u32, + 0xefcd_ab89u32, + 0x98ba_dcfeu32, + 0x1032_5476u32, + 0xc3d2_e1f0u32, + ]; + for block in message.chunks_exact(64) { + let mut words = [0u32; 80]; + for (index, word) in words.iter_mut().take(16).enumerate() { + let offset = index * 4; + *word = u32::from_be_bytes([ + block[offset], + block[offset + 1], + block[offset + 2], + block[offset + 3], + ]); + } + for index in 16..80 { + words[index] = + (words[index - 3] ^ words[index - 8] ^ words[index - 14] ^ words[index - 16]) + .rotate_left(1); + } + let [mut a, mut b, mut c, mut d, mut e] = state; + for (index, word) in words.iter().enumerate() { + let (function, constant) = match index { + 0..=19 => ((b & c) | ((!b) & d), 0x5a82_7999), + 20..=39 => (b ^ c ^ d, 0x6ed9_eba1), + 40..=59 => ((b & c) | (b & d) | (c & d), 0x8f1b_bcdc), + _ => (b ^ c ^ d, 0xca62_c1d6), + }; + let next = a + .rotate_left(5) + .wrapping_add(function) + .wrapping_add(e) + .wrapping_add(constant) + .wrapping_add(*word); + e = d; + d = c; + c = b.rotate_left(30); + b = a; + a = next; + } + state[0] = state[0].wrapping_add(a); + state[1] = state[1].wrapping_add(b); + state[2] = state[2].wrapping_add(c); + state[3] = state[3].wrapping_add(d); + state[4] = state[4].wrapping_add(e); + } + let mut digest = [0u8; 20]; + for (index, word) in state.iter().enumerate() { + digest[index * 4..index * 4 + 4].copy_from_slice(&word.to_be_bytes()); + } + digest +} + +#[cfg(all(windows, test))] +fn edge_safe_json_summary(value: &serde_json::Value) -> String { + value + .get("message") + .and_then(serde_json::Value::as_str) + .unwrap_or("bounded DevTools error") + .chars() + .take(256) + .collect() +} + +#[cfg(all(windows, test))] +fn edge_devtools_target(query: &WindowsEdgePortalDomQuery) -> Result<(String, String), String> { + let endpoint = format!("http://127.0.0.1:{}/json/list", query.devtools_port); + let targets: Vec = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(3)) + .build() + .map_err(|error| format!("Edge DevTools client setup failed: {error}"))? + .get(endpoint) + .send() + .and_then(reqwest::blocking::Response::error_for_status) + .map_err(|error| format!("Edge DevTools target discovery failed: {error}"))? + .json() + .map_err(|error| format!("Edge DevTools target list was invalid: {error}"))?; + if targets.len() > 64 { + return Err("Edge DevTools target list exceeded its bound".to_string()); + } + let matches = targets + .into_iter() + .filter(|target| { + target.get("type").and_then(serde_json::Value::as_str) == Some("page") + && target.get("url").and_then(serde_json::Value::as_str) == Some(query.url.as_str()) + && target.get("title").and_then(serde_json::Value::as_str) + == Some(query.document_title.as_str()) + }) + .collect::>(); + if matches.len() != 1 { + return Err(format!( + "Edge DevTools requires exactly one exact page target; found {}", + matches.len() + )); + } + let target = &matches[0]; + let target_id = target + .get("id") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Edge DevTools exact page target ID is missing".to_string())? + .to_string(); + let websocket_url = target + .get("webSocketDebuggerUrl") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Edge DevTools exact page WebSocket is missing".to_string())? + .to_string(); + Ok((target_id, websocket_url)) +} + +#[cfg(all(windows, test))] +fn edge_portal_capture_expression(query: &WindowsEdgePortalDomQuery) -> String { + let title = serde_json::to_string(&query.document_title).unwrap_or_default(); + let url = serde_json::to_string(&query.url).unwrap_or_default(); + let token = serde_json::to_string(&query.document_token).unwrap_or_default(); + let target_id = serde_json::to_string(&query.target_element_id).unwrap_or_default(); + let target_name = serde_json::to_string(&query.target_name).unwrap_or_default(); + let decoy_id = serde_json::to_string(&query.decoy_element_id).unwrap_or_default(); + let decoy_name = serde_json::to_string(&query.decoy_name).unwrap_or_default(); + let receipt_id = serde_json::to_string(&query.receipt_element_id).unwrap_or_default(); + format!( + r#"(() => {{ +const exact = (id) => {{ + const matches = document.querySelectorAll(`[id="${{CSS.escape(id)}}"]`); + return matches.length === 1 ? matches[0] : null; +}}; +const target = exact({target_id}); +const decoy = exact({decoy_id}); +const receipt = exact({receipt_id}); +const ok = document.title === {title} + && location.href === {url} + && document.documentElement.dataset.c5cDocument === {token} + && target instanceof HTMLInputElement + && target.getAttribute("aria-label") === {target_name} + && !target.disabled + && !target.readOnly + && decoy instanceof HTMLInputElement + && decoy.getAttribute("aria-label") === {decoy_name} + && receipt instanceof HTMLElement; +return {{ + ok, + url: location.href, + origin: location.origin, + documentTitle: document.title, + documentToken: document.documentElement.dataset.c5cDocument || "", + targetName: target?.getAttribute("aria-label") || "", + targetValue: target?.value || "", + semanticReceipt: receipt?.textContent || "", + decoyValue: decoy?.value || "" +}}; +}})()"# + ) +} + +#[cfg(all(windows, test))] +fn edge_portal_mutation_expression( + query: &WindowsEdgePortalDomQuery, + expected_before: &str, + expected_decoy: &str, + value: &str, +) -> String { + let title = serde_json::to_string(&query.document_title).unwrap_or_default(); + let url = serde_json::to_string(&query.url).unwrap_or_default(); + let token = serde_json::to_string(&query.document_token).unwrap_or_default(); + let target_id = serde_json::to_string(&query.target_element_id).unwrap_or_default(); + let target_name = serde_json::to_string(&query.target_name).unwrap_or_default(); + let decoy_id = serde_json::to_string(&query.decoy_element_id).unwrap_or_default(); + let decoy_name = serde_json::to_string(&query.decoy_name).unwrap_or_default(); + let receipt_id = serde_json::to_string(&query.receipt_element_id).unwrap_or_default(); + let expected_before = serde_json::to_string(expected_before).unwrap_or_default(); + let expected_decoy = serde_json::to_string(expected_decoy).unwrap_or_default(); + let value = serde_json::to_string(value).unwrap_or_default(); + format!( + r#"(() => {{ +const exact = (id) => {{ + const matches = document.querySelectorAll(`[id="${{CSS.escape(id)}}"]`); + return matches.length === 1 ? matches[0] : null; +}}; +const target = exact({target_id}); +const decoy = exact({decoy_id}); +const receipt = exact({receipt_id}); +const ready = document.title === {title} + && location.href === {url} + && document.documentElement.dataset.c5cDocument === {token} + && target instanceof HTMLInputElement + && target.getAttribute("aria-label") === {target_name} + && !target.disabled + && !target.readOnly + && target.value === {expected_before} + && decoy instanceof HTMLInputElement + && decoy.getAttribute("aria-label") === {decoy_name} + && decoy.value === {expected_decoy} + && receipt instanceof HTMLElement; +if (!ready) return {{ ok: false }}; +target.value = {value}; +target.dispatchEvent(new Event("input", {{ bubbles: true }})); +target.dispatchEvent(new Event("change", {{ bubbles: true }})); +return {{ + ok: true, + url: location.href, + origin: location.origin, + documentTitle: document.title, + documentToken: document.documentElement.dataset.c5cDocument || "", + targetName: target.getAttribute("aria-label") || "", + targetValue: target.value, + semanticReceipt: receipt.textContent || "", + decoyValue: decoy.value +}}; +}})()"# + ) +} + +#[cfg(all(windows, test))] +fn edge_runtime_value( + socket: &mut EdgeDevToolsSocket, + expression: String, +) -> Result { + let result = socket.command( + "Runtime.evaluate", + serde_json::json!({ + "expression": expression, + "returnByValue": true, + "awaitPromise": false, + }), + )?; + if result.get("exceptionDetails").is_some() { + return Err("Edge portal DOM evaluation returned an exception".to_string()); + } + result + .pointer("/result/value") + .cloned() + .ok_or_else(|| "Edge portal DOM evaluation returned no bounded value".to_string()) +} + +#[cfg(all(windows, test))] +fn edge_portal_dom_session( + query: &WindowsEdgePortalDomQuery, + expression: String, +) -> Result { + let parsed = reqwest::Url::parse(&query.url) + .map_err(|error| format!("Edge portal URL is invalid: {error}"))?; + if parsed.scheme() != "http" + || !parsed + .host_str() + .is_some_and(|host| host == "127.0.0.1" || host.eq_ignore_ascii_case("localhost")) + || !parsed.username().is_empty() + || parsed.password().is_some() + { + return Err("Edge portal DOM query requires an exact loopback HTTP URL".to_string()); + } + let (target_id, websocket_url) = edge_devtools_target(query)?; + let mut socket = EdgeDevToolsSocket::connect(&websocket_url, query.devtools_port)?; + let frame_tree = socket.command("Page.getFrameTree", serde_json::json!({}))?; + let frame_id = frame_tree + .pointer("/frameTree/frame/id") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Edge DevTools exact main frame ID is missing".to_string())? + .to_string(); + if frame_tree + .pointer("/frameTree/frame/url") + .and_then(serde_json::Value::as_str) + != Some(query.url.as_str()) + { + return Err("Edge DevTools main frame URL changed".to_string()); + } + let window = socket.command( + "Browser.getWindowForTarget", + serde_json::json!({ "targetId": target_id }), + )?; + let browser_window_id = window + .get("windowId") + .and_then(serde_json::Value::as_i64) + .ok_or_else(|| "Edge DevTools exact browser window ID is missing".to_string())?; + let value = edge_runtime_value(&mut socket, expression)?; + if value.get("ok").and_then(serde_json::Value::as_bool) != Some(true) { + return Err("Edge portal DOM identity failed exact revalidation".to_string()); + } + let string = |name: &str| -> Result { + value + .get(name) + .and_then(serde_json::Value::as_str) + .map(str::to_string) + .ok_or_else(|| format!("Edge portal DOM field {name} is missing")) + }; + let snapshot = WindowsEdgePortalDomSnapshot { + target_id, + frame_id, + browser_window_id, + url: string("url")?, + origin: string("origin")?, + document_title: string("documentTitle")?, + document_token: string("documentToken")?, + target_name: string("targetName")?, + target_value: string("targetValue")?, + semantic_receipt: string("semanticReceipt")?, + decoy_value: string("decoyValue")?, + }; + if snapshot.url != query.url + || snapshot.origin != parsed.origin().ascii_serialization() + || snapshot.document_title != query.document_title + || snapshot.document_token != query.document_token + || snapshot.target_name != query.target_name + || !snapshot.semantic_receipt.starts_with(&query.receipt_prefix) + { + return Err("Edge portal DOM snapshot changed its exact identity".to_string()); + } + Ok(snapshot) +} + +#[cfg(all(windows, test))] +pub(crate) fn capture_windows_edge_portal_dom( + query: &WindowsEdgePortalDomQuery, +) -> Result { + edge_portal_dom_session(query, edge_portal_capture_expression(query)) +} + +#[cfg(all(windows, test))] +pub(crate) fn mutate_windows_edge_portal_dom( + query: &WindowsEdgePortalDomQuery, + expected_target_id: &str, + expected_frame_id: &str, + expected_browser_window_id: i64, + expected_before: &str, + expected_decoy: &str, + value: &str, +) -> Result { + let snapshot = edge_portal_dom_session( + query, + edge_portal_mutation_expression(query, expected_before, expected_decoy, value), + )?; + if snapshot.target_id != expected_target_id + || snapshot.frame_id != expected_frame_id + || snapshot.browser_window_id != expected_browser_window_id + || snapshot.target_value != value + || snapshot.semantic_receipt != format!("{}{}", query.receipt_prefix, value) + || snapshot.decoy_value != expected_decoy + { + return Err( + "Edge portal mutation did not return the exact semantic receipt and frozen identity" + .to_string(), + ); + } + Ok(snapshot) +} + #[cfg(windows)] #[allow(dead_code)] fn bound_windows_accessibility_action_timeout( @@ -5832,10 +6449,11 @@ mod tests { #[cfg(windows)] use crate::kernel::capability::{ - bound_windows_accessibility_action_timeout, excel_grid_automation_id, - require_bound_excel_editor_identity, require_bound_excel_grid_identity, - require_bound_windows_process_identity, WindowsBoundComputerControlTarget, - WindowsBoundComputerScreenshotClient, + bound_windows_accessibility_action_timeout, capture_windows_edge_portal_dom, edge_sha1, + excel_grid_automation_id, require_bound_excel_editor_identity, + require_bound_excel_grid_identity, require_bound_windows_process_identity, + WindowsBoundComputerControlTarget, WindowsBoundComputerScreenshotClient, + WindowsEdgePortalDomQuery, }; use crate::kernel::capability::{ computer_control_action_contract_string, parse_computer_control_action, run_browser_browse, @@ -7236,6 +7854,37 @@ mod tests { ); } + #[cfg(windows)] + #[test] + fn edge_devtools_websocket_accept_hash_matches_rfc6455() { + use base64::Engine; + + assert_eq!( + base64::engine::general_purpose::STANDARD.encode(edge_sha1( + b"dGhlIHNhbXBsZSBub25jZQ==258EAFA5-E914-47DA-95CA-C5AB0DC85B11" + )), + "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=" + ); + } + + #[cfg(windows)] + #[test] + fn edge_portal_dom_rejects_non_loopback_url_before_target_discovery() { + let query = WindowsEdgePortalDomQuery { + devtools_port: 9, + url: "https://example.com/production".to_string(), + document_title: "forbidden production portal".to_string(), + document_token: "forbidden".to_string(), + target_element_id: "target".to_string(), + target_name: "Target".to_string(), + decoy_element_id: "decoy".to_string(), + decoy_name: "Decoy".to_string(), + receipt_element_id: "receipt".to_string(), + receipt_prefix: "receipt:".to_string(), + }; + assert!(capture_windows_edge_portal_dom(&query).is_err()); + } + #[test] fn computer_control_action_parser_accepts_click_and_hotkey_actions() { assert_eq!( diff --git a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs index bbf8aad..bc25252 100644 --- a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs +++ b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs @@ -653,6 +653,7 @@ pub fn accessibility_value_semantic_fingerprint(value: &str) -> Result "file-explorer", Self::Excel => "excel", + Self::Edge => "edge", Self::Generic => "generic", } } @@ -682,6 +686,7 @@ impl WindowsComputerUseTargetProfile { match self { Self::FileExplorer => "File Explorer", Self::Excel => "Excel", + Self::Edge => "Edge", Self::Generic => "Windows", } } @@ -737,9 +742,9 @@ fn current_windows_bounded_semantic_value( }; match profile { WindowsComputerUseTargetProfile::FileExplorer => selection().or_else(value), - WindowsComputerUseTargetProfile::Excel | WindowsComputerUseTargetProfile::Generic => { - value().or_else(selection) - } + WindowsComputerUseTargetProfile::Excel + | WindowsComputerUseTargetProfile::Edge + | WindowsComputerUseTargetProfile::Generic => value().or_else(selection), } } @@ -819,8 +824,8 @@ fn current_windows_selected_accessibility_descendant( use windows::Win32::UI::Accessibility::{ IUIAutomationGridItemPattern, IUIAutomationLegacyIAccessiblePattern, IUIAutomationSelectionItemPattern, IUIAutomationValuePattern, UIA_DataItemControlTypeId, - UIA_GridItemPatternId, UIA_LegacyIAccessiblePatternId, UIA_ListItemControlTypeId, - UIA_SelectionItemPatternId, UIA_ValuePatternId, + UIA_EditControlTypeId, UIA_GridItemPatternId, UIA_LegacyIAccessiblePatternId, + UIA_ListItemControlTypeId, UIA_SelectionItemPatternId, UIA_ValuePatternId, }; let mut pending = Vec::new(); @@ -918,9 +923,25 @@ fn current_windows_selected_accessibility_descendant( }; is_data_item && supports_value && exact_excel_target }, + WindowsComputerUseTargetProfile::Edge => unsafe { + let is_edit = element + .CurrentControlType() + .map(|control_type| control_type == UIA_EditControlTypeId) + .unwrap_or(false); + let supports_value = element + .GetCurrentPatternAs::(UIA_ValuePatternId) + .is_ok(); + let accepts_browser_chrome = matches!( + expected_target, + None | Some(WindowsBoundAccessibilityTarget::Any) + ); + is_edit && supports_value && accepts_browser_chrome + }, WindowsComputerUseTargetProfile::Generic => false, }; - if is_selected && exact_target_kind { + if (is_selected || profile == WindowsComputerUseTargetProfile::Edge) + && exact_target_kind + { return Some(element); } if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { @@ -1190,9 +1211,10 @@ fn capture_windows_redacted_state( target_profile, WindowsComputerUseTargetProfile::FileExplorer | WindowsComputerUseTargetProfile::Excel + | WindowsComputerUseTargetProfile::Edge ) { return Err( - "bound Windows accessibility observation supports only File Explorer or Excel" + "bound Windows accessibility observation supports only File Explorer, Excel, or Edge" .to_string(), ); } @@ -1442,8 +1464,8 @@ fn capture_windows_redacted_state( #[cfg(test)] mod tests { use std::collections::VecDeque; - use std::sync::atomic::{AtomicUsize, Ordering}; - use std::sync::Mutex; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; use chrono::Utc; use tempfile::tempdir; @@ -2633,6 +2655,1255 @@ mod tests { Ok(directory) } + #[cfg(windows)] + #[derive(Clone, Debug, Eq, PartialEq)] + struct EdgePortalLiveIdentity { + application_fingerprint: String, + window_handle: isize, + browser_process_id: u32, + devtools_port: u16, + target_id: String, + frame_id: String, + browser_window_id: i64, + profile_fingerprint: String, + tab_fingerprint: String, + url: String, + origin: String, + document_fingerprint: String, + target_fingerprint: String, + action_fingerprint: String, + target_value: String, + semantic_receipt: String, + decoy_value: String, + } + + #[cfg(windows)] + #[derive(Clone, Debug, Eq, PartialEq)] + struct EdgePortalContract { + application_fingerprint: String, + window_handle: isize, + browser_process_id: u32, + devtools_port: u16, + target_id: String, + frame_id: String, + browser_window_id: i64, + profile_fingerprint: String, + tab_fingerprint: String, + url: String, + origin: String, + document_fingerprint: String, + target_fingerprint: String, + action_fingerprint: String, + receipt_prefix: String, + decoy_value: String, + } + + #[cfg(windows)] + impl EdgePortalContract { + fn from_live(identity: &EdgePortalLiveIdentity, receipt_prefix: String) -> Self { + Self { + application_fingerprint: identity.application_fingerprint.clone(), + window_handle: identity.window_handle, + browser_process_id: identity.browser_process_id, + devtools_port: identity.devtools_port, + target_id: identity.target_id.clone(), + frame_id: identity.frame_id.clone(), + browser_window_id: identity.browser_window_id, + profile_fingerprint: identity.profile_fingerprint.clone(), + tab_fingerprint: identity.tab_fingerprint.clone(), + url: identity.url.clone(), + origin: identity.origin.clone(), + document_fingerprint: identity.document_fingerprint.clone(), + target_fingerprint: identity.target_fingerprint.clone(), + action_fingerprint: identity.action_fingerprint.clone(), + receipt_prefix, + decoy_value: identity.decoy_value.clone(), + } + } + } + + #[cfg(windows)] + fn validate_edge_portal_identity( + actual: &EdgePortalLiveIdentity, + expected: &EdgePortalContract, + ) -> Result { + let parsed = reqwest::Url::parse(&actual.url) + .map_err(|error| format!("Edge portal URL is invalid: {error}"))?; + if parsed.scheme() != "http" + || !parsed + .host_str() + .is_some_and(|host| host.eq_ignore_ascii_case("localhost") || host == "127.0.0.1") + || !parsed.username().is_empty() + || parsed.password().is_some() + { + return Err("Edge portal must remain on its exact loopback HTTP origin".to_string()); + } + if parsed.origin().ascii_serialization() != actual.origin { + return Err("Edge portal URL and origin identity disagree".to_string()); + } + let checks = [ + ( + actual.application_fingerprint == expected.application_fingerprint, + "Edge portal application changed", + ), + ( + actual.window_handle == expected.window_handle, + "Edge portal HWND changed", + ), + ( + actual.browser_process_id == expected.browser_process_id, + "Edge portal browser process changed", + ), + ( + actual.devtools_port == expected.devtools_port, + "Edge portal loopback DevTools endpoint changed", + ), + ( + actual.target_id == expected.target_id, + "Edge portal tab target changed", + ), + ( + actual.frame_id == expected.frame_id, + "Edge portal main frame changed", + ), + ( + actual.browser_window_id == expected.browser_window_id, + "Edge portal browser window identity changed", + ), + ( + actual.profile_fingerprint == expected.profile_fingerprint, + "Edge portal profile changed", + ), + ( + actual.tab_fingerprint == expected.tab_fingerprint, + "Edge portal tab changed", + ), + (actual.url == expected.url, "Edge portal URL changed"), + ( + actual.origin == expected.origin, + "Edge portal origin changed", + ), + ( + actual.document_fingerprint == expected.document_fingerprint, + "Edge portal document changed", + ), + ( + actual.target_fingerprint == expected.target_fingerprint, + "Edge portal target changed", + ), + ( + actual.action_fingerprint == expected.action_fingerprint, + "Edge portal action changed", + ), + ( + actual.decoy_value == expected.decoy_value, + "Edge portal wrong-field write was detected", + ), + ]; + for (matches, error) in checks { + if !matches { + return Err(error.to_string()); + } + } + if !actual + .semantic_receipt + .starts_with(&expected.receipt_prefix) + { + return Err( + "Edge portal semantic receipt is missing or belongs to another document" + .to_string(), + ); + } + Ok(fingerprint_parts(&[ + "edge-local-portal-semantic-receipt/v1", + &fingerprint_parts(&[&actual.target_value]), + &fingerprint_parts(&[&actual.semantic_receipt]), + ])) + } + + #[cfg(windows)] + fn require_edge_portal_effect_receipt( + actual: &EdgePortalLiveIdentity, + expected: &EdgePortalContract, + expected_value: &str, + expected_receipt: &str, + ) -> Result { + let semantic_fingerprint = validate_edge_portal_identity(actual, expected)?; + if actual.target_value != expected_value || actual.semantic_receipt != expected_receipt { + return Err( + "Edge portal did not return the exact field value and semantic receipt".to_string(), + ); + } + Ok(semantic_fingerprint) + } + + #[cfg(windows)] + fn bind_edge_portal_state( + mut state: RedactedComputerUseState, + identity: &EdgePortalLiveIdentity, + contract: &EdgePortalContract, + ) -> Result { + let semantic_fingerprint = validate_edge_portal_identity(identity, contract)?; + state.process_fingerprint = fingerprint_parts(&[ + "edge-local-portal-process/v1", + &state.process_fingerprint, + &identity.profile_fingerprint, + &identity.devtools_port.to_string(), + &identity.target_id, + ]); + state.application_fingerprint = fingerprint_parts(&[ + "edge-local-portal-application/v1", + &state.application_fingerprint, + &identity.application_fingerprint, + ]); + state.frame_fingerprint = fingerprint_parts(&[ + "edge-local-portal-frame/v1", + &state.frame_fingerprint, + &identity.profile_fingerprint, + &identity.tab_fingerprint, + &identity.target_id, + &identity.frame_id, + &identity.browser_window_id.to_string(), + &fingerprint_parts(&[&identity.url]), + &fingerprint_parts(&[&identity.origin]), + &identity.document_fingerprint, + ]); + state.target_fingerprint = fingerprint_parts(&[ + "edge-local-portal-target/v1", + &state.target_fingerprint, + &identity.target_fingerprint, + &identity.action_fingerprint, + ]); + state.semantic_fingerprint = Some(semantic_fingerprint); + state.safe_summary = + "Bound Edge local-portal field and independent semantic receipt are available." + .to_string(); + state.validate()?; + Ok(state) + } + + #[cfg(windows)] + fn deterministic_edge_portal_identity() -> EdgePortalLiveIdentity { + EdgePortalLiveIdentity { + application_fingerprint: fingerprint_parts(&["edge-application", "installed"]), + window_handle: 100, + browser_process_id: 41, + devtools_port: 43_125, + target_id: "tab-target-c5c".to_string(), + frame_id: "main-frame-c5c".to_string(), + browser_window_id: 7, + profile_fingerprint: fingerprint_parts(&["edge-profile", "c5c"]), + tab_fingerprint: fingerprint_parts(&["edge-tab", "c5c"]), + url: "http://127.0.0.1:43125/c5c/nonce".to_string(), + origin: "http://127.0.0.1:43125".to_string(), + document_fingerprint: fingerprint_parts(&["edge-document", "nonce"]), + target_fingerprint: fingerprint_parts(&["edge-target", "field"]), + action_fingerprint: fingerprint_parts(&["edge-action", "set", "field", "approved"]), + target_value: "before".to_string(), + semantic_receipt: "C5C receipt nonce:pending".to_string(), + decoy_value: "decoy-unchanged".to_string(), + } + } + + #[cfg(windows)] + #[test] + fn edge_portal_identity_rejects_profile_tab_url_origin_document_target_and_action_drift() { + let identity = deterministic_edge_portal_identity(); + let contract = EdgePortalContract::from_live(&identity, "C5C receipt nonce:".to_string()); + assert!(validate_edge_portal_identity(&identity, &contract).is_ok()); + + let mut cases = Vec::new(); + let mut changed = identity.clone(); + changed.application_fingerprint = fingerprint_parts(&["edge-application", "changed"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.window_handle += 1; + cases.push(changed); + let mut changed = identity.clone(); + changed.browser_process_id += 1; + cases.push(changed); + let mut changed = identity.clone(); + changed.devtools_port += 1; + cases.push(changed); + let mut changed = identity.clone(); + changed.target_id = "tab-target-other".to_string(); + cases.push(changed); + let mut changed = identity.clone(); + changed.frame_id = "main-frame-stale".to_string(); + cases.push(changed); + let mut changed = identity.clone(); + changed.browser_window_id += 1; + cases.push(changed); + let mut changed = identity.clone(); + changed.profile_fingerprint = fingerprint_parts(&["edge-profile", "other"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.tab_fingerprint = fingerprint_parts(&["edge-tab", "other"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.url = "http://127.0.0.1:43125/c5c/redirect".to_string(); + cases.push(changed); + let mut changed = identity.clone(); + changed.origin = "http://127.0.0.1:43126".to_string(); + cases.push(changed); + let mut changed = identity.clone(); + changed.document_fingerprint = fingerprint_parts(&["edge-document", "stale"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.target_fingerprint = fingerprint_parts(&["edge-target", "wrong-field"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.action_fingerprint = fingerprint_parts(&["edge-action", "mutated"]); + cases.push(changed); + let mut changed = identity.clone(); + changed.decoy_value = "wrong-field-write".to_string(); + cases.push(changed); + let mut changed = identity.clone(); + changed.semantic_receipt = "foreign receipt".to_string(); + cases.push(changed); + + for changed in cases { + assert!(validate_edge_portal_identity(&changed, &contract).is_err()); + } + } + + #[cfg(windows)] + #[test] + fn edge_portal_semantic_receipt_blocks_dom_or_screenshot_only_false_completion() { + let mut identity = deterministic_edge_portal_identity(); + let contract = EdgePortalContract::from_live(&identity, "C5C receipt nonce:".to_string()); + identity.target_value = "approved".to_string(); + assert!(require_edge_portal_effect_receipt( + &identity, + &contract, + "approved", + "C5C receipt nonce:approved", + ) + .is_err()); + identity.semantic_receipt = "C5C receipt nonce:approved".to_string(); + assert!(require_edge_portal_effect_receipt( + &identity, + &contract, + "approved", + "C5C receipt nonce:approved", + ) + .is_ok()); + } + + #[cfg(windows)] + #[derive(Clone)] + struct EdgePortalCaptureSpec { + edge_path: std::path::PathBuf, + window_handle: isize, + browser_process_id: u32, + devtools_port: u16, + profile: std::path::PathBuf, + url: String, + document_title: String, + document_token: String, + target_element_id: String, + target_name: String, + action_fingerprint: String, + receipt_element_id: String, + receipt_prefix: String, + decoy_element_id: String, + decoy_name: String, + } + + #[cfg(windows)] + fn c5c_installed_smoke_directory(name: &str) -> Result { + const ROOT_ENV: &str = "DEEPSEEK_AGENT_OS_C5C_SMOKE_ROOT"; + + let root = std::env::var_os(ROOT_ENV) + .filter(|value| !value.is_empty()) + .map(std::path::PathBuf::from) + .ok_or_else(|| { + format!( + "{ROOT_ENV} must name a fresh absolute authorized isolation root for installed C5C smokes" + ) + })?; + if !root.is_absolute() { + return Err(format!( + "{ROOT_ENV} must be an absolute authorized isolation root" + )); + } + let directory = root.join(name); + if directory.exists() + && std::fs::read_dir(&directory) + .map_err(|error| format!("C5C smoke directory is unreadable: {error}"))? + .next() + .is_some() + { + return Err(format!( + "C5C smoke directory must be fresh and empty: {}", + directory.display() + )); + } + std::fs::create_dir_all(&directory) + .map_err(|error| format!("C5C smoke directory creation failed: {error}"))?; + Ok(directory) + } + + #[cfg(windows)] + fn edge_executable() -> Result { + let program_files = std::env::var_os("ProgramFiles") + .map(std::path::PathBuf::from) + .unwrap_or_default(); + let program_files_x86 = std::env::var_os("ProgramFiles(x86)") + .map(std::path::PathBuf::from) + .unwrap_or_default(); + let local_app_data = std::env::var_os("LOCALAPPDATA") + .map(std::path::PathBuf::from) + .unwrap_or_default(); + [ + program_files.join("Microsoft/Edge/Application/msedge.exe"), + program_files_x86.join("Microsoft/Edge/Application/msedge.exe"), + local_app_data.join("Microsoft/Edge/Application/msedge.exe"), + ] + .into_iter() + .find(|candidate| candidate.is_file()) + .ok_or_else(|| "installed Microsoft Edge was not found".to_string()) + } + + #[cfg(windows)] + fn edge_application_and_profile_fingerprints( + browser_process_id: u32, + expected_edge_path: &std::path::Path, + profile: &std::path::Path, + ) -> Result<(String, String), String> { + use std::process::Command; + + let expected_edge_path = expected_edge_path + .canonicalize() + .map_err(|error| format!("installed Edge executable identity is invalid: {error}"))?; + let profile = profile + .canonicalize() + .map_err(|error| format!("isolated Edge profile is invalid: {error}"))?; + let script = format!( + r#" +$process = Get-CimInstance Win32_Process -Filter "ProcessId = {browser_process_id}" +if ($null -eq $process -or + [string]::IsNullOrWhiteSpace([string]$process.ExecutablePath) -or + [string]::IsNullOrWhiteSpace([string]$process.CommandLine)) {{ + throw 'exact Edge browser process command line is unavailable' +}} +[pscustomobject]@{{ + ExecutablePath = [string]$process.ExecutablePath + CommandLine = [string]$process.CommandLine +}} | ConvertTo-Json -Compress +"#, + ); + let output = Command::new("powershell.exe") + .args(["-NoProfile", "-Command", &script]) + .output() + .map_err(|error| format!("Edge process identity inspection failed: {error}"))?; + if !output.status.success() { + return Err(format!( + "Edge process identity inspection failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let process: serde_json::Value = serde_json::from_slice(&output.stdout) + .map_err(|error| format!("Edge process identity response is invalid: {error}"))?; + let executable_path = process + .get("ExecutablePath") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Edge process executable identity is missing".to_string())?; + let command_line = process + .get("CommandLine") + .and_then(serde_json::Value::as_str) + .filter(|value| !value.trim().is_empty()) + .ok_or_else(|| "Edge process command line identity is missing".to_string())?; + let executable_path = std::path::PathBuf::from(executable_path) + .canonicalize() + .map_err(|error| format!("running Edge executable identity is invalid: {error}"))?; + if !executable_path + .to_string_lossy() + .eq_ignore_ascii_case(&expected_edge_path.to_string_lossy()) + { + return Err("Edge browser process executable changed".to_string()); + } + let normalized_command = command_line.to_ascii_lowercase(); + let profile_text = profile.to_string_lossy(); + let normalized_profile = profile_text + .strip_prefix(r"\\?\") + .unwrap_or(&profile_text) + .to_ascii_lowercase(); + if !normalized_command.contains("--user-data-dir") + || !normalized_command.contains(&normalized_profile) + || !normalized_command.contains("--no-first-run") + || !normalized_command.contains("--remote-debugging-port=0") + { + return Err( + "Edge browser process is not bound to the exact isolated profile".to_string(), + ); + } + let executable_metadata = expected_edge_path.metadata().map_err(|error| { + format!("installed Edge executable metadata is unavailable: {error}") + })?; + Ok(( + fingerprint_parts(&[ + "edge-installed-application/v1", + &fingerprint_parts(&[&expected_edge_path.to_string_lossy().to_ascii_lowercase()]), + &executable_metadata.len().to_string(), + ]), + fingerprint_parts(&[ + "edge-isolated-profile/v1", + &fingerprint_parts(&[&normalized_profile]), + ]), + )) + } + + #[cfg(windows)] + fn normalized_edge_address_value(value: &str) -> Result { + let value = value.trim(); + if value.is_empty() { + return Err("Edge address bar value is empty".to_string()); + } + let candidate = if value.starts_with("http://") || value.starts_with("https://") { + value.to_string() + } else { + format!("http://{value}") + }; + reqwest::Url::parse(&candidate) + .map(|url| url.to_string()) + .map_err(|error| format!("Edge address bar value is invalid: {error}")) + } + + #[cfg(windows)] + fn capture_edge_portal_live_identity( + spec: &EdgePortalCaptureSpec, + ) -> Result { + use windows::Win32::Foundation::HWND; + use windows::Win32::System::Com::{ + CoCreateInstance, CoInitializeEx, CoUninitialize, CLSCTX_INPROC_SERVER, + COINIT_MULTITHREADED, + }; + use windows::Win32::UI::Accessibility::{ + CUIAutomation, IUIAutomation, IUIAutomationSelectionItemPattern, + IUIAutomationValuePattern, UIA_EditControlTypeId, UIA_SelectionItemPatternId, + UIA_TabItemControlTypeId, UIA_ValuePatternId, + }; + use windows::Win32::UI::WindowsAndMessaging::{GetClassNameW, GetWindowThreadProcessId}; + + struct ComGuard; + impl Drop for ComGuard { + fn drop(&mut self) { + unsafe { CoUninitialize() }; + } + } + + unsafe { CoInitializeEx(None, COINIT_MULTITHREADED) } + .ok() + .map_err(|error| format!("Edge portal COM initialization failed: {error}"))?; + let _guard = ComGuard; + let hwnd = HWND(spec.window_handle as _); + let mut actual_process_id = 0u32; + let thread_id = unsafe { GetWindowThreadProcessId(hwnd, Some(&mut actual_process_id)) }; + if thread_id == 0 || actual_process_id != spec.browser_process_id { + return Err("Edge portal exact HWND/PID binding changed".to_string()); + } + let mut class_buffer = [0u16; 256]; + let class_len = unsafe { GetClassNameW(hwnd, &mut class_buffer) }.max(0) as usize; + let window_class = String::from_utf16_lossy(&class_buffer[..class_len]); + if !window_class.eq_ignore_ascii_case("Chrome_WidgetWin_1") { + return Err("Edge portal HWND is not an exact Edge browser window".to_string()); + } + let (application_fingerprint, profile_fingerprint) = + edge_application_and_profile_fingerprints( + spec.browser_process_id, + &spec.edge_path, + &spec.profile, + )?; + + let automation: IUIAutomation = unsafe { + CoCreateInstance(&CUIAutomation, None, CLSCTX_INPROC_SERVER) + .map_err(|error| format!("Edge portal UI Automation setup failed: {error}"))? + }; + let root = unsafe { + automation + .ElementFromHandle(hwnd) + .map_err(|error| format!("Edge portal exact window inspection failed: {error}"))? + }; + let walker = unsafe { + automation + .RawViewWalker() + .map_err(|error| format!("Edge portal raw-view walker failed: {error}"))? + }; + let mut address_value = None; + let mut selected_tab = None; + let mut edit_diagnostics = Vec::new(); + let mut pending = Vec::new(); + if let Ok(child) = unsafe { walker.GetFirstChildElement(&root) } { + pending.push(child); + } + let mut visited = 0usize; + while let Some(element) = pending.pop() { + visited += 1; + if visited > 1_024 { + return Err( + "Edge browser-chrome accessibility traversal exceeded its bound".to_string(), + ); + } + let process_id = unsafe { element.CurrentProcessId() }.unwrap_or_default(); + let control_type = unsafe { element.CurrentControlType() }.ok(); + let name = unsafe { element.CurrentName() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let automation_id = unsafe { element.CurrentAutomationId() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let class_name = unsafe { element.CurrentClassName() } + .map(|value| value.to_string()) + .unwrap_or_default(); + let framework_id = unsafe { element.CurrentFrameworkId() } + .map(|value| value.to_string()) + .unwrap_or_default(); + + if process_id == spec.browser_process_id as i32 + && control_type == Some(UIA_EditControlTypeId) + { + let edit_value = unsafe { + element + .GetCurrentPatternAs::(UIA_ValuePatternId) + .and_then(|pattern| pattern.CurrentValue()) + } + .map(|value| value.to_string()) + .ok(); + if let Some(value) = edit_value.as_deref().filter(|_| edit_diagnostics.len() < 8) { + edit_diagnostics.push(format!( + "name={name:?}, automation_id={automation_id:?}, value={value:?}" + )); + } + let normalized_address = edit_value + .as_deref() + .and_then(|value| normalized_edge_address_value(value).ok()) + .filter(|value| value == &spec.url); + if automation_id == "addressEditBox" || normalized_address.is_some() { + let value = normalized_address.ok_or_else(|| { + "Edge address control did not expose the exact local-portal URL".to_string() + })?; + if address_value.replace(value).is_some() { + return Err("Edge portal exposed multiple address bars".to_string()); + } + } + } + + if process_id == spec.browser_process_id as i32 + && control_type == Some(UIA_TabItemControlTypeId) + && name == spec.document_title + { + let selected = unsafe { + element + .GetCurrentPatternAs::( + UIA_SelectionItemPatternId, + ) + .and_then(|selection| selection.CurrentIsSelected()) + } + .map(|value| value.as_bool()) + .unwrap_or(false); + if selected { + let fingerprint = fingerprint_parts(&[ + "edge-local-portal-tab/v1", + &process_id.to_string(), + &fingerprint_parts(&[&name]), + &fingerprint_parts(&[&automation_id]), + &fingerprint_parts(&[&class_name]), + &fingerprint_parts(&[&framework_id]), + ]); + if selected_tab.replace(fingerprint).is_some() { + return Err("Edge portal exposed multiple selected exact tabs".to_string()); + } + } + } + + if let Ok(child) = unsafe { walker.GetFirstChildElement(&element) } { + pending.push(child); + } + if let Ok(sibling) = unsafe { walker.GetNextSiblingElement(&element) } { + pending.push(sibling); + } + } + + let address_value = address_value.ok_or_else(|| { + format!( + "Edge portal address bar was not found; bounded Edit controls: {}", + edit_diagnostics.join("; ") + ) + })?; + if address_value != spec.url { + return Err("Edge portal address changed before DOM observation".to_string()); + } + let tab_fingerprint = selected_tab + .ok_or_else(|| "Edge portal exact selected tab was not found".to_string())?; + let query = crate::kernel::capability::WindowsEdgePortalDomQuery { + devtools_port: spec.devtools_port, + url: spec.url.clone(), + document_title: spec.document_title.clone(), + document_token: spec.document_token.clone(), + target_element_id: spec.target_element_id.clone(), + target_name: spec.target_name.clone(), + decoy_element_id: spec.decoy_element_id.clone(), + decoy_name: spec.decoy_name.clone(), + receipt_element_id: spec.receipt_element_id.clone(), + receipt_prefix: spec.receipt_prefix.clone(), + }; + let dom = crate::kernel::capability::capture_windows_edge_portal_dom(&query)?; + if dom.url != address_value { + return Err("Edge UI Automation and DevTools URL identities disagree".to_string()); + } + let document_fingerprint = fingerprint_parts(&[ + "edge-local-portal-document/v1", + &dom.target_id, + &dom.frame_id, + &dom.browser_window_id.to_string(), + &fingerprint_parts(&[&dom.document_title]), + &fingerprint_parts(&[&dom.document_token]), + &fingerprint_parts(&[&dom.url]), + ]); + let target_fingerprint = fingerprint_parts(&[ + "edge-local-portal-field/v1", + &dom.target_id, + &dom.frame_id, + &fingerprint_parts(&[&spec.target_element_id]), + &fingerprint_parts(&[&dom.target_name]), + ]); + Ok(EdgePortalLiveIdentity { + application_fingerprint, + window_handle: spec.window_handle, + browser_process_id: spec.browser_process_id, + devtools_port: spec.devtools_port, + target_id: dom.target_id, + frame_id: dom.frame_id, + browser_window_id: dom.browser_window_id, + profile_fingerprint, + tab_fingerprint, + url: dom.url, + origin: dom.origin, + document_fingerprint, + target_fingerprint, + action_fingerprint: spec.action_fingerprint.clone(), + target_value: dom.target_value, + semantic_receipt: dom.semantic_receipt, + decoy_value: dom.decoy_value, + }) + } + + #[cfg(windows)] + struct CorroboratedEdgePortalAccessibilityClient { + inner: WindowsBoundComputerUseAccessibilityClient, + spec: EdgePortalCaptureSpec, + contract: EdgePortalContract, + } + + #[cfg(windows)] + impl ComputerUseAccessibilityClient for CorroboratedEdgePortalAccessibilityClient { + fn capture_redacted_state(&self) -> Result { + let state = self.inner.capture_redacted_state()?; + let identity = capture_edge_portal_live_identity(&self.spec)?; + bind_edge_portal_state(state, &identity, &self.contract) + } + } + + #[cfg(windows)] + struct EdgePortalReceiptCorroboratingControlClient { + spec: EdgePortalCaptureSpec, + contract: EdgePortalContract, + expected_before: String, + expected_value: String, + expected_receipt: String, + } + + #[cfg(windows)] + impl ComputerControlClient for EdgePortalReceiptCorroboratingControlClient { + fn execute_control( + &self, + _target: &str, + action: &ComputerControlAction, + ) -> Result { + let ComputerControlAction::SetAccessibilityValue { value } = action else { + return Err( + "Edge local-portal control accepts only one exact DOM value action".to_string(), + ); + }; + if value != &self.expected_value { + return Err("Edge local-portal action value changed after approval".to_string()); + } + let query = crate::kernel::capability::WindowsEdgePortalDomQuery { + devtools_port: self.spec.devtools_port, + url: self.spec.url.clone(), + document_title: self.spec.document_title.clone(), + document_token: self.spec.document_token.clone(), + target_element_id: self.spec.target_element_id.clone(), + target_name: self.spec.target_name.clone(), + decoy_element_id: self.spec.decoy_element_id.clone(), + decoy_name: self.spec.decoy_name.clone(), + receipt_element_id: self.spec.receipt_element_id.clone(), + receipt_prefix: self.spec.receipt_prefix.clone(), + }; + let target_id = self.contract.target_id.clone(); + let frame_id = self.contract.frame_id.clone(); + let browser_window_id = self.contract.browser_window_id; + let expected_before = self.expected_before.clone(); + let expected_decoy = self.contract.decoy_value.clone(); + let value = value.clone(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + std::thread::spawn(move || { + let result = crate::kernel::capability::mutate_windows_edge_portal_dom( + &query, + &target_id, + &frame_id, + browser_window_id, + &expected_before, + &expected_decoy, + &value, + ); + let _ = sender.send(result); + }); + let snapshot = receiver + .recv_timeout(std::time::Duration::from_secs(8)) + .map_err(|error| { + format!( + "Edge local-portal one-shot DOM action timed out; the effect is unknown and automatic replay is forbidden: {error}" + ) + })? + .map_err(|error| { + format!( + "Edge local-portal one-shot DOM action returned an uncertain effect: {error}" + ) + })?; + if snapshot.semantic_receipt != self.expected_receipt { + return Err( + "Edge local-portal one-shot DOM action returned no exact receipt".to_string(), + ); + } + Ok(crate::kernel::capability::ComputerControlExecution { + summary: "Set one exact generated field in the isolated local Edge portal." + .to_string(), + }) + } + } + + #[cfg(windows)] + fn find_edge_window_for_profile(profile: &std::path::Path) -> Result<(isize, u32), String> { + use std::process::Command; + + let profile = powershell_literal(profile); + let script = format!( + r#" +$profile = [IO.Path]::GetFullPath({profile}) +$candidate = Get-CimInstance Win32_Process -Filter "Name = 'msedge.exe'" | Where-Object {{ + -not [string]::IsNullOrWhiteSpace([string]$_.CommandLine) -and + $_.CommandLine.IndexOf($profile, [StringComparison]::OrdinalIgnoreCase) -ge 0 +}} | ForEach-Object {{ + $process = Get-Process -Id $_.ProcessId -ErrorAction SilentlyContinue + if ($null -ne $process -and $process.MainWindowHandle -ne 0) {{ + [pscustomobject]@{{ ProcessId = [uint32]$_.ProcessId; Hwnd = [int64]$process.MainWindowHandle }} + }} +}} | Select-Object -First 1 +if ($null -eq $candidate) {{ throw 'isolated Edge window was not found' }} +[Console]::Out.Write("$($candidate.Hwnd)|$($candidate.ProcessId)") +"#, + ); + let output = Command::new("powershell.exe") + .args(["-NoProfile", "-Command", &script]) + .output() + .map_err(|error| format!("isolated Edge window discovery failed: {error}"))?; + if !output.status.success() { + return Err(format!( + "isolated Edge window discovery failed: {}", + String::from_utf8_lossy(&output.stderr).trim() + )); + } + let output = String::from_utf8_lossy(&output.stdout); + let (window_handle, process_id) = output + .trim() + .split_once('|') + .ok_or_else(|| "isolated Edge window identity is malformed".to_string())?; + Ok(( + window_handle + .parse::() + .map_err(|error| format!("isolated Edge HWND is invalid: {error}"))?, + process_id + .parse::() + .map_err(|error| format!("isolated Edge PID is invalid: {error}"))?, + )) + } + + #[cfg(windows)] + fn edge_profile_process_ids(profile: &std::path::Path) -> Vec { + use std::process::Command; + + let profile = powershell_literal(profile); + let script = format!( + r#" +$profile = [IO.Path]::GetFullPath({profile}) +Get-CimInstance Win32_Process -Filter "Name = 'msedge.exe'" | Where-Object {{ + -not [string]::IsNullOrWhiteSpace([string]$_.CommandLine) -and + $_.CommandLine.IndexOf($profile, [StringComparison]::OrdinalIgnoreCase) -ge 0 +}} | ForEach-Object {{ [Console]::Out.WriteLine([string]$_.ProcessId) }} +"#, + ); + Command::new("powershell.exe") + .args(["-NoProfile", "-Command", &script]) + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| { + String::from_utf8_lossy(&output.stdout) + .lines() + .filter_map(|line| line.trim().parse::().ok()) + .collect() + }) + .unwrap_or_default() + } + + #[cfg(windows)] + fn close_exact_edge_window(window_handle: isize) { + use windows::Win32::Foundation::{HWND, LPARAM, WPARAM}; + use windows::Win32::UI::WindowsAndMessaging::{PostMessageW, WM_CLOSE}; + + let _ = unsafe { + PostMessageW( + Some(HWND(window_handle as _)), + WM_CLOSE, + WPARAM(0), + LPARAM(0), + ) + }; + } + + #[cfg(windows)] + fn edge_devtools_port(profile: &std::path::Path) -> Result { + let contents = std::fs::read_to_string(profile.join("DevToolsActivePort")) + .map_err(|error| format!("isolated Edge DevTools endpoint is unavailable: {error}"))?; + let mut lines = contents.lines(); + let port = lines + .next() + .ok_or_else(|| "isolated Edge DevTools port is missing".to_string())? + .trim() + .parse::() + .map_err(|error| format!("isolated Edge DevTools port is invalid: {error}"))?; + let browser_path = lines + .next() + .filter(|value| value.starts_with("/devtools/browser/")) + .ok_or_else(|| "isolated Edge DevTools browser identity is missing".to_string())?; + if port == 0 || browser_path.len() > 256 { + return Err("isolated Edge DevTools endpoint exceeded its exact contract".to_string()); + } + Ok(port) + } + + #[cfg(windows)] + #[test] + #[ignore = "requires visible installed Edge with a fresh isolated profile and local portal"] + fn windows_edge_local_portal_isolated_value_smoke_verifies_exact_receipt() { + use std::io::{Read, Write}; + use std::net::{TcpListener, TcpStream}; + use std::process::Command; + use std::thread; + use std::time::Duration; + + use crate::kernel::capability::WindowsBoundComputerScreenshotClient; + + let directory = + c5c_installed_smoke_directory("edge-local-portal").expect("isolated Edge directory"); + let profile = directory.join("edge-profile"); + std::fs::create_dir(&profile).expect("isolated Edge profile is created"); + let nonce = Uuid::new_v4().simple().to_string(); + let document_title = format!("DS Agent C5C Portal {nonce}"); + let document_token = format!("c5c-document-{nonce}"); + let target_element_id = "c5c-target"; + let target_name = format!("C5C target field {nonce}"); + let decoy_element_id = "c5c-decoy"; + let decoy_name = format!("C5C decoy field {nonce}"); + let receipt_element_id = "c5c-receipt"; + let receipt_prefix = format!("C5C receipt {nonce}:"); + let before_value = "before"; + let decoy_value = "decoy-unchanged"; + let expected_value = "DS Agent C5C exact portal value"; + let expected_receipt = format!("{receipt_prefix}{expected_value}"); + let action_fingerprint = fingerprint_parts(&[ + "edge-local-portal-action/v1", + "set-accessibility-value", + &fingerprint_parts(&[&target_name]), + &fingerprint_parts(&[expected_value]), + ]); + + let listener = TcpListener::bind(("127.0.0.1", 0)).expect("local portal binds loopback"); + let address = listener.local_addr().expect("local portal address"); + listener + .set_nonblocking(true) + .expect("local portal is nonblocking"); + let url = format!("http://127.0.0.1:{}/c5c/{nonce}", address.port()); + let html = format!( + r#" + +{document_title} + +
+ + +

{receipt_prefix}pending

+
+ + +"# + ); + let response = Arc::new(format!( + "HTTP/1.1 200 OK\r\nContent-Type: text/html; charset=utf-8\r\nCache-Control: no-store\r\nConnection: close\r\nContent-Length: {}\r\n\r\n{}", + html.len(), + html + )); + let server_stop = Arc::new(AtomicBool::new(false)); + let server_stop_worker = Arc::clone(&server_stop); + let response_worker = Arc::clone(&response); + let server = thread::spawn(move || { + while !server_stop_worker.load(Ordering::SeqCst) { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0u8; 4_096]; + let _ = stream.read(&mut request); + let _ = stream.write_all(response_worker.as_bytes()); + let _ = stream.flush(); + } + Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { + thread::sleep(Duration::from_millis(10)); + } + Err(_) => break, + } + } + }); + + let edge_path = edge_executable().expect("installed Edge path"); + let mut edge = Command::new(&edge_path) + .arg(format!("--user-data-dir={}", profile.display())) + .args([ + "--no-first-run", + "--no-default-browser-check", + "--disable-background-mode", + "--disable-extensions", + "--disable-sync", + "--remote-debugging-port=0", + "--disable-features=msEdgeFirstRunExperience,msEdgeSidebarV2", + "--new-window", + ]) + .arg(&url) + .spawn() + .expect("isolated Edge starts"); + + let mut bound_window = None; + let mut devtools_port = None; + let mut last_diagnostic = "isolated Edge window discovery was not attempted".to_string(); + for _ in 0..60 { + if bound_window.is_none() { + match find_edge_window_for_profile(&profile) { + Ok(binding) => bound_window = Some(binding), + Err(error) => last_diagnostic = error, + } + } + if devtools_port.is_none() { + if let Ok(port) = edge_devtools_port(&profile) { + devtools_port = Some(port); + } + } + if bound_window.is_some() && devtools_port.is_some() { + break; + } + thread::sleep(Duration::from_millis(250)); + } + let result = (|| -> Result<(), String> { + let (window_handle, browser_process_id) = bound_window.ok_or_else(|| { + format!("Edge did not expose an isolated exact window: {last_diagnostic}") + })?; + let devtools_port = devtools_port + .ok_or_else(|| "Edge did not expose its isolated loopback endpoint".to_string())?; + let spec = EdgePortalCaptureSpec { + edge_path: edge_path.clone(), + window_handle, + browser_process_id, + devtools_port, + profile: profile.clone(), + url: url.clone(), + document_title: document_title.clone(), + document_token: document_token.clone(), + target_element_id: target_element_id.to_string(), + target_name: target_name.clone(), + action_fingerprint: action_fingerprint.clone(), + receipt_element_id: receipt_element_id.to_string(), + receipt_prefix: receipt_prefix.clone(), + decoy_element_id: decoy_element_id.to_string(), + decoy_name: decoy_name.clone(), + }; + let mut initial = None; + let mut last_capture = "Edge portal capture was not attempted".to_string(); + for _ in 0..40 { + match capture_edge_portal_live_identity(&spec) { + Ok(identity) + if identity.target_value == before_value + && identity.semantic_receipt == format!("{receipt_prefix}pending") + && identity.decoy_value == decoy_value => + { + initial = Some(identity); + break; + } + Ok(identity) => { + last_capture = format!( + "value={:?}, receipt={:?}, decoy={:?}", + identity.target_value, identity.semantic_receipt, identity.decoy_value + ) + } + Err(error) => last_capture = error, + } + thread::sleep(Duration::from_millis(250)); + } + let initial = initial.ok_or_else(|| { + format!("Edge local portal did not expose its exact identity: {last_capture}") + })?; + let contract = EdgePortalContract::from_live(&initial, receipt_prefix.clone()); + validate_edge_portal_identity(&initial, &contract)?; + let accessibility = CorroboratedEdgePortalAccessibilityClient { + inner: WindowsBoundComputerUseAccessibilityClient::new( + window_handle, + browser_process_id, + )?, + spec: spec.clone(), + contract: contract.clone(), + }; + let screenshot = WindowsBoundComputerScreenshotClient::new( + directory.join("screenshots"), + window_handle, + browser_process_id, + )?; + let store = EventStore::open(directory.join("edge-portal-smoke.db")) + .map_err(|error| error.to_string())?; + let observation = capture_computer_use_observation( + ComputerUseObservationPhase::PreAction, + &screenshot, + &accessibility, + )?; + let (_, observed) = persist_observed_computer_use_session( + &store, + None, + "Set one exact generated field in an isolated local Edge portal.".to_string(), + ComputerUseUndoCapability::None, + observation, + )?; + let mut expected_identity = initial.clone(); + expected_identity.target_value = expected_value.to_string(); + expected_identity.semantic_receipt = expected_receipt.clone(); + let expected_semantic = require_edge_portal_effect_receipt( + &expected_identity, + &contract, + expected_value, + &expected_receipt, + )?; + let bound = bind_computer_use_action( + &store, + observed.id, + ComputerControlAction::SetAccessibilityValue { + value: expected_value.to_string(), + }, + "Set the exact isolated local-portal field through loopback Edge DevTools DOM." + .to_string(), + ComputerUsePostcondition::TargetSemanticFingerprintEquals { + expected: expected_semantic, + }, + )?; + let approval_id = Uuid::new_v4(); + approve_computer_use_step( + &store, + bound.id, + approval_id, + &bound + .action + .as_ref() + .ok_or_else(|| "Edge portal smoke action is missing".to_string())? + .action_fingerprint, + ComputerUseApprovalActor::User, + )?; + let control = EdgePortalReceiptCorroboratingControlClient { + spec: spec.clone(), + contract: contract.clone(), + expected_before: before_value.to_string(), + expected_value: expected_value.to_string(), + expected_receipt: expected_receipt.clone(), + }; + let run = execute_ready_computer_use_step( + &store, + bound.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshot, + &accessibility, + &control, + )?; + if run.step.status != ComputerUseStepStatus::Verified + || run.step.action_start_count != 1 + { + return Err(format!( + "Edge portal smoke ended in {:?} with {} action starts: {}", + run.step.status, + run.step.action_start_count, + run.safe_error + .as_deref() + .unwrap_or("no exact semantic receipt was returned") + )); + } + let final_identity = capture_edge_portal_live_identity(&spec)?; + require_edge_portal_effect_receipt( + &final_identity, + &contract, + expected_value, + &expected_receipt, + )?; + Ok(()) + })(); + + if let Some((window_handle, _)) = bound_window { + close_exact_edge_window(window_handle); + } + for _ in 0..20 { + if edge_profile_process_ids(&profile).is_empty() { + break; + } + thread::sleep(Duration::from_millis(250)); + } + let remaining = edge_profile_process_ids(&profile); + for process_id in remaining { + let _ = Command::new("taskkill") + .args(["/PID", &process_id.to_string(), "/T", "/F"]) + .status(); + } + let _ = edge.kill(); + let _ = edge.wait(); + server_stop.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(address); + let _ = server.join(); + assert!( + edge_profile_process_ids(&profile).is_empty(), + "isolated Edge profile processes must be gone" + ); + result.expect("isolated Edge local-portal action verifies"); + } + #[cfg(windows)] fn select_file_in_exact_explorer_window( directory: &std::path::Path, From 89a39e7d32817c80c950969b0152b19cc543667f Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 03:03:40 +0800 Subject: [PATCH 4/5] Add C5D reliability gates --- .../src/kernel/computer_use_runtime.rs | 725 +++++++++++++++++- 1 file changed, 720 insertions(+), 5 deletions(-) diff --git a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs index bc25252..990300f 100644 --- a/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs +++ b/apps/desktop/src-tauri/src/kernel/computer_use_runtime.rs @@ -1556,7 +1556,7 @@ mod tests { struct FakeControlClient { calls: AtomicUsize, - fail: bool, + failure: Option<&'static str>, } struct InMemoryStepPersistence { @@ -1598,14 +1598,21 @@ mod tests { fn succeeding() -> Self { Self { calls: AtomicUsize::new(0), - fail: false, + failure: None, } } fn failing() -> Self { Self { calls: AtomicUsize::new(0), - fail: true, + failure: Some("fake input backend failed"), + } + } + + fn timing_out() -> Self { + Self { + calls: AtomicUsize::new(0), + failure: Some("deterministic control timeout"), } } } @@ -1617,8 +1624,8 @@ mod tests { action: &ComputerControlAction, ) -> Result { self.calls.fetch_add(1, Ordering::SeqCst); - if self.fail { - Err("fake input backend failed".to_string()) + if let Some(error) = self.failure { + Err(error.to_string()) } else { Ok(ComputerControlExecution { summary: action.audit_summary(), @@ -1734,6 +1741,31 @@ mod tests { (ready, approval_id) } + fn persist_action_started_for_test( + store: &EventStore, + step_id: Uuid, + approval_id: Uuid, + ) -> ComputerUseStep { + let mut step = store.get_computer_use_step(step_id).unwrap(); + let expected_revision = step.revision; + let action = step.action.clone().expect("ready step has exact action"); + step.mark_action_started( + approval_id, + &action.application_fingerprint, + &action.process_fingerprint, + &action.window_fingerprint, + &action.pre_window_title_fingerprint, + &action.frame_fingerprint, + &action.target_fingerprint, + Utc::now(), + ) + .unwrap(); + store + .update_computer_use_step(&step, expected_revision) + .unwrap(); + step + } + #[test] fn observation_retry_is_bounded_and_never_retries_an_action() { let screenshots = FakeScreenshotClient::new(COMPUTER_USE_OBSERVATION_MAX_ATTEMPTS); @@ -2364,6 +2396,313 @@ mod tests { assert_eq!(control.calls.load(Ordering::SeqCst), 1); } + #[test] + fn c5d_control_timeout_screenshot_failure_and_window_closure_never_replay() { + { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state("window", "target", Some("before")), + redacted_state("window", "target", Some("before")), + ]); + let control = FakeControlClient::timing_out(); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!(result.step.status, ComputerUseStepStatus::EffectUnknown); + assert_eq!(result.step.action_start_count, 1); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + assert!(result + .safe_error + .as_deref() + .is_some_and(|error| error.contains("control timeout"))); + assert!(execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .is_err()); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + } + + { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![ + redacted_state("window", "target", Some("before")), + redacted_state("window", "target", Some("before")), + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!(result.step.status, ComputerUseStepStatus::EffectUnknown); + assert_eq!(result.step.action_start_count, 1); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + assert!(execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .is_err()); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + } + + { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(3); + let accessibility = FakeAccessibilityClient::with_results(vec![ + Ok(redacted_state("window", "target", Some("before"))), + Ok(redacted_state("window", "target", Some("before"))), + Err("bound process/window closed after ActionStarted".to_string()), + Err("bound process/window closed after ActionStarted".to_string()), + ]); + let control = FakeControlClient::succeeding(); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + + let result = execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .unwrap(); + + assert_eq!(result.step.status, ComputerUseStepStatus::EffectUnknown); + assert_eq!(result.step.action_start_count, 1); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + assert!(execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &screenshots, + &accessibility, + &control, + ) + .is_err()); + assert_eq!(control.calls.load(Ordering::SeqCst), 1); + } + } + + #[test] + fn c5d_takeover_and_restart_after_action_started_are_inspected_without_replay() { + { + let store = EventStore::open_memory().unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![redacted_state( + "window", + "target", + Some("before"), + )]); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + persist_action_started_for_test(&store, ready.id, approval_id); + let taken_over = take_over_computer_use_step( + &store, + ready.id, + "User took over after durable ActionStarted.".to_string(), + ) + .unwrap(); + assert_eq!(taken_over.status, ComputerUseStepStatus::UserTakenOver); + assert_eq!(taken_over.action_start_count, 1); + let control = FakeControlClient::succeeding(); + assert!(execute_ready_computer_use_step( + &store, + ready.id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &FakeScreenshotClient::new(0), + &FakeAccessibilityClient::new(Vec::new()), + &control, + ) + .is_err()); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + + { + let directory = tempdir().unwrap(); + let path = directory.path().join("c5d-restart.db"); + let (step_id, approval_id) = { + let store = EventStore::open(&path).unwrap(); + let screenshots = FakeScreenshotClient::new(1); + let accessibility = FakeAccessibilityClient::new(vec![redacted_state( + "window", + "target", + Some("before"), + )]); + let (ready, approval_id) = + setup_ready_step(&store, &screenshots, &accessibility, "after"); + persist_action_started_for_test(&store, ready.id, approval_id); + (ready.id, approval_id) + }; + + let reopened = EventStore::open(&path).unwrap(); + let sweep = reopened + .recover_computer_use_steps_after_restart(Utc::now()) + .unwrap(); + assert_eq!(sweep.effect_unknown, 1); + let recovered = reopened.get_computer_use_step(step_id).unwrap(); + assert_eq!(recovered.status, ComputerUseStepStatus::EffectUnknown); + assert_eq!(recovered.action_start_count, 1); + let second = reopened + .recover_computer_use_steps_after_restart(Utc::now()) + .unwrap(); + assert_eq!(second.effect_unknown, 0); + let control = FakeControlClient::succeeding(); + assert!(execute_ready_computer_use_step( + &reopened, + step_id, + ComputerUseExecutionPermit { + approval_request_id: approval_id, + local_unlock_confirmed: true, + }, + &FakeScreenshotClient::new(0), + &FakeAccessibilityClient::new(Vec::new()), + &control, + ) + .is_err()); + assert_eq!(control.calls.load(Ordering::SeqCst), 0); + } + } + + #[cfg(windows)] + #[test] + fn c5d_deterministic_fault_injection_matrix_is_fail_closed() { + let matrix: [(&str, usize, fn()); 15] = [ + ( + "bounded observation retry and exhaustion", + 2, + observation_retry_is_bounded_and_never_retries_an_action, + ), + ( + "stale application process HWND and frame", + 4, + wrong_application_process_window_or_frame_stops_before_input, + ), + ( + "File Explorer folder and target drift", + 2, + file_explorer_folder_or_file_drift_stops_before_semantic_selection, + ), + ( + "Excel workbook sheet and cell drift", + 3, + excel_workbook_sheet_or_cell_drift_stops_before_semantic_write, + ), + ( + "Edge profile HWND PID target frame tab URL origin document action receipt and decoy drift", + 16, + edge_portal_identity_rejects_profile_tab_url_origin_document_target_and_action_drift, + ), + ( + "Edge DOM mutation without exact semantic receipt", + 1, + edge_portal_semantic_receipt_blocks_dom_or_screenshot_only_false_completion, + ), + ( + "expired approved observation", + 1, + stale_approved_observation_requires_replan_before_revalidation, + ), + ( + "approved action mutation", + 1, + action_mutation_after_approval_is_rejected_before_input, + ), + ( + "ambiguous control failure", + 1, + input_backend_failure_becomes_effect_unknown_and_is_not_replayed, + ), + ( + "control timeout screenshot failure and process window closure", + 3, + c5d_control_timeout_screenshot_failure_and_window_closure_never_replay, + ), + ( + "screenshot without semantic post receipt", + 1, + screenshot_only_post_state_stays_awaiting_verification, + ), + ( + "Excel write without exact cell receipt", + 1, + excel_semantic_write_without_a_cell_receipt_cannot_complete, + ), + ( + "corrupt semantic post receipt", + 1, + deterministic_postcondition_failure_is_distinct_from_action_failure, + ), + ( + "post-action target mutation", + 1, + post_action_target_change_becomes_effect_unknown_without_replay, + ), + ( + "takeover and restart after ActionStarted with recovery inspection", + 2, + c5d_takeover_and_restart_after_action_started_are_inspected_without_replay, + ), + ]; + let mut completed_cases = 0usize; + for (label, denominator, run) in matrix { + run(); + completed_cases += denominator; + eprintln!("C5D deterministic fault matrix passed: {label} ({denominator})"); + } + assert_eq!(completed_cases, 40); + } + #[test] fn screenshot_only_post_state_stays_awaiting_verification() { let directory = tempdir().unwrap(); @@ -2655,6 +2994,180 @@ mod tests { Ok(directory) } + #[cfg(windows)] + fn apply_c5d_window_change( + window_handle: isize, + expected_process_id: u32, + application: &str, + ) -> Result<(), String> { + const ENABLE_ENV: &str = "DEEPSEEK_AGENT_OS_C5D_WINDOW_CHANGE"; + const EVIDENCE_ENV: &str = "DEEPSEEK_AGENT_OS_C5D_WINDOW_CHANGE_EVIDENCE"; + const RUN_ID_ENV: &str = "DEEPSEEK_AGENT_OS_C5D_RUN_ID"; + + if std::env::var(ENABLE_ENV).ok().as_deref() != Some("1") { + return Ok(()); + } + let run_id = std::env::var(RUN_ID_ENV) + .map_err(|_| format!("{RUN_ID_ENV} is required for C5D window-change evidence"))?; + if run_id.is_empty() + || !run_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(format!("{RUN_ID_ENV} must be a safe non-empty label")); + } + let evidence_path = std::env::var_os(EVIDENCE_ENV) + .filter(|value| !value.is_empty()) + .map(std::path::PathBuf::from) + .ok_or_else(|| format!("{EVIDENCE_ENV} is required for C5D window-change evidence"))?; + if !evidence_path.is_absolute() || evidence_path.exists() { + return Err(format!( + "{EVIDENCE_ENV} must name a fresh absolute evidence file" + )); + } + let authorized_roots = [ + std::env::var_os("DEEPSEEK_AGENT_OS_C5B_SMOKE_ROOT"), + std::env::var_os("DEEPSEEK_AGENT_OS_C5C_SMOKE_ROOT"), + ] + .into_iter() + .flatten() + .filter(|value| !value.is_empty()) + .map(std::path::PathBuf::from) + .collect::>(); + if authorized_roots.is_empty() + || !authorized_roots + .iter() + .any(|root| root.is_absolute() && evidence_path.starts_with(root)) + { + return Err(format!( + "{EVIDENCE_ENV} must stay inside the active isolated C5B/C5C smoke root" + )); + } + let parent = evidence_path + .parent() + .ok_or_else(|| "C5D window-change evidence file has no parent".to_string())?; + std::fs::create_dir_all(parent) + .map_err(|error| format!("C5D evidence directory creation failed: {error}"))?; + + use std::thread; + use std::time::Duration; + use windows::Win32::Foundation::{HWND, RECT}; + use windows::Win32::UI::WindowsAndMessaging::{ + GetForegroundWindow, GetWindowRect, SetWindowPos, ShowWindow, SWP_NOACTIVATE, + SWP_NOOWNERZORDER, SWP_NOZORDER, SW_MINIMIZE, SW_RESTORE, + }; + + let hwnd = HWND(window_handle as _); + let before_process_id = windows_process_id_for_handle(window_handle)?; + if before_process_id != expected_process_id { + return Err( + "C5D bound HWND did not belong to the expected process before window change" + .to_string(), + ); + } + let mut before = RECT::default(); + unsafe { GetWindowRect(hwnd, &mut before) } + .map_err(|error| format!("C5D could not read the initial window rectangle: {error}"))?; + + let _ = unsafe { ShowWindow(hwnd, SW_MINIMIZE) }; + thread::sleep(Duration::from_millis(250)); + let foreground_while_minimized = unsafe { GetForegroundWindow() }; + let focus_loss_observed = foreground_while_minimized != hwnd; + if !focus_loss_observed { + return Err( + "C5D could not observe focus loss while the exact window was minimized".to_string(), + ); + } + + let _ = unsafe { ShowWindow(hwnd, SW_RESTORE) }; + thread::sleep(Duration::from_millis(300)); + let mut restored = RECT::default(); + unsafe { GetWindowRect(hwnd, &mut restored) }.map_err(|error| { + format!("C5D could not read the restored window rectangle: {error}") + })?; + let restored_width = restored.right - restored.left; + let restored_height = restored.bottom - restored.top; + if restored_width <= 0 || restored_height <= 0 { + return Err("C5D restored window rectangle was invalid".to_string()); + } + let changed_width = if restored_width > 800 { + restored_width - 53 + } else { + restored_width + 53 + }; + let changed_height = if restored_height > 600 { + restored_height - 37 + } else { + restored_height + 37 + }; + unsafe { + SetWindowPos( + hwnd, + None, + restored.left + 23, + restored.top + 19, + changed_width, + changed_height, + SWP_NOACTIVATE | SWP_NOOWNERZORDER | SWP_NOZORDER, + ) + } + .map_err(|error| format!("C5D exact window move/resize failed: {error}"))?; + thread::sleep(Duration::from_millis(350)); + + let mut after = RECT::default(); + unsafe { GetWindowRect(hwnd, &mut after) } + .map_err(|error| format!("C5D could not read the changed window rectangle: {error}"))?; + let moved = after.left != restored.left || after.top != restored.top; + let resized = (after.right - after.left) != restored_width + || (after.bottom - after.top) != restored_height; + if !moved || !resized { + return Err(format!( + "C5D exact window change was incomplete: moved={moved}, resized={resized}" + )); + } + let after_process_id = windows_process_id_for_handle(window_handle)?; + if after_process_id != expected_process_id { + return Err( + "C5D bound HWND did not belong to the expected process after window change" + .to_string(), + ); + } + + let evidence = serde_json::json!({ + "schema": "c5d-window-change-evidence/v1", + "run_id": run_id, + "application": application, + "window_handle": window_handle, + "process_id": expected_process_id, + "focus_loss_observed": focus_loss_observed, + "moved": moved, + "resized": resized, + "before": { + "left": before.left, + "top": before.top, + "right": before.right, + "bottom": before.bottom + }, + "restored": { + "left": restored.left, + "top": restored.top, + "right": restored.right, + "bottom": restored.bottom + }, + "after": { + "left": after.left, + "top": after.top, + "right": after.right, + "bottom": after.bottom + } + }); + let bytes = serde_json::to_vec_pretty(&evidence) + .map_err(|error| format!("C5D window evidence serialization failed: {error}"))?; + std::fs::write(&evidence_path, bytes) + .map_err(|error| format!("C5D window evidence write failed: {error}"))?; + Ok(()) + } + #[cfg(windows)] #[derive(Clone, Debug, Eq, PartialEq)] struct EdgePortalLiveIdentity { @@ -3837,6 +4350,7 @@ Get-CimInstance Win32_Process -Filter "Name = 'msedge.exe'" | Where-Object {{ .action_fingerprint, ComputerUseApprovalActor::User, )?; + apply_c5d_window_change(window_handle, browser_process_id, "edge-local-portal")?; let control = EdgePortalReceiptCorroboratingControlClient { spec: spec.clone(), contract: contract.clone(), @@ -4424,6 +4938,7 @@ if ($null -eq $window) {{ throw 'exact File Explorer HWND and LocationURL did no .action_fingerprint, ComputerUseApprovalActor::User, )?; + apply_c5d_window_change(window_handle, process_id, "file-explorer")?; let control = WindowsBoundComputerControlClient::new_file_explorer( window_handle, process_id, @@ -4663,6 +5178,7 @@ try {{ .action_fingerprint, ComputerUseApprovalActor::User, )?; + apply_c5d_window_change(window_handle, process_id, "excel")?; let control = ExcelObjectModelCorroboratingControlClient { inner: WindowsBoundComputerControlClient::new_excel( window_handle, @@ -4728,6 +5244,205 @@ try {{ run_result.expect("isolated Excel cell action verifies"); } + #[cfg(windows)] + #[test] + #[ignore = "requires an authorized interactive desktop with installed File Explorer, Excel, and Edge"] + fn windows_c5d_installed_reliability_matrix_records_thirty_exact_runs() { + use std::process::Command; + + const ROOT_ENV: &str = "DEEPSEEK_AGENT_OS_C5D_RELIABILITY_ROOT"; + const MATRIX_RUNS_PER_APP: usize = 10; + const EXPECTED_RUNS: usize = 30; + const DETERMINISTIC_CASES: usize = 40; + + let root = std::env::var_os(ROOT_ENV) + .filter(|value| !value.is_empty()) + .map(std::path::PathBuf::from) + .expect("C5D reliability root is required"); + assert!(root.is_absolute(), "C5D reliability root must be absolute"); + assert!( + !root.exists(), + "C5D reliability root must be fresh and absent: {}", + root.display() + ); + std::fs::create_dir_all(&root).expect("C5D reliability root is created"); + let report_path = root.join("c5d-reliability-matrix.json"); + let test_binary = std::env::current_exe().expect("current Rust test binary is available"); + + let fault_output = Command::new(&test_binary) + .arg( + "kernel::computer_use_runtime::tests::c5d_deterministic_fault_injection_matrix_is_fail_closed", + ) + .args(["--exact", "--nocapture", "--test-threads=1"]) + .output() + .expect("C5D deterministic fault child test starts"); + std::fs::write( + root.join("deterministic-fault-matrix.stdout.log"), + &fault_output.stdout, + ) + .expect("deterministic fault stdout is recorded"); + std::fs::write( + root.join("deterministic-fault-matrix.stderr.log"), + &fault_output.stderr, + ) + .expect("deterministic fault stderr is recorded"); + if !fault_output.status.success() { + let blocked = serde_json::json!({ + "schema": "c5d-reliability-matrix/v1", + "environment_profile": "current-authorized-interactive-windows-host", + "deterministic_fault_matrix": { + "declared_cases": DETERMINISTIC_CASES, + "passed": false, + "exit_code": fault_output.status.code() + }, + "installed_runs": [], + "summary": { + "attempted_runs": 0, + "completed_runs": 0, + "window_move_resize_attempts": 0, + "window_move_resize_recoveries": 0, + "wrong_target_writes": null, + "false_completions": null + } + }); + std::fs::write( + &report_path, + serde_json::to_vec_pretty(&blocked).expect("blocked report serializes"), + ) + .expect("blocked report is recorded"); + panic!("C5D deterministic fault matrix failed; installed runs were not started"); + } + + let applications = [ + ( + "file-explorer", + "DEEPSEEK_AGENT_OS_C5B_SMOKE_ROOT", + "kernel::computer_use_runtime::tests::windows_file_explorer_isolated_selection_smoke_verifies_exact_file", + ), + ( + "excel", + "DEEPSEEK_AGENT_OS_C5B_SMOKE_ROOT", + "kernel::computer_use_runtime::tests::windows_excel_isolated_cell_value_smoke_verifies_exact_outcome", + ), + ( + "edge-local-portal", + "DEEPSEEK_AGENT_OS_C5C_SMOKE_ROOT", + "kernel::computer_use_runtime::tests::windows_edge_local_portal_isolated_value_smoke_verifies_exact_receipt", + ), + ]; + let mut runs = Vec::with_capacity(EXPECTED_RUNS); + let mut completed_runs = 0usize; + let mut recovered_window_changes = 0usize; + + for iteration in 1..=MATRIX_RUNS_PER_APP { + for (application, smoke_root_env, test_name) in applications { + let run_id = format!("{application}-{iteration:02}"); + let run_root = root.join(&run_id); + std::fs::create_dir(&run_root).expect("fresh C5D run root is created"); + let window_evidence_path = run_root.join("window-change-evidence.json"); + let output = Command::new(&test_binary) + .arg(test_name) + .args(["--exact", "--ignored", "--nocapture", "--test-threads=1"]) + .env_remove("DEEPSEEK_AGENT_OS_C5B_SMOKE_ROOT") + .env_remove("DEEPSEEK_AGENT_OS_C5C_SMOKE_ROOT") + .env(smoke_root_env, &run_root) + .env("DEEPSEEK_AGENT_OS_C5D_WINDOW_CHANGE", "1") + .env("DEEPSEEK_AGENT_OS_C5D_RUN_ID", &run_id) + .env( + "DEEPSEEK_AGENT_OS_C5D_WINDOW_CHANGE_EVIDENCE", + &window_evidence_path, + ) + .output() + .expect("C5D installed child test starts"); + let stdout_path = run_root.join("test.stdout.log"); + let stderr_path = run_root.join("test.stderr.log"); + std::fs::write(&stdout_path, &output.stdout).expect("run stdout is recorded"); + std::fs::write(&stderr_path, &output.stderr).expect("run stderr is recorded"); + + let window_evidence = std::fs::read(&window_evidence_path) + .ok() + .and_then(|bytes| serde_json::from_slice::(&bytes).ok()); + let window_change_passed = window_evidence.as_ref().is_some_and(|evidence| { + evidence.get("schema").and_then(|value| value.as_str()) + == Some("c5d-window-change-evidence/v1") + && evidence.get("run_id").and_then(|value| value.as_str()) + == Some(run_id.as_str()) + && evidence.get("application").and_then(|value| value.as_str()) + == Some(application) + && evidence + .get("focus_loss_observed") + .and_then(|value| value.as_bool()) + == Some(true) + && evidence.get("moved").and_then(|value| value.as_bool()) == Some(true) + && evidence.get("resized").and_then(|value| value.as_bool()) == Some(true) + }); + let passed = output.status.success() && window_change_passed; + if passed { + completed_runs += 1; + recovered_window_changes += 1; + } + runs.push(serde_json::json!({ + "run_id": run_id, + "application": application, + "test_name": test_name, + "passed": passed, + "test_exit_code": output.status.code(), + "window_change_passed": window_change_passed, + "window_change_evidence": window_evidence, + "stdout": stdout_path + .strip_prefix(&root) + .expect("stdout stays in reliability root") + .to_string_lossy(), + "stderr": stderr_path + .strip_prefix(&root) + .expect("stderr stays in reliability root") + .to_string_lossy() + })); + } + } + + let completion_rate_percent = completed_runs * 100 / EXPECTED_RUNS; + let window_recovery_rate_percent = recovered_window_changes * 100 / EXPECTED_RUNS; + let all_exact_runs_passed = completed_runs == EXPECTED_RUNS; + let report = serde_json::json!({ + "schema": "c5d-reliability-matrix/v1", + "environment_profile": "current-authorized-interactive-windows-host", + "environment_profiles_authorized_and_present": 1, + "deterministic_fault_matrix": { + "declared_cases": DETERMINISTIC_CASES, + "passed": true, + "exit_code": fault_output.status.code() + }, + "installed_runs": runs, + "summary": { + "attempted_runs": EXPECTED_RUNS, + "completed_runs": completed_runs, + "completion_rate_percent": completion_rate_percent, + "window_move_resize_attempts": EXPECTED_RUNS, + "window_move_resize_recoveries": recovered_window_changes, + "window_recovery_rate_percent": window_recovery_rate_percent, + "wrong_target_writes": if all_exact_runs_passed { Some(0usize) } else { None }, + "false_completions": if all_exact_runs_passed { Some(0usize) } else { None } + }, + "required_gates": { + "attempted_runs": 30, + "minimum_completion_rate_percent": 85, + "minimum_window_recovery_rate_percent": 95, + "maximum_wrong_target_writes": 0, + "maximum_false_completions": 0 + } + }); + std::fs::write( + &report_path, + serde_json::to_vec_pretty(&report).expect("C5D reliability report serializes"), + ) + .expect("C5D reliability report is recorded"); + + assert_eq!(completed_runs, EXPECTED_RUNS); + assert!(completion_rate_percent >= 85); + assert!(window_recovery_rate_percent >= 95); + } + #[cfg(windows)] #[test] #[ignore = "requires a visible isolated Windows Notepad-like editor session"] From 88079c9b3d12587541c064ec2ea60c810b9f5467 Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 25 Jul 2026 03:51:29 +0800 Subject: [PATCH 5/5] release: prepare DS Agent v1.5.0 --- CODE_SIGNING_POLICY.md | 8 +- PRIVACY.md | 15 ++- README.md | 34 +++---- README.zh-CN.md | 29 +++--- SECURITY.md | 18 +++- apps/desktop/package.json | 2 +- apps/desktop/src-tauri/Cargo.lock | 2 +- apps/desktop/src-tauri/Cargo.toml | 2 +- .../src-tauri/src/kernel/app_update.rs | 6 +- apps/desktop/src-tauri/src/kernel/deepseek.rs | 4 +- apps/desktop/src-tauri/tauri.conf.json | 2 +- apps/desktop/src/App.tsx | 2 +- docs/INSTALLATION.md | 34 +++++-- docs/OPEN_SOURCE_RELEASE.md | 9 +- docs/RELEASE_NOTES_v1.5.0.md | 97 +++++++++++++++++++ package.json | 2 +- scripts/release-source-check.mjs | 84 ++++++++++++---- 17 files changed, 265 insertions(+), 85 deletions(-) create mode 100644 docs/RELEASE_NOTES_v1.5.0.md diff --git a/CODE_SIGNING_POLICY.md b/CODE_SIGNING_POLICY.md index 2cd7364..4584e84 100644 --- a/CODE_SIGNING_POLICY.md +++ b/CODE_SIGNING_POLICY.md @@ -1,10 +1,10 @@ # Code signing policy -Last updated: 2026-07-23 +Last updated: 2026-07-25 ## Current status -DS Agent `v1.4.0` is intentionally published unsigned. Both `ds-agent.exe` and +DS Agent `v1.5.0` is intentionally published unsigned. Both `ds-agent.exe` and the Windows x64 NSIS installer are expected to report Authenticode `NotSigned`. Windows may therefore display `Unknown publisher` or a Microsoft Defender SmartScreen warning. Users should download only over HTTPS from the official @@ -13,7 +13,7 @@ GitHub Release and verify the published SHA-256 before running the installer. The SignPath Foundation application is submitted and approval is pending. No DS Agent binary may be represented as SignPath-signed until the application is approved and a later-version artifact independently verifies as Authenticode -`Valid`. The project will not replace the immutable `v1.1.0`, `v1.2.0`, `v1.3.0`, or `v1.4.0` tag, +`Valid`. The project will not replace the immutable `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, or `v1.5.0` tag, Release, or asset if signing becomes available later. For releases accepted into that program: **Free code signing provided by @@ -57,7 +57,7 @@ them; guessed or placeholder identifiers are forbidden. ## Release verification -For the unsigned `v1.1.0`, `v1.2.0`, `v1.3.0`, and `v1.4.0` exceptions, maintainers verify and disclose the actual +For the unsigned `v1.1.0`, `v1.2.0`, `v1.3.0`, `v1.4.0`, and `v1.5.0` exceptions, maintainers verify and disclose the actual `NotSigned` status of both the application executable and installer. Evidence must bind the exact source commit, file name, product version, byte size, and SHA-256. The installer downloaded back from GitHub must match the reviewed diff --git a/PRIVACY.md b/PRIVACY.md index a0e318f..e7d6298 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,6 +1,6 @@ # Privacy Policy -Last updated: 2026-07-23 +Last updated: 2026-07-25 This policy describes the current published DS Agent desktop application and public project. DS Agent is local-first and does not operate a project cloud @@ -14,7 +14,7 @@ User-selected workspaces hold approved evidence, exports, reports, work packages, screenshots, and other artifacts. This information is not silently synced to a DS Agent-operated server. -The current stable `v1.4.0` accepts one user-supplied DeepSeek API key through +The current stable `v1.5.0` accepts one user-supplied DeepSeek API key through the onboarding screen and stores it in a dedicated Windows DPAPI-protected local vault. A process-environment key remains an explicit compatibility fallback and is never silently copied into that vault. The project does not @@ -31,13 +31,20 @@ coverage counts plus a redacted authorization intent; it does not receive the private capability proposal, a preparation/compiler command, or a local completion writer. -The v1.4.0 T1 Office verification engine keeps source identities, bounded fact +The v1.5.0 package preserves the T1 Office verification engine, which keeps source identities, bounded fact provenance, artifact hashes, render receipts, revision receipts, and continuation checkpoints locally in the selected workspace or application data. Ordinary chat does not yet automatically select or sequence the two T1 tools; their presence does not cause workspace files to be scanned or uploaded merely because the application is open. +The v1.5.0 durable Computer Use lifecycle stores bounded application, process, +window, frame, target, action, approval, screenshot-evidence, and semantic +receipt identities locally. Its installed reliability tests use generated +files, workbooks, data, fresh Edge profiles, an ephemeral loopback portal, and +isolated Event Stores. Those tests do not read stored browser logins, production +accounts, or the user's normal DS Agent AppData. + Uninstalling the application may not delete a user-selected workspace or every application-data file. Review and remove those local locations separately when you no longer want to retain them. @@ -74,7 +81,7 @@ the user or person operating the application: The optional local desktop bridge accepts only loopback addresses and is started and controlled by the user. DS Agent does not install or supervise that service. Production Microsoft and Google account registration and live -mail/calendar writes are disabled in `v1.4.0`; offline connector contracts do +mail/calendar writes are disabled in `v1.5.0`; offline connector contracts do not authorize a production account or external write. ## What can be included in a model request diff --git a/README.md b/README.md index 624fd77..e5f35a5 100644 --- a/README.md +++ b/README.md @@ -11,15 +11,14 @@

- v1.4.0 stable · - Download for Windows · + v1.5.0 stable · + Download for Windows · Apache-2.0

Created and maintained by Lee take.

-DS Agent is a DeepSeek-first local Windows agent for everyday work. Describe the -result you need in chat: it can organize office material, research with sources, +DS Agent is a DeepSeek-first local Windows agent for everyday work. Describe the result you need in chat: it can organize office material, research with sources, work with local files, run durable automations, and control desktop apps when you approve—while keeping steps, evidence, permissions, verification, and recovery visible. @@ -28,9 +27,7 @@ Under the hood, DS Agent is a local Agent Harness optimized for DeepSeek. The Harness gives model-backed work a durable, permissioned, and verifiable execution boundary on the local machine. -DS Agent began from a practical need: more colleagues were using DeepSeek in -daily work, yet it was difficult to find an Agent specifically optimized for -DeepSeek with strong local automation and a trustworthy execution boundary. +DS Agent began from a practical need: more colleagues were using DeepSeek in daily work, yet it was difficult to find an Agent optimized for DeepSeek with a trustworthy local execution boundary.

A real DS Agent v1.0.0 run turning meeting notes into a team execution checklist, with six completed verification steps @@ -132,7 +129,7 @@ does not claim completion from model confidence alone. Local files, browser actions, Office artifacts, and Computer Use are complete only when observable evidence satisfies the task's completion criteria. -In v1.4.0, DeepSeek may propose a bounded `GoalEnvelope`, but only the local +In v1.5.0, DeepSeek may propose a bounded `GoalEnvelope`, but only the local Kernel can validate and freeze it. For the same queued task, the Kernel alone derives the capability manifest, risk, and preview shown in one exact-task authorization card. Approval creates only exact authority; it does not execute @@ -140,8 +137,9 @@ a Tool, resume the task, or mark the Goal complete. Completion remains blocked until locally authoritative verifier evidence covers every frozen `done_when` condition and required artifact identity. -The v1.4.0 binary adds Kernel-authorized T1 Excel reconciliation, PowerPoint/render verification, and persisted goal-continuation checkpoints. These are production execution and verification primitives, but ordinary chat does not yet automatically select or sequence the two T1 tools. This release therefore -does not claim that a user can already run the complete one-sentence T1 Office workflow from the React chat UI. +The v1.5.0 binary preserves the T1 Excel/PowerPoint verification engine and adds a production-reachable durable Computer Use session/step UI and Kernel loop. +Exact installed evidence is limited to generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal; it does not claim general browser automation, stored-login reuse, or production-site completion. +The release remains honest: ordinary chat does not yet automatically select or sequence the two T1 tools or the complete T1 Office workflow. ## DeepSeek and DS Agent boundary @@ -166,7 +164,7 @@ high-risk action. See the full [model boundary](docs/AGENT_MODEL_BOUNDARY.md). reconciliation contracts validated with offline adversarial fake providers. Production Microsoft/Google account registration and live external-write -authority remain disabled in v1.4.0. The release does not sign in to real +authority remain disabled in v1.5.0. The release does not sign in to real accounts, send real email, or create, change, or cancel real calendar events. ## Why Rust @@ -179,7 +177,7 @@ remain thin; the Kernel and persistent projections own business state. ## Quick start -1. Download the [Windows x64 installer](https://github.com/Lee-take/dsagent/releases/download/v1.4.0/DS.Agent_1.4.0_x64-setup.exe). +1. Download the [Windows x64 installer](https://github.com/Lee-take/dsagent/releases/download/v1.5.0/DS.Agent_1.5.0_x64-setup.exe). 2. Enter your own valid DeepSeek API key in onboarding and run the explicit balance/model verification. The key is stored locally with Windows DPAPI. 3. Choose one local workspace and let the readiness doctor verify its managed @@ -191,7 +189,7 @@ A user-supplied DeepSeek API key is a required prerequisite. DS Agent does not bundle a shared key or bypass DeepSeek access requirements; use remains subject to DeepSeek's terms and account policies. -The v1.4.0 application executable and installer are Authenticode `NotSigned`. +The v1.5.0 application executable and installer are Authenticode `NotSigned`. Windows may display `Unknown publisher` or a Microsoft Defender SmartScreen warning. Download only over HTTPS from this repository, verify the SHA-256 in the GitHub Release, and read the [installation guide](docs/INSTALLATION.md) @@ -199,10 +197,10 @@ before running the installer. ## Code signing policy -DS Agent `v1.4.0` is intentionally unsigned. The SignPath Foundation application +DS Agent `v1.5.0` is intentionally unsigned. The SignPath Foundation application is submitted and approval is pending; no release is represented as signed. If the project is accepted, signing starts with a later new version and does not -replace the immutable v1.1.0, v1.2.0, v1.3.0, or v1.4.0 tag or asset. For releases accepted into the program: +replace the immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, or v1.5.0 tag or asset. For releases accepted into the program: **Free code signing provided by SignPath.io, certificate by SignPath Foundation.** See the full [code signing policy](CODE_SIGNING_POLICY.md) and [privacy policy](PRIVACY.md). @@ -220,8 +218,8 @@ example `D:\build-target\ds-agent-v1-release`. ## Stable release -- Release: [DS Agent v1.4.0](https://github.com/Lee-take/dsagent/releases/tag/v1.4.0) -- Installer: `DS.Agent_1.4.0_x64-setup.exe` +- Release: [DS Agent v1.5.0](https://github.com/Lee-take/dsagent/releases/tag/v1.5.0) +- Installer: `DS.Agent_1.5.0_x64-setup.exe` - Integrity: verify the final byte size and SHA-256 published in the GitHub Release before running the installer. - Onboarding: one user-supplied Key, Windows DPAPI storage, explicit DeepSeek @@ -235,6 +233,7 @@ example `D:\build-target\ds-agent-v1-release`. - Task authorization: one Kernel-derived exact-task card with manifest/risk/ preview binding, one user decision, per-capability audit, and exact revocation; approval does not execute or resume the task. +- Computer Use: production-reachable durable lifecycle and fail-closed execution; exact installed proof is limited to generated Explorer/Excel targets and an isolated Edge/loopback test portal, not general browsing. - T1 verification engine: exact source identities and provenance, non-overwriting XLSX/PPTX artifacts, actual local render evidence, bounded revisions, and persisted continuation checkpoints; ordinary chat does not yet automatically select or sequence this complete T1 path. ## Documentation @@ -242,6 +241,7 @@ example `D:\build-target\ds-agent-v1-release`. - [Installation](docs/INSTALLATION.md) - [DS Agent and DeepSeek boundary](docs/AGENT_MODEL_BOUNDARY.md) - [v1 architecture](docs/architecture/DS_AGENT_V1_ARCHITECTURE_PLAN.md) +- [v1.5.0 release notes](docs/RELEASE_NOTES_v1.5.0.md) - [v1.4.0 release notes](docs/RELEASE_NOTES_v1.4.0.md) - [v1.3.0 release notes](docs/RELEASE_NOTES_v1.3.0.md) - [v1.2.0 release notes](docs/RELEASE_NOTES_v1.2.0.md) diff --git a/README.zh-CN.md b/README.zh-CN.md index e34ac74..afdfcf7 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -11,8 +11,8 @@

- v1.4.0 正式稳定版 · - 下载 Windows 安装包 · + v1.5.0 正式稳定版 · + 下载 Windows 安装包 · Apache-2.0

@@ -118,15 +118,15 @@ Kernel 会持久保存任务和审核状态，只对失败步骤进行有边界 宣告完成。本地文件、浏览器操作、Office 产物和 Computer Use，只有在可观察证据满足任务 完成条件后才算真正完成。 -在 v1.4.0 中，DeepSeek 可以提出有界 `GoalEnvelope`，但只有本地 Kernel 能校验并冻结它。 +在 v1.5.0 中，DeepSeek 可以提出有界 `GoalEnvelope`，但只有本地 Kernel 能校验并冻结它。 对于同一个排队任务，也只有 Kernel 能派生 capability manifest、风险和 preview，并显示为 一张 exact-task 授权卡。批准只产生精确权限，不会执行 Tool、恢复任务或把 Goal 标记为完成； 只有本地权威 verifier 证据覆盖冻结目标的全部 `done_when` 和必需产物身份后，任务才可能 进入完成状态。 -v1.4.0 安装包新增了由 Kernel 授权的 T1 Excel 核对、PowerPoint/实际渲染验证，以及持久化的 -Goal continuation checkpoint。这些是生产执行与验证原语，但普通聊天目前还不会自动选择和串联这两个 T1 工具。因此，本版本不宣称用户已经能从 React 聊天界面用一句话自动完成整条 -T1 Office 工作流。 +v1.5.0 安装包保留 T1 Excel/PowerPoint 验证引擎，并新增生产可达的持久 Computer Use +会话/步骤 UI 与 Kernel 闭环。准确安装态证据只覆盖生成的 File Explorer、Excel 目标，以及使用全新 profile 和临时回环测试站点的已安装 Edge；本版本不宣称通用浏览器自动化、复用 +已登录状态或生产网站完成能力。普通聊天目前还不会自动选择和串联这两个 T1 工具，也不会自动完成整条 T1 Office 工作流。 ## DeepSeek 与 DS Agent 的工作边界 @@ -148,7 +148,7 @@ DeepSeek 可以提出动作，DS Agent 决定动作是否安全、是否允许 - 已通过离线对抗性 fake provider 验证的 Microsoft/Google 形态邮件、日历、同步、草稿、 外部变更和对账契约。 -v1.4.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限。当前正式版不会登录 +v1.5.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限。当前正式版不会登录 真实账号、发送真实邮件，也不会创建、修改或取消真实日历事件。 ## 为什么使用 Rust @@ -159,7 +159,7 @@ command 与 React UI 保持薄层，业务状态由 Kernel 和持久投影统一 ## 快速开始 -1. 下载 [Windows x64 安装包](https://github.com/Lee-take/dsagent/releases/download/v1.4.0/DS.Agent_1.4.0_x64-setup.exe)。 +1. 下载 [Windows x64 安装包](https://github.com/Lee-take/dsagent/releases/download/v1.5.0/DS.Agent_1.5.0_x64-setup.exe)。 2. 在首次设置中输入你自己的有效 DeepSeek API Key，并显式验证余额和模型；Key 使用 Windows DPAPI 保存在本机。 3. 选择一个本地工作目录，由 readiness doctor 检查受管目录和可写状态。 @@ -169,16 +169,16 @@ command 与 React UI 保持薄层，业务状态由 Kernel 和持久投影统一 用户自行提供有效的 DeepSeek API Key 是必备前提。DS Agent 不内置共享 Key，也不会 绕过 DeepSeek 的访问条件；实际使用仍须遵守 DeepSeek 的服务条款和账号规则。 -v1.4.0 应用程序和安装包的 Authenticode 状态均为 `NotSigned`。Windows 可能显示 +v1.5.0 应用程序和安装包的 Authenticode 状态均为 `NotSigned`。Windows 可能显示 `Unknown publisher`（未知发布者）或 Microsoft Defender SmartScreen 警告。请只通过 本仓库的 HTTPS 链接下载，运行前核对 GitHub Release 中的 SHA-256，并阅读 [安装指南](docs/INSTALLATION.md)。 ## Code signing policy（代码签名政策） -DS Agent `v1.4.0` 是如实披露的未签名版本。SignPath Foundation 申请已经提交、仍在等待 +DS Agent `v1.5.0` 是如实披露的未签名版本。SignPath Foundation 申请已经提交、仍在等待 审批，本版本不会被描述为已签名。若以后获批，只从后续新版本开始签名，不替换本版本不可 -移动的 tag 或资产，也不改写 v1.1.0、v1.2.0 或 v1.3.0。获准加入该计划的 Release 将遵循：**Free code signing provided by +移动的 tag 或资产，也不改写 v1.1.0、v1.2.0、v1.3.0 或 v1.4.0。获准加入该计划的 Release 将遵循：**Free code signing provided by SignPath.io, certificate by SignPath Foundation.** 完整说明见 [代码签名政策](CODE_SIGNING_POLICY.md)和[隐私政策](PRIVACY.md)。 @@ -195,8 +195,8 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev ## 正式稳定版 -- Release：[DS Agent v1.4.0](https://github.com/Lee-take/dsagent/releases/tag/v1.4.0) -- 安装包：`DS.Agent_1.4.0_x64-setup.exe` +- Release：[DS Agent v1.5.0](https://github.com/Lee-take/dsagent/releases/tag/v1.5.0) +- 安装包：`DS.Agent_1.5.0_x64-setup.exe` - 完整性：运行安装包前，核对 GitHub Release 中发布的最终字节数和 SHA-256。 - 首次设置：单一用户 Key、Windows DPAPI 本机存储、显式 DeepSeek 余额/V4 模型验证、 无密 readiness 和 workspace doctor。 @@ -206,6 +206,8 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev 必需产物身份的 fail-closed 证据门。 - 任务授权：一张由 Kernel 派生并绑定 manifest/risk/preview 的 exact-task 卡片、一次用户 决策、逐能力审计和精确撤销；批准不会执行或恢复任务。 +- Computer Use：生产可达的持久生命周期与 fail-closed 执行；准确安装态证明仅覆盖生成的 + Explorer/Excel 目标和隔离的 Edge/回环测试站点，不代表通用浏览能力。 - T1 验证引擎：准确来源身份和 provenance、禁止覆盖的 XLSX/PPTX、真实本地渲染证据、 有界修订和持久 continuation checkpoint；普通聊天尚不会自动选择和串联整条 T1 路径。 @@ -214,6 +216,7 @@ npx pnpm@9.15.9 --filter @deepseek-agent-os/desktop tauri:dev - [安装指南](docs/INSTALLATION.md) - [DS Agent 与 DeepSeek 的工作边界](docs/AGENT_MODEL_BOUNDARY.md) - [v1 架构计划](docs/architecture/DS_AGENT_V1_ARCHITECTURE_PLAN.md) +- [v1.5.0 发布说明](docs/RELEASE_NOTES_v1.5.0.md) - [v1.4.0 发布说明](docs/RELEASE_NOTES_v1.4.0.md) - [v1.3.0 发布说明](docs/RELEASE_NOTES_v1.3.0.md) - [v1.2.0 发布说明](docs/RELEASE_NOTES_v1.2.0.md) diff --git a/SECURITY.md b/SECURITY.md index 688e76b..127a28b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,6 +1,6 @@ # Security Policy -DS Agent is a local-first Windows desktop agent. DS Agent v1.4.0 is the current +DS Agent is a local-first Windows desktop agent. DS Agent v1.5.0 is the current published stable release and is not an official DeepSeek product. Security reports are welcome, especially around local credentials, permission gates, audit records, Computer Use boundaries, update integrity, code signing, and @@ -10,6 +10,7 @@ package import/export behavior. | Version | Supported | | --- | --- | +| 1.5.0 | Supported | | 1.4.0 | Supported | | 1.3.0 | Supported | | 1.2.0 | Supported | @@ -33,7 +34,7 @@ Include: ## Security Boundaries -- Current stable v1.4.0 stores one user-supplied DeepSeek API key in a dedicated +- Current stable v1.5.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault. A process-environment key is an explicit compatibility fallback and is not copied into the vault. Presence alone is never treated as verified readiness; balance and required V4 model checks produce only a @@ -51,11 +52,18 @@ Include: group, projection, manifest, preview, revision, fingerprint, hash, scopes, targets, and expiry. Approval grants only exact authority; it does not execute a Tool, resume a task, or mark a Goal complete. -- The v1.4.0 T1 reconciliation and PowerPoint executors require exact workspace - paths, non-overwriting output, grouped authorization, persisted artifact +- The v1.5.0 package preserves the T1 reconciliation and PowerPoint executors, + which require exact workspace paths, non-overwriting output, grouped authorization, persisted artifact identity, verifier evidence, and actual local render evidence. DeepSeek cannot approve those actions or mint their evidence or completion receipts. The ordinary chat UI does not yet automatically select or sequence these T1 tools. +- The v1.5.0 durable Computer Use lifecycle binds the application, process, + window, frame, target, action, approval, screenshots, and semantic receipt. + `ActionStarted` is persisted once before the effect; ambiguous outcomes fail + closed and are not replayed. Installed reliability evidence covers only + generated File Explorer and Excel targets plus installed Edge with a fresh + profile and an ephemeral loopback test portal. It is not evidence of general + browser control, stored-login reuse, or production-site completion. - `pnpm test:secrets` scans tracked and unignored repository files for live `sk-` style keys and non-empty `DEEPSEEK_API_KEY` assignments without printing candidate values. @@ -70,7 +78,7 @@ Include: memory. - Release identity must follow the [code signing policy](CODE_SIGNING_POLICY.md). An unsigned or invalidly signed artifact must not be represented as a signed - release. DS Agent v1.4.0 is explicitly disclosed as Authenticode `NotSigned`; + release. DS Agent v1.5.0 is explicitly disclosed as Authenticode `NotSigned`; Windows may show `Unknown publisher` or a Microsoft Defender SmartScreen warning. See also the [privacy policy](PRIVACY.md). diff --git a/apps/desktop/package.json b/apps/desktop/package.json index d82a16d..317cab8 100644 --- a/apps/desktop/package.json +++ b/apps/desktop/package.json @@ -1,7 +1,7 @@ { "name": "@deepseek-agent-os/desktop", "private": true, - "version": "1.4.0", + "version": "1.5.0", "author": "Lee take", "license": "Apache-2.0", "type": "module", diff --git a/apps/desktop/src-tauri/Cargo.lock b/apps/desktop/src-tauri/Cargo.lock index 03aadaf..a237687 100644 --- a/apps/desktop/src-tauri/Cargo.lock +++ b/apps/desktop/src-tauri/Cargo.lock @@ -794,7 +794,7 @@ dependencies = [ [[package]] name = "deepseek-agent-os-desktop" -version = "1.4.0" +version = "1.5.0" dependencies = [ "base64 0.22.1", "chrono", diff --git a/apps/desktop/src-tauri/Cargo.toml b/apps/desktop/src-tauri/Cargo.toml index d7d193d..e228557 100644 --- a/apps/desktop/src-tauri/Cargo.toml +++ b/apps/desktop/src-tauri/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "deepseek-agent-os-desktop" -version = "1.4.0" +version = "1.5.0" description = "Local-first DeepSeek AI work platform" edition = "2021" license = "Apache-2.0" diff --git a/apps/desktop/src-tauri/src/kernel/app_update.rs b/apps/desktop/src-tauri/src/kernel/app_update.rs index 04b4f3f..7dd3f54 100644 --- a/apps/desktop/src-tauri/src/kernel/app_update.rs +++ b/apps/desktop/src-tauri/src/kernel/app_update.rs @@ -17,8 +17,8 @@ pub(crate) const APP_UPDATE_RELEASES_API_URL: &str = const APP_UPDATE_RELEASE_DOWNLOAD_PATH_PREFIX: &str = "/Lee-take/dsagent/releases/download/"; const APP_UPDATE_LEGACY_RELEASE_DOWNLOAD_PATH_PREFIX: &str = "/Lee-take/deepseek-agent-os/releases/download/"; -const APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.4.0"; -const APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.4.0"; +const APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.5.0"; +const APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.5.0"; #[cfg(windows)] const WINDOWS_CREATE_NO_WINDOW: u32 = 0x08000000; @@ -653,7 +653,7 @@ mod tests { let status = update_status_from_releases(releases, app_update_current_version()); assert!(!status.update_available); - assert_eq!(status.current_version, "v1.4.0"); + assert_eq!(status.current_version, "v1.5.0"); assert_eq!(status.latest_version.as_deref(), Some("0.3.0")); assert!(status.asset_name.is_none()); } diff --git a/apps/desktop/src-tauri/src/kernel/deepseek.rs b/apps/desktop/src-tauri/src/kernel/deepseek.rs index 4f78e08..cec08b0 100644 --- a/apps/desktop/src-tauri/src/kernel/deepseek.rs +++ b/apps/desktop/src-tauri/src/kernel/deepseek.rs @@ -265,7 +265,7 @@ impl Drop for DeepSeekOperationsBriefingSynt impl HttpDeepSeekChatCompletionTransport { pub fn new() -> Result { let client = reqwest::blocking::Client::builder() - .user_agent("DS-Agent/1.4.0 deepseek-v4") + .user_agent("DS-Agent/1.5.0 deepseek-v4") .timeout(std::time::Duration::from_secs( DEEPSEEK_CHAT_HTTP_TIMEOUT_SECS, )) @@ -982,7 +982,7 @@ mod tests { assert_eq!(response.first_text(), Some("ok")); assert!(recorded.raw.starts_with("POST / HTTP/1.1")); - assert!(normalized_headers.contains("user-agent: ds-agent/1.4.0 deepseek-v4")); + assert!(normalized_headers.contains("user-agent: ds-agent/1.5.0 deepseek-v4")); } #[test] diff --git a/apps/desktop/src-tauri/tauri.conf.json b/apps/desktop/src-tauri/tauri.conf.json index 540edb6..6f98112 100644 --- a/apps/desktop/src-tauri/tauri.conf.json +++ b/apps/desktop/src-tauri/tauri.conf.json @@ -2,7 +2,7 @@ "$schema": "https://schema.tauri.app/config/2", "productName": "DS Agent", "mainBinaryName": "ds-agent", - "version": "1.4.0", + "version": "1.5.0", "identifier": "ai.deepseek-agent-os.desktop", "build": { "beforeDevCommand": "npx pnpm@9.15.9 dev", diff --git a/apps/desktop/src/App.tsx b/apps/desktop/src/App.tsx index 135b467..14e1b37 100644 --- a/apps/desktop/src/App.tsx +++ b/apps/desktop/src/App.tsx @@ -224,7 +224,7 @@ const fallbackOnboardingReadiness: OnboardingReadinessProjection = { message_key: "onboarding.workspace.workspace_missing", }, version: { - current_version: "1.4.0", + current_version: "1.5.0", status: "current", blocking: false, message_key: "onboarding.version.current", diff --git a/docs/INSTALLATION.md b/docs/INSTALLATION.md index 03ff386..5e58567 100644 --- a/docs/INSTALLATION.md +++ b/docs/INSTALLATION.md @@ -9,14 +9,14 @@ project does not depend on the maintainer's local directories. The published Windows release provides a normal NSIS setup executable: ```text -DS.Agent_1.4.0_x64-setup.exe +DS.Agent_1.5.0_x64-setup.exe ``` -Version `1.4.0` is the current published stable release. Earlier commits, tags, +Version `1.5.0` is the current published stable release. Earlier commits, tags, Releases, and assets remain immutable. Both `ds-agent.exe` and the installer are Authenticode `NotSigned`, so Windows may show `Unknown publisher` or a Microsoft Defender SmartScreen warning. Download only over HTTPS from the official GitHub -Release and verify its byte size and SHA-256 against the `v1.4.0` Release before +Release and verify its byte size and SHA-256 against the `v1.5.0` Release before running it. The installer embeds the Microsoft WebView2 bootstrapper and runs it silently when the target machine needs the WebView2 runtime; users do not need Node.js, pnpm, Rust, or a @@ -27,12 +27,12 @@ not the workspace, evidence folder, export folder, or event database location. ## Code signing policy and verification -DS Agent `v1.4.0` is intentionally unsigned. Its HTTPS source, immutable tag, +DS Agent `v1.5.0` is intentionally unsigned. Its HTTPS source, immutable tag, exact byte size, and SHA-256 are the verification route for this asset. The SignPath Foundation application is submitted and approval is pending; no -publisher, certificate, or signed status is claimed for v1.4.0. If signing is +publisher, certificate, or signed status is claimed for v1.5.0. If signing is approved later, it starts with a new version and does not replace this tag or -asset or any immutable v1.1.0, v1.2.0, or v1.3.0 publication. +asset or any immutable v1.1.0, v1.2.0, v1.3.0, or v1.4.0 publication. For releases accepted into the open-source signing program: **Free code signing provided by SignPath.io, certificate by SignPath Foundation.** The complete @@ -43,12 +43,12 @@ describes current local data and user-triggered network behavior. On Windows, inspect a downloaded installer without launching it: ```powershell -Get-AuthenticodeSignature .\DS.Agent_1.4.0_x64-setup.exe | +Get-AuthenticodeSignature .\DS.Agent_1.5.0_x64-setup.exe | Select-Object Status, StatusMessage, SignerCertificate, TimeStamperCertificate -Get-FileHash .\DS.Agent_1.4.0_x64-setup.exe -Algorithm SHA256 +Get-FileHash .\DS.Agent_1.5.0_x64-setup.exe -Algorithm SHA256 ``` -For `v1.4.0`, `Status` is expected to be `NotSigned`. An unexpected signer, +For `v1.5.0`, `Status` is expected to be `NotSigned`. An unexpected signer, signature identity, hash, version, source, or asset mismatch is a stop condition. For a future release explicitly documented as signed, any status other than `Valid` is also a stop condition. @@ -84,12 +84,19 @@ reject, and revoke operate only on the Kernel-issued task/group/manifest/preview intent. Approval grants exact authority only; it does not execute a Tool, resume the task, or mark the Goal complete. -The v1.4.0 package contains Kernel-authorized T1 Excel reconciliation and +The v1.5.0 package preserves Kernel-authorized T1 Excel reconciliation and PowerPoint/render executors plus persisted continuation checkpoints. The ordinary chat UI does not yet automatically select or sequence these two T1 tools, so this release is not represented as a one-sentence Office workflow. The existing general Office create/open/update actions remain separate. +The package also exposes a production durable Computer Use session/step +lifecycle through the right-rail UI and Kernel. Its exact installed reliability +evidence covers generated File Explorer and Excel targets plus installed Edge +using a fresh profile and an ephemeral loopback test portal. That evidence does +not authorize or promise arbitrary websites, stored browser logins, production +portals, secure/UAC desktops, CAPTCHA, payment, or administrator authentication. + Good first tasks to try: - `根据我的证据文件夹，生成一份经营简报，并导出 HTML 和 PDF。` @@ -290,6 +297,13 @@ Web search depends on the selected model route: Screen inspection and computer control are permissioned Computer Use features. +- Durable steps bind the exact application, process, window, frame, target, + action, approval, screenshots, and semantic receipt. `ActionStarted` is + persisted once before an effect; ambiguous results fail closed and are not + replayed automatically. +- The v1.5.0 File Explorer, Excel, and Edge/loopback cases are installed-test + evidence for generated isolated targets, not a general browser or arbitrary + desktop-application support promise. - Optional local bridge routes use the configured local bridge service when a local loopback HTTP bridge is available. - DeepSeek and custom routes use local Windows or macOS screen and input paths. diff --git a/docs/OPEN_SOURCE_RELEASE.md b/docs/OPEN_SOURCE_RELEASE.md index 7b6c335..7725fe0 100644 --- a/docs/OPEN_SOURCE_RELEASE.md +++ b/docs/OPEN_SOURCE_RELEASE.md @@ -69,7 +69,7 @@ Ship a buildable local-first desktop Agent OS preview that demonstrates: fresh, distinct `CARGO_TARGET_DIR` builds with identical application and installer byte sizes and SHA-256 values. - The immutable `v1.0.2` installer remains unsigned. `v1.1.0`, `v1.2.0`, - `v1.3.0`, and `v1.4.0` are explicitly disclosed unsigned exceptions: both the application executable and + `v1.3.0`, `v1.4.0`, and `v1.5.0` are explicitly disclosed unsigned exceptions: both the application executable and NSIS installer must read back as `NotSigned`, and each Release must warn about `Unknown publisher` and Microsoft Defender SmartScreen while binding the HTTPS asset to its exact source, version, byte size, and SHA-256. A later @@ -119,7 +119,8 @@ Ship a buildable local-first desktop Agent OS preview that demonstrates: exports or packaged assets do not enter generated source archives. - `.env.example` documents local DeepSeek and optional local bridge environment variables without storing secret values. -- `docs/RELEASE_NOTES_v1.4.0.md` is the current stable release note. +- `docs/RELEASE_NOTES_v1.5.0.md` is the current stable release note. + `docs/RELEASE_NOTES_v1.4.0.md` preserves the immutable Step 4 release evidence. `docs/RELEASE_NOTES_v1.3.0.md` preserves the immutable Step 3 release evidence. `docs/RELEASE_NOTES_v1.2.0.md` preserves the immutable Step 2 release evidence. `docs/RELEASE_NOTES_v1.1.0.md`, `docs/RELEASE_NOTES_v1.0.2.md`, `docs/RELEASE_NOTES_v1.0.1.md`, @@ -166,6 +167,10 @@ testing, signing, and user support. approval and audit records only. - Do not claim the T1 Office engine as an ordinary-chat one-sentence workflow until the product UI can actually select, sequence, and complete it. +- Describe the v1.5.0 production-reachable durable Computer Use lifecycle + separately from its exact installed-test evidence. File Explorer, Excel, and + Edge/ephemeral-loopback cases do not prove general browser or production-site + automation. - Do not hide high-risk Computer Use limitations. - Do not add broad feature work outside the existing DeepSeek-first workflows, permissions, memory, Windows setup behavior, and Operations Briefing scope. diff --git a/docs/RELEASE_NOTES_v1.5.0.md b/docs/RELEASE_NOTES_v1.5.0.md new file mode 100644 index 0000000..9d586cb --- /dev/null +++ b/docs/RELEASE_NOTES_v1.5.0.md @@ -0,0 +1,97 @@ +# DS Agent v1.5.0 + +`v1.5.0` is a backward-compatible minor release that packages the Step 5 +durable, verified Computer Use golden slice. Package, desktop, Tauri, Cargo, +updater, and installer metadata are `1.5.0` / `v1.5.0`. + +## Production-reachable durable Computer Use + +- The installed app exposes one Kernel-owned Computer Use session and step + lifecycle through the existing right-rail UI and Tauri commands. A user can + start from a recent successful screenshot, bind one exact action and + postcondition, approve it through the existing capability policy, re-observe, + run once, take over, cancel, or re-observe after a safe stop. +- Every action binds the exact application, process, window, title, frame, + target, semantic state, approval request, action fingerprint, screenshot + evidence, and postcondition. The Kernel revalidates those identities + immediately before execution. +- `ActionStarted` is persisted before the external effect and can be recorded + only once. A control error, timeout, screenshot failure, process or window + closure, restart, takeover, corrupt evidence, or ambiguous post-action state + becomes fail-closed `EffectUnknown` or recovery inspection and is never + replayed automatically. +- A screenshot, click, navigation, or mutation cannot prove completion by + itself. Verification requires the exact post-action semantic receipt bound to + the same action revision. DeepSeek remains advisory and cannot approve an + action or mint authority, evidence, verification, or completion. + +## Exact installed-test boundary + +The Step 5 reliability evidence is deliberately narrower than general desktop +or browser automation. It covers generated, isolated targets only: + +- one exact generated File Explorer folder and item; +- one exact generated Excel workbook, worksheet, cell, value, and sentinels; +- one installed Edge window using a fresh profile and one test-declared, + ephemeral loopback portal with an exact tab, URL, origin, document, field, + decoy, action, and semantic receipt. + +The Edge DOM helper is compiled only for Windows tests. This release does not +claim arbitrary website control, stored-login reuse, a production portal or +tenant, or general browser completion. The named File Explorer, Excel, and +Edge/local-portal cases are exact installed-test evidence; they are not a +promise that every application, workbook, website, or desktop layout is +supported. + +The final Step 5 matrix records 30 installed application runs: 10 File Explorer, +10 Excel, and 10 Edge/local-portal. All 30 complete, all 30 recover after the +declared window move/resize/focus change, wrong-target writes are zero, and +false completions are zero. A separate deterministic matrix covers 40 fault +cases, including stale identities, redirects, target/action drift, control and +screenshot failures, missing or corrupt receipts, closure, restart, takeover, +and recovery inspection. + +## Preserved boundaries + +`v1.5.0` preserves the v1.4.0 T1 Office engine, exact-task grouped +authorization, Goal completion receipts, Event Store invariants, migration and +recovery compatibility, updater integrity, and source-only release guards. + +`docs/templates/operations-briefing-smoke-evidence` remains deterministic local +test material. The bundled smoke files are marked as +`SMOKE SAMPLE evidence for local verification only` and +`Replace before operational use`; replace them before operational use. The +desktop seed action continues to use blank operator templates under +`docs/templates/operations-briefing-evidence`. + +This release does not add Step 6 persistent automation/recovery work, production +Microsoft or Google accounts, live mail/calendar writes, a production portal, +stored browser login reuse, broad browser automation, secure/UAC desktop +control, CAPTCHA, payment or administrator authentication, VM/profile claims, +Headless/ACP, or arbitrary executable extensions. + +## Reproducible Windows package + +The final release gate compares the application and installer from two fresh, +distinct `CARGO_TARGET_DIR` builds byte-for-byte. The GitHub Release records the +exact main/tag commit, file name, byte size, SHA-256, version, and truthful +signature state, then independently downloads and re-verifies the published +asset. + +## Unsigned release + +Both `ds-agent.exe` and `DS.Agent_1.5.0_x64-setup.exe` are intentionally +Authenticode `NotSigned`; there is no signer. Windows may show `Unknown +publisher` or Microsoft Defender SmartScreen. Download only over HTTPS from the +official [`v1.5.0` GitHub Release](https://github.com/Lee-take/dsagent/releases/tag/v1.5.0) +and verify the exact byte size and SHA-256 published there before running it. + +The SignPath Foundation application remains submitted and approval is pending. +This release is not represented as signed or SignPath-approved. If signing +becomes available later, it begins with a subsequent new version and does not +replace the immutable v1.1.0, v1.2.0, v1.3.0, v1.4.0, or v1.5.0 tag, Release, +or asset. + +No real API key, production account, paid API, production tenant, installed DS +Agent overwrite, current user AppData mutation, or external production target +is required for this release verification. diff --git a/package.json b/package.json index 4c14ea7..c430b00 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "deepseek-agent-os", "private": true, - "version": "1.4.0", + "version": "1.5.0", "author": "Lee take", "license": "Apache-2.0", "description": "Local-first DeepSeek AI work platform with background runs, permissioned tools, auditable evidence, and recovery.", diff --git a/scripts/release-source-check.mjs b/scripts/release-source-check.mjs index e16494e..65cf6a7 100644 --- a/scripts/release-source-check.mjs +++ b/scripts/release-source-check.mjs @@ -4,7 +4,7 @@ import { spawnSync } from "node:child_process"; import { existsSync, readFileSync, statSync } from "node:fs"; import path from "node:path"; -const expectedVersion = "1.4.0"; +const expectedVersion = "1.5.0"; const maxSourceFileBytes = 2 * 1024 * 1024; const binaryReleaseExtensions = new Set([ ".appimage", @@ -52,6 +52,7 @@ const requiredDocs = [ "CODE_SIGNING_POLICY.md", "PRIVACY.md", "docs/INSTALLATION.md", + "docs/RELEASE_NOTES_v1.5.0.md", "docs/RELEASE_NOTES_v1.4.0.md", "docs/RELEASE_NOTES_v1.3.0.md", "docs/RELEASE_NOTES_v1.2.0.md", @@ -89,6 +90,7 @@ const publicReleaseCopyFiles = [ "apps/desktop/package.json", "docs/INSTALLATION.md", "docs/OPEN_SOURCE_RELEASE.md", + "docs/RELEASE_NOTES_v1.5.0.md", "docs/RELEASE_NOTES_v1.4.0.md", "docs/RELEASE_NOTES_v1.3.0.md", "docs/RELEASE_NOTES_v1.2.0.md", @@ -734,7 +736,7 @@ function checkRequiredDocs() { for (const [phrase, label] of [ ["One Kernel. Modular capabilities. Verifiable execution.", "English README product promise"], ["DS Agent is a local Agent Harness optimized for DeepSeek", "English README DeepSeek-first positioning"], - ["v1.4.0 stable", "English README formal stable status"], + ["v1.5.0 stable", "English README formal stable status"], ["The key difference is that Memory, Automation, Computer Use, parallel Subagents, and Skills are not isolated plugins", "English README unified Kernel thesis"], ["contract-first modular Harness architecture", "English README modular architecture"], ["Five core capabilities, one engineering philosophy", "English README five-capability framing"], @@ -746,7 +748,8 @@ function checkRequiredDocs() { ["goal → done-when contract → context → plan → permission → execution → evidence → verification → recovery", "English README Loop Engineering contract"], ["DeepSeek and DS Agent boundary", "English README model boundary"], ["Why Rust", "English README Rust rationale"], - ["Production Microsoft/Google account registration and live external-write authority remain disabled in v1.4.0", "English README connector authority boundary"], + ["Production Microsoft/Google account registration and live external-write authority remain disabled in v1.5.0", "English README connector authority boundary"], + ["Exact installed evidence is limited to generated File Explorer and Excel targets plus installed Edge with a fresh profile and an ephemeral loopback test portal", "English README exact Computer Use evidence boundary"], ["Approval creates only exact authority; it does not execute a Tool, resume the task, or mark the Goal complete", "English README grouped authorization non-execution boundary"], ["ordinary chat does not yet automatically select or sequence the two T1 tools", "English README T1 reachability boundary"], ["README.zh-CN.md", "English README language switch"], @@ -757,7 +760,7 @@ function checkRequiredDocs() { for (const [phrase, label] of [ ["一个 Kernel，模块化能力，统一可信执行。", "Chinese README product promise"], ["DS Agent 是专门为 DeepSeek 优化的本地 Agent Harness", "Chinese README DeepSeek-first positioning"], - ["v1.4.0 正式稳定版", "Chinese README formal stable status"], + ["v1.5.0 正式稳定版", "Chinese README formal stable status"], ["真正的差异点是：记忆、自动化执行、Computer Use、Subagent 并行协作和技能", "Chinese README unified Kernel thesis"], ["契约优先的模块化 Harness 架构", "Chinese README modular architecture"], ["五项核心能力，一套工程理念", "Chinese README five-capability framing"], @@ -769,7 +772,8 @@ function checkRequiredDocs() { ["目标 → 完成条件 → 上下文 → 规划 → 权限 → 执行 → 证据 → 验证 → 恢复", "Chinese README Loop Engineering contract"], ["DeepSeek 与 DS Agent 的工作边界", "Chinese README model boundary"], ["为什么使用 Rust", "Chinese README Rust rationale"], - ["v1.4.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限", "Chinese README connector authority boundary"], + ["v1.5.0 仍未开放生产 Microsoft/Google 账号注册和真实外部写入权限", "Chinese README connector authority boundary"], + ["准确安装态证据只覆盖生成的 File Explorer、Excel 目标，以及使用全新 profile 和临时回环测试站点的已安装 Edge", "Chinese README exact Computer Use evidence boundary"], ["批准只产生精确权限，不会执行 Tool、恢复任务或把 Goal 标记为完成", "Chinese README grouped authorization non-execution boundary"], ["普通聊天目前还不会自动选择和串联这两个 T1 工具", "Chinese README T1 reachability boundary"], ["README.md", "Chinese README language switch"], @@ -1619,14 +1623,14 @@ function checkPublicReleaseCopyPositioning() { checkTextIncludes( "apps/desktop/src-tauri/src/kernel/app_update.rs", readText("apps/desktop/src-tauri/src/kernel/app_update.rs"), - 'APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.4.0"', - "app updater User-Agent v1.4.0", + 'APP_UPDATE_USER_AGENT: &str = "DS-Agent-Updater/1.5.0"', + "app updater User-Agent v1.5.0", ); checkTextIncludes( "apps/desktop/src-tauri/src/kernel/app_update.rs", readText("apps/desktop/src-tauri/src/kernel/app_update.rs"), - 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.4.0"', - "app updater current release tag v1.4.0", + 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v1.5.0"', + "app updater current release tag v1.5.0", ); checkTextDoesNotInclude( "apps/desktop/src-tauri/src/kernel/app_update.rs", @@ -1634,6 +1638,48 @@ function checkPublicReleaseCopyPositioning() { 'APP_UPDATE_CURRENT_RELEASE_TAG: &str = "v0.9.0"', "app updater current release tag must not regress to v0.9.0", ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "Package, desktop, Tauri, Cargo, updater, and installer metadata are `1.5.0` / `v1.5.0`.", + "v1.5.0 stable release notes version identity", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "Both `ds-agent.exe` and `DS.Agent_1.5.0_x64-setup.exe` are intentionally Authenticode `NotSigned`", + "v1.5.0 dual-artifact unsigned disclosure", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "The Edge DOM helper is compiled only for Windows tests", + "v1.5.0 test-only Edge DOM boundary", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "The named File Explorer, Excel, and Edge/local-portal cases are exact installed-test evidence; they are not a promise that every application, workbook, website, or desktop layout is supported", + "v1.5.0 installed-test reachability boundary", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "DeepSeek remains advisory and cannot approve an action or mint authority, evidence, verification, or completion", + "v1.5.0 model authority boundary", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "This release does not add Step 6 persistent automation/recovery work", + "v1.5.0 Step 6 exclusion", + ); + checkTextIncludesCollapsed( + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), + "The final Step 5 matrix records 30 installed application runs: 10 File Explorer, 10 Excel, and 10 Edge/local-portal", + "v1.5.0 installed reliability denominator", + ); checkTextIncludesCollapsed( "docs/RELEASE_NOTES_v1.4.0.md", readText("docs/RELEASE_NOTES_v1.4.0.md"), @@ -3487,7 +3533,7 @@ function checkGovernanceDocs() { checkTextIncludesCollapsed( "SECURITY.md", securityPolicy, - "DS Agent v1.4.0 is the current published stable release", + "DS Agent v1.5.0 is the current published stable release", "SECURITY.md current stable scope", ); checkTextIncludes( @@ -3517,7 +3563,7 @@ function checkGovernanceDocs() { checkTextIncludesCollapsed( "SECURITY.md", securityPolicy, - "Current stable v1.4.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault", + "Current stable v1.5.0 stores one user-supplied DeepSeek API key in a dedicated Windows DPAPI vault", "SECURITY.md current stable credential boundary", ); checkTextIncludesCollapsed( @@ -3578,7 +3624,7 @@ function checkCodeSigningAndPrivacyPolicies() { for (const [phrase, label] of [ ["# Code signing policy", "code-signing policy title"], - ["DS Agent `v1.4.0` is intentionally published unsigned", "code-signing current unsigned status"], + ["DS Agent `v1.5.0` is intentionally published unsigned", "code-signing current unsigned status"], ["The SignPath Foundation application is submitted and approval is pending", "code-signing pending application status"], ["Free code signing provided by", "SignPath acknowledgement"], ["certificate by", "SignPath certificate acknowledgement"], @@ -3601,7 +3647,7 @@ function checkCodeSigningAndPrivacyPolicies() { for (const [phrase, label] of [ ["# Privacy Policy", "privacy policy title"], ["does not operate a project cloud backend, advertising service, or project analytics or telemetry service", "no project telemetry service"], - ["The current stable `v1.4.0` accepts one user-supplied DeepSeek API key", "privacy current credential behavior"], + ["The current stable `v1.5.0` accepts one user-supplied DeepSeek API key", "privacy current credential behavior"], ["dedicated Windows DPAPI-protected local vault", "privacy DPAPI storage boundary"], ["https://cdn.deepseek.com/policies/en-US/deepseek-privacy-policy.html", "DeepSeek privacy disclosure"], ["DuckDuckGo's privacy policy", "web-search privacy disclosure"], @@ -3609,7 +3655,7 @@ function checkCodeSigningAndPrivacyPolicies() { ["Hugging Face", "skill-source privacy disclosure"], ["WebView2 data and privacy documentation", "WebView2 privacy disclosure"], ["accepts only loopback addresses", "local bridge privacy boundary"], - ["Production Microsoft and Google account registration and live mail/calendar writes are disabled in `v1.4.0`", "disabled connector privacy boundary"], + ["Production Microsoft and Google account registration and live mail/calendar writes are disabled in `v1.5.0`", "disabled connector privacy boundary"], ["not uploaded merely because the application is open", "no passive content upload"], ]) { checkTextIncludesCollapsed("PRIVACY.md", privacyPolicy, phrase, label); @@ -3622,7 +3668,7 @@ function checkCodeSigningAndPrivacyPolicies() { checkTextIncludes(filePath, content, "NotSigned", `${filePath} unsigned Authenticode disclosure`); checkTextIncludes(filePath, content, "Unknown publisher", `${filePath} unknown-publisher warning`); checkTextIncludes(filePath, content, "SmartScreen", `${filePath} SmartScreen warning`); - checkTextIncludes(filePath, content, "docs/RELEASE_NOTES_v1.4.0.md", `${filePath} v1.4.0 release-notes link`); + checkTextIncludes(filePath, content, "docs/RELEASE_NOTES_v1.5.0.md", `${filePath} v1.5.0 release-notes link`); checkTextDoesNotInclude(filePath, content, "12,716,857 bytes", `${filePath} no stale v1.0.1 asset size`); checkTextDoesNotInclude(filePath, content, "469C4EFA54F4C94A6E37D28C9C88D331B26E1770C6792DC93D02B451640E2A6F", `${filePath} no stale v1.0.1 asset SHA-256`); } @@ -4232,10 +4278,10 @@ function checkReproducibleWindowsReleaseBuild() { "open-source release requires byte-reproducible Windows candidates", ); checkTextIncludesCollapsed( - "docs/RELEASE_NOTES_v1.4.0.md", - readText("docs/RELEASE_NOTES_v1.4.0.md"), + "docs/RELEASE_NOTES_v1.5.0.md", + readText("docs/RELEASE_NOTES_v1.5.0.md"), "The final release gate compares the application and installer from two fresh, distinct `CARGO_TARGET_DIR` builds byte-for-byte", - "v1.4.0 release notes disclose reproducibility gate", + "v1.5.0 release notes disclose reproducibility gate", ); } @@ -4374,7 +4420,7 @@ function checkSmokeScriptReleaseLabels() { const rustRuntimeExpectations = [ [ "apps/desktop/src-tauri/src/kernel/deepseek.rs", - "DS-Agent/1.4.0 deepseek-v4", + "DS-Agent/1.5.0 deepseek-v4", "DeepSeek runtime User-Agent release label", ], [