feat: computer use, session/workspace tooling, and remote SSH integration

src/apps/cli/src/agent/agentic_system.rs

@@ -62,6 +62,7 @@ pub async fn init_agentic_system() -> Result<AgenticSystem> {
         tool_registry,
         tool_state_manager,
         None,
+        None,
     ));
 
     let stream_processor = Arc::new(execution::StreamProcessor::new(event_queue.clone()));

src/apps/desktop/Cargo.toml

@@ -47,6 +47,17 @@ urlencoding = { workspace = true }
 reqwest = { workspace = true }
 thiserror = "1.0"
 futures = { workspace = true }
+async-trait = { workspace = true }
+screenshots = "0.8"
+enigo = "0.2"
+image = { version = "0.24", default-features = false, features = ["png", "jpeg"] }
+resvg = { version = "0.47.0", default-features = false }
+fontdue = "0.9"
+
+[target.'cfg(target_os = "macos")'.dependencies]
+core-foundation = "0.9"
+core-graphics = "0.23"
+dispatch = "0.2"
 
 [target.'cfg(windows)'.dependencies]
 win32job = { workspace = true }

src/apps/desktop/src/api/agentic_api.rs

@@ -6,15 +6,14 @@ use std::sync::Arc;
 use tauri::{AppHandle, State};
 
 use crate::api::app_state::AppState;
+use crate::api::session_storage_path::desktop_effective_session_storage_path;
 use bitfun_core::agentic::coordination::{
     AssistantBootstrapBlockReason, AssistantBootstrapEnsureOutcome, AssistantBootstrapSkipReason,
     ConversationCoordinator, DialogScheduler, DialogSubmissionPolicy, DialogTriggerSource,
 };
 use bitfun_core::agentic::core::*;
 use bitfun_core::agentic::image_analysis::ImageContextData;
 use bitfun_core::agentic::tools::image_context::get_image_context;
-use bitfun_core::service::remote_ssh::workspace_state::get_effective_session_path;
-
 #[derive(Debug, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct CreateSessionRequest {
@@ -24,6 +23,8 @@ pub struct CreateSessionRequest {
     pub workspace_path: String,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
     pub config: Option<SessionConfigDTO>,
 }
 
@@ -40,6 +41,8 @@ pub struct SessionConfigDTO {
     pub model_name: Option<String>,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
 }
 
 #[derive(Debug, Serialize)]
@@ -84,6 +87,8 @@ pub struct EnsureCoordinatorSessionRequest {
     pub workspace_path: String,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -157,6 +162,8 @@ pub struct DeleteSessionRequest {
     pub workspace_path: String,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -166,6 +173,8 @@ pub struct RestoreSessionRequest {
     pub workspace_path: String,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -174,6 +183,8 @@ pub struct ListSessionsRequest {
     pub workspace_path: String,
     #[serde(default)]
     pub remote_connection_id: Option<String>,
+    #[serde(default)]
+    pub remote_ssh_host: Option<String>,
 }
 
 #[derive(Debug, Deserialize)]
@@ -214,6 +225,12 @@ pub async fn create_session(
             .as_ref()
             .and_then(|c| norm_conn(c.remote_connection_id.clone()))
     });
+    let remote_ssh_host = norm_conn(request.remote_ssh_host.clone()).or_else(|| {
+        request
+            .config
+            .as_ref()
+            .and_then(|c| norm_conn(c.remote_ssh_host.clone()))
+    });
 
     let config = request
         .config
@@ -227,11 +244,13 @@ pub async fn create_session(
             compression_threshold: c.compression_threshold.unwrap_or(0.8),
             workspace_path: Some(request.workspace_path.clone()),
             remote_connection_id: remote_conn.clone(),
+            remote_ssh_host: remote_ssh_host.clone(),
             model_id: c.model_name,
         })
         .unwrap_or(SessionConfig {
             workspace_path: Some(request.workspace_path.clone()),
             remote_connection_id: remote_conn.clone(),
+            remote_ssh_host: remote_ssh_host.clone(),
             ..Default::default()
         });
 
@@ -269,6 +288,7 @@ pub async fn update_session_model(
 #[tauri::command]
 pub async fn ensure_coordinator_session(
     coordinator: State<'_, Arc<ConversationCoordinator>>,
+    app_state: State<'_, AppState>,
     request: EnsureCoordinatorSessionRequest,
 ) -> Result<(), String> {
     let session_id = request.session_id.trim();
@@ -288,7 +308,13 @@ pub async fn ensure_coordinator_session(
         return Err("workspace_path is required when the session is not loaded".to_string());
     }
 
-    let effective = get_effective_session_path(wp, request.remote_connection_id.as_deref()).await;
+    let effective = desktop_effective_session_storage_path(
+        &app_state,
+        wp,
+        request.remote_connection_id.as_deref(),
+        request.remote_ssh_host.as_deref(),
+    )
+    .await;
     coordinator
         .restore_session(&effective, session_id)
         .await
@@ -487,10 +513,16 @@ pub async fn cancel_tool(
 #[tauri::command]
 pub async fn delete_session(
     coordinator: State<'_, Arc<ConversationCoordinator>>,
+    app_state: State<'_, AppState>,
     request: DeleteSessionRequest,
 ) -> Result<(), String> {
-    let effective_path =
-        get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await;
+    let effective_path = desktop_effective_session_storage_path(
+        &app_state,
+        &request.workspace_path,
+        request.remote_connection_id.as_deref(),
+        request.remote_ssh_host.as_deref(),
+    )
+    .await;
     coordinator
         .delete_session(&effective_path, &request.session_id)
         .await
@@ -500,10 +532,16 @@ pub async fn delete_session(
 #[tauri::command]
 pub async fn restore_session(
     coordinator: State<'_, Arc<ConversationCoordinator>>,
+    app_state: State<'_, AppState>,
     request: RestoreSessionRequest,
 ) -> Result<SessionResponse, String> {
-    let effective_path =
-        get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await;
+    let effective_path = desktop_effective_session_storage_path(
+        &app_state,
+        &request.workspace_path,
+        request.remote_connection_id.as_deref(),
+        request.remote_ssh_host.as_deref(),
+    )
+    .await;
     let session = coordinator
         .restore_session(&effective_path, &request.session_id)
         .await
@@ -515,11 +553,16 @@ pub async fn restore_session(
 #[tauri::command]
 pub async fn list_sessions(
     coordinator: State<'_, Arc<ConversationCoordinator>>,
+    app_state: State<'_, AppState>,
     request: ListSessionsRequest,
 ) -> Result<Vec<SessionResponse>, String> {
-    // Map remote workspace path to local session storage path
-    let effective_path =
-        get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await;
+    let effective_path = desktop_effective_session_storage_path(
+        &app_state,
+        &request.workspace_path,
+        request.remote_connection_id.as_deref(),
+        request.remote_ssh_host.as_deref(),
+    )
+    .await;
     let summaries = coordinator
         .list_sessions(&effective_path)
         .await
@@ -726,13 +769,15 @@ fn message_to_dto(message: Message) -> MessageDTO {
             result,
             result_for_assistant,
             is_error: _,
+            image_attachments,
         } => {
             serde_json::json!({
                 "type": "tool_result",
                 "tool_id": tool_id,
                 "tool_name": tool_name,
                 "result": result,
                 "result_for_assistant": result_for_assistant,
+                "has_image_attachments": image_attachments.as_ref().is_some_and(|a| !a.is_empty()),
             })
         }
         MessageContent::Mixed {

src/apps/desktop/src/api/app_state.rs

@@ -50,6 +50,8 @@ pub struct RemoteWorkspace {
     pub connection_id: String,
     pub connection_name: String,
     pub remote_path: String,
+    #[serde(default)]
+    pub ssh_host: String,
 }
 
 pub struct AppState {
@@ -211,6 +213,7 @@ impl AppState {
                         connection_id: first.connection_id.clone(),
                         remote_path: first.remote_path.clone(),
                         connection_name: first.connection_name.clone(),
+                        ssh_host: first.ssh_host.clone(),
                     };
                     *remote_workspace_clone.write().await = Some(app_workspace);
                 }
@@ -345,6 +348,7 @@ impl AppState {
                 connection_id: workspace.connection_id.clone(),
                 remote_path: workspace.remote_path.clone(),
                 connection_name: workspace.connection_name.clone(),
+                ssh_host: workspace.ssh_host.clone(),
             };
             if let Err(e) = manager.set_remote_workspace(core_workspace).await {
                 log::warn!("Failed to persist remote workspace: {}", e);
@@ -370,6 +374,7 @@ impl AppState {
             workspace.remote_path.clone(),
             workspace.connection_id.clone(),
             workspace.connection_name.clone(),
+            workspace.ssh_host.clone(),
         ).await;
         state_manager
             .set_active_connection_hint(Some(workspace.connection_id.clone()))

src/apps/desktop/src/api/commands.rs

@@ -27,10 +27,17 @@ fn remote_workspace_from_info(info: &WorkspaceInfo) -> Option<crate::api::Remote
     let rp = bitfun_core::service::remote_ssh::normalize_remote_workspace_path(
         &info.root_path.to_string_lossy(),
     );
+    let ssh_host = info
+        .metadata
+        .get("sshHost")
+        .and_then(|v| v.as_str())
+        .unwrap_or("")
+        .to_string();
     Some(crate::api::RemoteWorkspace {
         connection_id: cid,
         remote_path: rp,
         connection_name: name,
+        ssh_host,
     })
 }
 
@@ -57,6 +64,9 @@ pub struct OpenRemoteWorkspaceRequest {
     pub remote_path: String,
     pub connection_id: String,
     pub connection_name: String,
+    /// SSH config `host` (DNS or alias). When set, used for session mirror paths even if not connected.
+    #[serde(default)]
+    pub ssh_host: Option<String>,
 }
 
 #[derive(Debug, Deserialize, Default)]
@@ -737,10 +747,45 @@ pub async fn open_remote_workspace(
     request: OpenRemoteWorkspaceRequest,
 ) -> Result<WorkspaceInfoDto, String> {
     use bitfun_core::service::remote_ssh::normalize_remote_workspace_path;
+    use bitfun_core::service::remote_ssh::workspace_state::remote_workspace_stable_id;
     use bitfun_core::service::workspace::WorkspaceCreateOptions;
 
     let remote_path = normalize_remote_workspace_path(&request.remote_path);
 
+    let mut ssh_host = request
+        .ssh_host
+        .as_deref()
+        .map(str::trim)
+        .filter(|s| !s.is_empty())
+        .map(|s| s.to_string());
+
+    if ssh_host.is_none() {
+        if let Ok(mgr) = state.get_ssh_manager_async().await {
+            ssh_host = mgr
+                .get_saved_host_for_connection_id(&request.connection_id)
+                .await;
+        }
+    }
+    if ssh_host.is_none() {
+        if let Ok(mgr) = state.get_ssh_manager_async().await {
+            ssh_host = mgr
+                .get_connection_config(&request.connection_id)
+                .await
+                .map(|c| c.host)
+                .map(|h| h.trim().to_string())
+                .filter(|s| !s.is_empty());
+        }
+    }
+    let ssh_host = ssh_host.unwrap_or_else(|| {
+        warn!(
+            "open_remote_workspace: no ssh host from request, saved profile, or active connection; using connection_name (may not match session mirror): connection_id={}",
+            request.connection_id
+        );
+        request.connection_name.clone()
+    });
+
+    let stable_workspace_id = remote_workspace_stable_id(&ssh_host, &remote_path);
+
     let display_name = remote_path
         .split('/')
         .filter(|s| !s.is_empty())
@@ -761,6 +806,8 @@ pub async fn open_remote_workspace(
         description: None,
         tags: Vec::new(),
         remote_connection_id: Some(request.connection_id.clone()),
+        remote_ssh_host: Some(ssh_host.clone()),
+        stable_workspace_id: Some(stable_workspace_id),
     };
 
     match state
@@ -777,6 +824,10 @@ pub async fn open_remote_workspace(
                 "connectionName".to_string(),
                 serde_json::Value::String(request.connection_name.clone()),
             );
+            workspace_info.metadata.insert(
+                "sshHost".to_string(),
+                serde_json::Value::String(ssh_host.clone()),
+            );
 
             {
                 let manager = state.workspace_service.get_manager();
@@ -795,6 +846,7 @@ pub async fn open_remote_workspace(
                 connection_id: request.connection_id.clone(),
                 connection_name: request.connection_name.clone(),
                 remote_path: remote_path.clone(),
+                ssh_host: ssh_host.clone(),
             };
             if let Err(e) = state.set_remote_workspace(remote_workspace).await {
                 warn!("Failed to set remote workspace state: {}", e);
@@ -1268,6 +1320,8 @@ pub async fn scan_workspace_info(
             assistant_id: None,
             display_name: None,
             remote_connection_id: None,
+            remote_ssh_host: None,
+            stable_workspace_id: None,
         },
     )
     .await