diff --git a/src/apps/cli/src/agent/agentic_system.rs b/src/apps/cli/src/agent/agentic_system.rs index 640514bf..d8c41baf 100644 --- a/src/apps/cli/src/agent/agentic_system.rs +++ b/src/apps/cli/src/agent/agentic_system.rs @@ -62,6 +62,7 @@ pub async fn init_agentic_system() -> Result { tool_registry, tool_state_manager, None, + None, )); let stream_processor = Arc::new(execution::StreamProcessor::new(event_queue.clone())); diff --git a/src/apps/desktop/Cargo.toml b/src/apps/desktop/Cargo.toml index 0f65bc87..8a7878a6 100644 --- a/src/apps/desktop/Cargo.toml +++ b/src/apps/desktop/Cargo.toml @@ -47,6 +47,17 @@ urlencoding = { workspace = true } reqwest = { workspace = true } thiserror = "1.0" futures = { workspace = true } +async-trait = { workspace = true } +screenshots = "0.8" +enigo = "0.2" +image = { version = "0.24", default-features = false, features = ["png", "jpeg"] } +resvg = { version = "0.47.0", default-features = false } +fontdue = "0.9" + +[target.'cfg(target_os = "macos")'.dependencies] +core-foundation = "0.9" +core-graphics = "0.23" +dispatch = "0.2" [target.'cfg(windows)'.dependencies] win32job = { workspace = true } diff --git a/src/apps/desktop/assets/computer_use_pointer.svg b/src/apps/desktop/assets/computer_use_pointer.svg new file mode 100644 index 00000000..9ba8a4b6 --- /dev/null +++ b/src/apps/desktop/assets/computer_use_pointer.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/src/apps/desktop/assets/fonts/Inter-Regular.ttf b/src/apps/desktop/assets/fonts/Inter-Regular.ttf new file mode 100644 index 00000000..047c92f6 Binary files /dev/null and b/src/apps/desktop/assets/fonts/Inter-Regular.ttf differ diff --git a/src/apps/desktop/src/api/agentic_api.rs b/src/apps/desktop/src/api/agentic_api.rs index 199cc290..32ad50f5 100644 --- a/src/apps/desktop/src/api/agentic_api.rs +++ b/src/apps/desktop/src/api/agentic_api.rs @@ -6,6 +6,7 @@ use std::sync::Arc; use tauri::{AppHandle, State}; use crate::api::app_state::AppState; +use crate::api::session_storage_path::desktop_effective_session_storage_path; use bitfun_core::agentic::coordination::{ AssistantBootstrapBlockReason, AssistantBootstrapEnsureOutcome, AssistantBootstrapSkipReason, ConversationCoordinator, DialogScheduler, DialogSubmissionPolicy, DialogTriggerSource, @@ -13,8 +14,6 @@ use bitfun_core::agentic::coordination::{ use bitfun_core::agentic::core::*; use bitfun_core::agentic::image_analysis::ImageContextData; use bitfun_core::agentic::tools::image_context::get_image_context; -use bitfun_core::service::remote_ssh::workspace_state::get_effective_session_path; - #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct CreateSessionRequest { @@ -24,6 +23,8 @@ pub struct CreateSessionRequest { pub workspace_path: String, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, pub config: Option, } @@ -40,6 +41,8 @@ pub struct SessionConfigDTO { pub model_name: Option, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, } #[derive(Debug, Serialize)] @@ -84,6 +87,8 @@ pub struct EnsureCoordinatorSessionRequest { pub workspace_path: String, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, } #[derive(Debug, Deserialize)] @@ -157,6 +162,8 @@ pub struct DeleteSessionRequest { pub workspace_path: String, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, } #[derive(Debug, Deserialize)] @@ -166,6 +173,8 @@ pub struct RestoreSessionRequest { pub workspace_path: String, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, } #[derive(Debug, Deserialize)] @@ -174,6 +183,8 @@ pub struct ListSessionsRequest { pub workspace_path: String, #[serde(default)] pub remote_connection_id: Option, + #[serde(default)] + pub remote_ssh_host: Option, } #[derive(Debug, Deserialize)] @@ -214,6 +225,12 @@ pub async fn create_session( .as_ref() .and_then(|c| norm_conn(c.remote_connection_id.clone())) }); + let remote_ssh_host = norm_conn(request.remote_ssh_host.clone()).or_else(|| { + request + .config + .as_ref() + .and_then(|c| norm_conn(c.remote_ssh_host.clone())) + }); let config = request .config @@ -227,11 +244,13 @@ pub async fn create_session( compression_threshold: c.compression_threshold.unwrap_or(0.8), workspace_path: Some(request.workspace_path.clone()), remote_connection_id: remote_conn.clone(), + remote_ssh_host: remote_ssh_host.clone(), model_id: c.model_name, }) .unwrap_or(SessionConfig { workspace_path: Some(request.workspace_path.clone()), remote_connection_id: remote_conn.clone(), + remote_ssh_host: remote_ssh_host.clone(), ..Default::default() }); @@ -269,6 +288,7 @@ pub async fn update_session_model( #[tauri::command] pub async fn ensure_coordinator_session( coordinator: State<'_, Arc>, + app_state: State<'_, AppState>, request: EnsureCoordinatorSessionRequest, ) -> Result<(), String> { let session_id = request.session_id.trim(); @@ -288,7 +308,13 @@ pub async fn ensure_coordinator_session( return Err("workspace_path is required when the session is not loaded".to_string()); } - let effective = get_effective_session_path(wp, request.remote_connection_id.as_deref()).await; + let effective = desktop_effective_session_storage_path( + &app_state, + wp, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; coordinator .restore_session(&effective, session_id) .await @@ -487,10 +513,16 @@ pub async fn cancel_tool( #[tauri::command] pub async fn delete_session( coordinator: State<'_, Arc>, + app_state: State<'_, AppState>, request: DeleteSessionRequest, ) -> Result<(), String> { - let effective_path = - get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let effective_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; coordinator .delete_session(&effective_path, &request.session_id) .await @@ -500,10 +532,16 @@ pub async fn delete_session( #[tauri::command] pub async fn restore_session( coordinator: State<'_, Arc>, + app_state: State<'_, AppState>, request: RestoreSessionRequest, ) -> Result { - let effective_path = - get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let effective_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let session = coordinator .restore_session(&effective_path, &request.session_id) .await @@ -515,11 +553,16 @@ pub async fn restore_session( #[tauri::command] pub async fn list_sessions( coordinator: State<'_, Arc>, + app_state: State<'_, AppState>, request: ListSessionsRequest, ) -> Result, String> { - // Map remote workspace path to local session storage path - let effective_path = - get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let effective_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let summaries = coordinator .list_sessions(&effective_path) .await @@ -726,6 +769,7 @@ fn message_to_dto(message: Message) -> MessageDTO { result, result_for_assistant, is_error: _, + image_attachments, } => { serde_json::json!({ "type": "tool_result", @@ -733,6 +777,7 @@ fn message_to_dto(message: Message) -> MessageDTO { "tool_name": tool_name, "result": result, "result_for_assistant": result_for_assistant, + "has_image_attachments": image_attachments.as_ref().is_some_and(|a| !a.is_empty()), }) } MessageContent::Mixed { diff --git a/src/apps/desktop/src/api/app_state.rs b/src/apps/desktop/src/api/app_state.rs index 1d2f46b8..c00d3c45 100644 --- a/src/apps/desktop/src/api/app_state.rs +++ b/src/apps/desktop/src/api/app_state.rs @@ -50,6 +50,8 @@ pub struct RemoteWorkspace { pub connection_id: String, pub connection_name: String, pub remote_path: String, + #[serde(default)] + pub ssh_host: String, } pub struct AppState { @@ -211,6 +213,7 @@ impl AppState { connection_id: first.connection_id.clone(), remote_path: first.remote_path.clone(), connection_name: first.connection_name.clone(), + ssh_host: first.ssh_host.clone(), }; *remote_workspace_clone.write().await = Some(app_workspace); } @@ -345,6 +348,7 @@ impl AppState { connection_id: workspace.connection_id.clone(), remote_path: workspace.remote_path.clone(), connection_name: workspace.connection_name.clone(), + ssh_host: workspace.ssh_host.clone(), }; if let Err(e) = manager.set_remote_workspace(core_workspace).await { log::warn!("Failed to persist remote workspace: {}", e); @@ -370,6 +374,7 @@ impl AppState { workspace.remote_path.clone(), workspace.connection_id.clone(), workspace.connection_name.clone(), + workspace.ssh_host.clone(), ).await; state_manager .set_active_connection_hint(Some(workspace.connection_id.clone())) diff --git a/src/apps/desktop/src/api/commands.rs b/src/apps/desktop/src/api/commands.rs index bc242a71..c352cd0c 100644 --- a/src/apps/desktop/src/api/commands.rs +++ b/src/apps/desktop/src/api/commands.rs @@ -27,10 +27,17 @@ fn remote_workspace_from_info(info: &WorkspaceInfo) -> Option, } #[derive(Debug, Deserialize, Default)] @@ -737,10 +747,45 @@ pub async fn open_remote_workspace( request: OpenRemoteWorkspaceRequest, ) -> Result { use bitfun_core::service::remote_ssh::normalize_remote_workspace_path; + use bitfun_core::service::remote_ssh::workspace_state::remote_workspace_stable_id; use bitfun_core::service::workspace::WorkspaceCreateOptions; let remote_path = normalize_remote_workspace_path(&request.remote_path); + let mut ssh_host = request + .ssh_host + .as_deref() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + + if ssh_host.is_none() { + if let Ok(mgr) = state.get_ssh_manager_async().await { + ssh_host = mgr + .get_saved_host_for_connection_id(&request.connection_id) + .await; + } + } + if ssh_host.is_none() { + if let Ok(mgr) = state.get_ssh_manager_async().await { + ssh_host = mgr + .get_connection_config(&request.connection_id) + .await + .map(|c| c.host) + .map(|h| h.trim().to_string()) + .filter(|s| !s.is_empty()); + } + } + let ssh_host = ssh_host.unwrap_or_else(|| { + warn!( + "open_remote_workspace: no ssh host from request, saved profile, or active connection; using connection_name (may not match session mirror): connection_id={}", + request.connection_id + ); + request.connection_name.clone() + }); + + let stable_workspace_id = remote_workspace_stable_id(&ssh_host, &remote_path); + let display_name = remote_path .split('/') .filter(|s| !s.is_empty()) @@ -761,6 +806,8 @@ pub async fn open_remote_workspace( description: None, tags: Vec::new(), remote_connection_id: Some(request.connection_id.clone()), + remote_ssh_host: Some(ssh_host.clone()), + stable_workspace_id: Some(stable_workspace_id), }; match state @@ -777,6 +824,10 @@ pub async fn open_remote_workspace( "connectionName".to_string(), serde_json::Value::String(request.connection_name.clone()), ); + workspace_info.metadata.insert( + "sshHost".to_string(), + serde_json::Value::String(ssh_host.clone()), + ); { let manager = state.workspace_service.get_manager(); @@ -795,6 +846,7 @@ pub async fn open_remote_workspace( connection_id: request.connection_id.clone(), connection_name: request.connection_name.clone(), remote_path: remote_path.clone(), + ssh_host: ssh_host.clone(), }; if let Err(e) = state.set_remote_workspace(remote_workspace).await { warn!("Failed to set remote workspace state: {}", e); @@ -1268,6 +1320,8 @@ pub async fn scan_workspace_info( assistant_id: None, display_name: None, remote_connection_id: None, + remote_ssh_host: None, + stable_workspace_id: None, }, ) .await diff --git a/src/apps/desktop/src/api/computer_use_api.rs b/src/apps/desktop/src/api/computer_use_api.rs new file mode 100644 index 00000000..4f674186 --- /dev/null +++ b/src/apps/desktop/src/api/computer_use_api.rs @@ -0,0 +1,102 @@ +//! Tauri commands for Computer use (permissions + settings deep links). + +use crate::api::app_state::AppState; +use crate::computer_use::DesktopComputerUseHost; +use bitfun_core::agentic::tools::computer_use_host::ComputerUseHost; +use bitfun_core::service::config::types::AIConfig; +use serde::{Deserialize, Serialize}; +use tauri::State; + +#[derive(Debug, Clone, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ComputerUseStatusResponse { + pub computer_use_enabled: bool, + pub accessibility_granted: bool, + pub screen_capture_granted: bool, + pub platform_note: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ComputerUseOpenSettingsRequest { + /// `accessibility` | `screen_capture` + pub pane: String, +} + +#[tauri::command] +pub async fn computer_use_get_status( + state: State<'_, AppState>, +) -> Result { + let ai: AIConfig = state + .config_service + .get_config(Some("ai")) + .await + .map_err(|e| e.to_string())?; + + let host = DesktopComputerUseHost::new(); + let snap = host + .permission_snapshot() + .await + .map_err(|e| e.to_string())?; + + Ok(ComputerUseStatusResponse { + computer_use_enabled: ai.computer_use_enabled, + accessibility_granted: snap.accessibility_granted, + screen_capture_granted: snap.screen_capture_granted, + platform_note: snap.platform_note, + }) +} + +#[tauri::command] +pub async fn computer_use_request_permissions() -> Result<(), String> { + let host = DesktopComputerUseHost::new(); + host.request_accessibility_permission() + .await + .map_err(|e| e.to_string())?; + host.request_screen_capture_permission() + .await + .map_err(|e| e.to_string())?; + Ok(()) +} + +#[tauri::command] +pub async fn computer_use_open_system_settings( + request: ComputerUseOpenSettingsRequest, +) -> Result<(), String> { + #[cfg(target_os = "macos")] + { + let url = match request.pane.as_str() { + "accessibility" => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_Accessibility" + } + "screen_capture" => { + "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" + } + _ => return Err(format!("Unknown settings pane: {}", request.pane)), + }; + std::process::Command::new("open") + .arg(url) + .status() + .map_err(|e| e.to_string())?; + return Ok(()); + } + #[cfg(target_os = "windows")] + { + let _ = request; + return Err("Open system settings is not wired for Windows yet.".to_string()); + } + #[cfg(target_os = "linux")] + { + let _ = request; + return Err("Open system settings: use your desktop environment privacy settings.".to_string()); + } + #[cfg(not(any( + target_os = "macos", + target_os = "windows", + target_os = "linux" + )))] + { + let _ = request; + Err("Unsupported platform.".to_string()) + } +} diff --git a/src/apps/desktop/src/api/dto.rs b/src/apps/desktop/src/api/dto.rs index 0ee04a31..37cc36cc 100644 --- a/src/apps/desktop/src/api/dto.rs +++ b/src/apps/desktop/src/api/dto.rs @@ -1,6 +1,8 @@ //! DTO Module -use bitfun_core::service::remote_ssh::normalize_remote_workspace_path; +use bitfun_core::service::remote_ssh::{ + normalize_remote_workspace_path, LOCAL_WORKSPACE_SSH_HOST, +}; use bitfun_core::service::workspace::manager::WorkspaceKind; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -73,6 +75,8 @@ pub struct WorkspaceInfoDto { pub connection_id: Option, #[serde(skip_serializing_if = "Option::is_none")] pub connection_name: Option, + #[serde(rename = "sshHost", skip_serializing_if = "Option::is_none")] + pub ssh_host: Option, } impl WorkspaceInfoDto { @@ -89,6 +93,18 @@ impl WorkspaceInfoDto { .get("connectionName") .and_then(|v| v.as_str()) .map(|s| s.to_string()); + let ssh_host = info + .metadata + .get("sshHost") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + .or_else(|| { + if matches!(info.workspace_kind, WorkspaceKind::Remote) { + None + } else { + Some(LOCAL_WORKSPACE_SSH_HOST.to_string()) + } + }); let root_path = if matches!(info.workspace_kind, WorkspaceKind::Remote) { normalize_remote_workspace_path(&info.root_path.to_string_lossy()) @@ -122,6 +138,7 @@ impl WorkspaceInfoDto { .map(WorkspaceWorktreeInfoDto::from_workspace_worktree_info), connection_id, connection_name, + ssh_host, } } } diff --git a/src/apps/desktop/src/api/mod.rs b/src/apps/desktop/src/api/mod.rs index be8d7a43..2cc3bfdd 100644 --- a/src/apps/desktop/src/api/mod.rs +++ b/src/apps/desktop/src/api/mod.rs @@ -8,6 +8,7 @@ pub mod browser_api; pub mod btw_api; pub mod clipboard_file_api; pub mod commands; +pub mod computer_use_api; pub mod config_api; pub mod context_upload_api; pub mod cron_api; @@ -26,6 +27,7 @@ pub mod project_context_api; pub mod remote_connect_api; pub mod runtime_api; pub mod session_api; +pub mod session_storage_path; pub mod skill_api; pub mod snapshot_service; pub mod ssh_api; diff --git a/src/apps/desktop/src/api/session_api.rs b/src/apps/desktop/src/api/session_api.rs index eaae3e91..574bd9b5 100644 --- a/src/apps/desktop/src/api/session_api.rs +++ b/src/apps/desktop/src/api/session_api.rs @@ -1,8 +1,9 @@ //! Session persistence API +use crate::api::app_state::AppState; +use crate::api::session_storage_path::desktop_effective_session_storage_path; use bitfun_core::agentic::persistence::PersistenceManager; use bitfun_core::infrastructure::PathManager; -use bitfun_core::service::remote_ssh::workspace_state::get_effective_session_path; use bitfun_core::service::session::{ DialogTurnData, SessionMetadata, SessionTranscriptExport, SessionTranscriptExportOptions, }; @@ -15,6 +16,8 @@ pub struct ListPersistedSessionsRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -24,6 +27,8 @@ pub struct LoadSessionTurnsRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub limit: Option, } @@ -33,6 +38,8 @@ pub struct SaveSessionTurnRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -41,6 +48,8 @@ pub struct SaveSessionMetadataRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -49,6 +58,8 @@ pub struct ExportSessionTranscriptRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, #[serde(default = "default_tools")] pub tools: bool, #[serde(default)] @@ -69,6 +80,8 @@ pub struct DeletePersistedSessionRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -77,6 +90,8 @@ pub struct TouchSessionActivityRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -85,14 +100,23 @@ pub struct LoadPersistedSessionMetadataRequest { pub workspace_path: String, #[serde(default, skip_serializing_if = "Option::is_none")] pub remote_connection_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub remote_ssh_host: Option, } #[tauri::command] pub async fn list_persisted_sessions( request: ListPersistedSessionsRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result, String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -105,9 +129,16 @@ pub async fn list_persisted_sessions( #[tauri::command] pub async fn load_session_turns( request: LoadSessionTurnsRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result, String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -127,9 +158,16 @@ pub async fn load_session_turns( #[tauri::command] pub async fn save_session_turn( request: SaveSessionTurnRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result<(), String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -142,9 +180,16 @@ pub async fn save_session_turn( #[tauri::command] pub async fn save_session_metadata( request: SaveSessionMetadataRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result<(), String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -157,9 +202,16 @@ pub async fn save_session_metadata( #[tauri::command] pub async fn export_session_transcript( request: ExportSessionTranscriptRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -181,9 +233,16 @@ pub async fn export_session_transcript( #[tauri::command] pub async fn delete_persisted_session( request: DeletePersistedSessionRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result<(), String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -196,9 +255,16 @@ pub async fn delete_persisted_session( #[tauri::command] pub async fn touch_session_activity( request: TouchSessionActivityRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result<(), String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; @@ -211,9 +277,16 @@ pub async fn touch_session_activity( #[tauri::command] pub async fn load_persisted_session_metadata( request: LoadPersistedSessionMetadataRequest, + app_state: State<'_, AppState>, path_manager: State<'_, Arc>, ) -> Result, String> { - let workspace_path = get_effective_session_path(&request.workspace_path, request.remote_connection_id.as_deref()).await; + let workspace_path = desktop_effective_session_storage_path( + &app_state, + &request.workspace_path, + request.remote_connection_id.as_deref(), + request.remote_ssh_host.as_deref(), + ) + .await; let manager = PersistenceManager::new(path_manager.inner().clone()) .map_err(|e| format!("Failed to create persistence manager: {}", e))?; diff --git a/src/apps/desktop/src/api/session_storage_path.rs b/src/apps/desktop/src/api/session_storage_path.rs new file mode 100644 index 00000000..18048842 --- /dev/null +++ b/src/apps/desktop/src/api/session_storage_path.rs @@ -0,0 +1,34 @@ +//! Shared desktop resolution of on-disk session roots for remote workspaces. + +use crate::api::app_state::AppState; +use bitfun_core::service::remote_ssh::workspace_state::get_effective_session_path; + +pub async fn desktop_effective_session_storage_path( + app_state: &AppState, + workspace_path: &str, + remote_connection_id: Option<&str>, + remote_ssh_host: Option<&str>, +) -> std::path::PathBuf { + let conn = remote_connection_id.map(str::trim).filter(|s| !s.is_empty()); + let host_from_request = remote_ssh_host + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(|s| s.to_string()); + let mut host_owned = host_from_request.clone(); + if host_owned.is_none() { + if let Some(cid) = conn { + host_owned = app_state + .workspace_service + .remote_ssh_host_for_remote_workspace(cid, workspace_path) + .await; + } + } + if host_owned.is_none() { + if let Some(cid) = conn { + if let Ok(mgr) = app_state.get_ssh_manager_async().await { + host_owned = mgr.get_saved_host_for_connection_id(cid).await; + } + } + } + get_effective_session_path(workspace_path, conn, host_owned.as_deref()).await +} diff --git a/src/apps/desktop/src/api/ssh_api.rs b/src/apps/desktop/src/api/ssh_api.rs index ecbca5c0..8572cae8 100644 --- a/src/apps/desktop/src/api/ssh_api.rs +++ b/src/apps/desktop/src/api/ssh_api.rs @@ -341,10 +341,17 @@ pub async fn remote_open_workspace( let connections = manager.get_saved_connections().await; let conn = connections.iter().find(|c| c.id == connection_id); + let ssh_host = manager + .get_connection_config(&connection_id) + .await + .map(|c| c.host) + .unwrap_or_default(); + let workspace = crate::api::RemoteWorkspace { connection_id: connection_id.clone(), connection_name: conn.map(|c| c.name.clone()).unwrap_or_default(), remote_path: remote_path.clone(), + ssh_host, }; state.set_remote_workspace(workspace).await diff --git a/src/apps/desktop/src/api/tool_api.rs b/src/apps/desktop/src/api/tool_api.rs index 11336521..3268ae3a 100644 --- a/src/apps/desktop/src/api/tool_api.rs +++ b/src/apps/desktop/src/api/tool_api.rs @@ -99,6 +99,7 @@ fn build_tool_context(workspace_path: Option<&str>) -> ToolUseContext { options: None, response_state: None, image_context_provider: Some(Arc::new(create_image_context_provider())), + computer_use_host: None, subagent_parent_info: None, cancellation_token: None, workspace_services: None, @@ -297,7 +298,9 @@ pub async fn execute_tool(request: ToolExecutionRequest) -> Result data.clone(), + bitfun_core::agentic::tools::framework::ToolResult::Result { data, .. } => { + data.clone() + } bitfun_core::agentic::tools::framework::ToolResult::Progress { content, .. } => content.clone(), bitfun_core::agentic::tools::framework::ToolResult::StreamChunk { data, .. } => data.clone(), }).collect::>() diff --git a/src/apps/desktop/src/computer_use/desktop_host.rs b/src/apps/desktop/src/computer_use/desktop_host.rs new file mode 100644 index 00000000..c078c87b --- /dev/null +++ b/src/apps/desktop/src/computer_use/desktop_host.rs @@ -0,0 +1,1871 @@ +//! Cross-platform `ComputerUseHost` via `screenshots` + `enigo`. + +use async_trait::async_trait; +use bitfun_core::agentic::tools::computer_use_host::{ + ComputerScreenshot, ComputerUseHost, ComputerUseImageContentRect, + ComputerUseNavigateQuadrant, ComputerUseNavigationRect, ComputerUsePermissionSnapshot, + ComputerUseScreenshotParams, ComputerUseScreenshotRefinement, ScreenshotCropCenter, + COMPUTER_USE_QUADRANT_CLICK_READY_MAX_LONG_EDGE, COMPUTER_USE_QUADRANT_EDGE_EXPAND_PX, +}; +use bitfun_core::util::errors::{BitFunError, BitFunResult}; +use enigo::{ + Axis, Button, Coordinate, Direction, Enigo, Key, Keyboard, Mouse, Settings, +}; +use fontdue::{Font, FontSettings}; +use image::codecs::jpeg::JpegEncoder; +use image::{DynamicImage, Rgb, RgbImage}; +use log::warn; +use resvg::tiny_skia::{Pixmap, Transform}; +use resvg::usvg; +use screenshots::display_info::DisplayInfo; +use screenshots::Screen; +use std::sync::{Mutex, OnceLock}; +use std::time::Duration; + +/// Default pointer overlay; replace `assets/computer_use_pointer.svg` and rebuild to customize. +/// Hotspot in SVG user space must stay at **(0,0)** (arrow tip). +const POINTER_OVERLAY_SVG: &str = include_str!("../../assets/computer_use_pointer.svg"); + +#[derive(Debug)] +struct PointerPixmapCache { + w: u32, + h: u32, + /// Premultiplied RGBA8 (`tiny-skia` / `resvg` format). + rgba: Vec, +} + +static POINTER_PIXMAP_CACHE: OnceLock> = OnceLock::new(); + +fn pointer_pixmap_cache() -> Option<&'static PointerPixmapCache> { + POINTER_PIXMAP_CACHE + .get_or_init(|| match rasterize_pointer_svg(POINTER_OVERLAY_SVG, 0.3375) { + Ok(p) => Some(p), + Err(e) => { + warn!( + "computer_use: pointer SVG rasterize failed ({}); using fallback cross", + e + ); + None + } + }) + .as_ref() +} + +fn rasterize_pointer_svg(svg: &str, scale: f32) -> Result { + let opt = usvg::Options::default(); + let tree = usvg::Tree::from_str(svg, &opt).map_err(|e| e.to_string())?; + let size = tree.size(); + let w = ((size.width() * scale).ceil() as u32).max(1); + let h = ((size.height() * scale).ceil() as u32).max(1); + let mut pixmap = Pixmap::new(w, h).ok_or_else(|| "pixmap allocation failed".to_string())?; + resvg::render( + &tree, + Transform::from_scale(scale, scale), + &mut pixmap.as_mut(), + ); + Ok(PointerPixmapCache { + w, + h, + rgba: pixmap.data().to_vec(), + }) +} + +/// Alpha-composite premultiplied RGBA onto `img` with SVG (0,0) at `(cx, cy)`. +fn blend_pointer_pixmap(img: &mut RgbImage, cx: i32, cy: i32, p: &PointerPixmapCache) { + let iw = img.width() as i32; + let ih = img.height() as i32; + for row in 0..p.h { + for col in 0..p.w { + let i = ((row * p.w + col) * 4) as usize; + if i + 3 >= p.rgba.len() { + break; + } + let pr = p.rgba[i]; + let pg = p.rgba[i + 1]; + let pb = p.rgba[i + 2]; + let pa = p.rgba[i + 3] as u32; + if pa == 0 { + continue; + } + let px = cx + col as i32; + let py = cy + row as i32; + if px < 0 || py < 0 || px >= iw || py >= ih { + continue; + } + let dst = img.get_pixel(px as u32, py as u32); + let inv = 255 - pa; + let nr = (pr as u32 + dst[0] as u32 * inv / 255).min(255) as u8; + let ng = (pg as u32 + dst[1] as u32 * inv / 255).min(255) as u8; + let nb = (pb as u32 + dst[2] as u32 * inv / 255).min(255) as u8; + img.put_pixel(px as u32, py as u32, Rgb([nr, ng, nb])); + } + } +} + +fn draw_pointer_fallback_cross(img: &mut RgbImage, cx: i32, cy: i32) { + const ARM: i32 = 2; + const OUTLINE: Rgb = Rgb([255, 255, 255]); + const CORE: Rgb = Rgb([40, 40, 48]); + let w = img.width() as i32; + let h = img.height() as i32; + let mut plot = |x: i32, y: i32, c: Rgb| { + if x >= 0 && x < w && y >= 0 && y < h { + img.put_pixel(x as u32, y as u32, c); + } + }; + for t in -ARM..=ARM { + for k in -1..=1 { + plot(cx + t, cy + k, OUTLINE); + plot(cx + k, cy + t, OUTLINE); + } + } + for t in -ARM..=ARM { + plot(cx + t, cy, CORE); + plot(cx, cy + t, CORE); + } +} + +// ── Computer-use coordinate grid (100 px step): lines + anti-aliased axis labels (Inter OFL) ── + +const COORD_GRID_DEFAULT_STEP: u32 = 100; +const COORD_GRID_MAJOR_STEP: u32 = 500; +/// Logical scale knob; mapped to TTF pixel size for `fontdue` (`scale * 3.5`). +const COORD_LABEL_SCALE: i32 = 11; + +/// Inter (OFL); variable font from google/fonts OFL tree. +const COORD_AXIS_FONT_TTF: &[u8] = include_bytes!("../../assets/fonts/Inter-Regular.ttf"); + +static COORD_AXIS_FONT: OnceLock = OnceLock::new(); + +fn coord_axis_font() -> &'static Font { + COORD_AXIS_FONT.get_or_init(|| { + Font::from_bytes(COORD_AXIS_FONT_TTF, FontSettings::default()) + .expect("Inter TTF embedded for computer-use axis labels") + }) +} + +#[inline] +fn coord_label_px() -> f32 { + COORD_LABEL_SCALE as f32 * 3.5 +} + +/// Alpha-blend grayscale coverage onto `img` (baseline-anchored glyph). +fn coord_blit_glyph( + img: &mut RgbImage, + baseline_x: i32, + baseline_y: i32, + metrics: &fontdue::Metrics, + bitmap: &[u8], + fg: Rgb, +) { + let w = metrics.width; + let h = metrics.height; + if w == 0 || h == 0 { + return; + } + let iw = img.width() as i32; + let ih = img.height() as i32; + let xmin = metrics.xmin as i32; + let ymin = metrics.ymin as i32; + for row in 0..h { + for col in 0..w { + let alpha = bitmap[row * w + col] as u32; + if alpha == 0 { + continue; + } + let px = baseline_x + xmin + col as i32; + let py = baseline_y + ymin + row as i32; + if px < 0 || py < 0 || px >= iw || py >= ih { + continue; + } + let dst = img.get_pixel(px as u32, py as u32); + let inv = 255u32.saturating_sub(alpha); + let nr = ((fg[0] as u32 * alpha + dst[0] as u32 * inv) / 255).min(255) as u8; + let ng = ((fg[1] as u32 * alpha + dst[1] as u32 * inv) / 255).min(255) as u8; + let nb = ((fg[2] as u32 * alpha + dst[2] as u32 * inv) / 255).min(255) as u8; + img.put_pixel(px as u32, py as u32, Rgb([nr, ng, nb])); + } + } +} + +/// Axis numerals: synthetic bold via small 2×2 offset stack (still Inter Regular source). +fn coord_blit_glyph_bold( + img: &mut RgbImage, + baseline_x: i32, + baseline_y: i32, + metrics: &fontdue::Metrics, + bitmap: &[u8], + fg: Rgb, +) { + coord_blit_glyph(img, baseline_x, baseline_y, metrics, bitmap, fg); + coord_blit_glyph(img, baseline_x + 1, baseline_y, metrics, bitmap, fg); + coord_blit_glyph(img, baseline_x, baseline_y + 1, metrics, bitmap, fg); + coord_blit_glyph(img, baseline_x + 1, baseline_y + 1, metrics, bitmap, fg); +} + +fn coord_measure_str_width(text: &str, px: f32) -> i32 { + let font = coord_axis_font(); + let mut adv = 0f32; + for c in text.chars() { + adv += font.metrics(c, px).advance_width; + } + adv.ceil() as i32 +} + +/// Left-to-right string on one baseline. +fn coord_draw_text_h(img: &mut RgbImage, mut baseline_x: i32, baseline_y: i32, text: &str, fg: Rgb, px: f32) { + let font = coord_axis_font(); + for c in text.chars() { + let (m, bmp) = font.rasterize(c, px); + coord_blit_glyph_bold(img, baseline_x, baseline_y, &m, &bmp, fg); + baseline_x += m.advance_width.ceil() as i32; + } +} + +/// Vertically center a horizontal digit string on tick `py`. +fn coord_draw_u32_h_centered(img: &mut RgbImage, lx: i32, py: i32, n: u32, fg: Rgb, px: f32) { + let s = n.to_string(); + let font = coord_axis_font(); + let (m_rep, _) = font.rasterize('8', px); + let text_h = m_rep.height as i32; + let baseline_y = py - (m_rep.ymin + text_h / 2); + coord_draw_text_h(img, lx, baseline_y, &s, fg, px); +} + +#[inline] +fn coord_plot(img: &mut RgbImage, x: i32, y: i32, c: Rgb) { + let w = img.width() as i32; + let h = img.height() as i32; + if x >= 0 && x < w && y >= 0 && y < h { + img.put_pixel(x as u32, y as u32, c); + } +} + +fn coord_digit_block_width(digit_count: usize, px: f32) -> i32 { + if digit_count == 0 { + return 0; + } + let s: String = std::iter::repeat('8').take(digit_count).collect(); + coord_measure_str_width(&s, px) +} + +/// Height of a vertical digit stack (top-to-bottom) for `nd` decimal digits. +fn coord_vertical_digit_stack_height(nd: usize, px: f32) -> i32 { + if nd == 0 { + return 0; + } + let font = coord_axis_font(); + let gap = (px * 0.22).ceil().max(1.0) as i32; + let mut tot = 0i32; + for _ in 0..nd { + let (m, _) = font.rasterize('8', px); + tot += m.height as i32 + gap; + } + tot - gap +} + +/// Draw decimal `n` with digits stacked **top-to-bottom** (high-order digit at top). +/// Column is centered on `center_x` (tick position); narrow horizontal footprint for dense x-axis ticks. +fn coord_draw_u32_vertical_stack( + img: &mut RgbImage, + center_x: i32, + top_y: i32, + n: u32, + fg: Rgb, + px: f32, +) { + let s = n.to_string(); + let font = coord_axis_font(); + let gap = (px * 0.22).ceil().max(1.0) as i32; + let mut ty = top_y; + for c in s.chars() { + let (m, bmp) = font.rasterize(c, px); + let top_left_x = center_x - m.width as i32 / 2; + let top_left_y = ty; + let baseline_x = top_left_x - m.xmin as i32; + let baseline_y = top_left_y - m.ymin as i32; + coord_blit_glyph_bold(img, baseline_x, baseline_y, &m, &bmp, fg); + ty += m.height as i32 + gap; + } +} + +fn content_grid_step(min_side: u32) -> u32 { + if min_side < 240 { + 25u32 + } else if min_side < 480 { + 50u32 + } else { + COORD_GRID_DEFAULT_STEP + } +} + +/// Symmetric white margins (left = right, top = bottom) for ruler labels outside the capture. +/// `ruler_origin_*` is the **full-capture native** pixel index of the content’s top-left (0,0 for full screen; crop `x0,y0` for point crops) so label digit width fits large coordinates. +fn computer_use_margins( + cw: u32, + ch: u32, + ruler_origin_x: u32, + ruler_origin_y: u32, +) -> (u32, u32) { + if cw < 2 || ch < 2 { + return (0, 0); + } + let px = coord_label_px(); + let tick_len = 14i32; + let pad = 12i32; + let max_val_x = ruler_origin_x.saturating_add(cw.saturating_sub(1)); + let max_val_y = ruler_origin_y.saturating_add(ch.saturating_sub(1)); + let nd_x = (max_val_x.max(1).ilog10() as usize + 1).max(4); + let nd_y = (max_val_y.max(1).ilog10() as usize + 1).max(4); + let nd = nd_x.max(nd_y); + let ml = (coord_digit_block_width(nd, px) + tick_len + pad).max(0) as u32; + // Top/bottom: x-axis labels are vertical stacks — need height for `nd_x` digits. + let x_stack_h = coord_vertical_digit_stack_height(nd_x, px); + let mt = (x_stack_h + tick_len + pad).max(0) as u32; + (ml, mt) +} + +/// White border, grid lines on the capture only, numeric labels in the margin. +/// `ruler_origin_x/y`: **full-capture native** index of content pixel (0,0) — for a point crop, pass the crop’s `x0,y0` so tick labels match the same **whole-screen bitmap** space as a full-screen shot (not 0..crop_width only). +fn compose_computer_use_frame( + content: RgbImage, + ruler_origin_x: u32, + ruler_origin_y: u32, +) -> (RgbImage, u32, u32) { + let cw = content.width(); + let ch = content.height(); + if cw < 2 || ch < 2 { + return (content, 0, 0); + } + let grid_step = content_grid_step(cw.min(ch)); + let (ml, mt) = computer_use_margins(cw, ch, ruler_origin_x, ruler_origin_y); + let mr = ml; + let mb = mt; + let tw = ml + cw + mr; + let th = mt + ch + mb; + let label_px = coord_label_px(); + let tick_len = 14i32; + let pad = 12i32; + + let mut out = RgbImage::new(tw, th); + for p in out.pixels_mut() { + *p = Rgb([255u8, 255, 255]); + } + for yy in 0..ch { + for xx in 0..cw { + out.put_pixel(ml + xx, mt + yy, *content.get_pixel(xx, yy)); + } + } + + let grid = Rgb([52, 52, 68]); + let grid_major = Rgb([95, 95, 118]); + let tick = Rgb([180, 130, 40]); + // Coordinate numerals in white margins — saturated red for visibility. + let label = Rgb([200, 32, 40]); + + let cl = ml as i32; + let ct = mt as i32; + let cr = (ml + cw - 1) as i32; + let cb = (mt + ch - 1) as i32; + let wi = tw as i32; + let hi = th as i32; + + let mut gx = grid_step as i32; + while gx < cw as i32 { + let major = (gx as u32) % COORD_GRID_MAJOR_STEP == 0; + let thick = if major { 2 } else { 1 }; + let c = if major { grid_major } else { grid }; + for t in 0..thick { + let px = cl + gx + t; + if px >= cl && px <= cr { + for py in ct..=cb { + coord_plot(&mut out, px, py, c); + } + } + } + gx += grid_step as i32; + } + + let mut gy = grid_step as i32; + while gy < ch as i32 { + let major = (gy as u32) % COORD_GRID_MAJOR_STEP == 0; + let thick = if major { 2 } else { 1 }; + let c = if major { grid_major } else { grid }; + for t in 0..thick { + let py = ct + gy + t; + if py >= ct && py <= cb { + for px in cl..=cr { + coord_plot(&mut out, px, py, c); + } + } + } + gy += grid_step as i32; + } + + let top_label_y = pad.max(2); + for gxc in (0..cw as i32).step_by(grid_step as usize) { + let tick_x = cl + gxc; + for k in 0..tick_len.min(ct.max(1)) { + coord_plot(&mut out, tick_x, ct - 1 - k, tick); + } + let val = ruler_origin_x.saturating_add(gxc.max(0) as u32); + let col_w = coord_measure_str_width("8", label_px).max(1); + let cx = tick_x.clamp(col_w / 2 + 2, wi - col_w / 2 - 2); + coord_draw_u32_vertical_stack(&mut out, cx, top_label_y, val, label, label_px); + } + + let bot_label_y = cb + tick_len + 4; + for gxc in (0..cw as i32).step_by(grid_step as usize) { + let tick_x = cl + gxc; + for k in 0..tick_len { + let y = cb + 1 + k; + if y < hi { + coord_plot(&mut out, tick_x, y, tick); + } + } + let val = ruler_origin_x.saturating_add(gxc.max(0) as u32); + let col_w = coord_measure_str_width("8", label_px).max(1); + let cx = tick_x.clamp(col_w / 2 + 2, wi - col_w / 2 - 2); + coord_draw_u32_vertical_stack(&mut out, cx, bot_label_y, val, label, label_px); + } + + let left_numbers_x = pad.max(2); + for gyc in (0..ch as i32).step_by(grid_step as usize) { + let py = ct + gyc; + for k in 0..tick_len.min(cl.max(1)) { + coord_plot(&mut out, cl - 1 - k, py, tick); + } + let val = ruler_origin_y.saturating_add(gyc.max(0) as u32); + let s = val.to_string(); + let dw = coord_measure_str_width(&s, label_px); + let lx = left_numbers_x.min(cl - dw - 2).max(2); + coord_draw_u32_h_centered(&mut out, lx, py, val, label, label_px); + } + + let right_text_x = cr + tick_len + 4; + for gyc in (0..ch as i32).step_by(grid_step as usize) { + let py = ct + gyc; + for k in 0..tick_len { + let x = cr + 1 + k; + if x < wi { + coord_plot(&mut out, x, py, tick); + } + } + let val = ruler_origin_y.saturating_add(gyc.max(0) as u32); + let s = val.to_string(); + let dw = coord_measure_str_width(&s, label_px); + let lx = right_text_x.min(wi - dw - 2).max(2); + coord_draw_u32_h_centered(&mut out, lx, py, val, label, label_px); + } + + (out, ml, mt) +} + +/// JPEG quality for computer-use screenshots. Native display resolution is preserved (no downscale) +/// so `coordinate_mode` \"image\" pixel indices match the screen capture 1:1. Very large displays +/// increase request payload size; if the API rejects the image, lower quality or split workflows may be needed. +const JPEG_QUALITY: u8 = 75; + +/// Half extent from the crop center in **native** capture pixels (target region up to 500×500). +const POINT_CROP_HALF_PX: u32 = 250; + +#[inline] +fn clamp_center_to_native(cx: u32, cy: u32, nw: u32, nh: u32) -> (u32, u32) { + if nw == 0 || nh == 0 { + return (0, 0); + } + let cx = cx.min(nw - 1); + let cy = cy.min(nh - 1); + (cx, cy) +} + +/// Top-left and size of the native crop rectangle around `(cx, cy)`, clamped to the bitmap (≤500×500 when the display is large enough). +fn crop_rect_around_point_native(cx: u32, cy: u32, nw: u32, nh: u32) -> (u32, u32, u32, u32) { + let (cx, cy) = clamp_center_to_native(cx, cy, nw, nh); + if nw == 0 || nh == 0 { + return (0, 0, 1, 1); + } + let edge = POINT_CROP_HALF_PX.saturating_mul(2); + let tw = edge.min(nw).max(1); + let th = edge.min(nh).max(1); + let mut x0 = cx.saturating_sub(POINT_CROP_HALF_PX); + let mut y0 = cy.saturating_sub(POINT_CROP_HALF_PX); + if x0.saturating_add(tw) > nw { + x0 = nw.saturating_sub(tw); + } + if y0.saturating_add(th) > nh { + y0 = nh.saturating_sub(th); + } + (x0, y0, tw, th) +} + +#[inline] +fn full_navigation_rect(nw: u32, nh: u32) -> ComputerUseNavigationRect { + ComputerUseNavigationRect { + x0: 0, + y0: 0, + width: nw.max(1), + height: nh.max(1), + } +} + +fn intersect_navigation_rect( + a: ComputerUseNavigationRect, + b: ComputerUseNavigationRect, +) -> Option { + let ax1 = a.x0.saturating_add(a.width); + let ay1 = a.y0.saturating_add(a.height); + let bx1 = b.x0.saturating_add(b.width); + let by1 = b.y0.saturating_add(b.height); + let x0 = a.x0.max(b.x0); + let y0 = a.y0.max(b.y0); + let x1 = ax1.min(bx1); + let y1 = ay1.min(by1); + if x0 >= x1 || y0 >= y1 { + return None; + } + Some(ComputerUseNavigationRect { + x0, + y0, + width: x1 - x0, + height: y1 - y0, + }) +} + +/// Expand `r` by `pad` pixels left/up/right/down, clamped to `0..max_w` × `0..max_h`. +fn expand_navigation_rect_edges( + r: ComputerUseNavigationRect, + pad: u32, + max_w: u32, + max_h: u32, +) -> ComputerUseNavigationRect { + let x0 = r.x0.saturating_sub(pad); + let y0 = r.y0.saturating_sub(pad); + let x1 = r + .x0 + .saturating_add(r.width) + .saturating_add(pad) + .min(max_w); + let y1 = r + .y0 + .saturating_add(r.height) + .saturating_add(pad) + .min(max_h); + let width = x1.saturating_sub(x0).max(1); + let height = y1.saturating_sub(y0).max(1); + ComputerUseNavigationRect { + x0, + y0, + width, + height, + } +} + +fn quadrant_split_rect( + r: ComputerUseNavigationRect, + q: ComputerUseNavigateQuadrant, +) -> ComputerUseNavigationRect { + let hw = r.width / 2; + let hh = r.height / 2; + let rw = r.width - hw; + let rh = r.height - hh; + match q { + ComputerUseNavigateQuadrant::TopLeft => ComputerUseNavigationRect { + x0: r.x0, + y0: r.y0, + width: hw, + height: hh, + }, + ComputerUseNavigateQuadrant::TopRight => ComputerUseNavigationRect { + x0: r.x0 + hw, + y0: r.y0, + width: rw, + height: hh, + }, + ComputerUseNavigateQuadrant::BottomLeft => ComputerUseNavigationRect { + x0: r.x0, + y0: r.y0 + hh, + width: hw, + height: rh, + }, + ComputerUseNavigateQuadrant::BottomRight => ComputerUseNavigationRect { + x0: r.x0 + hw, + y0: r.y0 + hh, + width: rw, + height: rh, + }, + } +} + +/// macOS: map JPEG/bitmap pixels to/from **CoreGraphics global display coordinates** (same as +/// `CGDisplayBounds` / `CGEventGetLocation`): origin at the **top-left of the main display**, Y +/// increases **downward**. Not AppKit bottom-left / Y-up. +#[cfg(target_os = "macos")] +#[derive(Clone, Copy, Debug)] +struct MacPointerGeo { + disp_ox: f64, + disp_oy: f64, + disp_w: f64, + disp_h: f64, + full_px_w: u32, + full_px_h: u32, + crop_x0: u32, + crop_y0: u32, +} + +#[cfg(target_os = "macos")] +impl MacPointerGeo { + fn from_display(full_w: u32, full_h: u32, d: &DisplayInfo) -> Self { + Self { + disp_ox: d.x as f64, + disp_oy: d.y as f64, + disp_w: d.width as f64, + disp_h: d.height as f64, + full_px_w: full_w, + full_px_h: full_h, + crop_x0: 0, + crop_y0: 0, + } + } + + fn with_crop(mut self, x0: u32, y0: u32) -> Self { + self.crop_x0 = x0; + self.crop_y0 = y0; + self + } + + /// Map **continuous** framebuffer pixel center `(cx, cy)` (0.5 = middle of left/top pixel) to CG global. + fn full_pixel_center_to_global_f64(&self, cx: f64, cy: f64) -> BitFunResult<(f64, f64)> { + if self.disp_w <= 0.0 || self.disp_h <= 0.0 || self.full_px_w == 0 || self.full_px_h == 0 { + return Err(BitFunError::tool("Invalid macOS pointer geometry.".to_string())); + } + let px_w = self.full_px_w as f64; + let px_h = self.full_px_h as f64; + let max_cx = (self.full_px_w.saturating_sub(1) as f64) + 0.5; + let max_cy = (self.full_px_h.saturating_sub(1) as f64) + 0.5; + let cx = cx.clamp(0.5, max_cx); + let cy = cy.clamp(0.5, max_cy); + let gx = self.disp_ox + (cx / px_w) * self.disp_w; + let gy = self.disp_oy + (cy / px_h) * self.disp_h; + Ok((gx, gy)) + } + + /// `CGEventGetLocation` global mouse -> full-buffer pixel; then optional crop to view. + fn global_to_view_pixel(&self, mx: f64, my: f64, view_w: u32, view_h: u32) -> Option<(i32, i32)> { + if self.disp_w <= 0.0 || self.disp_h <= 0.0 || self.full_px_w == 0 || self.full_px_h == 0 { + return None; + } + let lx = mx - self.disp_ox; + let ly = my - self.disp_oy; + if lx < 0.0 || lx >= self.disp_w || ly < 0.0 || ly >= self.disp_h { + return None; + } + let full_ix = ((lx / self.disp_w) * self.full_px_w as f64).floor() as i32; + let full_iy = ((ly / self.disp_h) * self.full_px_h as f64).floor() as i32; + let full_ix = full_ix.clamp(0, self.full_px_w.saturating_sub(1) as i32); + let full_iy = full_iy.clamp(0, self.full_px_h.saturating_sub(1) as i32); + let vx = full_ix - self.crop_x0 as i32; + let vy = full_iy - self.crop_y0 as i32; + if vx >= 0 && vy >= 0 && (vx as u32) < view_w && (vy as u32) < view_h { + Some((vx, vy)) + } else { + None + } + } +} + +#[derive(Clone, Copy, Debug)] +struct PointerMap { + /// Composed JPEG size (includes white margin). + image_w: u32, + image_h: u32, + /// Top-left of capture inside the JPEG. + content_origin_x: u32, + content_origin_y: u32, + /// Native capture pixel size (the screen bitmap, no margin). + content_w: u32, + content_h: u32, + native_w: u32, + native_h: u32, + origin_x: i32, + origin_y: i32, + #[cfg(target_os = "macos")] + macos_geo: Option, +} + +impl PointerMap { + /// Continuous mapping: **composed JPEG** pixel `(x,y)` -> global (macOS CG). + fn map_image_to_global_f64(&self, x: i32, y: i32) -> BitFunResult<(f64, f64)> { + if self.image_w == 0 + || self.image_h == 0 + || self.content_w == 0 + || self.content_h == 0 + || self.native_w == 0 + || self.native_h == 0 + { + return Err(BitFunError::tool( + "Invalid screenshot coordinate map (zero dimension).".to_string(), + )); + } + let ox = self.content_origin_x as i32; + let oy = self.content_origin_y as i32; + let cx_img = x - ox; + let cy_img = y - oy; + let max_cx = self.content_w.saturating_sub(1) as i32; + let max_cy = self.content_h.saturating_sub(1) as i32; + let cx_img = cx_img.clamp(0, max_cx) as f64; + let cy_img = cy_img.clamp(0, max_cy) as f64; + let cw = self.content_w as f64; + let ch = self.content_h as f64; + let nw = self.native_w as f64; + let nh = self.native_h as f64; + + #[cfg(target_os = "macos")] + if let Some(g) = self.macos_geo { + let cx = g.crop_x0 as f64 + (cx_img + 0.5) * nw / cw; + let cy = g.crop_y0 as f64 + (cy_img + 0.5) * nh / ch; + return g.full_pixel_center_to_global_f64(cx, cy); + } + + let center_full_x = self.origin_x as f64 + (cx_img + 0.5) * nw / cw; + let center_full_y = self.origin_y as f64 + (cy_img + 0.5) * nh / ch; + Ok((center_full_x, center_full_y)) + } + + /// Normalized 0..=1000 maps to the **capture** (same as pre-margin bitmap; independent of ruler padding). + fn map_normalized_to_global_f64(&self, x: i32, y: i32) -> BitFunResult<(f64, f64)> { + if self.native_w == 0 || self.native_h == 0 { + return Err(BitFunError::tool( + "Invalid screenshot coordinate map (zero native dimension).".to_string(), + )); + } + let nw = self.native_w as f64; + let nh = self.native_h as f64; + let tx = (x.clamp(0, 1000) as f64) / 1000.0; + let ty = (y.clamp(0, 1000) as f64) / 1000.0; + + #[cfg(target_os = "macos")] + if let Some(g) = self.macos_geo { + let cx = g.crop_x0 as f64 + tx * (nw - 1.0).max(0.0) + 0.5; + let cy = g.crop_y0 as f64 + ty * (nh - 1.0).max(0.0) + 0.5; + return g.full_pixel_center_to_global_f64(cx, cy); + } + + let gx = self.origin_x as f64 + tx * (nw - 1.0).max(0.0) + 0.5; + let gy = self.origin_y as f64 + ty * (nh - 1.0).max(0.0) + 0.5; + Ok((gx, gy)) + } +} + +/// What the last tool `screenshot` implied for **plain** follow-up captures (no crop / no `navigate_quadrant`). +/// **PointCrop** is not reused for plain refresh: the next bare `screenshot` shows the **full display** again so +/// "full" is never stuck at ~500×500 after a point crop. **Quadrant** plain refresh keeps the current drill tile. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ComputerUseNavFocus { + FullDisplay, + Quadrant { + rect: ComputerUseNavigationRect, + }, + PointCrop { + rect: ComputerUseNavigationRect, + }, +} + +pub struct DesktopComputerUseHost { + last_pointer_map: Mutex>, + /// When true, a fresh `screenshot_display` is required before `click` and before `key_chord` that sends Return/Enter + /// (set after pointer moves / click; cleared after screenshot). + click_needs_fresh_screenshot: Mutex, + /// Last `screenshot_display` scope (full screen vs point crop) for tool hints and click rules. + last_shot_refinement: Mutex>, + /// Drill / crop context for the next `screenshot` (see [`ComputerUseNavFocus`]). + navigation_focus: Mutex>, +} + +impl std::fmt::Debug for DesktopComputerUseHost { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("DesktopComputerUseHost").finish_non_exhaustive() + } +} + +impl DesktopComputerUseHost { + pub fn new() -> Self { + Self { + last_pointer_map: Mutex::new(None), + click_needs_fresh_screenshot: Mutex::new(true), + last_shot_refinement: Mutex::new(None), + navigation_focus: Mutex::new(None), + } + } + + fn refinement_from_shot(shot: &ComputerScreenshot) -> ComputerUseScreenshotRefinement { + use ComputerUseScreenshotRefinement as R; + if let Some(c) = shot.screenshot_crop_center { + return R::RegionAroundPoint { + center_x: c.x, + center_y: c.y, + }; + } + let Some(nav) = shot.navigation_native_rect else { + return R::FullDisplay; + }; + let full = nav.x0 == 0 + && nav.y0 == 0 + && nav.width == shot.native_width + && nav.height == shot.native_height; + if full { + R::FullDisplay + } else { + R::QuadrantNavigation { + x0: nav.x0, + y0: nav.y0, + width: nav.width, + height: nav.height, + click_ready: shot.quadrant_navigation_click_ready, + } + } + } + + fn ensure_input_automation_allowed() -> BitFunResult<()> { + #[cfg(target_os = "macos")] + { + if macos::ax_trusted() { + return Ok(()); + } + let exe = std::env::current_exe() + .map(|p| p.display().to_string()) + .unwrap_or_else(|_| "(unknown path)".to_string()); + return Err(BitFunError::tool(format!( + "macOS Accessibility is not enabled for this executable. System Settings > Privacy & Security > Accessibility: add and enable BitFun. Development builds use the debug binary at: {}", + exe + ))); + } + #[cfg(not(target_os = "macos"))] + { + Ok(()) + } + } + + fn with_enigo(f: F) -> BitFunResult + where + F: FnOnce(&mut Enigo) -> BitFunResult, + { + Self::ensure_input_automation_allowed()?; + let settings = Settings::default(); + let mut enigo = Enigo::new(&settings) + .map_err(|e| BitFunError::tool(format!("enigo init: {}", e)))?; + f(&mut enigo) + } + + /// Enigo on macOS uses Text Input Source / AppKit paths that must run on the main queue. + /// Tokio `spawn_blocking` threads are not main; dispatch there hits `dispatch_assert_queue_fail`. + fn run_enigo_job(job: F) -> BitFunResult + where + F: FnOnce(&mut Enigo) -> BitFunResult + Send, + T: Send, + { + #[cfg(target_os = "macos")] + { + macos::run_on_main_for_enigo(|| Self::with_enigo(job)) + } + #[cfg(not(target_os = "macos"))] + { + Self::with_enigo(job) + } + } + + /// Absolute pointer move in Quartz global **points** with full float precision (avoids enigo integer truncation). + #[cfg(target_os = "macos")] + fn post_mouse_moved_cg_global(x: f64, y: f64) -> BitFunResult<()> { + use core_graphics::event::{CGEvent, CGEventTapLocation, CGEventType, CGMouseButton}; + use core_graphics::event_source::{CGEventSource, CGEventSourceStateID}; + use core_graphics::geometry::CGPoint; + + let source = CGEventSource::new(CGEventSourceStateID::CombinedSessionState).map_err(|_| { + BitFunError::tool("CGEventSource create failed (mouse_move)".to_string()) + })?; + let pt = CGPoint { x, y }; + let ev = CGEvent::new_mouse_event( + source, + CGEventType::MouseMoved, + pt, + CGMouseButton::Left, + ) + .map_err(|_| BitFunError::tool("CGEvent MouseMoved failed".to_string()))?; + ev.post(CGEventTapLocation::HID); + Ok(()) + } + + fn map_button(s: &str) -> BitFunResult - ))} + ); + })} )} , diff --git a/src/web-ui/src/app/components/NavPanel/NavPanel.scss b/src/web-ui/src/app/components/NavPanel/NavPanel.scss index a9c47d56..bdb8f01a 100644 --- a/src/web-ui/src/app/components/NavPanel/NavPanel.scss +++ b/src/web-ui/src/app/components/NavPanel/NavPanel.scss @@ -871,7 +871,12 @@ $_section-header-height: 24px; &__workspace-menu { position: fixed; - min-width: 156px; + /* Shrink-wrap to widest row; cap width for long paths + viewport on small screens */ + display: inline-block; + width: fit-content; + max-width: min(440px, calc(100vw - 32px)); + vertical-align: top; + box-sizing: border-box; padding: $size-gap-1 0; background: var(--color-bg-elevated, #1e1e22); border: 1px solid var(--border-subtle, rgba(255, 255, 255, 0.08)); @@ -917,8 +922,10 @@ $_section-header-height: 24px; flex-shrink: 0; } - span { + /* Only the label row grows; nested spans inside --main must NOT get flex:1 or text splits across columns */ + > span { flex: 1; + min-width: 0; } &--workspace { @@ -933,6 +940,37 @@ $_section-header-height: 24px; white-space: nowrap; } + &__workspace-menu-item--workspace &__workspace-menu-item-main { + display: flex; + align-items: baseline; + gap: 0.3em; + white-space: nowrap; + } + + &__workspace-menu-item-host { + /* flex-shrink 0: host is not squeezed; max-width ~26ch fits 255.255.255.255 + margin in typical UI fonts */ + flex: 0 0 auto; + max-width: min(26ch, 52%); + overflow: hidden; + text-overflow: ellipsis; + color: var(--color-text-muted); + font-size: 0.92em; + } + + &__workspace-menu-item-host-sep { + flex: 0 0 auto; + color: var(--color-text-muted); + opacity: 0.55; + font-size: 0.92em; + } + + &__workspace-menu-item-name { + flex: 1 1 auto; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + } + &__workspace-menu-divider { height: 1px; margin: $size-gap-1 0; diff --git a/src/web-ui/src/app/components/NavPanel/sections/sessions/SessionsSection.tsx b/src/web-ui/src/app/components/NavPanel/sections/sessions/SessionsSection.tsx index 7545bdb2..56575039 100644 --- a/src/web-ui/src/app/components/NavPanel/sections/sessions/SessionsSection.tsx +++ b/src/web-ui/src/app/components/NavPanel/sections/sessions/SessionsSection.tsx @@ -24,7 +24,10 @@ import { selectActiveBtwSessionTab, } from '@/flow_chat/services/openBtwSession'; import { resolveSessionRelationship } from '@/flow_chat/utils/sessionMetadata'; -import { compareSessionsForDisplay } from '@/flow_chat/utils/sessionOrdering'; +import { + compareSessionsForDisplay, + sessionBelongsToWorkspaceNavRow, +} from '@/flow_chat/utils/sessionOrdering'; import { stateMachineManager } from '@/flow_chat/state-machine'; import { SessionExecutionState } from '@/flow_chat/state-machine/types'; import './SessionsSection.scss'; @@ -52,6 +55,8 @@ interface SessionsSectionProps { workspacePath?: string; /** Remote SSH: same `workspacePath` on different hosts must filter by this (see Session.remoteConnectionId). */ remoteConnectionId?: string | null; + /** Remote SSH: disambiguates same path on different hosts; when set with matching session host, connectionId may differ. */ + remoteSshHost?: string | null; isActiveWorkspace?: boolean; showCreateActions?: boolean; /** When set (e.g. assistant workspace), session row tooltip includes this assistant name. */ @@ -64,6 +69,7 @@ const SessionsSection: React.FC = ({ workspaceId, workspacePath, remoteConnectionId = null, + remoteSshHost = null, isActiveWorkspace = true, assistantLabel, showSessionModeIcon = true, @@ -121,7 +127,7 @@ const SessionsSection: React.FC = ({ useEffect(() => { setExpandLevel(0); - }, [workspaceId, workspacePath, remoteConnectionId]); + }, [workspaceId, workspacePath, remoteConnectionId, remoteSshHost]); useEffect(() => { if (!openMenuSessionId) return; @@ -140,18 +146,12 @@ const SessionsSection: React.FC = ({ Array.from(flowChatState.sessions.values()) .filter((s: Session) => { if (workspacePath) { - if (s.workspacePath !== workspacePath) return false; - const wsConn = remoteConnectionId?.trim() ?? ''; - const sessConn = s.remoteConnectionId?.trim() ?? ''; - if (wsConn.length > 0 || sessConn.length > 0) { - return sessConn === wsConn; - } - return true; + return sessionBelongsToWorkspaceNavRow(s, workspacePath, remoteConnectionId, remoteSshHost); } return !s.workspacePath; }) .sort(compareSessionsForDisplay), - [flowChatState.sessions, workspacePath, remoteConnectionId] + [flowChatState.sessions, workspacePath, remoteConnectionId, remoteSshHost] ); const { topLevelSessions, childrenByParent } = useMemo(() => { diff --git a/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx b/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx index ba7cbd89..fd4b9c09 100644 --- a/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx +++ b/src/web-ui/src/app/components/NavPanel/sections/workspaces/WorkspaceItem.tsx @@ -489,6 +489,7 @@ const WorkspaceItem: React.FC = ({ workspaceId={workspace.id} workspacePath={workspace.rootPath} remoteConnectionId={isRemoteWorkspace(workspace) ? workspace.connectionId : null} + remoteSshHost={isRemoteWorkspace(workspace) ? workspace.sshHost : null} isActiveWorkspace={isActive} assistantLabel={workspaceDisplayName} /> @@ -668,6 +669,7 @@ const WorkspaceItem: React.FC = ({ workspaceId={workspace.id} workspacePath={workspace.rootPath} remoteConnectionId={isRemoteWorkspace(workspace) ? workspace.connectionId : null} + remoteSshHost={isRemoteWorkspace(workspace) ? workspace.sshHost : null} isActiveWorkspace={isActive} /> diff --git a/src/web-ui/src/app/layout/AppLayout.tsx b/src/web-ui/src/app/layout/AppLayout.tsx index 68df4ed0..056de355 100644 --- a/src/web-ui/src/app/layout/AppLayout.tsx +++ b/src/web-ui/src/app/layout/AppLayout.tsx @@ -40,7 +40,14 @@ interface AppLayoutProps { const AppLayout: React.FC = ({ className = '' }) => { const { t } = useI18n('components'); - const { currentWorkspace, hasWorkspace, openWorkspace, recentWorkspaces, loading } = useWorkspaceContext(); + const { + currentWorkspace, + hasWorkspace, + openWorkspace, + switchWorkspace, + recentWorkspaces, + loading, + } = useWorkspaceContext(); const sshContext = useContext(SSHContext); /** When SSH finishes connecting, re-run FlowChat init (first run may have skipped while disconnected). */ const remoteSshFlowChatKey = @@ -68,13 +75,13 @@ const AppLayout: React.FC = ({ className = '' }) => { if (autoOpenAttemptedRef.current || loading) return; if (!hasWorkspace && recentWorkspaces.length > 0) { autoOpenAttemptedRef.current = true; - openWorkspace(recentWorkspaces[0].rootPath).catch(err => { + switchWorkspace(recentWorkspaces[0]).catch(err => { log.warn('Auto-open recent workspace failed', err); }); } else { autoOpenAttemptedRef.current = true; } - }, [hasWorkspace, loading, recentWorkspaces, openWorkspace]); + }, [hasWorkspace, loading, recentWorkspaces, switchWorkspace]); // Dialog state (previously in TitleBar) const [showNewProjectDialog, setShowNewProjectDialog] = useState(false); @@ -153,22 +160,8 @@ const AppLayout: React.FC = ({ className = '' }) => { const initializeFlowChat = async () => { if (!currentWorkspace?.rootPath) return; - // Skip initialization for remote workspaces that are not yet SSH-connected. - // On startup, password-auth remote workspaces cannot auto-reconnect and will - // be removed from the sidebar shortly by SSHRemoteProvider. Attempting to - // initialize FlowChat for them would fail with a misleading error notification. - if (currentWorkspace.workspaceKind === WorkspaceKind.Remote && currentWorkspace.connectionId) { - const { sshApi } = await import('@/features/ssh-remote/sshApi'); - const connected = await sshApi.isConnected(currentWorkspace.connectionId).catch(() => false); - if (!connected) { - log.warn('Skipping FlowChat initialization: remote workspace not connected', { - rootPath: currentWorkspace.rootPath, - connectionId: currentWorkspace.connectionId, - }); - return; - } - } - + // Remote session index and turns live under ~/.bitfun/remote_ssh/... (local disk). + // Always initialize FlowChat so historical sessions list even when SSH is not connected yet. try { const explicitPreferredMode = sessionStorage.getItem('bitfun:flowchat:preferredMode') || @@ -188,6 +181,9 @@ const AppLayout: React.FC = ({ className = '' }) => { initializationPreferredMode, currentWorkspace.workspaceKind === WorkspaceKind.Remote ? currentWorkspace.connectionId + : undefined, + currentWorkspace.workspaceKind === WorkspaceKind.Remote + ? currentWorkspace.sshHost : undefined ); @@ -260,6 +256,7 @@ const AppLayout: React.FC = ({ className = '' }) => { currentWorkspace?.rootPath, currentWorkspace?.workspaceKind, currentWorkspace?.connectionId, + currentWorkspace?.sshHost, remoteSshFlowChatKey, ensureAssistantBootstrapForWorkspace, t, diff --git a/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts b/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts index 4642c9ac..c9dbe5b6 100644 --- a/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts +++ b/src/web-ui/src/app/scenes/settings/settingsTabSearchContent.ts @@ -43,6 +43,10 @@ export const SETTINGS_TAB_SEARCH_CONTENT: Record { {displayRecentWorkspaces.length > 0 ? (
- {displayRecentWorkspaces.map(ws => ( - + {displayRecentWorkspaces.map(ws => { + const { hostPrefix, folderLabel, tooltip } = getRecentWorkspaceLineParts(ws); + return ( + - ))} + ); + })}
) : (

{t('welcomeScene.noRecentWorkspaces')}

diff --git a/src/web-ui/src/features/ssh-remote/SSHRemoteProvider.tsx b/src/web-ui/src/features/ssh-remote/SSHRemoteProvider.tsx index 61eb4276..f1b845f3 100644 --- a/src/web-ui/src/features/ssh-remote/SSHRemoteProvider.tsx +++ b/src/web-ui/src/features/ssh-remote/SSHRemoteProvider.tsx @@ -13,6 +13,21 @@ import { normalizeRemoteWorkspacePath } from '@/shared/utils/pathUtils'; const log = createLogger('SSHRemoteProvider'); +/** Match opened `WorkspaceInfo` so list_sessions maps to ~/.bitfun/remote_ssh/... */ +function sshHostForRemoteWorkspace(connectionId: string, remotePath: string): string | undefined { + const norm = normalizeRemoteWorkspacePath(remotePath); + const cid = connectionId.trim(); + for (const w of workspaceManager.getState().openedWorkspaces.values()) { + if (w.workspaceKind !== WorkspaceKind.Remote) continue; + if ((w.connectionId ?? '').trim() !== cid) continue; + if (normalizeRemoteWorkspacePath(w.rootPath) === norm) { + const h = w.sshHost?.trim(); + if (h) return h; + } + } + return undefined; +} + export type ConnectionStatus = 'disconnected' | 'connecting' | 'connected' | 'error'; interface SSHContextValue { @@ -176,6 +191,7 @@ export const SSHRemoteProvider: React.FC = ({ children } connectionId: result.connectionId, connectionName: savedConn.name, remotePath: workspace.remotePath, + sshHost: reconnectConfig.host?.trim() || workspace.sshHost?.trim() || undefined, }; log.info('Successfully reconnected to remote workspace', { @@ -222,6 +238,7 @@ export const SSHRemoteProvider: React.FC = ({ children } connectionId: ws.connectionId, connectionName: ws.connectionName || 'Remote', remotePath: rp, + sshHost: ws.sshHost?.trim() || undefined, }); } @@ -273,7 +290,14 @@ export const SSHRemoteProvider: React.FC = ({ children } await workspaceManager.openRemoteWorkspace(workspace).catch(() => {}); } // Re-initialize sessions now that the workspace is registered in the state manager - void flowChatStore.initializeFromDisk(workspace.remotePath).catch(() => {}); + void flowChatStore + .initializeFromDisk( + workspace.remotePath, + workspace.connectionId, + workspace.sshHost?.trim() || + sshHostForRemoteWorkspace(workspace.connectionId, workspace.remotePath) + ) + .catch(() => {}); continue; } @@ -297,7 +321,17 @@ export const SSHRemoteProvider: React.FC = ({ children } await workspaceManager.openRemoteWorkspace(result.workspace).catch(() => {}); } // Re-initialize sessions now that the workspace is registered in the state manager - void flowChatStore.initializeFromDisk(result.workspace.remotePath).catch(() => {}); + void flowChatStore + .initializeFromDisk( + result.workspace.remotePath, + result.workspace.connectionId, + result.workspace.sshHost?.trim() || + sshHostForRemoteWorkspace( + result.workspace.connectionId, + result.workspace.remotePath + ) + ) + .catch(() => {}); } else { // Reconnection failed (or skipped for password auth) — remove the workspace // from the sidebar. Password-auth workspaces can never auto-reconnect, and @@ -434,6 +468,7 @@ export const SSHRemoteProvider: React.FC = ({ children } connectionId, connectionName: connName, remotePath, + sshHost: connectionConfig?.host?.trim() || undefined, }; setRemoteWorkspace(remoteWs); setShowFileBrowser(false); diff --git a/src/web-ui/src/features/ssh-remote/types.ts b/src/web-ui/src/features/ssh-remote/types.ts index e256d787..0698a092 100644 --- a/src/web-ui/src/features/ssh-remote/types.ts +++ b/src/web-ui/src/features/ssh-remote/types.ts @@ -67,6 +67,8 @@ export interface RemoteWorkspace { connectionId: string; connectionName: string; remotePath: string; + /** SSH `host` from connection profile; required for correct local session mirror paths. */ + sshHost?: string; } export interface SSHConfigEntry { diff --git a/src/web-ui/src/flow_chat/hooks/useFlowChat.ts b/src/web-ui/src/flow_chat/hooks/useFlowChat.ts index 97884bc5..586baf46 100644 --- a/src/web-ui/src/flow_chat/hooks/useFlowChat.ts +++ b/src/web-ui/src/flow_chat/hooks/useFlowChat.ts @@ -19,6 +19,7 @@ import { aiExperienceConfigService } from '@/infrastructure/config/services'; import { configManager } from '@/infrastructure/config/services/ConfigManager'; import { useI18n } from '@/infrastructure/i18n'; import { useCurrentWorkspace } from '@/infrastructure/contexts/WorkspaceContext'; +import { WorkspaceKind } from '@/shared/types'; import { generateTempTitle } from '../utils/titleUtils'; import { createLogger } from '@/shared/utils/logger'; @@ -60,7 +61,7 @@ async function getModelContextWindow(modelName?: string): Promise { export const useFlowChat = () => { const { t } = useI18n('flow-chat'); - const { workspacePath } = useCurrentWorkspace(); + const { workspacePath, workspace } = useCurrentWorkspace(); const [state, setState] = useState(flowChatStore.getState()); const processingLock = useRef(false); @@ -102,10 +103,16 @@ export const useFlowChat = () => { const maxContextTokens = await getModelContextWindow(config?.modelName); + const isRemote = workspace?.workspaceKind === WorkspaceKind.Remote; + const remoteConnectionId = isRemote ? workspace?.connectionId : undefined; + const remoteSshHost = isRemote ? workspace?.sshHost : undefined; + const response = await agentAPI.createSession({ sessionName, agentType: 'agentic', // Default to agentic; can change via mode selector. workspacePath, + remoteConnectionId, + remoteSshHost, config: { modelName: config?.modelName || 'default', enableTools: true, @@ -113,6 +120,8 @@ export const useFlowChat = () => { autoCompact: true, maxContextTokens: maxContextTokens, enableContextCompression: true, + remoteConnectionId, + remoteSshHost, } }); @@ -134,13 +143,19 @@ export const useFlowChat = () => { sessionName, maxContextTokens, undefined, - workspacePath + workspacePath, + remoteConnectionId, + remoteSshHost ); return response.sessionId; } catch (error) { log.error('Failed to create session', { error }); + + const isRemoteFb = workspace?.workspaceKind === WorkspaceKind.Remote; + const remoteConnectionIdFb = isRemoteFb ? workspace?.connectionId : undefined; + const remoteSshHostFb = isRemoteFb ? workspace?.sshHost : undefined; // Fallback to a frontend-only session without Terminal. const sessionId = `session_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; @@ -169,14 +184,16 @@ export const useFlowChat = () => { sessionName, undefined, undefined, - workspacePath + workspacePath, + remoteConnectionIdFb, + remoteSshHostFb ); log.warn('Using fallback mode without Terminal'); return sessionId; } - }, [t, workspacePath]); + }, [t, workspacePath, workspace]); const switchSession = useCallback(async (sessionId: string) => { try { diff --git a/src/web-ui/src/flow_chat/services/BtwThreadService.ts b/src/web-ui/src/flow_chat/services/BtwThreadService.ts index 5e85b1f7..b6f91ad5 100644 --- a/src/web-ui/src/flow_chat/services/BtwThreadService.ts +++ b/src/web-ui/src/flow_chat/services/BtwThreadService.ts @@ -95,18 +95,22 @@ export async function startBtwThread(params: { const modelName = parentSession?.config?.modelName || 'default'; const childSessionName = buildChildSessionName(question); const remoteConnectionId = parentSession?.remoteConnectionId; + const remoteSshHost = parentSession?.remoteSshHost; const created = await agentAPI.createSession({ sessionName: childSessionName, agentType, workspacePath, remoteConnectionId, + remoteSshHost, config: { modelName, enableTools: false, safeMode: true, autoCompact: true, enableContextCompression: true, + remoteConnectionId, + remoteSshHost, }, }); @@ -127,7 +131,8 @@ export async function startBtwThread(params: { parentTurnIndex, }, }, - remoteConnectionId + remoteConnectionId, + remoteSshHost ); flowChatStore.updateSessionRelationship(childSessionId, { parentSessionId, sessionKind: 'btw' }); flowChatStore.updateSessionBtwOrigin(childSessionId, { diff --git a/src/web-ui/src/flow_chat/services/FlowChatManager.ts b/src/web-ui/src/flow_chat/services/FlowChatManager.ts index dc7d1c94..05a2a5ce 100644 --- a/src/web-ui/src/flow_chat/services/FlowChatManager.ts +++ b/src/web-ui/src/flow_chat/services/FlowChatManager.ts @@ -13,7 +13,10 @@ import { AgentService } from '../../shared/services/agent-service'; import { stateMachineManager } from '../state-machine'; import { EventBatcher } from './EventBatcher'; import { createLogger } from '@/shared/utils/logger'; -import { compareSessionsForDisplay } from '../utils/sessionOrdering'; +import { + compareSessionsForDisplay, + sessionBelongsToWorkspaceNavRow, +} from '../utils/sessionOrdering'; import type { FlowChatContext, SessionConfig, DialogTurn } from './flow-chat-manager/types'; import type { FlowToolItem, FlowTextItem, ModelRound } from '../types/flow-chat'; @@ -73,20 +76,34 @@ export class FlowChatManager { async initialize( workspacePath: string, preferredMode?: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { await this.initializeEventListeners(); - await this.context.flowChatStore.initializeFromDisk(workspacePath, remoteConnectionId); - - const wsConn = remoteConnectionId?.trim() ?? ''; - const sessionMatchesWorkspace = (session: { workspacePath?: string; remoteConnectionId?: string }) => { - if ((session.workspacePath || workspacePath) !== workspacePath) return false; - const sc = session.remoteConnectionId?.trim() ?? ''; - if (wsConn.length > 0 || sc.length > 0) { - return sc === wsConn; - } - return true; + await this.context.flowChatStore.initializeFromDisk( + workspacePath, + remoteConnectionId, + remoteSshHost + ); + + const sessionMatchesWorkspace = (session: { + workspacePath?: string; + remoteConnectionId?: string; + remoteSshHost?: string; + }) => { + const sp = session.workspacePath || workspacePath; + if (sp !== workspacePath) return false; + return sessionBelongsToWorkspaceNavRow( + { + workspacePath: sp, + remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, + }, + workspacePath, + remoteConnectionId, + remoteSshHost + ); }; const state = this.context.flowChatStore.getState(); @@ -120,7 +137,8 @@ export class FlowChatManager { latestSession.sessionId, workspacePath, undefined, - latestSession.remoteConnectionId + latestSession.remoteConnectionId, + latestSession.remoteSshHost ); } @@ -178,12 +196,16 @@ export class FlowChatManager { ensureAssistantBootstrap?: boolean; /** When set, only removes/reinits sessions for this SSH connection (same path, different hosts). */ remoteConnectionId?: string | null; + /** Disambiguates remote workspaces that share the same `workspacePath` (e.g. `/` on different hosts). */ + remoteSshHost?: string | null; } ): Promise { const remoteConnectionId = options?.remoteConnectionId; + const remoteSshHost = options?.remoteSshHost; const removedSessionIds = this.context.flowChatStore.removeSessionsByWorkspace( workspacePath, - remoteConnectionId + remoteConnectionId, + remoteSshHost ); removedSessionIds.forEach(sessionId => { @@ -200,29 +222,32 @@ export class FlowChatManager { const hasHistoricalSessions = await this.initialize( workspacePath, options.preferredMode, - remoteConnectionId ?? undefined + remoteConnectionId ?? undefined, + remoteSshHost ?? undefined ); const state = this.context.flowChatStore.getState(); const activeSession = state.activeSessionId ? state.sessions.get(state.activeSessionId) ?? null : null; - const wsConn = remoteConnectionId?.trim() ?? ''; const hasActiveWorkspaceSession = !!activeSession && - (activeSession.workspacePath || workspacePath) === workspacePath && - (() => { - const sc = activeSession.remoteConnectionId?.trim() ?? ''; - if (wsConn.length > 0 || sc.length > 0) { - return sc === wsConn; - } - return true; - })(); + sessionBelongsToWorkspaceNavRow( + { + workspacePath: activeSession.workspacePath || workspacePath, + remoteConnectionId: activeSession.remoteConnectionId, + remoteSshHost: activeSession.remoteSshHost, + }, + workspacePath, + remoteConnectionId, + remoteSshHost + ); if (!hasHistoricalSessions || !hasActiveWorkspaceSession) { await this.createChatSession( { workspacePath, ...(remoteConnectionId ? { remoteConnectionId } : {}), + ...(remoteSshHost ? { remoteSshHost } : {}), }, options.preferredMode ); diff --git a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts index 1c55057f..a80e7398 100644 --- a/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts +++ b/src/web-ui/src/flow_chat/services/flow-chat-manager/EventHandlerModule.ts @@ -217,7 +217,10 @@ function handleSessionCreated(context: FlowChatContext, event: any): void { sessionId, sessionName || 'Remote Session', agentType || 'agentic', - resolveExternalSessionWorkspacePath(context, event) + resolveExternalSessionWorkspacePath(context, event), + undefined, + extractEventRemoteConnectionId(event), + extractEventRemoteSshHost(event) ); } @@ -234,6 +237,24 @@ function resolveExternalSessionWorkspacePath( return candidate || undefined; } +function extractEventRemoteConnectionId(event?: Record | null): string | undefined { + if (!event) return undefined; + const id = + (typeof event.remoteConnectionId === 'string' && event.remoteConnectionId) || + (typeof event.remote_connection_id === 'string' && event.remote_connection_id) || + undefined; + return id?.trim() || undefined; +} + +function extractEventRemoteSshHost(event?: Record | null): string | undefined { + if (!event) return undefined; + const h = + (typeof event.remoteSshHost === 'string' && event.remoteSshHost) || + (typeof event.remote_ssh_host === 'string' && event.remote_ssh_host) || + undefined; + return h?.trim() || undefined; +} + /** * Handle session title generated event (from AI auto-generation) */ @@ -312,7 +333,10 @@ function handleImageAnalysisStarted(context: FlowChatContext, event: ImageAnalys sessionId, 'Remote Session', 'agentic', - resolveExternalSessionWorkspacePath(context, event as any) + resolveExternalSessionWorkspacePath(context, event as any), + undefined, + extractEventRemoteConnectionId(event as any), + extractEventRemoteSshHost(event as any) ); session = store.getState().sessions.get(sessionId); } @@ -466,7 +490,10 @@ function handleDialogTurnStarted(context: FlowChatContext, event: any): void { sessionId, 'Remote Session', 'agentic', - resolveExternalSessionWorkspacePath(context, event) + resolveExternalSessionWorkspacePath(context, event), + undefined, + extractEventRemoteConnectionId(event), + extractEventRemoteSshHost(event) ); } diff --git a/src/web-ui/src/flow_chat/services/flow-chat-manager/PersistenceModule.ts b/src/web-ui/src/flow_chat/services/flow-chat-manager/PersistenceModule.ts index 84bbf304..ceb58563 100644 --- a/src/web-ui/src/flow_chat/services/flow-chat-manager/PersistenceModule.ts +++ b/src/web-ui/src/flow_chat/services/flow-chat-manager/PersistenceModule.ts @@ -233,7 +233,12 @@ async function performSaveDialogTurnToDisk( const turnIndex = dialogTurn.backendTurnIndex ?? session.dialogTurns.indexOf(dialogTurn); const turnData = convertDialogTurnToBackendFormat(dialogTurn, turnIndex); - await sessionAPI.saveSessionTurn(turnData, workspacePath, session.remoteConnectionId); + await sessionAPI.saveSessionTurn( + turnData, + workspacePath, + session.remoteConnectionId, + session.remoteSshHost + ); await updateSessionMetadata(context, sessionId); @@ -407,7 +412,8 @@ export async function updateSessionMetadata( existingMetadata = await sessionAPI.loadSessionMetadata( sessionId, workspacePath, - session.remoteConnectionId + session.remoteConnectionId, + session.remoteSshHost ); } catch { // ignore @@ -415,7 +421,12 @@ export async function updateSessionMetadata( const metadata = buildSessionMetadata(session, existingMetadata); - await sessionAPI.saveSessionMetadata(metadata, workspacePath, session.remoteConnectionId); + await sessionAPI.saveSessionMetadata( + metadata, + workspacePath, + session.remoteConnectionId, + session.remoteSshHost + ); } catch (error) { log.warn('Failed to update session metadata', { sessionId, error }); } @@ -427,14 +438,16 @@ export async function updateSessionMetadata( export async function touchSessionActivity( sessionId: string, workspacePath?: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { const { sessionAPI } = await import('@/infrastructure/api'); await sessionAPI.touchSessionActivity( sessionId, requireWorkspacePath(sessionId, workspacePath), - remoteConnectionId + remoteConnectionId, + remoteSshHost ); } catch (error) { log.debug('Failed to touch session activity', { sessionId, error }); diff --git a/src/web-ui/src/flow_chat/services/flow-chat-manager/SessionModule.ts b/src/web-ui/src/flow_chat/services/flow-chat-manager/SessionModule.ts index 00333a65..841500e6 100644 --- a/src/web-ui/src/flow_chat/services/flow-chat-manager/SessionModule.ts +++ b/src/web-ui/src/flow_chat/services/flow-chat-manager/SessionModule.ts @@ -66,9 +66,15 @@ const resolveSessionWorkspace = ( if (!workspacePath) return null; const state = workspaceManager.getState(); - const pathMatches = Array.from(state.openedWorkspaces.values()).filter( - workspace => workspace.rootPath === workspacePath - ); + const pathMatches = Array.from(state.openedWorkspaces.values()).filter(workspace => { + if (workspace.rootPath !== workspacePath) return false; + if (workspace.workspaceKind !== WorkspaceKind.Remote) return true; + const cid = config?.remoteConnectionId?.trim(); + const host = config?.remoteSshHost?.trim(); + if (cid && workspace.connectionId !== cid) return false; + if (host && (workspace.sshHost?.trim() ?? '') !== host) return false; + return true; + }); if (pathMatches.length === 0) { return state.currentWorkspace; } @@ -80,6 +86,11 @@ const resolveSessionWorkspace = ( const byConn = pathMatches.find(w => w.connectionId === configCid); if (byConn) return byConn; } + const configHost = config?.remoteSshHost?.trim(); + if (configHost) { + const byHost = pathMatches.find(w => (w.sshHost?.trim() ?? '') === configHost); + if (byHost) return byHost; + } const cur = state.currentWorkspace; if (cur && pathMatches.some(w => w.id === cur.id)) { return cur; @@ -157,6 +168,10 @@ export async function createChatSession( } const remoteConnectionId = workspace?.workspaceKind === WorkspaceKind.Remote ? workspace.connectionId : undefined; + const remoteSshHost = + workspace?.workspaceKind === WorkspaceKind.Remote + ? workspace.sshHost?.trim() || undefined + : undefined; const agentType = resolveAgentType(mode, workspace); const sessionMode = normalizeSessionDisplayMode(agentType, workspace); const creationKey = @@ -188,6 +203,7 @@ export async function createChatSession( agentType, workspacePath, remoteConnectionId, + remoteSshHost, config: { modelName: config.modelName || 'auto', enableTools: true, @@ -195,6 +211,8 @@ export async function createChatSession( autoCompact: true, maxContextTokens: maxContextTokens, enableContextCompression: true, + remoteConnectionId, + remoteSshHost, } }); @@ -206,7 +224,8 @@ export async function createChatSession( maxContextTokens, agentType, workspacePath, - remoteConnectionId + remoteConnectionId, + remoteSshHost ); return response.sessionId; @@ -248,11 +267,17 @@ export async function switchChatSession( sessionId, workspacePath, undefined, - session.remoteConnectionId + session.remoteConnectionId, + session.remoteSshHost ); try { - await agentAPI.restoreSession(sessionId, workspacePath, session.remoteConnectionId); + await agentAPI.restoreSession( + sessionId, + workspacePath, + session.remoteConnectionId, + session.remoteSshHost + ); context.flowChatStore.setState(prev => { const newSessions = new Map(prev.sessions); @@ -272,10 +297,13 @@ export async function switchChatSession( agentType: currentSession.mode || 'agentic', workspacePath, remoteConnectionId: currentSession.remoteConnectionId, + remoteSshHost: currentSession.remoteSshHost, config: { modelName: currentSession.config.modelName || 'auto', enableTools: true, - safeMode: true + safeMode: true, + remoteConnectionId: currentSession.remoteConnectionId, + remoteSshHost: currentSession.remoteSshHost, } }); @@ -302,7 +330,8 @@ export async function switchChatSession( touchSessionActivity( sessionId, session?.workspacePath, - session?.remoteConnectionId + session?.remoteConnectionId, + session?.remoteSshHost ).catch(error => { log.debug('Failed to touch session activity', { sessionId, error }); }); @@ -376,6 +405,7 @@ export async function ensureBackendSession( sessionId, workspacePath, remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, }); clearHistoricalFlag(); } catch (e: any) { @@ -395,10 +425,13 @@ export async function ensureBackendSession( agentType: session.mode || 'agentic', workspacePath, remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, config: { modelName: session.config.modelName || 'auto', enableTools: true, - safeMode: true + safeMode: true, + remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, } }); clearHistoricalFlag(); @@ -425,10 +458,13 @@ export async function retryCreateBackendSession( agentType: session.mode || 'agentic', workspacePath, remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, config: { modelName: session.config.modelName || 'auto', enableTools: true, - safeMode: true + safeMode: true, + remoteConnectionId: session.remoteConnectionId, + remoteSshHost: session.remoteSshHost, } }); } diff --git a/src/web-ui/src/flow_chat/services/openBtwSession.ts b/src/web-ui/src/flow_chat/services/openBtwSession.ts index 597f817a..211b1a1f 100644 --- a/src/web-ui/src/flow_chat/services/openBtwSession.ts +++ b/src/web-ui/src/flow_chat/services/openBtwSession.ts @@ -86,7 +86,7 @@ export async function openMainSession( sessionId: string, options?: { workspaceId?: string; - activateWorkspace?: (workspaceId: string) => Promise | void; + activateWorkspace?: (workspaceId: string) => void | Promise; } ): Promise { useSceneStore.getState().openScene('session'); diff --git a/src/web-ui/src/flow_chat/store/FlowChatStore.ts b/src/web-ui/src/flow_chat/store/FlowChatStore.ts index 1bb5c3d8..6a5ea74b 100644 --- a/src/web-ui/src/flow_chat/store/FlowChatStore.ts +++ b/src/web-ui/src/flow_chat/store/FlowChatStore.ts @@ -175,7 +175,8 @@ export class FlowChatStore { maxContextTokens?: number, mode?: string, workspacePath?: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): void { import('../state-machine').then(({ stateMachineManager }) => { stateMachineManager.getOrCreate(sessionId); @@ -198,6 +199,7 @@ export class FlowChatStore { mode: mode || 'agentic', workspacePath, remoteConnectionId, + remoteSshHost, parentSessionId: relationship.parentSessionId, sessionKind: relationship.sessionKind, btwThreads: [], @@ -225,7 +227,8 @@ export class FlowChatStore { mode: string, workspacePath?: string, meta?: { parentSessionId?: string; sessionKind?: SessionKind; btwOrigin?: Session['btwOrigin'] }, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): void { import('../state-machine').then(({ stateMachineManager }) => { stateMachineManager.getOrCreate(sessionId); @@ -253,6 +256,7 @@ export class FlowChatStore { isHistorical: false, workspacePath, remoteConnectionId, + remoteSshHost, parentSessionId: relationship.parentSessionId, sessionKind: relationship.sessionKind, btwThreads: [], @@ -521,12 +525,18 @@ export class FlowChatStore { const { agentAPI } = await import('@/infrastructure/api'); const deleteResults = await Promise.allSettled( sessionIdsToDelete.map(async id => { - const workspacePath = this.state.sessions.get(id)?.workspacePath; + const sess = this.state.sessions.get(id); + const workspacePath = sess?.workspacePath; if (!workspacePath) { throw new Error(`Workspace path not found for session ${id}`); } - await agentAPI.deleteSession(id, workspacePath); + await agentAPI.deleteSession( + id, + workspacePath, + sess?.remoteConnectionId, + sess?.remoteSshHost + ); }) ); @@ -635,14 +645,23 @@ export class FlowChatStore { }); } - public removeSessionsByWorkspace(workspacePath: string, remoteConnectionId?: string | null): string[] { + public removeSessionsByWorkspace( + workspacePath: string, + remoteConnectionId?: string | null, + remoteSshHost?: string | null + ): string[] { const wsConn = remoteConnectionId?.trim() ?? ''; + const wsHost = remoteSshHost?.trim() ?? ''; const removedSessionIds = Array.from(this.state.sessions.values()) .filter(session => { if (session.workspacePath !== workspacePath) return false; const sc = session.remoteConnectionId?.trim() ?? ''; if (wsConn.length > 0 || sc.length > 0) { - return sc === wsConn; + if (sc !== wsConn) return false; + } + const sh = session.remoteSshHost?.trim() ?? ''; + if (wsHost.length > 0 || sh.length > 0) { + return sh === wsHost; } return true; }) @@ -1253,14 +1272,16 @@ export class FlowChatStore { const metadata = await sessionAPI.loadSessionMetadata( sessionId, workspacePath, - session.remoteConnectionId + session.remoteConnectionId, + session.remoteSshHost ); const nextMetadata = buildSessionMetadata(session, metadata); await sessionAPI.saveSessionMetadata( nextMetadata, workspacePath, - session.remoteConnectionId + session.remoteConnectionId, + session.remoteSshHost ); } catch (error) { log.error('Failed to sync session title', { sessionId, error }); @@ -1451,7 +1472,12 @@ export class FlowChatStore { status: 'cancelled' as const }; - await sessionAPI.saveSessionTurn(turnData, workspacePath, session.remoteConnectionId); + await sessionAPI.saveSessionTurn( + turnData, + workspacePath, + session.remoteConnectionId, + session.remoteSshHost + ); } catch (error) { log.error('Failed to save cancelled dialog turn', { sessionId, turnId, error }); } @@ -1462,11 +1488,15 @@ export class FlowChatStore { * Initialize by loading persisted session metadata from disk * Clears sessions from other workspaces, then loads sessions for the target workspace. */ - public async initializeFromDisk(workspacePath: string, remoteConnectionId?: string): Promise { + public async initializeFromDisk( + workspacePath: string, + remoteConnectionId?: string, + remoteSshHost?: string + ): Promise { try { const { sessionAPI } = await import('@/infrastructure/api'); - const sessions = await sessionAPI.listSessions(workspacePath, remoteConnectionId); - + const sessions = await sessionAPI.listSessions(workspacePath, remoteConnectionId, remoteSshHost); + const { stateMachineManager } = await import('../state-machine'); sessions.forEach(metadata => { stateMachineManager.getOrCreate(metadata.sessionId); @@ -1540,8 +1570,8 @@ export class FlowChatStore { maxContextTokens, mode: validatedAgentType, workspacePath: (metadata as any).workspacePath || workspacePath, - remoteConnectionId: - (metadata as any).remoteConnectionId || remoteConnectionId, + remoteConnectionId: metadata.remoteConnectionId || remoteConnectionId, + remoteSshHost: metadata.remoteSshHost || remoteSshHost, parentSessionId: relationship.parentSessionId, sessionKind: relationship.sessionKind, btwThreads: [], @@ -1571,7 +1601,8 @@ export class FlowChatStore { sessionId: string, workspacePath: string, limit?: number, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { const { stateMachineManager } = await import('../state-machine'); @@ -1579,7 +1610,7 @@ export class FlowChatStore { try { const { agentAPI } = await import('@/infrastructure/api'); - await agentAPI.restoreSession(sessionId, workspacePath, remoteConnectionId); + await agentAPI.restoreSession(sessionId, workspacePath, remoteConnectionId, remoteSshHost); } catch (error) { log.warn('Backend session restore failed (may be new session)', { sessionId, error }); } @@ -1589,7 +1620,8 @@ export class FlowChatStore { sessionId, workspacePath, limit, - remoteConnectionId + remoteConnectionId, + remoteSshHost ); const dialogTurns = this.convertToDialogTurns(turns); diff --git a/src/web-ui/src/flow_chat/tool-cards/TerminalToolCard.tsx b/src/web-ui/src/flow_chat/tool-cards/TerminalToolCard.tsx index fcbb33b0..6356d0ac 100644 --- a/src/web-ui/src/flow_chat/tool-cards/TerminalToolCard.tsx +++ b/src/web-ui/src/flow_chat/tool-cards/TerminalToolCard.tsx @@ -289,11 +289,58 @@ export const TerminalToolCard: React.FC = ({ createTerminalTab(terminalSessionId, terminalName); }, [terminalSessionId]); - const output = toolResult?.result?.output || ''; - const exitCode = toolResult?.result?.exit_code ?? 0; - const workingDir = toolResult?.result?.working_directory || ''; - const executionTimeMs = toolResult?.result?.execution_time_ms; - const wasInterrupted = toolResult?.result?.interrupted || false; + const { + output, + exitCode, + workingDir, + executionTimeMs, + wasInterrupted, + } = useMemo(() => { + const raw = toolResult?.result; + let rec: Record | null = null; + if (raw != null && typeof raw === 'string') { + try { + rec = JSON.parse(raw) as Record; + } catch { + rec = null; + } + } else if (raw != null && typeof raw === 'object') { + rec = raw as Record; + } + + if (!rec) { + return { + output: '', + exitCode: 0, + workingDir: '', + executionTimeMs: undefined as number | undefined, + wasInterrupted: false, + }; + } + + const stdout = typeof rec.stdout === 'string' ? rec.stdout : ''; + const stderr = typeof rec.stderr === 'string' ? rec.stderr : ''; + const combinedOut = [stdout, stderr].filter((s) => s.length > 0).join('\n'); + const outputField = typeof rec.output === 'string' ? rec.output : ''; + const output = outputField || combinedOut; + + const exitRaw = rec.exit_code; + const exitCode = typeof exitRaw === 'number' ? exitRaw : 0; + + const workingDir = + typeof rec.working_directory === 'string' ? rec.working_directory : ''; + + const execInResult = + typeof rec.execution_time_ms === 'number' ? rec.execution_time_ms : undefined; + const durationInResult = + typeof rec.duration_ms === 'number' ? rec.duration_ms : undefined; + const executionTimeMs = + execInResult ?? durationInResult ?? toolResult?.duration_ms; + + const wasInterrupted = Boolean(rec.interrupted); + + return { output, exitCode, workingDir, executionTimeMs, wasInterrupted }; + }, [toolResult?.result, toolResult?.duration_ms]); const isLoading = status === 'preparing' || status === 'streaming' || status === 'running'; const isFailed = status === 'error'; diff --git a/src/web-ui/src/flow_chat/types/flow-chat.ts b/src/web-ui/src/flow_chat/types/flow-chat.ts index 03cb7d4e..7ffdf685 100644 --- a/src/web-ui/src/flow_chat/types/flow-chat.ts +++ b/src/web-ui/src/flow_chat/types/flow-chat.ts @@ -190,6 +190,9 @@ export interface Session { /** SSH remote: same `workspacePath` on different hosts must not share coordinator/persistence. */ remoteConnectionId?: string; + /** SSH config host for `~/.bitfun/remote_ssh/{host}/...` session paths when disconnected. */ + remoteSshHost?: string; + /** * Optional parent session id for hierarchical sessions. * Used by /btw "side threads" and potentially other derived sessions. @@ -234,6 +237,7 @@ export interface SessionConfig { workspacePath?: string; /** Disambiguates sessions when multiple remote workspaces share the same `workspacePath`. */ remoteConnectionId?: string; + remoteSshHost?: string; } export interface QueuedMessage { diff --git a/src/web-ui/src/flow_chat/utils/sessionMetadata.ts b/src/web-ui/src/flow_chat/utils/sessionMetadata.ts index 3d164d8d..69885ace 100644 --- a/src/web-ui/src/flow_chat/utils/sessionMetadata.ts +++ b/src/web-ui/src/flow_chat/utils/sessionMetadata.ts @@ -194,6 +194,8 @@ export function buildSessionMetadata( | 'config' | 'createdAt' | 'workspacePath' + | 'remoteConnectionId' + | 'remoteSshHost' | 'todos' | 'dialogTurns' | 'sessionKind' @@ -245,5 +247,8 @@ export function buildSessionMetadata( ), todos: session.todos || existingMetadata?.todos || [], workspacePath: session.workspacePath || existingMetadata?.workspacePath, + remoteConnectionId: + session.remoteConnectionId ?? existingMetadata?.remoteConnectionId, + remoteSshHost: session.remoteSshHost ?? existingMetadata?.remoteSshHost, }; } diff --git a/src/web-ui/src/flow_chat/utils/sessionOrdering.ts b/src/web-ui/src/flow_chat/utils/sessionOrdering.ts index bce27712..d4353795 100644 --- a/src/web-ui/src/flow_chat/utils/sessionOrdering.ts +++ b/src/web-ui/src/flow_chat/utils/sessionOrdering.ts @@ -1,4 +1,63 @@ import type { Session } from '../types/flow-chat'; +import { normalizeRemoteWorkspacePath } from '@/shared/utils/pathUtils'; + +/** Extract `host` from our saved form `ssh-{user}@{host}:{port}` (used when metadata omits `remoteSshHost`). */ +function hostFromSshConnectionId(connectionId: string): string | null { + const t = connectionId.trim(); + const m = t.match(/^ssh-[^@]+@(.+):(\d+)$/); + return m ? m[1].trim().toLowerCase() : null; +} + +/** Row-level SSH host: prefer workspace metadata, else parse from `connectionId` (sidebar may lack `sshHost`). */ +function effectiveWorkspaceSshHost( + remoteSshHost?: string | null, + remoteConnectionId?: string | null +): string { + const h = remoteSshHost?.trim().toLowerCase() ?? ''; + if (h) return h; + return hostFromSshConnectionId(remoteConnectionId?.trim() ?? '') ?? ''; +} + +/** + * Whether a persisted session belongs to a nav row for this workspace. + * Remote mirror lists sessions by host+path on disk; metadata `workspacePath` / `remoteSshHost` can be stale, + * so we must match by SSH host (from metadata or embedded in connection id) before rejecting on path alone. + */ +export function sessionBelongsToWorkspaceNavRow( + session: Pick, + workspacePath: string, + remoteConnectionId?: string | null, + remoteSshHost?: string | null +): boolean { + const wp = normalizeRemoteWorkspacePath(workspacePath); + const sp = normalizeRemoteWorkspacePath(session.workspacePath || workspacePath); + + const wsConn = remoteConnectionId?.trim() ?? ''; + const sessConn = session.remoteConnectionId?.trim() ?? ''; + const wsHostEff = effectiveWorkspaceSshHost(remoteSshHost, remoteConnectionId); + const sessHost = session.remoteSshHost?.trim().toLowerCase() ?? ''; + const sessConnHost = hostFromSshConnectionId(sessConn); + const wsConnHost = hostFromSshConnectionId(wsConn); + + if (wsHostEff.length > 0) { + if (sessHost === wsHostEff) { + return true; + } + if (sessConnHost === wsHostEff) { + return true; + } + if (sessConnHost && wsConnHost && sessConnHost === wsConnHost) { + return sp === wp; + } + } + + if (sp !== wp) return false; + + if (wsConn.length > 0 || sessConn.length > 0) { + return sessConn === wsConn; + } + return true; +} export function getSessionSortTimestamp(session: Pick): number { return session.lastFinishedAt ?? session.createdAt; diff --git a/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts b/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts index 5bb823a7..4628cca6 100644 --- a/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/AgentAPI.ts @@ -24,6 +24,7 @@ export interface SessionConfig { enableContextCompression?: boolean; compressionThreshold?: number; remoteConnectionId?: string; + remoteSshHost?: string; } @@ -33,6 +34,7 @@ export interface CreateSessionRequest { agentType: string; workspacePath: string; remoteConnectionId?: string; + remoteSshHost?: string; config?: SessionConfig; } @@ -212,10 +214,15 @@ export class AgentAPI { } - async deleteSession(sessionId: string, workspacePath: string, remoteConnectionId?: string): Promise { + async deleteSession( + sessionId: string, + workspacePath: string, + remoteConnectionId?: string, + remoteSshHost?: string + ): Promise { try { await api.invoke('delete_session', { - request: { sessionId, workspacePath, remoteConnectionId } + request: { sessionId, workspacePath, remoteConnectionId, remoteSshHost } }); } catch (error) { throw createTauriCommandError('delete_session', error, { sessionId, workspacePath }); @@ -223,10 +230,15 @@ export class AgentAPI { } - async restoreSession(sessionId: string, workspacePath: string, remoteConnectionId?: string): Promise { + async restoreSession( + sessionId: string, + workspacePath: string, + remoteConnectionId?: string, + remoteSshHost?: string + ): Promise { try { return await api.invoke('restore_session', { - request: { sessionId, workspacePath, remoteConnectionId }, + request: { sessionId, workspacePath, remoteConnectionId, remoteSshHost }, }); } catch (error) { throw createTauriCommandError('restore_session', error, { sessionId, workspacePath }); @@ -241,6 +253,7 @@ export class AgentAPI { sessionId: string; workspacePath: string; remoteConnectionId?: string; + remoteSshHost?: string; }): Promise { try { await api.invoke('ensure_coordinator_session', { request }); @@ -259,10 +272,14 @@ export class AgentAPI { - async listSessions(workspacePath: string, remoteConnectionId?: string): Promise { + async listSessions( + workspacePath: string, + remoteConnectionId?: string, + remoteSshHost?: string + ): Promise { try { return await api.invoke('list_sessions', { - request: { workspacePath, remoteConnectionId }, + request: { workspacePath, remoteConnectionId, remoteSshHost }, }); } catch (error) { throw createTauriCommandError('list_sessions', error, { workspacePath }); diff --git a/src/web-ui/src/infrastructure/api/service-api/GlobalAPI.ts b/src/web-ui/src/infrastructure/api/service-api/GlobalAPI.ts index 324451c3..2b758760 100644 --- a/src/web-ui/src/infrastructure/api/service-api/GlobalAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/GlobalAPI.ts @@ -56,6 +56,8 @@ export interface WorkspaceInfo { worktree?: WorkspaceWorktreeInfo | null; connectionId?: string; connectionName?: string; + /** With `rootPath`, forms logical key `{sshHost}:{rootPath}`; local uses `localhost`. */ + sshHost?: string; } export interface UpdateAppStatusRequest { @@ -70,6 +72,8 @@ export interface OpenRemoteWorkspaceRequest { remotePath: string; connectionId: string; connectionName: string; + /** Passed through to Rust so session files map to ~/.bitfun/remote_ssh/{host}/... before/during connect. */ + sshHost?: string; } export interface CreateAssistantWorkspaceRequest {} @@ -143,13 +147,29 @@ export class GlobalAPI { } } - async openRemoteWorkspace(remotePath: string, connectionId: string, connectionName: string): Promise { + async openRemoteWorkspace( + remotePath: string, + connectionId: string, + connectionName: string, + sshHost?: string + ): Promise { try { + const h = sshHost?.trim(); return await api.invoke('open_remote_workspace', { - request: { remotePath, connectionId, connectionName } + request: { + remotePath, + connectionId, + connectionName, + ...(h ? { sshHost: h } : {}), + }, }); } catch (error) { - throw createTauriCommandError('open_remote_workspace', error, { remotePath, connectionId, connectionName }); + throw createTauriCommandError('open_remote_workspace', error, { + remotePath, + connectionId, + connectionName, + sshHost, + }); } } diff --git a/src/web-ui/src/infrastructure/api/service-api/SessionAPI.ts b/src/web-ui/src/infrastructure/api/service-api/SessionAPI.ts index d225984f..cfea56ff 100644 --- a/src/web-ui/src/infrastructure/api/service-api/SessionAPI.ts +++ b/src/web-ui/src/infrastructure/api/service-api/SessionAPI.ts @@ -3,17 +3,31 @@ import { api } from './ApiClient'; import { createTauriCommandError } from '../errors/TauriCommandError'; import type { SessionMetadata, DialogTurnData } from '@/shared/types/session-history'; -function remoteConnField(remoteConnectionId?: string): Record { - return remoteConnectionId ? { remote_connection_id: remoteConnectionId } : {}; +function remoteSessionFields( + remoteConnectionId?: string, + remoteSshHost?: string +): Record { + const o: Record = {}; + if (remoteConnectionId) { + o.remote_connection_id = remoteConnectionId; + } + if (remoteSshHost) { + o.remote_ssh_host = remoteSshHost; + } + return o; } export class SessionAPI { - async listSessions(workspacePath: string, remoteConnectionId?: string): Promise { + async listSessions( + workspacePath: string, + remoteConnectionId?: string, + remoteSshHost?: string + ): Promise { try { return await api.invoke('list_persisted_sessions', { request: { workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { @@ -25,13 +39,14 @@ export class SessionAPI { sessionId: string, workspacePath: string, limit?: number, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { const request: Record = { session_id: sessionId, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), }; if (limit !== undefined) { @@ -49,14 +64,15 @@ export class SessionAPI { async saveSessionTurn( turnData: DialogTurnData, workspacePath: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { await api.invoke('save_session_turn', { request: { turn_data: turnData, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { @@ -67,14 +83,15 @@ export class SessionAPI { async saveSessionMetadata( metadata: SessionMetadata, workspacePath: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { await api.invoke('save_session_metadata', { request: { metadata, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { @@ -85,14 +102,15 @@ export class SessionAPI { async deleteSession( sessionId: string, workspacePath: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { await api.invoke('delete_persisted_session', { request: { session_id: sessionId, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { @@ -103,14 +121,15 @@ export class SessionAPI { async touchSessionActivity( sessionId: string, workspacePath: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { await api.invoke('touch_session_activity', { request: { session_id: sessionId, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { @@ -121,14 +140,15 @@ export class SessionAPI { async loadSessionMetadata( sessionId: string, workspacePath: string, - remoteConnectionId?: string + remoteConnectionId?: string, + remoteSshHost?: string ): Promise { try { return await api.invoke('load_persisted_session_metadata', { request: { session_id: sessionId, workspace_path: workspacePath, - ...remoteConnField(remoteConnectionId), + ...remoteSessionFields(remoteConnectionId, remoteSshHost), } }); } catch (error) { diff --git a/src/web-ui/src/infrastructure/config/components/SessionConfig.tsx b/src/web-ui/src/infrastructure/config/components/SessionConfig.tsx index 34ac684b..d9635dc5 100644 --- a/src/web-ui/src/infrastructure/config/components/SessionConfig.tsx +++ b/src/web-ui/src/infrastructure/config/components/SessionConfig.tsx @@ -32,8 +32,17 @@ import './DebugConfig.scss'; const log = createLogger('SessionConfig'); +const IS_TAURI_DESKTOP = typeof window !== 'undefined' && '__TAURI__' in window; + const AGENT_SESSION_TITLE = 'session-title-func-agent'; +type ComputerUseStatusPayload = { + computerUseEnabled: boolean; + accessibilityGranted: boolean; + screenCaptureGranted: boolean; + platformNote: string | null; +}; + const SessionConfig: React.FC = () => { const { t } = useTranslation('settings/session-config'); const { t: tTools } = useTranslation('settings/agentic-tools'); @@ -50,6 +59,12 @@ const SessionConfig: React.FC = () => { const [confirmationTimeout, setConfirmationTimeout] = useState(''); const [toolExecConfigLoading, setToolExecConfigLoading] = useState(false); + const [computerUseEnabled, setComputerUseEnabled] = useState(false); + const [computerUseAccess, setComputerUseAccess] = useState(false); + const [computerUseScreen, setComputerUseScreen] = useState(false); + const [computerUseNote, setComputerUseNote] = useState(null); + const [computerUseBusy, setComputerUseBusy] = useState(false); + // ── Debug mode config state ────────────────────────────────────────────── const [debugConfig, setDebugConfig] = useState(DEFAULT_DEBUG_MODE_CONFIG); const [debugHasChanges, setDebugHasChanges] = useState(false); @@ -61,6 +76,22 @@ const SessionConfig: React.FC = () => { loadAllData(); }, []); + const refreshComputerUseStatus = useCallback(async (): Promise => { + if (!IS_TAURI_DESKTOP) return false; + try { + const { invoke } = await import('@tauri-apps/api/core'); + const s = await invoke('computer_use_get_status'); + setComputerUseEnabled(s.computerUseEnabled); + setComputerUseAccess(s.accessibilityGranted); + setComputerUseScreen(s.screenCaptureGranted); + setComputerUseNote(s.platformNote ?? null); + return true; + } catch (error) { + log.error('computer_use_get_status failed', error); + return false; + } + }, []); + const loadAllData = async () => { setIsLoading(true); try { @@ -72,6 +103,7 @@ const SessionConfig: React.FC = () => { execTimeout, confirmTimeout, debugConfigData, + computerUseCfg, ] = await Promise.all([ aiExperienceConfigService.getSettingsAsync(), configManager.getConfig('ai.models') || [], @@ -80,6 +112,7 @@ const SessionConfig: React.FC = () => { configManager.getConfig('ai.tool_execution_timeout_secs'), configManager.getConfig('ai.tool_confirmation_timeout_secs'), configManager.getConfig('ai.debug_mode_config'), + configManager.getConfig('ai.computer_use_enabled'), ]); setSettings(loadedSettings); @@ -89,6 +122,13 @@ const SessionConfig: React.FC = () => { setExecutionTimeout(execTimeout != null ? String(execTimeout) : ''); setConfirmationTimeout(confirmTimeout != null ? String(confirmTimeout) : ''); if (debugConfigData) setDebugConfig(debugConfigData); + + if (IS_TAURI_DESKTOP) { + const ok = await refreshComputerUseStatus(); + if (!ok) setComputerUseEnabled(computerUseCfg ?? false); + } else { + setComputerUseEnabled(computerUseCfg ?? false); + } } catch (error) { log.error('Failed to load session config data', error); setSettings(await aiExperienceConfigService.getSettingsAsync()); @@ -169,6 +209,52 @@ const SessionConfig: React.FC = () => { } }; + const handleComputerUseEnabledChange = async (checked: boolean) => { + setComputerUseBusy(true); + setComputerUseEnabled(checked); + try { + await configManager.setConfig('ai.computer_use_enabled', checked); + const { globalEventBus } = await import('@/infrastructure/event-bus'); + globalEventBus.emit('mode:config:updated'); + notificationService.success( + checked ? t('messages.saveSuccess') : t('messages.saveSuccess'), + { duration: 2000 } + ); + await refreshComputerUseStatus(); + } catch (error) { + log.error('Failed to save computer_use_enabled', error); + notificationService.error(t('messages.saveFailed')); + setComputerUseEnabled(!checked); + } finally { + setComputerUseBusy(false); + } + }; + + const handleComputerUseRequestPermissions = async () => { + setComputerUseBusy(true); + try { + const { invoke } = await import('@tauri-apps/api/core'); + await invoke('computer_use_request_permissions'); + await refreshComputerUseStatus(); + notificationService.success(t('messages.saveSuccess'), { duration: 2000 }); + } catch (error) { + log.error('computer_use_request_permissions failed', error); + notificationService.error(t('messages.saveFailed')); + } finally { + setComputerUseBusy(false); + } + }; + + const handleComputerUseOpenSettings = async (pane: 'accessibility' | 'screen_capture') => { + try { + const { invoke } = await import('@tauri-apps/api/core'); + await invoke('computer_use_open_system_settings', { request: { pane } }); + } catch (error) { + log.error('computer_use_open_system_settings failed', error); + notificationService.error(t('messages.saveFailed')); + } + }; + const handleToolTimeoutChange = async (type: 'execution' | 'confirmation', value: string) => { const configKey = type === 'execution' ? 'ai.tool_execution_timeout_secs' : 'ai.tool_confirmation_timeout_secs'; @@ -410,6 +496,59 @@ const SessionConfig: React.FC = () => { + {/* ── Computer use (desktop) ─────────────────────────────── */} + + {IS_TAURI_DESKTOP ? ( + <> + +
+ handleComputerUseEnabledChange(e.target.checked)} + disabled={computerUseBusy} + size="small" + /> +
+ + {computerUseNote ? ( + + {computerUseNote} + + ) : null} + +
+ {computerUseAccess ? t('computerUse.granted') : t('computerUse.notGranted')} + + +
+ + +
+ {computerUseScreen ? t('computerUse.granted') : t('computerUse.notGranted')} + +
+ + + + + + ) : null} + + {/* ── Debug mode settings ───────────────────────────────── */} { try { this.setLoading(true); @@ -422,6 +428,7 @@ class WorkspaceManager { remotePath, remoteWorkspace.connectionId, remoteWorkspace.connectionName, + remoteWorkspace.sshHost, ); const [recentWorkspaces, openedWorkspaces] = await Promise.all([ @@ -756,6 +763,20 @@ class WorkspaceManager { return this.setActiveWorkspace(workspace.id); } + if (isRemoteWorkspace(workspace)) { + const connectionId = workspace.connectionId?.trim() ?? ''; + const connectionName = workspace.connectionName?.trim() || connectionId; + if (!connectionId) { + throw new Error('Remote workspace is missing connectionId; reconnect via SSH first.'); + } + return this.openRemoteWorkspace({ + connectionId, + connectionName, + remotePath: workspace.rootPath, + sshHost: workspace.sshHost, + }); + } + return this.openWorkspace(workspace.rootPath); } diff --git a/src/web-ui/src/locales/en-US/settings/session-config.json b/src/web-ui/src/locales/en-US/settings/session-config.json index 9c24094c..a7eac84b 100644 --- a/src/web-ui/src/locales/en-US/settings/session-config.json +++ b/src/web-ui/src/locales/en-US/settings/session-config.json @@ -12,6 +12,21 @@ "sectionTitle": "Tool execution behavior", "sectionDescription": "Confirmation and timeout settings when tools run in a session." }, + "computerUse": { + "sectionTitle": "Computer use (Claw)", + "sectionDescription": "Let the assistant capture the screen and control the mouse and keyboard in BitFun desktop. Requires macOS Accessibility and Screen Recording (or equivalent on other platforms). Screenshots in tool results need a primary model with Anthropic API format.", + "enable": "Enable Computer use", + "enableDesc": "When off, the ComputerUse tool stays disabled even in Claw mode.", + "accessibility": "Accessibility", + "screenCapture": "Screen recording", + "granted": "Granted", + "notGranted": "Not granted", + "request": "Request", + "openSettings": "Open System Settings", + "refreshStatus": "Refresh status", + "desktopOnly": "Computer use settings are only available in the BitFun desktop app.", + "platformNote": "Note" + }, "model": { "label": "Model", "primary": "Primary Model", diff --git a/src/web-ui/src/locales/zh-CN/settings/session-config.json b/src/web-ui/src/locales/zh-CN/settings/session-config.json index 6ea4512d..a3bb18ca 100644 --- a/src/web-ui/src/locales/zh-CN/settings/session-config.json +++ b/src/web-ui/src/locales/zh-CN/settings/session-config.json @@ -12,6 +12,21 @@ "sectionTitle": "工具执行行为", "sectionDescription": "本会话中 AI 调用工具时的确认与超时策略。" }, + "computerUse": { + "sectionTitle": "Computer use（助理 Claw）", + "sectionDescription": "在 BitFun 桌面端允许助理截取屏幕并控制键鼠。macOS 需「辅助功能」与「屏幕录制」权限。截图型工具结果需主模型为 Anthropic API 格式。", + "enable": "启用 Computer use", + "enableDesc": "关闭时，即使在 Claw 模式下也不会启用 ComputerUse 工具。", + "accessibility": "辅助功能", + "screenCapture": "屏幕录制", + "granted": "已授权", + "notGranted": "未授权", + "request": "请求授权", + "openSettings": "打开系统设置", + "refreshStatus": "刷新状态", + "desktopOnly": "Computer use 仅在 BitFun 桌面应用中可用。", + "platformNote": "说明" + }, "model": { "label": "模型", "primary": "主力模型", diff --git a/src/web-ui/src/shared/types/global-state.ts b/src/web-ui/src/shared/types/global-state.ts index 19a57aca..1affca8d 100644 --- a/src/web-ui/src/shared/types/global-state.ts +++ b/src/web-ui/src/shared/types/global-state.ts @@ -99,6 +99,11 @@ export interface WorkspaceInfo { worktree?: WorkspaceWorktreeInfo | null; connectionId?: string; connectionName?: string; + /** + * Logical workspace host for stable scoping: `{sshHost}:{rootPath}`. + * Local / assistant workspaces use `localhost` (from backend); remote uses SSH config host. + */ + sshHost?: string; } export function isRemoteWorkspace(workspace: WorkspaceInfo | null | undefined): boolean { @@ -164,7 +169,12 @@ export interface GlobalStateAPI { openWorkspace(path: string): Promise; - openRemoteWorkspace(remotePath: string, connectionId: string, connectionName: string): Promise; + openRemoteWorkspace( + remotePath: string, + connectionId: string, + connectionName: string, + sshHost?: string + ): Promise; createAssistantWorkspace(): Promise; deleteAssistantWorkspace(workspaceId: string): Promise; resetAssistantWorkspace(workspaceId: string): Promise; @@ -292,6 +302,9 @@ function mapWorkspaceInfo(workspace: APIWorkspaceInfo): WorkspaceInfo { worktree: mapWorkspaceWorktree(workspace.worktree), connectionId: workspace.connectionId, connectionName: workspace.connectionName, + sshHost: + workspace.sshHost ?? + (workspace.workspaceKind?.toLowerCase() !== 'remote' ? 'localhost' : undefined), }; } @@ -340,8 +353,15 @@ export function createGlobalStateAPI(): GlobalStateAPI { return mapWorkspaceInfo(await globalAPI.openWorkspace(path)); }, - async openRemoteWorkspace(remotePath: string, connectionId: string, connectionName: string): Promise { - return mapWorkspaceInfo(await globalAPI.openRemoteWorkspace(remotePath, connectionId, connectionName)); + async openRemoteWorkspace( + remotePath: string, + connectionId: string, + connectionName: string, + sshHost?: string + ): Promise { + return mapWorkspaceInfo( + await globalAPI.openRemoteWorkspace(remotePath, connectionId, connectionName, sshHost) + ); }, async createAssistantWorkspace(): Promise { diff --git a/src/web-ui/src/shared/types/session-history.ts b/src/web-ui/src/shared/types/session-history.ts index 31c4efee..ff9c48ea 100644 --- a/src/web-ui/src/shared/types/session-history.ts +++ b/src/web-ui/src/shared/types/session-history.ts @@ -31,6 +31,8 @@ export interface SessionMetadata { customMetadata?: SessionCustomMetadata; todos?: any[]; workspacePath?: string; + remoteConnectionId?: string; + remoteSshHost?: string; } export type SessionStatus = 'active' | 'archived' | 'completed'; diff --git a/src/web-ui/src/shared/utils/recentWorkspaceDisplay.ts b/src/web-ui/src/shared/utils/recentWorkspaceDisplay.ts new file mode 100644 index 00000000..c63bd802 --- /dev/null +++ b/src/web-ui/src/shared/utils/recentWorkspaceDisplay.ts @@ -0,0 +1,48 @@ +import { WorkspaceKind, type WorkspaceInfo } from '@/shared/types'; + +/** Remote POSIX path without leading slash (e.g. `/mnt/vdb/lfs` → `mnt/vdb/lfs`) for list readability. */ +function compactRemotePathForRecentList(rootPath: string, displayName: string): string { + let s = rootPath.replace(/\\/g, '/'); + while (s.includes('//')) { + s = s.replace('//', '/'); + } + if (s === '/' || s.trim() === '') { + return displayName.trim() || '/'; + } + const trimmed = s.replace(/\/+$/, ''); + const body = trimmed.startsWith('/') ? trimmed.slice(1) : trimmed; + return body.trim() || displayName; +} + +export type RecentWorkspaceLineParts = { + /** SSH host (or connection display name) for remote; omit for local. */ + hostPrefix: string | null; + folderLabel: string; + /** Tooltip: `host:path` for remote, path for local. */ + tooltip: string; +}; + +/** + * Labels for recent-workspace rows: remote shows host (VS Code–style disambiguation). + */ +export function getRecentWorkspaceLineParts(workspace: WorkspaceInfo): RecentWorkspaceLineParts { + if (workspace.workspaceKind !== WorkspaceKind.Remote) { + return { + hostPrefix: null, + folderLabel: workspace.name, + tooltip: workspace.rootPath, + }; + } + const host = + workspace.sshHost?.trim() || + workspace.connectionName?.trim() || + null; + const tooltipBase = host ?? workspace.connectionId?.trim() ?? ''; + const tooltip = + tooltipBase.length > 0 ? `${tooltipBase}:${workspace.rootPath}` : workspace.rootPath; + return { + hostPrefix: host, + folderLabel: compactRemotePathForRecentList(workspace.rootPath, workspace.name), + tooltip, + }; +} diff --git a/src/web-ui/src/tools/workspace/components/WorkspaceManager.css b/src/web-ui/src/tools/workspace/components/WorkspaceManager.css index 1260bb4b..e1711966 100644 --- a/src/web-ui/src/tools/workspace/components/WorkspaceManager.css +++ b/src/web-ui/src/tools/workspace/components/WorkspaceManager.css @@ -151,6 +151,12 @@ line-height: 1.3; } +.workspace-name__ssh-host { + font-weight: 500; + color: rgba(255, 255, 255, 0.55); + margin-right: 2px; +} + .workspace-path { font-size: 14px; color: rgba(255, 255, 255, 0.7); diff --git a/src/web-ui/src/tools/workspace/components/WorkspaceManager.tsx b/src/web-ui/src/tools/workspace/components/WorkspaceManager.tsx index 95c116c7..0cdafcd0 100644 --- a/src/web-ui/src/tools/workspace/components/WorkspaceManager.tsx +++ b/src/web-ui/src/tools/workspace/components/WorkspaceManager.tsx @@ -5,6 +5,7 @@ import { WorkspaceInfo, WorkspaceKind, WorkspaceType } from '../../../shared/typ import { Modal } from '@/component-library'; import { i18nService } from '@/infrastructure/i18n'; import { createLogger } from '@/shared/utils/logger'; +import { getRecentWorkspaceLineParts } from '@/shared/utils/recentWorkspaceDisplay'; import './WorkspaceManager.css'; const log = createLogger('WorkspaceManager'); @@ -224,7 +225,19 @@ const WorkspaceManager: React.FC = ({ {getWorkspaceIcon(workspace)}
-
{getWorkspaceDisplayName(workspace)}
+
+ {(() => { + const { hostPrefix } = getRecentWorkspaceLineParts(workspace); + return ( + <> + {hostPrefix ? ( + {hostPrefix} · + ) : null} + {getWorkspaceDisplayName(workspace)} + + ); + })()} +
{workspace.rootPath}
{workspace.workspaceType}