feat(tasks): add isolated cloud computer use

products/tasks/backend/facade/api.py

@@ -3500,6 +3500,7 @@ def create_task(team_id: int, user_id: int | None, *, validated_data: dict) -> c
     pending_user_message = (validated_data.pop("pending_user_message", None) or "").strip() or None
     pending_user_artifact_ids = validated_data.pop("pending_user_artifact_ids", None) or []
     warm_auto_publish = validated_data.pop("auto_publish", None)
+    computer_use = validated_data.pop("computer_use", False)
 
     if user_id is not None:
         validated_data["created_by"] = User.objects.get(id=user_id)
@@ -3508,6 +3509,7 @@ def create_task(team_id: int, user_id: int | None, *, validated_data: dict) -> c
         warm_branch_provided
         and validated_data["origin_product"] == Task.OriginProduct.USER_CREATED
         and validated_data.get("repository")
+        and not computer_use
         and user_id is not None
     ):
         warm_run = _find_idling_warm_run(
@@ -4056,7 +4058,7 @@ def run_task(
     pending_user_message = validated_data.get("pending_user_message")
     pending_user_artifact_ids = validated_data.get("pending_user_artifact_ids") or []
 
-    if not resume_from_run_id:
+    if not resume_from_run_id and not validated_data.get("computer_use", False):
         warm_run = _idling_warm_run_for_task(task)
         if warm_run is not None and (branch or None) == (warm_run.branch or None):
             warm_state = warm_run.state or {}
@@ -4134,6 +4136,9 @@ def run_task(
     if rtk_enabled is not None:
         extra_state = extra_state or {}
         extra_state["rtk_enabled"] = rtk_enabled
+    if validated_data.get("computer_use") is True:
+        extra_state = extra_state or {}
+        extra_state["computer_use"] = True
 
     if resume_from_run_id:
         previous_run = task.runs.filter(id=resume_from_run_id).first()

products/tasks/backend/logic/services/docker_sandbox.py

@@ -55,6 +55,7 @@
     SandboxStatus,
     SandboxTemplate,
     build_agent_runtime_env_prefix,
+    build_computer_use_env_prefix,
     parse_sandbox_repo_mount_map,
     redact_sandbox_command,
     wait_for_health_check,
@@ -800,6 +801,7 @@ def _build_agent_server_command(
         event_ingest_keep_stream_open: bool = False,
         repo_ready_file: str | None = None,
         rtk_enabled: bool = True,
+        computer_use: bool = False,
     ) -> str:
         # The host proxy URL (e.g. localhost:8003) is unreachable from inside the container;
         # rewrite it the same way POSTHOG_API_URL is for Docker sandboxes.
@@ -821,6 +823,7 @@ def _build_agent_server_command(
         # Only append when opted in: agent-server builds without the option reject unknown
         # flags, so default runs (and resumes of old snapshots) must not see it.
         auto_publish_flag = " --autoPublish true" if auto_publish else ""
+        computer_use_flag = " --computerUse true" if computer_use else ""
         branch_flag = f" --baseBranch {shlex.quote(branch)}" if branch else ""
         repo_flag = f" --repositoryPath {shlex.quote(repo_path)}" if repo_path else ""
         domains_flag = f" --allowedDomains {shlex.quote(','.join(allowed_domains))}" if allowed_domains else ""
@@ -832,9 +835,11 @@ def _build_agent_server_command(
         unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS)
         server_cmd = (
             f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} "
-            f"{env_prefix}./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} "
+            f"{build_computer_use_env_prefix(computer_use)}{env_prefix}"
+            f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} "
             f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}"
-            f"{create_pr_flag}{auto_publish_flag}{branch_flag}{mcp_servers_arg}{relay_mcp_servers_arg}"
+            f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}"
+            f"{relay_mcp_servers_arg}"
             f"{domains_flag}{repo_ready_flag}"
         )
 
@@ -892,6 +897,7 @@ def start_agent_server(
         repo_ready_file: str | None = None,
         wait_for_health: bool = True,
         rtk_enabled: bool = True,
+        computer_use: bool = False,
     ) -> None:
         """Start the agent-server HTTP server in the sandbox.
 
@@ -931,6 +937,15 @@ def start_agent_server(
             logger.warning(f"Installed agent-server in sandbox {self.id} predates --autoPublish; starting review-first")
             auto_publish = False
 
+        if computer_use:
+            desktop_result = self.start_virtual_desktop(restricted_egress=allowed_domains is not None)
+            if desktop_result.exit_code != 0:
+                raise SandboxExecutionError(
+                    "Virtual desktop failed to start",
+                    {"sandbox_id": self.id, "stderr": desktop_result.stderr},
+                    cause=RuntimeError(desktop_result.stderr or "virtual desktop command returned non-zero exit"),
+                )
+
         command = self._build_agent_server_command(
             repo_path,
             task_id,
@@ -953,6 +968,7 @@ def start_agent_server(
             event_ingest_keep_stream_open=event_ingest_keep_stream_open,
             repo_ready_file=repo_ready_file,
             rtk_enabled=rtk_enabled,
+            computer_use=computer_use,
         )
 
         logger.info(f"Starting agent-server in sandbox {self.id} for {repository or 'no-repo'}")

products/tasks/backend/logic/services/modal_sandbox.py

@@ -83,6 +83,7 @@
     WORKING_DIR,
     SandboxBase,
     build_agent_runtime_env_prefix,
+    build_computer_use_env_prefix,
     redact_sandbox_command,
     wait_for_health_check,
 )
@@ -952,6 +953,7 @@ def _build_agent_server_command(
         event_ingest_keep_stream_open: bool = False,
         repo_ready_file: str | None = None,
         rtk_enabled: bool = True,
+        computer_use: bool = False,
     ) -> str:
         env_prefix = build_agent_runtime_env_prefix(
             interaction_origin=interaction_origin,
@@ -969,6 +971,7 @@ def _build_agent_server_command(
         # Only append when opted in: agent-server builds without the option reject unknown
         # flags, so default runs (and resumes of old snapshots) must not see it.
         auto_publish_flag = " --autoPublish true" if auto_publish else ""
+        computer_use_flag = " --computerUse true" if computer_use else ""
         repo_flag = f" --repositoryPath {shlex.quote(repo_path)}" if repo_path else ""
         branch_flag = f" --baseBranch {shlex.quote(branch)}" if branch else ""
         domains_flag = f" --allowedDomains {shlex.quote(','.join(allowed_domains))}" if allowed_domains else ""
@@ -980,9 +983,11 @@ def _build_agent_server_command(
         unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS)
         server_cmd = (
             f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} "
-            f"{env_prefix}./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} "
+            f"{build_computer_use_env_prefix(computer_use)}{env_prefix}"
+            f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} "
             f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}"
-            f"{create_pr_flag}{auto_publish_flag}{branch_flag}{mcp_servers_arg}{relay_mcp_servers_arg}"
+            f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}"
+            f"{relay_mcp_servers_arg}"
             f"{domains_flag}{repo_ready_flag}"
         )
 
@@ -1073,6 +1078,7 @@ def start_agent_server(
         repo_ready_file: str | None = None,
         wait_for_health: bool = True,
         rtk_enabled: bool = True,
+        computer_use: bool = False,
     ) -> None:
         """Start the agent-server HTTP server in the sandbox.
 
@@ -1111,6 +1117,15 @@ def start_agent_server(
             logger.warning(f"Installed agent-server in sandbox {self.id} predates --autoPublish; starting review-first")
             auto_publish = False
 
+        if computer_use:
+            desktop_result = self.start_virtual_desktop(restricted_egress=allowed_domains is not None)
+            if desktop_result.exit_code != 0:
+                raise SandboxExecutionError(
+                    "Virtual desktop failed to start",
+                    {"sandbox_id": self.id, "stderr": desktop_result.stderr},
+                    cause=RuntimeError(desktop_result.stderr or "virtual desktop command returned non-zero exit"),
+                )
+
         command = self._build_agent_server_command(
             repo_path,
             task_id,
@@ -1133,6 +1148,7 @@ def start_agent_server(
             event_ingest_keep_stream_open=event_ingest_keep_stream_open,
             repo_ready_file=repo_ready_file,
             rtk_enabled=rtk_enabled,
+            computer_use=computer_use,
         )
 
         logger.info(f"Starting agent-server in sandbox {self.id} for {repository or 'no-repo'}")

products/tasks/backend/logic/services/sandbox.py

@@ -28,6 +28,7 @@
 from pydantic import BaseModel, model_validator
 
 from products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIR, SNAPSHOT_KIND_FILESYSTEM, SnapshotKind
+from products.tasks.backend.logic.services.agentsh import ENV_FILE, ENV_WRAPPER_SCRIPT, build_exec_prefix
 from products.tasks.backend.logic.services.sandbox_config import (
     BURSTABLE_REQUEST_CPU_CORES,
     BURSTABLE_REQUEST_MEMORY_MB,
@@ -164,6 +165,12 @@ def redact_sandbox_command(command: str) -> str:
     return SENSITIVE_AGENT_RUNTIME_ENV_PATTERN.sub(r"\g<name>=<redacted>", command)
 
 
+def build_computer_use_env_prefix(computer_use: bool) -> str:
+    if not computer_use:
+        return ""
+    return "DISPLAY=:99 GTK_A11Y=none "
+
+
 def build_agent_runtime_env_prefix(
     *,
     interaction_origin: str | None = None,
@@ -250,6 +257,12 @@ def agent_server_supports_auto_publish(self) -> bool:
         result = self.execute("grep -q autoPublish /scripts/node_modules/.bin/agent-server", timeout_seconds=10)
         return result.exit_code == 0
 
+    def start_virtual_desktop(self, *, restricted_egress: bool = False) -> ExecutionResult:
+        command = "DISPLAY=:99 /usr/local/bin/start-virtual-desktop"
+        if restricted_egress:
+            command = f"cd /scripts && env -0 > {ENV_FILE} && {build_exec_prefix()} {ENV_WRAPPER_SCRIPT} {command}"
+        return self.execute(command, timeout_seconds=30)
+
     def clone_repository(
         self,
         repository: str,
@@ -334,6 +347,7 @@ def start_agent_server(
         repo_ready_file: str | None = None,
         wait_for_health: bool = True,
         rtk_enabled: bool = True,
+        computer_use: bool = False,
     ) -> None:
         """Start the agent-server HTTP server in the sandbox.
 

products/tasks/backend/logic/services/tests/test_modal_sandbox.py

@@ -448,6 +448,48 @@ def test_start_agent_server_success_without_domains_skips_agentsh(self, mock_san
         assert "agentsh exec" not in command
         assert "nohup" in command
 
+    def test_start_agent_server_starts_virtual_desktop_when_enabled(self, mock_sandbox: Any):
+        mock_sandbox.execute = MagicMock(
+            return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None),
+        )
+
+        mock_sandbox.start_agent_server(
+            repository="posthog/posthog",
+            task_id="task-123",
+            run_id="run-456",
+            computer_use=True,
+            wait_for_health=False,
+        )
+
+        commands = [call.args[0] for call in mock_sandbox.execute.call_args_list]
+        assert "DISPLAY=:99 /usr/local/bin/start-virtual-desktop" in commands
+        launch_command = next(command for command in commands if "--taskId" in command)
+        assert "DISPLAY=:99" in launch_command
+        assert "WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS" not in launch_command
+        assert "GTK_A11Y=none" in launch_command
+        assert "--computerUse true" in launch_command
+
+    def test_start_agent_server_starts_restricted_desktop_inside_agentsh(self, mock_sandbox: Any) -> None:
+        mock_sandbox.execute = MagicMock(
+            return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None),
+        )
+
+        with patch.object(mock_sandbox, "_setup_agentsh"):
+            mock_sandbox.start_agent_server(
+                repository="posthog/posthog",
+                task_id="task-123",
+                run_id="run-456",
+                computer_use=True,
+                allowed_domains=["github.com"],
+                wait_for_health=False,
+            )
+
+        desktop_command = next(
+            call.args[0] for call in mock_sandbox.execute.call_args_list if "start-virtual-desktop" in call.args[0]
+        )
+        assert "agentsh exec" in desktop_command
+        assert "/tmp/agentsh-env-wrapper.sh" in desktop_command
+
     def test_start_agent_server_waits_for_repository_before_launch(self, mock_sandbox: Any):
         mock_sandbox.execute = MagicMock(
             return_value=ExecutionResult(stdout="ok:1", stderr="", exit_code=0, error=None),

products/tasks/backend/presentation/serializers.py

@@ -1777,6 +1777,11 @@ class TaskRunCreateRequestSerializer(ImportedMcpServersFieldMixin, RelayedMcpSer
             "follows the server-side default (enabled); false opts this run out."
         ),
     )
+    computer_use = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text="Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.",
+    )
 
     def validate(self, attrs):
         errors: dict[str, str] = {}
@@ -1953,6 +1958,11 @@ class TaskRunBootstrapCreateRequestSerializer(
             "follows the server-side default (enabled); false opts this run out."
         ),
     )
+    computer_use = serializers.BooleanField(
+        required=False,
+        default=False,
+        help_text="Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.",
+    )
     home_quick_action = serializers.CharField(
         required=False,
         default=None,

products/tasks/backend/sandbox/images/Dockerfile.sandbox-base

@@ -10,7 +10,7 @@
 ENV GH_TELEMETRY=false
 
 # Install system packages (expanded for coding environments)
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
     # Core tools
     curl \
@@ -38,6 +38,19 @@
     postgresql-client \
     mysql-client \
     redis-tools \
+    # Virtual desktop tools
+    dbus-x11 \
+    epiphany-browser \
+    imagemagick \
+    libgtk-3-bin \
+    openbox \
+    python3-xdg \
+    wmctrl \
+    x11-utils \
+    xdotool \
+    xfonts-base \
+    xterm \
+    xvfb \
     # System libraries often needed by Python packages
     libssl-dev \
     libffi-dev \
@@ -56,12 +69,12 @@
     && rm -rf /var/lib/apt/lists/*
 
 # Install Node.js 24.x
 RUN curl -fsSL https://deb.nodesource.com/setup_24.x | bash - && \
     apt-get install -y --no-install-recommends nodejs && \
     rm -rf /var/lib/apt/lists/*
 
 # Install additional language package managers
 RUN npm install -g yarn pnpm typescript ts-node nodemon
 
 # Install ruff and ty
 # uv comes from its pinned, registry-verified official image (same pattern as the
@@ -85,7 +98,7 @@
 
 # Install agentsh for runtime egress policy enforcement
 ARG AGENTSH_TAG=v0.18.3
 RUN set -eux; \
     version="${AGENTSH_TAG#v}"; \
     arch="$(dpkg --print-architecture)"; \
     case "$arch" in \
@@ -111,7 +124,7 @@
 # developer instructions instead. POSTHOG_RTK=0 (set per run from the task processing
 # context) opts a run out of both.
 ARG RTK_VERSION=0.43.0
 RUN set -eux; \
     arch="$(dpkg --print-architecture)"; \
     case "$arch" in \
       amd64) rtk_asset="rtk-x86_64-unknown-linux-musl.tar.gz"; \
@@ -136,7 +149,7 @@
 # PostHog-side image changes that keep the agent version unchanged.
 ARG AGENT_VERSION=latest
 ARG COMMIT_HASH
 RUN mkdir -p /scripts && \
     cd /scripts && \
     npm init -y && \
     CACHE_BUST=${COMMIT_HASH} npm install "@posthog/agent@${AGENT_VERSION}"
@@ -165,6 +178,9 @@
 
 ENV PYTHONPATH="/tmp/workspace"
 
+COPY products/tasks/backend/sandbox/images/start-virtual-desktop.sh /usr/local/bin/start-virtual-desktop
+RUN chmod +x /usr/local/bin/start-virtual-desktop
+
 WORKDIR /tmp/workspace
 
 RUN python3 --version && \

products/tasks/backend/sandbox/images/start-virtual-desktop.sh

@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+export DISPLAY="${DISPLAY:-:99}"
+
+if xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then
+    exit 0
+fi
+
+rm -f "/tmp/.X${DISPLAY#:}-lock"
+nohup Xvfb "$DISPLAY" -screen 0 1440x900x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1 &
+
+for _ in $(seq 1 100); do
+    if xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then
+        nohup openbox-session >/tmp/openbox.log 2>&1 &
+        for _ in $(seq 1 50); do
+            wmctrl -m >/dev/null 2>&1 && exit 0
+            sleep 0.1
+        done
+        echo "Window manager failed to start" >&2
+        exit 1
+    fi
+    sleep 0.1
+done
+
+echo "Virtual display failed to start" >&2
+exit 1

products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py

@@ -112,6 +112,10 @@ def auto_publish(self) -> bool:
         """User-opted auto-publish: the agent pushes and opens a draft PR on completion."""
         return (self.state or {}).get("auto_publish") is True
 
+    @property
+    def computer_use(self) -> bool:
+        return (self.state or {}).get("computer_use") is True
+
     @property
     def has_github_credentials(self) -> bool:
         return self.github_integration_id is not None or self.github_user_integration_id is not None
@@ -359,6 +363,14 @@ def _is_modal_vm_sandbox_enabled(
     custom_image_available: bool = False,
     state: dict | None = None,
 ) -> bool:
+    if (state or {}).get("computer_use") is True:
+        log_with_activity_context(
+            "modal_vm_sandbox_required_for_computer_use",
+            run_id=run_id,
+            use_modal_vm_sandbox=True,
+        )
+        return True
+
     if allowed_domains is not None:
         log_with_activity_context(
             "modal_vm_sandbox_skipped_restricted_egress",

products/tasks/backend/temporal/process_task/activities/start_agent_server.py

@@ -338,6 +338,7 @@ def _invoke_start_agent_server(
             repo_ready_file=repo_ready_file,
             wait_for_health=wait_for_health,
             rtk_enabled=ctx.rtk_enabled,
+            computer_use=ctx.computer_use,
         )
 
         # Mark startup-time token issuance so follow-ups within the next