From d4ea4344de4da3c6dc40fcd788d8213f1c044a1a Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 17:15:37 +0300 Subject: [PATCH 1/5] feat(tasks): add isolated cloud computer use Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- products/tasks/backend/facade/api.py | 5 ++- .../backend/logic/services/docker_sandbox.py | 19 ++++++++- .../backend/logic/services/modal_sandbox.py | 19 ++++++++- .../tasks/backend/logic/services/sandbox.py | 4 ++ .../services/tests/test_modal_sandbox.py | 19 +++++++++ .../tasks/backend/presentation/serializers.py | 10 +++++ .../sandbox/images/Dockerfile.sandbox-base | 14 +++++++ .../sandbox/images/start-virtual-desktop.sh | 27 +++++++++++++ .../activities/get_task_processing_context.py | 4 ++ .../activities/start_agent_server.py | 1 + .../tests/test_start_agent_server.py | 40 +++++++++++++++++++ products/tasks/backend/tests/test_api.py | 15 +++++++ .../tasks/backend/tests/test_warm_facade.py | 9 +++++ 13 files changed, 181 insertions(+), 5 deletions(-) create mode 100644 products/tasks/backend/sandbox/images/start-virtual-desktop.sh diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py index 735ddc2033af..9540a4c20a85 100644 --- a/products/tasks/backend/facade/api.py +++ b/products/tasks/backend/facade/api.py @@ -4056,7 +4056,7 @@ def run_task( pending_user_message = validated_data.get("pending_user_message") pending_user_artifact_ids = validated_data.get("pending_user_artifact_ids") or [] - if not resume_from_run_id: + if not resume_from_run_id and not validated_data.get("computer_use", False): warm_run = _idling_warm_run_for_task(task) if warm_run is not None and (branch or None) == (warm_run.branch or None): warm_state = warm_run.state or {} @@ -4134,6 +4134,9 @@ def run_task( if rtk_enabled is not None: extra_state = extra_state or {} extra_state["rtk_enabled"] = rtk_enabled + if validated_data.get("computer_use") is True: + extra_state = extra_state or {} + extra_state["computer_use"] = True if resume_from_run_id: previous_run = task.runs.filter(id=resume_from_run_id).first() diff --git a/products/tasks/backend/logic/services/docker_sandbox.py b/products/tasks/backend/logic/services/docker_sandbox.py index ac8ac638d115..082d05fae6b6 100644 --- a/products/tasks/backend/logic/services/docker_sandbox.py +++ b/products/tasks/backend/logic/services/docker_sandbox.py @@ -800,6 +800,7 @@ def _build_agent_server_command( event_ingest_keep_stream_open: bool = False, repo_ready_file: str | None = None, rtk_enabled: bool = True, + computer_use: bool = False, ) -> str: # The host proxy URL (e.g. localhost:8003) is unreachable from inside the container; # rewrite it the same way POSTHOG_API_URL is for Docker sandboxes. @@ -821,6 +822,7 @@ def _build_agent_server_command( # Only append when opted in: agent-server builds without the option reject unknown # flags, so default runs (and resumes of old snapshots) must not see it. auto_publish_flag = " --autoPublish true" if auto_publish else "" + computer_use_flag = " --computerUse true" if computer_use else "" branch_flag = f" --baseBranch {shlex.quote(branch)}" if branch else "" repo_flag = f" --repositoryPath {shlex.quote(repo_path)}" if repo_path else "" domains_flag = f" --allowedDomains {shlex.quote(','.join(allowed_domains))}" if allowed_domains else "" @@ -832,9 +834,11 @@ def _build_agent_server_command( unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS) server_cmd = ( f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} " - f"{env_prefix}./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " + f"{'DISPLAY=:99 ' if computer_use else ''}{env_prefix}" + f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}" - f"{create_pr_flag}{auto_publish_flag}{branch_flag}{mcp_servers_arg}{relay_mcp_servers_arg}" + f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}" + f"{relay_mcp_servers_arg}" f"{domains_flag}{repo_ready_flag}" ) @@ -892,6 +896,7 @@ def start_agent_server( repo_ready_file: str | None = None, wait_for_health: bool = True, rtk_enabled: bool = True, + computer_use: bool = False, ) -> None: """Start the agent-server HTTP server in the sandbox. @@ -931,6 +936,15 @@ def start_agent_server( logger.warning(f"Installed agent-server in sandbox {self.id} predates --autoPublish; starting review-first") auto_publish = False + if computer_use: + desktop_result = self.start_virtual_desktop() + if desktop_result.exit_code != 0: + raise SandboxExecutionError( + "Virtual desktop failed to start", + {"sandbox_id": self.id, "stderr": desktop_result.stderr}, + cause=RuntimeError(desktop_result.stderr or "virtual desktop command returned non-zero exit"), + ) + command = self._build_agent_server_command( repo_path, task_id, @@ -953,6 +967,7 @@ def start_agent_server( event_ingest_keep_stream_open=event_ingest_keep_stream_open, repo_ready_file=repo_ready_file, rtk_enabled=rtk_enabled, + computer_use=computer_use, ) logger.info(f"Starting agent-server in sandbox {self.id} for {repository or 'no-repo'}") diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py index a91a0f8b43e9..742f8c627ce0 100644 --- a/products/tasks/backend/logic/services/modal_sandbox.py +++ b/products/tasks/backend/logic/services/modal_sandbox.py @@ -952,6 +952,7 @@ def _build_agent_server_command( event_ingest_keep_stream_open: bool = False, repo_ready_file: str | None = None, rtk_enabled: bool = True, + computer_use: bool = False, ) -> str: env_prefix = build_agent_runtime_env_prefix( interaction_origin=interaction_origin, @@ -969,6 +970,7 @@ def _build_agent_server_command( # Only append when opted in: agent-server builds without the option reject unknown # flags, so default runs (and resumes of old snapshots) must not see it. auto_publish_flag = " --autoPublish true" if auto_publish else "" + computer_use_flag = " --computerUse true" if computer_use else "" repo_flag = f" --repositoryPath {shlex.quote(repo_path)}" if repo_path else "" branch_flag = f" --baseBranch {shlex.quote(branch)}" if branch else "" domains_flag = f" --allowedDomains {shlex.quote(','.join(allowed_domains))}" if allowed_domains else "" @@ -980,9 +982,11 @@ def _build_agent_server_command( unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS) server_cmd = ( f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} " - f"{env_prefix}./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " + f"{'DISPLAY=:99 ' if computer_use else ''}{env_prefix}" + f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}" - f"{create_pr_flag}{auto_publish_flag}{branch_flag}{mcp_servers_arg}{relay_mcp_servers_arg}" + f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}" + f"{relay_mcp_servers_arg}" f"{domains_flag}{repo_ready_flag}" ) @@ -1073,6 +1077,7 @@ def start_agent_server( repo_ready_file: str | None = None, wait_for_health: bool = True, rtk_enabled: bool = True, + computer_use: bool = False, ) -> None: """Start the agent-server HTTP server in the sandbox. @@ -1111,6 +1116,15 @@ def start_agent_server( logger.warning(f"Installed agent-server in sandbox {self.id} predates --autoPublish; starting review-first") auto_publish = False + if computer_use: + desktop_result = self.start_virtual_desktop() + if desktop_result.exit_code != 0: + raise SandboxExecutionError( + "Virtual desktop failed to start", + {"sandbox_id": self.id, "stderr": desktop_result.stderr}, + cause=RuntimeError(desktop_result.stderr or "virtual desktop command returned non-zero exit"), + ) + command = self._build_agent_server_command( repo_path, task_id, @@ -1133,6 +1147,7 @@ def start_agent_server( event_ingest_keep_stream_open=event_ingest_keep_stream_open, repo_ready_file=repo_ready_file, rtk_enabled=rtk_enabled, + computer_use=computer_use, ) logger.info(f"Starting agent-server in sandbox {self.id} for {repository or 'no-repo'}") diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.py index b55e80aaf546..76ecaca834f3 100644 --- a/products/tasks/backend/logic/services/sandbox.py +++ b/products/tasks/backend/logic/services/sandbox.py @@ -250,6 +250,9 @@ def agent_server_supports_auto_publish(self) -> bool: result = self.execute("grep -q autoPublish /scripts/node_modules/.bin/agent-server", timeout_seconds=10) return result.exit_code == 0 + def start_virtual_desktop(self) -> ExecutionResult: + return self.execute("DISPLAY=:99 /usr/local/bin/start-virtual-desktop", timeout_seconds=30) + def clone_repository( self, repository: str, @@ -334,6 +337,7 @@ def start_agent_server( repo_ready_file: str | None = None, wait_for_health: bool = True, rtk_enabled: bool = True, + computer_use: bool = False, ) -> None: """Start the agent-server HTTP server in the sandbox. diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py index 803c8b4a5f96..8571157b91cc 100644 --- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py +++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py @@ -448,6 +448,25 @@ def test_start_agent_server_success_without_domains_skips_agentsh(self, mock_san assert "agentsh exec" not in command assert "nohup" in command + def test_start_agent_server_starts_virtual_desktop_when_enabled(self, mock_sandbox: Any): + mock_sandbox.execute = MagicMock( + return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None), + ) + + mock_sandbox.start_agent_server( + repository="posthog/posthog", + task_id="task-123", + run_id="run-456", + computer_use=True, + wait_for_health=False, + ) + + commands = [call.args[0] for call in mock_sandbox.execute.call_args_list] + assert "DISPLAY=:99 /usr/local/bin/start-virtual-desktop" in commands + launch_command = next(command for command in commands if "--taskId" in command) + assert "DISPLAY=:99" in launch_command + assert "--computerUse true" in launch_command + def test_start_agent_server_waits_for_repository_before_launch(self, mock_sandbox: Any): mock_sandbox.execute = MagicMock( return_value=ExecutionResult(stdout="ok:1", stderr="", exit_code=0, error=None), diff --git a/products/tasks/backend/presentation/serializers.py b/products/tasks/backend/presentation/serializers.py index b6529de32a77..de7ec789f2b8 100644 --- a/products/tasks/backend/presentation/serializers.py +++ b/products/tasks/backend/presentation/serializers.py @@ -1777,6 +1777,11 @@ class TaskRunCreateRequestSerializer(ImportedMcpServersFieldMixin, RelayedMcpSer "follows the server-side default (enabled); false opts this run out." ), ) + computer_use = serializers.BooleanField( + required=False, + default=False, + help_text="Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.", + ) def validate(self, attrs): errors: dict[str, str] = {} @@ -1953,6 +1958,11 @@ class TaskRunBootstrapCreateRequestSerializer( "follows the server-side default (enabled); false opts this run out." ), ) + computer_use = serializers.BooleanField( + required=False, + default=False, + help_text="Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.", + ) home_quick_action = serializers.CharField( required=False, default=None, diff --git a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base index 6fc629e0dabe..51394db3cd52 100644 --- a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base +++ b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base @@ -38,6 +38,17 @@ RUN apt-get update && \ postgresql-client \ mysql-client \ redis-tools \ + # Virtual desktop tools + dbus-x11 \ + epiphany-browser \ + imagemagick \ + libgtk-3-bin \ + openbox \ + wmctrl \ + x11-utils \ + xdotool \ + xterm \ + xvfb \ # System libraries often needed by Python packages libssl-dev \ libffi-dev \ @@ -165,6 +176,9 @@ ENV IS_SANDBOX=1 ENV PYTHONPATH="/tmp/workspace" +COPY products/tasks/backend/sandbox/images/start-virtual-desktop.sh /usr/local/bin/start-virtual-desktop +RUN chmod +x /usr/local/bin/start-virtual-desktop + WORKDIR /tmp/workspace RUN python3 --version && \ diff --git a/products/tasks/backend/sandbox/images/start-virtual-desktop.sh b/products/tasks/backend/sandbox/images/start-virtual-desktop.sh new file mode 100644 index 000000000000..e4645aeda715 --- /dev/null +++ b/products/tasks/backend/sandbox/images/start-virtual-desktop.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -euo pipefail + +export DISPLAY="${DISPLAY:-:99}" + +if xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then + exit 0 +fi + +rm -f "/tmp/.X${DISPLAY#:}-lock" +nohup Xvfb "$DISPLAY" -screen 0 1440x900x24 -nolisten tcp -ac >/tmp/xvfb.log 2>&1 & + +for _ in $(seq 1 100); do + if xdpyinfo -display "$DISPLAY" >/dev/null 2>&1; then + nohup openbox-session >/tmp/openbox.log 2>&1 & + for _ in $(seq 1 50); do + wmctrl -m >/dev/null 2>&1 && exit 0 + sleep 0.1 + done + echo "Window manager failed to start" >&2 + exit 1 + fi + sleep 0.1 +done + +echo "Virtual display failed to start" >&2 +exit 1 diff --git a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py index 589453ce3268..da759e597395 100644 --- a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py @@ -112,6 +112,10 @@ def auto_publish(self) -> bool: """User-opted auto-publish: the agent pushes and opens a draft PR on completion.""" return (self.state or {}).get("auto_publish") is True + @property + def computer_use(self) -> bool: + return (self.state or {}).get("computer_use") is True + @property def has_github_credentials(self) -> bool: return self.github_integration_id is not None or self.github_user_integration_id is not None diff --git a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py index 6cc63b2b83b7..9adabef9fc7e 100644 --- a/products/tasks/backend/temporal/process_task/activities/start_agent_server.py +++ b/products/tasks/backend/temporal/process_task/activities/start_agent_server.py @@ -338,6 +338,7 @@ def _invoke_start_agent_server( repo_ready_file=repo_ready_file, wait_for_health=wait_for_health, rtk_enabled=ctx.rtk_enabled, + computer_use=ctx.computer_use, ) # Mark startup-time token issuance so follow-ups within the next diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py index 29773c1cff61..ffbe30c3e1bc 100644 --- a/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py +++ b/products/tasks/backend/temporal/process_task/activities/tests/test_start_agent_server.py @@ -181,6 +181,46 @@ async def test_start_agent_server_uses_captured_sandbox_event_ingest_flag(mocker assert sandbox.start_agent_server.call_args.kwargs["event_ingest_token"] == "event-ingest-token" +async def test_start_agent_server_forwards_computer_use(mocker) -> None: + context = _context(state={"computer_use": True}) + sandbox = mocker.Mock() + sandbox.execute.return_value = ExecutionResult(stdout="", stderr="", exit_code=0) + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.Sandbox.get_by_id", + return_value=sandbox, + ) + mocker.patch("products.tasks.backend.temporal.process_task.activities.start_agent_server.emit_agent_log") + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.Task.objects.select_related" + ).return_value.get.return_value = mocker.Mock(created_by_id=None) + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.create_oauth_access_token_for_run", + return_value="oauth-token", + ) + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.get_sandbox_ph_mcp_configs", + return_value=[], + ) + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.TaskRun.objects.filter", + ).return_value.first.return_value = mocker.Mock(state={}, imported_mcp_servers=None) + mocker.patch( + "products.tasks.backend.temporal.process_task.activities.start_agent_server.create_sandbox_event_ingest_token", + return_value=None, + ) + + await start_agent_server( + StartAgentServerInput( + context=context, + sandbox_id="sandbox-id", + sandbox_url="https://sandbox.example", + sandbox_connect_token="connect-token", + ) + ) + + assert sandbox.start_agent_server.call_args.kwargs["computer_use"] is True + + async def test_start_agent_server_forwards_imported_and_relayed_mcp_servers(mocker) -> None: context = _context() sandbox = mocker.Mock() diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py index 74f91d274337..5ad70a07a291 100644 --- a/products/tasks/backend/tests/test_api.py +++ b/products/tasks/backend/tests/test_api.py @@ -2348,6 +2348,21 @@ def test_run_endpoint_persists_rtk_enabled(self, rtk_enabled, mock_workflow): assert task_run.state["rtk_enabled"] is rtk_enabled mock_workflow.assert_called_once() + @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow") + def test_run_endpoint_persists_computer_use(self, mock_workflow): + task = self.create_task() + + response = self.client.post( + f"/api/projects/@current/tasks/{task.id}/run/", + {"computer_use": True}, + format="json", + ) + + assert response.status_code == status.HTTP_200_OK + task_run = TaskRun.objects.get(id=response.json()["latest_run"]["id"]) + assert task_run.state["computer_use"] is True + mock_workflow.assert_called_once() + @patch("products.tasks.backend.temporal.client.execute_task_processing_workflow") def test_run_endpoint_omits_rtk_enabled_when_not_set(self, mock_workflow): task = self.create_task() diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py index f50c602d49ff..74ef458b1e7b 100644 --- a/products/tasks/backend/tests/test_warm_facade.py +++ b/products/tasks/backend/tests/test_warm_facade.py @@ -196,6 +196,15 @@ def test_does_not_overwrite_existing_warm_description(self): warm_task.refresh_from_db() assert warm_task.description == "already there" + def test_computer_use_creates_a_new_cold_task(self): + warm_task, _ = self._warm_run() + with patch(f"{TITLE_SRC}.generate_task_title", return_value="T"): + dto = self._create(computer_use=True) + + assert str(dto.id) != str(warm_task.id) + created_run = TaskRun.objects.exclude(task=warm_task).get() + assert created_run.state["computer_use"] is True + def test_branch_mismatch_creates_a_new_cold_task(self): warm_task, _ = self._warm_run(branch="main") with patch(f"{TITLE_SRC}.generate_task_title", return_value="T"): From ea9082471bf0f71cc9eb38272e804c67a36b6c77 Mon Sep 17 00:00:00 2001 From: "tests-posthog[bot]" <250237707+tests-posthog[bot]@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:20:19 +0000 Subject: [PATCH 2/5] chore: update OpenAPI generated types --- products/tasks/frontend/generated/api.schemas.ts | 6 ++++++ products/tasks/frontend/generated/api.zod.ts | 15 +++++++++++++++ services/mcp/src/api/generated.ts | 6 ++++++ 3 files changed, 27 insertions(+) diff --git a/products/tasks/frontend/generated/api.schemas.ts b/products/tasks/frontend/generated/api.schemas.ts index 2570bb1e67da..5ea57f66cdaf 100644 --- a/products/tasks/frontend/generated/api.schemas.ts +++ b/products/tasks/frontend/generated/api.schemas.ts @@ -1355,6 +1355,8 @@ export interface ClaudeTaskRunCreateSchemaApi { * @nullable */ rtk_enabled?: boolean | null + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean } /** @@ -1465,6 +1467,8 @@ export interface CodexTaskRunCreateSchemaApi { * @nullable */ rtk_enabled?: boolean | null + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean } export interface TaskRunResumeRequestSchemaApi { @@ -1787,6 +1791,8 @@ export interface TaskRunBootstrapCreateRequestApi { * @nullable */ rtk_enabled?: boolean | null + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean /** * Label of the Home-tab quick action that started this run (e.g. 'Fix CI'), surfaced on the workstream. * @maxLength 120 diff --git a/products/tasks/frontend/generated/api.zod.ts b/products/tasks/frontend/generated/api.zod.ts index ba994811eba0..80767fc6c9b5 100644 --- a/products/tasks/frontend/generated/api.zod.ts +++ b/products/tasks/frontend/generated/api.zod.ts @@ -795,6 +795,7 @@ export const tasksRunCreateBodyOneBranchMax = 255 export const tasksRunCreateBodyOnePendingUserArtifactIdsItemMax = 128 +export const tasksRunCreateBodyOneComputerUseDefault = false export const tasksRunCreateBodyTwoImportedMcpServersItemNameMax = 64 export const tasksRunCreateBodyTwoImportedMcpServersItemUrlMax = 2048 @@ -810,6 +811,7 @@ export const tasksRunCreateBodyTwoBranchMax = 255 export const tasksRunCreateBodyTwoPendingUserArtifactIdsItemMax = 128 +export const tasksRunCreateBodyTwoComputerUseDefault = false export const tasksRunCreateBodyThreeModeDefault = `background` export const tasksRunCreateBodyThreeBranchMax = 255 @@ -949,6 +951,10 @@ export const TasksRunCreateBody = /* @__PURE__ */ zod.union([ .describe( 'Whether rtk command-output compression is enabled for this run. Omitted or null follows the server-side default (enabled); false opts this run out.' ), + computer_use: zod + .boolean() + .default(tasksRunCreateBodyOneComputerUseDefault) + .describe('Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.'), }) .describe('Request body for creating a new task run'), zod @@ -1086,6 +1092,10 @@ export const TasksRunCreateBody = /* @__PURE__ */ zod.union([ .describe( 'Whether rtk command-output compression is enabled for this run. Omitted or null follows the server-side default (enabled); false opts this run out.' ), + computer_use: zod + .boolean() + .default(tasksRunCreateBodyTwoComputerUseDefault) + .describe('Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.'), }) .describe('Request body for creating a new task run'), zod.object({ @@ -1354,6 +1364,7 @@ export const tasksRunsCreateBodyEnvironmentDefault = `local` export const tasksRunsCreateBodyModeDefault = `background` export const tasksRunsCreateBodyBranchMax = 255 +export const tasksRunsCreateBodyComputerUseDefault = false export const tasksRunsCreateBodyHomeQuickActionMax = 120 export const TasksRunsCreateBody = /* @__PURE__ */ zod @@ -1481,6 +1492,10 @@ export const TasksRunsCreateBody = /* @__PURE__ */ zod .describe( 'Whether rtk command-output compression is enabled for this run. Omitted or null follows the server-side default (enabled); false opts this run out.' ), + computer_use: zod + .boolean() + .default(tasksRunsCreateBodyComputerUseDefault) + .describe('Whether this cloud run may control an isolated virtual Linux desktop in its sandbox.'), home_quick_action: zod .string() .max(tasksRunsCreateBodyHomeQuickActionMax) diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts index 90c84e4f3cbf..604b8113690f 100644 --- a/services/mcp/src/api/generated.ts +++ b/services/mcp/src/api/generated.ts @@ -14134,6 +14134,8 @@ export namespace Schemas { * @nullable */ rtk_enabled?: boolean | null; + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean; } export type ClickhouseEventProperties = { [key: string]: unknown }; @@ -14504,6 +14506,8 @@ export namespace Schemas { * @nullable */ rtk_enabled?: boolean | null; + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean; } export type PropertyGroupOperator = typeof PropertyGroupOperator[keyof typeof PropertyGroupOperator]; @@ -61654,6 +61658,8 @@ export namespace Schemas { * @nullable */ rtk_enabled?: boolean | null; + /** Whether this cloud run may control an isolated virtual Linux desktop in its sandbox. */ + computer_use?: boolean; /** * Label of the Home-tab quick action that started this run (e.g. 'Fix CI'), surfaced on the workstream. * @maxLength 120 From 16b6bdba7a05d430791164feb13d880ad428a90c Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 17:36:46 +0300 Subject: [PATCH 3/5] fix(tasks): make cloud desktop runtime reliable Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- products/tasks/backend/logic/services/docker_sandbox.py | 3 ++- products/tasks/backend/logic/services/modal_sandbox.py | 3 ++- products/tasks/backend/logic/services/sandbox.py | 6 ++++++ .../backend/logic/services/tests/test_modal_sandbox.py | 2 ++ .../tasks/backend/sandbox/images/Dockerfile.sandbox-base | 2 ++ 5 files changed, 14 insertions(+), 2 deletions(-) diff --git a/products/tasks/backend/logic/services/docker_sandbox.py b/products/tasks/backend/logic/services/docker_sandbox.py index 082d05fae6b6..fae91bad9cde 100644 --- a/products/tasks/backend/logic/services/docker_sandbox.py +++ b/products/tasks/backend/logic/services/docker_sandbox.py @@ -55,6 +55,7 @@ SandboxStatus, SandboxTemplate, build_agent_runtime_env_prefix, + build_computer_use_env_prefix, parse_sandbox_repo_mount_map, redact_sandbox_command, wait_for_health_check, @@ -834,7 +835,7 @@ def _build_agent_server_command( unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS) server_cmd = ( f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} " - f"{'DISPLAY=:99 ' if computer_use else ''}{env_prefix}" + f"{build_computer_use_env_prefix(computer_use)}{env_prefix}" f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}" f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}" diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py index 742f8c627ce0..a8fb0c029803 100644 --- a/products/tasks/backend/logic/services/modal_sandbox.py +++ b/products/tasks/backend/logic/services/modal_sandbox.py @@ -83,6 +83,7 @@ WORKING_DIR, SandboxBase, build_agent_runtime_env_prefix, + build_computer_use_env_prefix, redact_sandbox_command, wait_for_health_check, ) @@ -982,7 +983,7 @@ def _build_agent_server_command( unset_flags = "".join(f"-u {name} " for name in SANDBOX_AGENT_LAUNCH_UNSET_ENV_VARS) server_cmd = ( f"env {unset_flags}BASH_ENV={shlex.quote(BASH_ENV_SCRIPT)} " - f"{'DISPLAY=:99 ' if computer_use else ''}{env_prefix}" + f"{build_computer_use_env_prefix(computer_use)}{env_prefix}" f"./node_modules/.bin/agent-server --port {AGENT_SERVER_PORT}{repo_flag} " f"--taskId {shlex.quote(task_id)} --runId {shlex.quote(run_id)} --mode {shlex.quote(mode)}" f"{create_pr_flag}{auto_publish_flag}{computer_use_flag}{branch_flag}{mcp_servers_arg}" diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.py index 76ecaca834f3..1b818a6b84e1 100644 --- a/products/tasks/backend/logic/services/sandbox.py +++ b/products/tasks/backend/logic/services/sandbox.py @@ -164,6 +164,12 @@ def redact_sandbox_command(command: str) -> str: return SENSITIVE_AGENT_RUNTIME_ENV_PATTERN.sub(r"\g=", command) +def build_computer_use_env_prefix(computer_use: bool) -> str: + if not computer_use: + return "" + return "DISPLAY=:99 WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1 GTK_A11Y=none " + + def build_agent_runtime_env_prefix( *, interaction_origin: str | None = None, diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py index 8571157b91cc..c8b2d714b054 100644 --- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py +++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py @@ -465,6 +465,8 @@ def test_start_agent_server_starts_virtual_desktop_when_enabled(self, mock_sandb assert "DISPLAY=:99 /usr/local/bin/start-virtual-desktop" in commands launch_command = next(command for command in commands if "--taskId" in command) assert "DISPLAY=:99" in launch_command + assert "WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1" in launch_command + assert "GTK_A11Y=none" in launch_command assert "--computerUse true" in launch_command def test_start_agent_server_waits_for_repository_before_launch(self, mock_sandbox: Any): diff --git a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base index 51394db3cd52..49630c5cfff5 100644 --- a/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base +++ b/products/tasks/backend/sandbox/images/Dockerfile.sandbox-base @@ -44,9 +44,11 @@ RUN apt-get update && \ imagemagick \ libgtk-3-bin \ openbox \ + python3-xdg \ wmctrl \ x11-utils \ xdotool \ + xfonts-base \ xterm \ xvfb \ # System libraries often needed by Python packages From 14f36a4747dea89bf2ca8a6926f4189647390670 Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 18:02:10 +0300 Subject: [PATCH 4/5] fix(tasks): secure cloud computer runtime Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- products/tasks/backend/facade/api.py | 1 + .../backend/logic/services/docker_sandbox.py | 2 +- .../backend/logic/services/modal_sandbox.py | 2 +- .../tasks/backend/logic/services/sandbox.py | 10 +++++--- .../services/tests/test_modal_sandbox.py | 23 ++++++++++++++++++- .../activities/get_task_processing_context.py | 8 +++++++ .../tests/test_get_task_processing_context.py | 19 +++++++++++++++ .../tasks/backend/tests/test_warm_facade.py | 2 +- 8 files changed, 60 insertions(+), 7 deletions(-) diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py index 9540a4c20a85..44630bcc5779 100644 --- a/products/tasks/backend/facade/api.py +++ b/products/tasks/backend/facade/api.py @@ -3508,6 +3508,7 @@ def create_task(team_id: int, user_id: int | None, *, validated_data: dict) -> c warm_branch_provided and validated_data["origin_product"] == Task.OriginProduct.USER_CREATED and validated_data.get("repository") + and not validated_data.get("computer_use", False) and user_id is not None ): warm_run = _find_idling_warm_run( diff --git a/products/tasks/backend/logic/services/docker_sandbox.py b/products/tasks/backend/logic/services/docker_sandbox.py index fae91bad9cde..7d0981e1db43 100644 --- a/products/tasks/backend/logic/services/docker_sandbox.py +++ b/products/tasks/backend/logic/services/docker_sandbox.py @@ -938,7 +938,7 @@ def start_agent_server( auto_publish = False if computer_use: - desktop_result = self.start_virtual_desktop() + desktop_result = self.start_virtual_desktop(restricted_egress=allowed_domains is not None) if desktop_result.exit_code != 0: raise SandboxExecutionError( "Virtual desktop failed to start", diff --git a/products/tasks/backend/logic/services/modal_sandbox.py b/products/tasks/backend/logic/services/modal_sandbox.py index a8fb0c029803..edb2f36926fc 100644 --- a/products/tasks/backend/logic/services/modal_sandbox.py +++ b/products/tasks/backend/logic/services/modal_sandbox.py @@ -1118,7 +1118,7 @@ def start_agent_server( auto_publish = False if computer_use: - desktop_result = self.start_virtual_desktop() + desktop_result = self.start_virtual_desktop(restricted_egress=allowed_domains is not None) if desktop_result.exit_code != 0: raise SandboxExecutionError( "Virtual desktop failed to start", diff --git a/products/tasks/backend/logic/services/sandbox.py b/products/tasks/backend/logic/services/sandbox.py index 1b818a6b84e1..e6d7cd1d3a46 100644 --- a/products/tasks/backend/logic/services/sandbox.py +++ b/products/tasks/backend/logic/services/sandbox.py @@ -28,6 +28,7 @@ from pydantic import BaseModel, model_validator from products.tasks.backend.constants import DEFAULT_SANDBOX_WORKING_DIR, SNAPSHOT_KIND_FILESYSTEM, SnapshotKind +from products.tasks.backend.logic.services.agentsh import ENV_FILE, ENV_WRAPPER_SCRIPT, build_exec_prefix from products.tasks.backend.logic.services.sandbox_config import ( BURSTABLE_REQUEST_CPU_CORES, BURSTABLE_REQUEST_MEMORY_MB, @@ -167,7 +168,7 @@ def redact_sandbox_command(command: str) -> str: def build_computer_use_env_prefix(computer_use: bool) -> str: if not computer_use: return "" - return "DISPLAY=:99 WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1 GTK_A11Y=none " + return "DISPLAY=:99 GTK_A11Y=none " def build_agent_runtime_env_prefix( @@ -256,8 +257,11 @@ def agent_server_supports_auto_publish(self) -> bool: result = self.execute("grep -q autoPublish /scripts/node_modules/.bin/agent-server", timeout_seconds=10) return result.exit_code == 0 - def start_virtual_desktop(self) -> ExecutionResult: - return self.execute("DISPLAY=:99 /usr/local/bin/start-virtual-desktop", timeout_seconds=30) + def start_virtual_desktop(self, *, restricted_egress: bool = False) -> ExecutionResult: + command = "DISPLAY=:99 /usr/local/bin/start-virtual-desktop" + if restricted_egress: + command = f"cd /scripts && env -0 > {ENV_FILE} && {build_exec_prefix()} {ENV_WRAPPER_SCRIPT} {command}" + return self.execute(command, timeout_seconds=30) def clone_repository( self, diff --git a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py index c8b2d714b054..0b8be66bbc2b 100644 --- a/products/tasks/backend/logic/services/tests/test_modal_sandbox.py +++ b/products/tasks/backend/logic/services/tests/test_modal_sandbox.py @@ -465,10 +465,31 @@ def test_start_agent_server_starts_virtual_desktop_when_enabled(self, mock_sandb assert "DISPLAY=:99 /usr/local/bin/start-virtual-desktop" in commands launch_command = next(command for command in commands if "--taskId" in command) assert "DISPLAY=:99" in launch_command - assert "WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS=1" in launch_command + assert "WEBKIT_DISABLE_SANDBOX_THIS_IS_DANGEROUS" not in launch_command assert "GTK_A11Y=none" in launch_command assert "--computerUse true" in launch_command + def test_start_agent_server_starts_restricted_desktop_inside_agentsh(self, mock_sandbox: Any) -> None: + mock_sandbox.execute = MagicMock( + return_value=ExecutionResult(stdout="", stderr="", exit_code=0, error=None), + ) + + with patch.object(mock_sandbox, "_setup_agentsh"): + mock_sandbox.start_agent_server( + repository="posthog/posthog", + task_id="task-123", + run_id="run-456", + computer_use=True, + allowed_domains=["github.com"], + wait_for_health=False, + ) + + desktop_command = next( + call.args[0] for call in mock_sandbox.execute.call_args_list if "start-virtual-desktop" in call.args[0] + ) + assert "agentsh exec" in desktop_command + assert "/tmp/agentsh-env-wrapper.sh" in desktop_command + def test_start_agent_server_waits_for_repository_before_launch(self, mock_sandbox: Any): mock_sandbox.execute = MagicMock( return_value=ExecutionResult(stdout="ok:1", stderr="", exit_code=0, error=None), diff --git a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py index da759e597395..1997ea525c86 100644 --- a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py @@ -363,6 +363,14 @@ def _is_modal_vm_sandbox_enabled( custom_image_available: bool = False, state: dict | None = None, ) -> bool: + if (state or {}).get("computer_use") is True: + log_with_activity_context( + "modal_vm_sandbox_required_for_computer_use", + run_id=run_id, + use_modal_vm_sandbox=True, + ) + return True + if allowed_domains is not None: log_with_activity_context( "modal_vm_sandbox_skipped_restricted_egress", diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py index 4ad44e04e10b..6dadecfc6fcc 100644 --- a/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py @@ -730,6 +730,25 @@ def test_modal_vm_sandbox_state_override_skips_flag_check(self, origin_product, payload_mock.assert_not_called() + def test_computer_use_requires_modal_vm_even_with_restricted_egress(self) -> None: + with patch( + VM_FLAG_PAYLOAD_TARGET, + return_value=None, + ) as payload_mock: + assert ( + _is_modal_vm_sandbox_enabled( + distinct_id="distinct-id", + organization_id="organization-id", + run_id="run-id", + origin_product="user_created", + allowed_domains=["github.com"], + state={"computer_use": True}, + ) + is True + ) + + payload_mock.assert_not_called() + def test_modal_vm_sandbox_restricted_egress_forces_gvisor(self): with patch( VM_FLAG_PAYLOAD_TARGET, diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py index 74ef458b1e7b..daf5df4171e5 100644 --- a/products/tasks/backend/tests/test_warm_facade.py +++ b/products/tasks/backend/tests/test_warm_facade.py @@ -196,7 +196,7 @@ def test_does_not_overwrite_existing_warm_description(self): warm_task.refresh_from_db() assert warm_task.description == "already there" - def test_computer_use_creates_a_new_cold_task(self): + def test_computer_use_creates_a_new_cold_task(self) -> None: warm_task, _ = self._warm_run() with patch(f"{TITLE_SRC}.generate_task_title", return_value="T"): dto = self._create(computer_use=True) From 44a6a0ef379cb574cefb71b4c4b7a4fec997ffc1 Mon Sep 17 00:00:00 2001 From: Richard Solomou Date: Sun, 19 Jul 2026 18:32:55 +0300 Subject: [PATCH 5/5] fix(tasks): keep computer use out of task fields Generated-By: PostHog Code Task-Id: c6d78416-85c8-4304-bd69-e309562468c4 --- products/tasks/backend/facade/api.py | 3 ++- products/tasks/backend/tests/test_warm_facade.py | 2 -- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py index 44630bcc5779..302ede7d0114 100644 --- a/products/tasks/backend/facade/api.py +++ b/products/tasks/backend/facade/api.py @@ -3500,6 +3500,7 @@ def create_task(team_id: int, user_id: int | None, *, validated_data: dict) -> c pending_user_message = (validated_data.pop("pending_user_message", None) or "").strip() or None pending_user_artifact_ids = validated_data.pop("pending_user_artifact_ids", None) or [] warm_auto_publish = validated_data.pop("auto_publish", None) + computer_use = validated_data.pop("computer_use", False) if user_id is not None: validated_data["created_by"] = User.objects.get(id=user_id) @@ -3508,7 +3509,7 @@ def create_task(team_id: int, user_id: int | None, *, validated_data: dict) -> c warm_branch_provided and validated_data["origin_product"] == Task.OriginProduct.USER_CREATED and validated_data.get("repository") - and not validated_data.get("computer_use", False) + and not computer_use and user_id is not None ): warm_run = _find_idling_warm_run( diff --git a/products/tasks/backend/tests/test_warm_facade.py b/products/tasks/backend/tests/test_warm_facade.py index daf5df4171e5..c662fa8a9bad 100644 --- a/products/tasks/backend/tests/test_warm_facade.py +++ b/products/tasks/backend/tests/test_warm_facade.py @@ -202,8 +202,6 @@ def test_computer_use_creates_a_new_cold_task(self) -> None: dto = self._create(computer_use=True) assert str(dto.id) != str(warm_task.id) - created_run = TaskRun.objects.exclude(task=warm_task).get() - assert created_run.state["computer_use"] is True def test_branch_mismatch_creates_a_new_cold_task(self): warm_task, _ = self._warm_run(branch="main")