From 8475cf0c4149cf6459dbfde30f395ae8f14947d5 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:04 +0100 Subject: [PATCH 01/26] feat(tasks): inject agent otel telemetry config into cloud sandboxes Companion to PostHog/code#posthog-code/agent-run-otel-telemetry, which adds OTLP export of cloud-run metadata (logs + APM spans) to the agent-server. New optional settings SANDBOX_AGENT_OTEL_LOGS_URL/_TOKEN and SANDBOX_AGENT_OTEL_TRACES_URL are mapped to POSTHOG_AGENT_OTEL_* sandbox env vars by a shared helper called from both env assembly paths (fresh provisioning and snapshot resume). The names are deliberately not standard OTEL_* so OTel SDKs in user code running in the sandbox never auto-export into the telemetry project; the keys are reserved so user sandbox env vars cannot override them. Local dev is covered by adding the URLs to the agentsh debug firewall settings and the Docker localhost rewrite list; prod egress is already allowed via *.posthog.com. Telemetry stays off unless URL + token are both set; the traces URL additionally enables APM spans. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 15 +++++++ posthog/settings/temporal.py | 8 ++++ products/tasks/backend/constants.py | 3 ++ .../tasks/backend/logic/services/agentsh.py | 2 + .../backend/logic/services/docker_sandbox.py | 2 + .../activities/provision_sandbox.py | 3 ++ .../tests/test_provision_sandbox.py | 39 +++++++++++++++++++ .../temporal/process_task/tests/test_utils.py | 24 +++++++++++- .../backend/temporal/process_task/utils.py | 20 ++++++++++ 9 files changed, 115 insertions(+), 1 deletion(-) diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index c515f5900842..a8d3ac381e3e 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -177,6 +177,21 @@ SANDBOX_MCP_URL=https://.ngrok-free.app/mcp `SANDBOX_MCP_URL` overrides the `host.docker.internal` default (which only resolves from local Docker sandboxes, not Modal). Without it, sandbox agents can't reach the MCP server and lose access to the PostHog `execute-sql`, query, and tool-calling stack. +### Agent run telemetry (optional) + +To ship agent-server run metadata to PostHog Logs, set both of the first two; the third additionally produces one APM trace per run (root `task_run` span, a `turn` span per prompt, a `tool_call:` span per tool call) with trace/span ids stamped on the log records: + +```bash +SANDBOX_AGENT_OTEL_LOGS_URL=http://localhost:8000/i/v1/logs # or https://us.i.posthog.com/i/v1/logs +SANDBOX_AGENT_OTEL_LOGS_TOKEN= +SANDBOX_AGENT_OTEL_TRACES_URL=http://localhost:8000/i/v1/traces # optional, enables APM spans +``` + +They're injected into the sandbox as `POSTHOG_AGENT_OTEL_LOGS_URL`/`_TOKEN`/`POSTHOG_AGENT_OTEL_TRACES_URL` (deliberately not standard `OTEL_*` names, so OTel SDKs in user code don't auto-export into the telemetry project). +The agent-server exports run/turn/tool lifecycle metadata (never message content or tool arguments), tagged with `run_id`/`task_id`/`team_id`/`user_id`/`distinct_id` resource attributes and `service.name=posthog-code-agent`. +Telemetry stays off when either of the first two vars is unset. +For local Docker sandboxes the localhost URLs are rewritten to `host.docker.internal` automatically; local ingestion requires the `capture-logs` service to be running. + ### MCP server `.env` `MODAL_DOCKER` (and the local Docker provider) both depend on the MCP server running at `localhost:8787`. The server reads its config from `services/mcp/.env` — without it, things like `POSTHOG_API_BASE_URL`, the UI-apps token, and analytics keys are missing and the server will either refuse to start or return broken responses to the sandbox. diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 12f4eefb50f7..9d8eca0a02fd 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -51,6 +51,14 @@ SANDBOX_LLM_GATEWAY_URL: str | None = get_from_env("SANDBOX_LLM_GATEWAY_URL", None, optional=True) SANDBOX_MCP_URL: str | None = get_from_env("SANDBOX_MCP_URL", None, optional=True) +# OTLP destinations for agent-server run telemetry (PostHog Logs/APM). +# Full ingest URLs (e.g. https://us.i.posthog.com/i/v1/logs and .../i/v1/traces) +# plus the project API key of the telemetry project. Telemetry stays off unless +# URL + token are set; the traces URL additionally enables APM spans. +SANDBOX_AGENT_OTEL_LOGS_URL: str | None = get_from_env("SANDBOX_AGENT_OTEL_LOGS_URL", None, optional=True) +SANDBOX_AGENT_OTEL_LOGS_TOKEN: str | None = get_from_env("SANDBOX_AGENT_OTEL_LOGS_TOKEN", None, optional=True) +SANDBOX_AGENT_OTEL_TRACES_URL: str | None = get_from_env("SANDBOX_AGENT_OTEL_TRACES_URL", None, optional=True) + # client_id of the OAuthApplication used to mint the access token the PostHog setup wizard # uses when it runs inside a task sandbox (the "run the wizard in the cloud" onboarding path). # It must be the wizard's own app so the LLM gateway authorizes the token like a normal wizard diff --git a/products/tasks/backend/constants.py b/products/tasks/backend/constants.py index 8dc2f7548933..bb6469737018 100644 --- a/products/tasks/backend/constants.py +++ b/products/tasks/backend/constants.py @@ -365,6 +365,9 @@ def vm_sandbox_allowed_origins(*, distinct_id: str, organization_id: str) -> set "GH_TOKEN", "LLM_GATEWAY_URL", "POSTHOG_RESUME_RUN_ID", + "POSTHOG_AGENT_OTEL_LOGS_URL", + "POSTHOG_AGENT_OTEL_LOGS_TOKEN", + "POSTHOG_AGENT_OTEL_TRACES_URL", "CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC", "DISABLE_TELEMETRY", "DISABLE_ERROR_REPORTING", diff --git a/products/tasks/backend/logic/services/agentsh.py b/products/tasks/backend/logic/services/agentsh.py index 93b64cc67704..058ec7cb42a3 100644 --- a/products/tasks/backend/logic/services/agentsh.py +++ b/products/tasks/backend/logic/services/agentsh.py @@ -64,6 +64,8 @@ def _port_from_url(url: str | None) -> int | None: "SANDBOX_API_URL", "SANDBOX_LLM_GATEWAY_URL", "SANDBOX_MCP_URL", + "SANDBOX_AGENT_OTEL_LOGS_URL", + "SANDBOX_AGENT_OTEL_TRACES_URL", ) diff --git a/products/tasks/backend/logic/services/docker_sandbox.py b/products/tasks/backend/logic/services/docker_sandbox.py index 598ae9db63bd..30455c4f0296 100644 --- a/products/tasks/backend/logic/services/docker_sandbox.py +++ b/products/tasks/backend/logic/services/docker_sandbox.py @@ -84,6 +84,8 @@ { "POSTHOG_API_URL", "POSTHOG_SITE_URL", + "POSTHOG_AGENT_OTEL_LOGS_URL", + "POSTHOG_AGENT_OTEL_TRACES_URL", "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT", "OTEL_EXPORTER_OTLP_ENDPOINT", } diff --git a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py index a121568ae5dc..1cc2ee477533 100644 --- a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py +++ b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py @@ -45,6 +45,7 @@ get_sandbox_api_url, get_sandbox_github_token, get_sandbox_name_for_task, + get_sandbox_otel_env_vars, get_sandbox_snapshot_metadata, get_task_run_credential_user, parse_run_state, @@ -344,6 +345,8 @@ def _build_environment_variables( # the cache key) and cost attribution. Rely on Temporal retries instead. environment_variables["POSTHOG_DISABLE_MODEL_FALLBACK"] = "1" + environment_variables.update(get_sandbox_otel_env_vars()) + if ctx.allowed_domains is not None: environment_variables.update(NETWORK_RESTRICTED_AGENT_ENV) diff --git a/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py b/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py index 20eebaba58b3..5125b5a21920 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py +++ b/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py @@ -197,3 +197,42 @@ def test_build_environment_variables_disables_telemetry_when_restricted( assert all(env.get(k) == "1" for k in keys) else: assert not (keys & env.keys()) + + +@patch(f"{_PROVISION}.get_git_identity_env_vars", return_value={}) +@patch(f"{_PROVISION}.get_sandbox_jwt_public_key", return_value="pub") +@patch(f"{_PROVISION}.get_sandbox_api_url", return_value="https://api.example") +@pytest.mark.parametrize( + "url, token, traces_url, expected_keys", + [ + ( + "https://us.i.posthog.com/i/v1/logs", + "phc_telemetry", + "https://us.i.posthog.com/i/v1/traces", + {"POSTHOG_AGENT_OTEL_LOGS_URL", "POSTHOG_AGENT_OTEL_LOGS_TOKEN", "POSTHOG_AGENT_OTEL_TRACES_URL"}, + ), + ( + "https://us.i.posthog.com/i/v1/logs", + "phc_telemetry", + None, + {"POSTHOG_AGENT_OTEL_LOGS_URL", "POSTHOG_AGENT_OTEL_LOGS_TOKEN"}, + ), + ("https://us.i.posthog.com/i/v1/logs", None, None, set()), + (None, "phc_telemetry", None, set()), + # Traces alone are useless without the logs pair carrying the token. + (None, None, "https://us.i.posthog.com/i/v1/traces", set()), + ], +) +def test_build_environment_variables_injects_otel_env_only_when_fully_configured( + _api, _jwt, _git, url, token, traces_url, expected_keys +): + ctx = _context() + + with override_settings( + SANDBOX_AGENT_OTEL_LOGS_URL=url, + SANDBOX_AGENT_OTEL_LOGS_TOKEN=token, + SANDBOX_AGENT_OTEL_TRACES_URL=traces_url, + ): + env = _build_environment_variables(ctx, MagicMock(), "", "access-token") + + assert {key for key in env if key.startswith("POSTHOG_AGENT_OTEL_")} == expected_keys diff --git a/products/tasks/backend/temporal/process_task/tests/test_utils.py b/products/tasks/backend/temporal/process_task/tests/test_utils.py index 44769040e084..a804ce6de179 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_utils.py +++ b/products/tasks/backend/temporal/process_task/tests/test_utils.py @@ -1,6 +1,6 @@ from unittest.mock import MagicMock, patch -from django.test import TestCase +from django.test import SimpleTestCase, TestCase, override_settings from parameterized import parameterized @@ -16,6 +16,7 @@ McpServerConfig, RunState, build_imported_mcp_server_configs, + build_sandbox_environment_variables, get_git_identity_env_vars, get_github_credential_source, get_imported_mcp_server_configs, @@ -1066,3 +1067,24 @@ def test_collision_detection_is_case_insensitive(self): ) assert get_relayed_mcp_server_names(task_run, {"grafana"}) == ["Playwright", "internal-cli"] + +class TestBuildSandboxEnvironmentVariables(SimpleTestCase): + @patch( + "products.tasks.backend.logic.services.connection_token.get_sandbox_jwt_public_key", + return_value="pub", + ) + @patch( + "products.tasks.backend.temporal.process_task.utils.get_sandbox_api_url", + return_value="https://api.example", + ) + def test_snapshot_resume_env_includes_otel_config_when_configured(self, _api, _jwt) -> None: + with override_settings( + SANDBOX_AGENT_OTEL_LOGS_URL="https://us.i.posthog.com/i/v1/logs", + SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_telemetry", + SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", + ): + env = build_sandbox_environment_variables(None, "access-token", 1) + + assert env["POSTHOG_AGENT_OTEL_LOGS_URL"] == "https://us.i.posthog.com/i/v1/logs" + assert env["POSTHOG_AGENT_OTEL_LOGS_TOKEN"] == "phc_telemetry" + assert env["POSTHOG_AGENT_OTEL_TRACES_URL"] == "https://us.i.posthog.com/i/v1/traces" diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.py index 242f9e67a667..788d8db0318d 100644 --- a/products/tasks/backend/temporal/process_task/utils.py +++ b/products/tasks/backend/temporal/process_task/utils.py @@ -1148,6 +1148,26 @@ def build_sandbox_environment_variables( if settings.SANDBOX_LLM_GATEWAY_URL: env_vars["LLM_GATEWAY_URL"] = settings.SANDBOX_LLM_GATEWAY_URL + env_vars.update(get_sandbox_otel_env_vars()) + + return env_vars + + +def get_sandbox_otel_env_vars() -> dict[str, str]: + """OTLP config for agent-server run telemetry (PostHog Logs/APM). + + Deliberately POSTHOG_-prefixed rather than the standard OTEL_* names: the + sandbox env is inherited by user processes, and standard OTEL_* vars would + make any OTel SDK in user code auto-export into our telemetry project. + """ + if not (settings.SANDBOX_AGENT_OTEL_LOGS_URL and settings.SANDBOX_AGENT_OTEL_LOGS_TOKEN): + return {} + env_vars = { + "POSTHOG_AGENT_OTEL_LOGS_URL": settings.SANDBOX_AGENT_OTEL_LOGS_URL, + "POSTHOG_AGENT_OTEL_LOGS_TOKEN": settings.SANDBOX_AGENT_OTEL_LOGS_TOKEN, + } + if settings.SANDBOX_AGENT_OTEL_TRACES_URL: + env_vars["POSTHOG_AGENT_OTEL_TRACES_URL"] = settings.SANDBOX_AGENT_OTEL_TRACES_URL return env_vars From 2701d1d8abbf127fe042c75331946ec167aa1235 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:06 +0100 Subject: [PATCH 02/26] chore(tasks): add implementation report for agent run telemetry Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 143 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100644 REPORT.md diff --git a/REPORT.md b/REPORT.md new file mode 100644 index 000000000000..ffbd4ee16e78 --- /dev/null +++ b/REPORT.md @@ -0,0 +1,143 @@ +# Agent run telemetry: PostHog Code cloud tasks → PostHog Logs + APM + +This branch is one half of a two-repo change; the companion branch is `posthog-code/agent-run-otel-telemetry` in `PostHog/code`. +This repo (`PostHog/posthog`) carries the configuration/injection side; `PostHog/code` carries the telemetry emitter inside the agent. + +## Goal + +Cloud task runs executed by PostHog Code should be observable in PostHog itself: +every run's lifecycle metadata delivered as OTel **logs** into the Logs product, and one OTel **trace** per run (root span, per-turn spans, per-tool-call spans) into APM, with logs and spans cross-linked via `trace_id`/`span_id`. + +Hard requirements: + +- **Filterable per user**: the Logs UI must answer "show me everything cloud runs did for this user" directly, so `user_id`/`distinct_id` (plus `team_id`, `task_id`, `run_id`) are OTel resource attributes, which PostHog Logs facets via `resource_fingerprint`. +- **Cloud tasks only** for now; desktop local runs do not export session telemetry. +- **Metadata only**: the S3 session log remains the source of truth for full transcripts. Telemetry never carries prompts, agent message/thought text, tool arguments, or tool output. + +## Background: what existed before + +- Agent session logs flow from the sandbox `agent-server` through `SessionLogWriter` to the Django endpoint `POST /api/projects/{team}/tasks/{task}/runs/{run}/append_log/`, which appends NDJSON to S3 (`TaskRun.append_log`, 30-day TTL). Two more delivery legs exist: an NDJSON event-ingest stream and SSE to connected clients. +- A February attempt at OTel logs export (`OtelLogWriter`, commits `6abadc79` → `99a3aea8` → `8876c8fb` in `PostHog/code`) was unwired, and its default endpoint `/i/v1/agent-logs` does not exist in the ingest service; it would 404 today. +- The correct ingest is the `capture-logs` Rust service (`rust/capture-logs/`): `POST /i/v1/logs` and `POST /i/v1/traces`, OTLP http/protobuf or http/JSON, auth `Authorization: Bearer `, 2 MB request cap, billed by uncompressed bytes, severity normalized to lowercase, prod host `https://us.i.posthog.com`. +- Working dogfood precedents followed here: the desktop Electron transport (`posthog-code-desktop` → `/i/v1/logs`), the engineering-analytics CI log emitter, the streamlit sandbox proxy (env-injected OTLP config), and the plugin-server metrics exporter (telemetry off unless URL + token are both set). + +## Architecture + +``` +sandbox (agentsh) +└── agent-server (PostHog/code, packages/agent) + ├── ACP streams (tapped) ──► SessionLogWriter ──► Django append_log ──► S3 (product log, unchanged) + │ │ + │ └─ sink ──► OtelRunTelemetry + │ ├─ log records ──► POST {POSTHOG_AGENT_OTEL_LOGS_URL} + │ └─ RunTraceBuilder spans ──► POST {POSTHOG_AGENT_OTEL_TRACES_URL} + └── terminal error events ──────────────────────► mirrored into OtelRunTelemetry directly + +capture-logs (Rust) ──► Kafka ──► ClickHouse (logs / trace_spans) ──► Logs + APM UI +``` + +Telemetry is emitted **from inside the sandbox** by the agent-server process, deliberately independent of the Django/S3 product path: if `append_log` is degraded, telemetry still flows, and a broken product log pipeline is exactly the failure telemetry must capture. +Egress works because `*.posthog.com` is in the agentsh `INFRASTRUCTURE_DOMAINS` allowlist (prod) and the new `SANDBOX_AGENT_OTEL_*_URL` hosts join the DEBUG-only firewall list (local dev). + +## What ships per run + +### Log records (service.name=posthog-code-agent) + +Resource attributes on every record: `service.name`, `service.version` (agent version), `run_id`, `task_id`, `team_id`, `user_id`, `distinct_id`, `device_type` (`cloud`), `adapter` (`claude`/`codex`), `run_mode` (`interactive`/`background`). + +Exported events (allowlist, everything else is dropped): + +| Event | Severity | Notable attributes | +| --- | --- | --- | +| `_posthog/run_started` | info | `agent_version`, `session_id` | +| `_posthog/sdk_session` | info | `adapter`, `session_id` | +| `_posthog/usage_update` (and the `session/update` variant) | info | `tokens_input/output/cached_read/cached_write`, `cost_usd` | +| `_posthog/turn_complete` | info | `stop_reason` | +| `_posthog/task_complete` | info | `stop_reason` | +| `_posthog/error` | **error** | `error_source`, `stop_reason`, message in body (capped) | +| `_posthog/console` | mapped from level | agent-server internal logs | +| `_posthog/progress` | info | `progress_group/step/status` | +| `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | +| `_posthog/mode_change`, `_posthog/compact_boundary` | info | | +| `_posthog/permission_request/response/resolved` | info | `request_id`, `tool_call_id` (identifiers only; tool content excluded) | +| `session/update: tool_call` | info | `tool_call_id`, `tool_kind`, `tool_status` (no title, no rawInput) | +| `session/update: tool_call_update` (terminal only) | info / warn on `failed` | `tool_call_id`, `tool_status` | + +Deliberately dropped: `agent_message`, `agent_message_chunk`, `agent_thought_chunk`, `user_message`, `session/prompt` bodies, in-progress `tool_call_update` snapshots (they re-send the growing tool input/output), `available_commands_update`, and any unknown method (fail-closed allowlist). +Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `log_attributes` faceting table only indexes key/value pairs under 256 chars). + +### APM trace (one per run) + +- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup; status OK on `task_complete`, ERROR with the error message on `_posthog/error`. +- `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. +- `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. +- Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; an error cascades ERROR status through open tool and turn spans to the root. +- Every log record is emitted under the OTel context of the span it belongs to (tool logs on the tool span, lifecycle logs on the root), so `trace_id`/`span_id` land in the `logs` table columns and the UI links Logs ⇄ trace waterfall. + +### Delivery timing + +Telemetry is near-realtime, not end-of-turn: records are created the moment each notification flows through the writer and batched for at most 2 s (`BatchLogRecordProcessor` / `BatchSpanProcessor`, `scheduledDelayMillis` 2000). +Spans export when they end (tools mid-turn, turns at `turn_complete`, root at cleanup). +Flush safety nets: an explicit flush after a terminal error in `signalTaskComplete`, a full shutdown-flush in `cleanupSession` (which SIGTERM reaches via `stop()`), so sandbox teardown cannot eat the tail of a run's telemetry. + +## Changes in PostHog/code (companion branch) + +- `packages/agent/src/otel-telemetry.ts` (renamed from `otel-log-writer.ts`): `OtelRunTelemetry`, the single `SessionLogSink` owning the OTLP log exporter and (when a traces URL is configured) the `RunTraceBuilder`. Contains the pure `mapNotificationToLogRecord()` allowlist mapper. Fixes the dead `/i/v1/agent-logs` default. Never throws into the run; ignores entries for other sessions. +- `packages/agent/src/otel-trace-builder.ts` (new): `RunTraceBuilder`, the span state machine described above; `handle(entry)` returns the context each log record should attach to. +- `packages/agent/src/otel-attributes.ts` (new): shared pure helpers (`strAttr`, `numAttr`, `usageAttributes`, truncation, caps). +- `packages/agent/src/session-log-writer.ts`: optional `sinks: SessionLogSink[]`, teed in `appendRawLine` after the entry is built; a throwing sink warns once and can never break product log persistence; message chunks never reach sinks. `SessionContext` moved here from the otel module. +- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, shuts it down in `cleanupSession`, flushes after terminal errors, and mirrors `enqueueTaskTerminalEvent` payloads into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). +- `packages/agent/src/server/bin.ts` + `server/types.ts`: zod-validated env `POSTHOG_AGENT_OTEL_LOGS_URL`, `POSTHOG_AGENT_OTEL_LOGS_TOKEN`, `POSTHOG_AGENT_OTEL_TRACES_URL` → `AgentServerConfig.otelLogsUrl/otelLogsToken/otelTracesUrl`. Telemetry is off unless the logs pair is set; spans additionally require the traces URL (per-signal kill switch). +- `packages/agent/src/types.ts`: deleted the dead `OtelTransportConfig`/`AgentConfig.otelTransport` left over from the February attempt. +- Dependencies: `@opentelemetry/api`, `@opentelemetry/sdk-trace-base`, `@opentelemetry/exporter-trace-otlp-http`, version-aligned with the existing logs SDK (0.208.x experimental / 2.x stable line). +- Tests (71 passing): parameterized log-mapping matrix, a hard privacy test asserting exported payloads never contain tool args/titles/output, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, and four trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade, orphan export on shutdown). +- `packages/agent/README.md`: documents the env vars and behavior. + +## Changes in PostHog/posthog (this branch) + +- `posthog/settings/temporal.py`: new optional settings `SANDBOX_AGENT_OTEL_LOGS_URL`, `SANDBOX_AGENT_OTEL_LOGS_TOKEN`, `SANDBOX_AGENT_OTEL_TRACES_URL` (all default unset = telemetry off). +- `products/tasks/backend/temporal/process_task/utils.py`: `get_sandbox_otel_env_vars()` maps those settings to the sandbox env vars, gated on the logs pair; called from **both** env assembly paths so fresh provisioning and snapshot-resume behave identically: + - `activities/provision_sandbox.py` `_build_environment_variables` + - `utils.py` `build_sandbox_environment_variables` (used by `create_sandbox_from_snapshot`) +- `products/tasks/backend/constants.py`: the three env keys added to `RESERVED_SANDBOX_ENVIRONMENT_VARIABLE_KEYS` so user-supplied SandboxEnvironment vars cannot override them. +- `products/tasks/backend/logic/services/agentsh.py`: `SANDBOX_AGENT_OTEL_LOGS_URL`/`SANDBOX_AGENT_OTEL_TRACES_URL` added to `_DEBUG_SANDBOX_URL_SETTINGS` so local-dev hosts pass the agentsh syscall firewall (prod egress already covered by `*.posthog.com`). +- `products/tasks/backend/logic/services/docker_sandbox.py`: `POSTHOG_AGENT_OTEL_LOGS_URL`/`POSTHOG_AGENT_OTEL_TRACES_URL` added to `_DOCKER_URL_ENV_KEYS` so localhost URLs are rewritten to `host.docker.internal` for local Docker sandboxes. +- Tests: parameterized gating matrix on `_build_environment_variables` (5 rows: full config, logs-only, partial configs, traces-without-logs all correctly gated) and a `SimpleTestCase` wiring guard on the snapshot-resume path. +- `docs/internal/sandboxes-setup-guide.md`: local-dev setup section for the new settings. + +## Key design decisions + +1. **Emit from the sandbox, not from Django.** Independence from the product log path (see Architecture), matching the streamlit sandbox precedent. The Django-tee alternative would add an outbound call to a hot API path and go dark precisely when `append_log` breaks. +2. **`POSTHOG_`-prefixed env vars instead of standard `OTEL_*` names.** The sandbox env is inherited by the user's own processes (their tests, their apps). Standard `OTEL_EXPORTER_OTLP_*` vars would make any OTel SDK in user code silently auto-export the user's telemetry into our internal project. Custom names mean only agent-server reads the config. +3. **`service.name=posthog-code-agent`, not `posthog-code`.** `service.name` identifies the emitting process, not the product: the desktop app already ships its process logs as `posthog-code-desktop`, and `service_name` is both the primary Logs UI facet and part of the ClickHouse sort key `(team_id, service_name, timestamp)`, so component-level names keep streams separable and queries narrow. House pattern matches (`posthog-django-*`, `node-*`, `github-ci-logs`). +4. **Fail-closed allowlist for content.** Only known lifecycle events are exported; unknown methods are dropped. This is a privacy boundary (customer prompts/repo content must not reach the telemetry project) and a cost control (logs are billed by bytes; in-progress tool snapshots re-send growing output). +5. **Generic `SessionLogSink` instead of hardcoding OTel into `SessionLogWriter`.** The February attempt was removed partly because of hard coupling; the sink interface keeps the writer single-purpose, is desktop-neutral (no sinks wired there), and isolates sink failures. +6. **Terminal-error mirror.** `enqueueTaskTerminalEvent` feeds only the event-ingest stream, bypassing `SessionLogWriter`; without the explicit mirror the most important record (run failed) would be missing from telemetry. +7. **Per-signal kill switch.** Logs and spans have separate URLs; unsetting the traces URL disables spans without touching logs, and unsetting either of the logs pair disables everything. +8. **Token exposure is acceptable by design.** The sandbox receives a project API key of the telemetry project: a write-only, public-by-design key class (the same class that ships in client SDKs), far weaker than the `POSTHOG_PERSONAL_API_KEY` already present in the sandbox. Worst case is junk telemetry writes; `capture-logs` has a token drop list as the kill switch. + +## Verification + +- `PostHog/code`: 71 tests pass in the agent package (including the new telemetry suite), `tsc --noEmit` clean via turbo, biome clean on all touched files (one pre-existing warning untouched). The package's pre-existing test failures in this environment (missing Postgres/git fixtures) were confirmed byte-identical with and without these changes by running the failing files against a stashed tree. +- `PostHog/posthog`: 21 tests pass across `test_provision_sandbox.py` and the new `TestBuildSandboxEnvironmentVariables`; `ruff check`/`format` clean on all touched files. DB-dependent suites in this sandbox fail identically with and without the change (no Postgres available). +- Local-dev routing verified: Caddy serves `/i/v1/logs`/`/i/v1/traces` on `localhost:8000` and proxies to `capture-logs`, and the Docker URL rewrite covers the new vars. + +## Rollout runbook (what remains) + +1. Choose the destination telemetry project and create/locate its project API key. Recommendation: the shared internal project where `posthog-code-desktop` logs and Code analytics already land, so desktop and cloud correlate in one Logs view (separable by `service_name`). +2. Set in prod US: + - `SANDBOX_AGENT_OTEL_LOGS_URL=https://us.i.posthog.com/i/v1/logs` + - `SANDBOX_AGENT_OTEL_LOGS_TOKEN=` + - `SANDBOX_AGENT_OTEL_TRACES_URL=https://us.i.posthog.com/i/v1/traces` +3. Run one cloud task; verify in the destination project: Logs filtered by `service.name=posthog-code-agent` (facet by `distinct_id`/`user_id`/`run_id`), and the APM trace for the run (`task_run` → `turn` → `tool_call:*` waterfall, logs linked from spans). +4. Add a saved Logs view and alerts (error severity on the service; volume anomaly), and watch billed bytes for a week; the event allowlist and body caps are the tuning knobs. +5. Optional follow-ups: unify the desktop transport with the new telemetry module; consider a shared `product` resource attribute across `posthog-code-*` services; sampling if volume warrants. + +## Configuration reference + +| Where | Name | Meaning | +| --- | --- | --- | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | +| Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | From 4374beac0e304331bee0f433c3271474cebb7ae1 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:08 +0100 Subject: [PATCH 03/26] chore(tasks): sync implementation report with telemetry review fixes Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/REPORT.md b/REPORT.md index ffbd4ee16e78..d71388637f6e 100644 --- a/REPORT.md +++ b/REPORT.md @@ -68,7 +68,7 @@ Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `l ### APM trace (one per run) -- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup; status OK on `task_complete`, ERROR with the error message on `_posthog/error`. +- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status OK once a turn ends cleanly with `end_turn` (the sandbox never emits `task_complete` for successful runs; the terminal "completed" status is decided by the workflow outside, so a clean turn end is the in-sandbox success signal), ERROR with the error message on `_posthog/error` (an error always wins, a later turn completion cannot flip it back), unset when the run ends without either (cancelled or timed out). - `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. - `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. - Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; an error cascades ERROR status through open tool and turn spans to the root. @@ -86,7 +86,7 @@ Flush safety nets: an explicit flush after a terminal error in `signalTaskComple - `packages/agent/src/otel-trace-builder.ts` (new): `RunTraceBuilder`, the span state machine described above; `handle(entry)` returns the context each log record should attach to. - `packages/agent/src/otel-attributes.ts` (new): shared pure helpers (`strAttr`, `numAttr`, `usageAttributes`, truncation, caps). - `packages/agent/src/session-log-writer.ts`: optional `sinks: SessionLogSink[]`, teed in `appendRawLine` after the entry is built; a throwing sink warns once and can never break product log persistence; message chunks never reach sinks. `SessionContext` moved here from the otel module. -- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, shuts it down in `cleanupSession`, flushes after terminal errors, and mirrors `enqueueTaskTerminalEvent` payloads into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). +- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, shuts it down in `cleanupSession`, flushes after terminal errors, and mirrors `enqueueTaskTerminalEvent` payloads into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). Fatal crashes (`reportFatalError`, the uncaught-exception/unhandled-rejection path) also mirror an error record (`error_source=agent_server_crash`) and shut telemetry down, so hard process deaths reach the telemetry project instead of vanishing. - `packages/agent/src/server/bin.ts` + `server/types.ts`: zod-validated env `POSTHOG_AGENT_OTEL_LOGS_URL`, `POSTHOG_AGENT_OTEL_LOGS_TOKEN`, `POSTHOG_AGENT_OTEL_TRACES_URL` → `AgentServerConfig.otelLogsUrl/otelLogsToken/otelTracesUrl`. Telemetry is off unless the logs pair is set; spans additionally require the traces URL (per-signal kill switch). - `packages/agent/src/types.ts`: deleted the dead `OtelTransportConfig`/`AgentConfig.otelTransport` left over from the February attempt. - Dependencies: `@opentelemetry/api`, `@opentelemetry/sdk-trace-base`, `@opentelemetry/exporter-trace-otlp-http`, version-aligned with the existing logs SDK (0.208.x experimental / 2.x stable line). @@ -112,7 +112,7 @@ Flush safety nets: an explicit flush after a terminal error in `signalTaskComple 3. **`service.name=posthog-code-agent`, not `posthog-code`.** `service.name` identifies the emitting process, not the product: the desktop app already ships its process logs as `posthog-code-desktop`, and `service_name` is both the primary Logs UI facet and part of the ClickHouse sort key `(team_id, service_name, timestamp)`, so component-level names keep streams separable and queries narrow. House pattern matches (`posthog-django-*`, `node-*`, `github-ci-logs`). 4. **Fail-closed allowlist for content.** Only known lifecycle events are exported; unknown methods are dropped. This is a privacy boundary (customer prompts/repo content must not reach the telemetry project) and a cost control (logs are billed by bytes; in-progress tool snapshots re-send growing output). 5. **Generic `SessionLogSink` instead of hardcoding OTel into `SessionLogWriter`.** The February attempt was removed partly because of hard coupling; the sink interface keeps the writer single-purpose, is desktop-neutral (no sinks wired there), and isolates sink failures. -6. **Terminal-error mirror.** `enqueueTaskTerminalEvent` feeds only the event-ingest stream, bypassing `SessionLogWriter`; without the explicit mirror the most important record (run failed) would be missing from telemetry. +6. **Terminal-error mirrors.** Two paths bypass `SessionLogWriter` and are mirrored into telemetry explicitly: `enqueueTaskTerminalEvent` (agent-server-sourced run errors, which feed only the event-ingest stream) and `reportFatalError` (unrecoverable crashes, which mark the run failed via the API with no session log involvement). Without the mirrors the most important records, failed and crashed runs, would be missing from telemetry. 7. **Per-signal kill switch.** Logs and spans have separate URLs; unsetting the traces URL disables spans without touching logs, and unsetting either of the logs pair disables everything. 8. **Token exposure is acceptable by design.** The sandbox receives a project API key of the telemetry project: a write-only, public-by-design key class (the same class that ships in client SDKs), far weaker than the `POSTHOG_PERSONAL_API_KEY` already present in the sandbox. Worst case is junk telemetry writes; `capture-logs` has a token drop list as the kill switch. From 756f2e7ab91797a9de7463b545eaad10778b0caa Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:10 +0100 Subject: [PATCH 04/26] chore(tasks): sync implementation report with telemetry console-export and root-status fixes Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/REPORT.md b/REPORT.md index d71388637f6e..eb4df168abef 100644 --- a/REPORT.md +++ b/REPORT.md @@ -55,7 +55,6 @@ Exported events (allowlist, everything else is dropped): | `_posthog/turn_complete` | info | `stop_reason` | | `_posthog/task_complete` | info | `stop_reason` | | `_posthog/error` | **error** | `error_source`, `stop_reason`, message in body (capped) | -| `_posthog/console` | mapped from level | agent-server internal logs | | `_posthog/progress` | info | `progress_group/step/status` | | `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | | `_posthog/mode_change`, `_posthog/compact_boundary` | info | | @@ -63,12 +62,12 @@ Exported events (allowlist, everything else is dropped): | `session/update: tool_call` | info | `tool_call_id`, `tool_kind`, `tool_status` (no title, no rawInput) | | `session/update: tool_call_update` (terminal only) | info / warn on `failed` | `tool_call_id`, `tool_status` | -Deliberately dropped: `agent_message`, `agent_message_chunk`, `agent_thought_chunk`, `user_message`, `session/prompt` bodies, in-progress `tool_call_update` snapshots (they re-send the growing tool input/output), `available_commands_update`, and any unknown method (fail-closed allowlist). +Deliberately dropped: `agent_message`, `agent_message_chunk`, `agent_thought_chunk`, `user_message`, `session/prompt` bodies, in-progress `tool_call_update` snapshots (they re-send the growing tool input/output), `available_commands_update`, `_posthog/console` (free-text agent-server diagnostics interpolate arbitrary data — e.g. the prompt preview logged on user-message handling and stringified extension params — so exporting them would leak content; they stay in the S3 log and event-ingest stream), and any unknown method (fail-closed allowlist). Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `log_attributes` faceting table only indexes key/value pairs under 256 chars). ### APM trace (one per run) -- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status OK once a turn ends cleanly with `end_turn` (the sandbox never emits `task_complete` for successful runs; the terminal "completed" status is decided by the workflow outside, so a clean turn end is the in-sandbox success signal), ERROR with the error message on `_posthog/error` (an error always wins, a later turn completion cannot flip it back), unset when the run ends without either (cancelled or timed out). +- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR with the error message on `_posthog/error` (an error always wins, later turn completions cannot flip it back) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. - `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. - `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. - Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; an error cascades ERROR status through open tool and turn spans to the root. From 781f591acc420bdf154ddd2effa9f02c5ae2de7b Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:12 +0100 Subject: [PATCH 05/26] chore(tasks): sync implementation report with telemetry hardening fixes Mirrors the PostHog/code telemetry changes: run errors export provenance only (generic body, error_source/stop_reason attributes - the raw message stays in the session log and the run's error_message), flush/shutdown are best-effort and per-signal independent with a 5s export timeout, a run error marks still-open tool spans as interrupted, and the deprecated OtelTransportConfig stubs are kept for published-package compatibility. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/REPORT.md b/REPORT.md index eb4df168abef..0de82e8fc284 100644 --- a/REPORT.md +++ b/REPORT.md @@ -54,7 +54,7 @@ Exported events (allowlist, everything else is dropped): | `_posthog/usage_update` (and the `session/update` variant) | info | `tokens_input/output/cached_read/cached_write`, `cost_usd` | | `_posthog/turn_complete` | info | `stop_reason` | | `_posthog/task_complete` | info | `stop_reason` | -| `_posthog/error` | **error** | `error_source`, `stop_reason`, message in body (capped) | +| `_posthog/error` | **error** | `error_source`, `stop_reason`; body is the generic "run error" — the raw message is free text that can embed prompt/repo content, so it stays in the session log and the run's `error_message` | | `_posthog/progress` | info | `progress_group/step/status` | | `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | | `_posthog/mode_change`, `_posthog/compact_boundary` | info | | @@ -67,10 +67,10 @@ Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `l ### APM trace (one per run) -- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR with the error message on `_posthog/error` (an error always wins, later turn completions cannot flip it back) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. +- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR on `_posthog/error` (an error always wins, later turn completions cannot flip it back; `error_source` lands as a root-span attribute while the raw message is withheld — see the error row above) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. - `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. - `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. -- Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; an error cascades ERROR status through open tool and turn spans to the root. +- Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; a run error cascades ERROR status to the open turn and the root, and closes still-open tool spans as ERROR with `tool_status=interrupted` so APM never shows a healthy-looking active tool under a failed run. - Every log record is emitted under the OTel context of the span it belongs to (tool logs on the tool span, lifecycle logs on the root), so `trace_id`/`span_id` land in the `logs` table columns and the UI links Logs ⇄ trace waterfall. ### Delivery timing @@ -78,6 +78,7 @@ Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `l Telemetry is near-realtime, not end-of-turn: records are created the moment each notification flows through the writer and batched for at most 2 s (`BatchLogRecordProcessor` / `BatchSpanProcessor`, `scheduledDelayMillis` 2000). Spans export when they end (tools mid-turn, turns at `turn_complete`, root at cleanup). Flush safety nets: an explicit flush after a terminal error in `signalTaskComplete`, a full shutdown-flush in `cleanupSession` (which SIGTERM reaches via `stop()`), so sandbox teardown cannot eat the tail of a run's telemetry. +Flush and shutdown are best-effort and per-signal independent (`Promise.allSettled`), and every export is capped at 5 s (`exportTimeoutMillis`, down from the SDK's 30 s default), so a rejecting or hanging traces endpoint can neither starve log delivery nor hold up session cleanup. ## Changes in PostHog/code (companion branch) @@ -87,9 +88,9 @@ Flush safety nets: an explicit flush after a terminal error in `signalTaskComple - `packages/agent/src/session-log-writer.ts`: optional `sinks: SessionLogSink[]`, teed in `appendRawLine` after the entry is built; a throwing sink warns once and can never break product log persistence; message chunks never reach sinks. `SessionContext` moved here from the otel module. - `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, shuts it down in `cleanupSession`, flushes after terminal errors, and mirrors `enqueueTaskTerminalEvent` payloads into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). Fatal crashes (`reportFatalError`, the uncaught-exception/unhandled-rejection path) also mirror an error record (`error_source=agent_server_crash`) and shut telemetry down, so hard process deaths reach the telemetry project instead of vanishing. - `packages/agent/src/server/bin.ts` + `server/types.ts`: zod-validated env `POSTHOG_AGENT_OTEL_LOGS_URL`, `POSTHOG_AGENT_OTEL_LOGS_TOKEN`, `POSTHOG_AGENT_OTEL_TRACES_URL` → `AgentServerConfig.otelLogsUrl/otelLogsToken/otelTracesUrl`. Telemetry is off unless the logs pair is set; spans additionally require the traces URL (per-signal kill switch). -- `packages/agent/src/types.ts`: deleted the dead `OtelTransportConfig`/`AgentConfig.otelTransport` left over from the February attempt. +- `packages/agent/src/types.ts`: the February `OtelTransportConfig`/`AgentConfig.otelTransport` remain as `@deprecated`, ignored stubs — `@posthog/agent` is a published package, so removing exported types is an API break reserved for a major. - Dependencies: `@opentelemetry/api`, `@opentelemetry/sdk-trace-base`, `@opentelemetry/exporter-trace-otlp-http`, version-aligned with the existing logs SDK (0.208.x experimental / 2.x stable line). -- Tests (71 passing): parameterized log-mapping matrix, a hard privacy test asserting exported payloads never contain tool args/titles/output, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, and four trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade, orphan export on shutdown). +- Tests (74 passing): parameterized log-mapping matrix, hard privacy tests asserting exported payloads never contain tool args/titles/output or raw error messages, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, and five trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade incl. interrupted tools, orphan export on shutdown, log shutdown isolated from a failing traces endpoint). - `packages/agent/README.md`: documents the env vars and behavior. ## Changes in PostHog/posthog (this branch) @@ -117,7 +118,7 @@ Flush safety nets: an explicit flush after a terminal error in `signalTaskComple ## Verification -- `PostHog/code`: 71 tests pass in the agent package (including the new telemetry suite), `tsc --noEmit` clean via turbo, biome clean on all touched files (one pre-existing warning untouched). The package's pre-existing test failures in this environment (missing Postgres/git fixtures) were confirmed byte-identical with and without these changes by running the failing files against a stashed tree. +- `PostHog/code`: 74 tests pass in the agent package (including the new telemetry suite), `tsc --noEmit` clean via turbo, biome clean on all touched files (one pre-existing warning untouched). The package's pre-existing test failures in this environment (missing Postgres/git fixtures) were confirmed byte-identical with and without these changes by running the failing files against a stashed tree. - `PostHog/posthog`: 21 tests pass across `test_provision_sandbox.py` and the new `TestBuildSandboxEnvironmentVariables`; `ruff check`/`format` clean on all touched files. DB-dependent suites in this sandbox fail identically with and without the change (no Postgres available). - Local-dev routing verified: Caddy serves `/i/v1/logs`/`/i/v1/traces` on `localhost:8000` and proxies to `capture-logs`, and the Docker URL rewrite covers the new vars. From b8ce9b02ffd2302d28bfa59d05f6453cbbc11858 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:15 +0100 Subject: [PATCH 06/26] chore(tasks): sync implementation report with telemetry root-span terminal-point fix Mirrors the PostHog/code change: the task_run root span is now ended and exported at the run's in-process terminal point (background prompt settled, terminal failure, or interactive close) instead of relying on sandbox teardown, which never delivers SIGTERM to the exec'd agent-server process. Documents the interactive hard-teardown limitation. Stacked on posthog-code/agent-run-otel-telemetry (pure fast-forward). Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/REPORT.md b/REPORT.md index 0de82e8fc284..f87e086e1e76 100644 --- a/REPORT.md +++ b/REPORT.md @@ -67,7 +67,7 @@ Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `l ### APM trace (one per run) -- `task_run` root span (kind SERVER): opened at session init, closed at session cleanup. Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR on `_posthog/error` (an error always wins, later turn completions cannot flip it back; `error_source` lands as a root-span attribute while the raw message is withheld — see the error row above) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. +- `task_run` root span (kind SERVER): opened at session init, closed at the run's in-process terminal point — a background run's prompt settling (`finalizeRunTelemetry`), a terminal failure (`signalTaskComplete`), or session cleanup (interactive `close`). Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR on `_posthog/error` (an error always wins, later turn completions cannot flip it back; `error_source` lands as a root-span attribute while the raw message is withheld — see the error row above) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. - `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. - `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. - Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; a run error cascades ERROR status to the open turn and the root, and closes still-open tool spans as ERROR with `tool_status=interrupted` so APM never shows a healthy-looking active tool under a failed run. @@ -76,8 +76,8 @@ Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `l ### Delivery timing Telemetry is near-realtime, not end-of-turn: records are created the moment each notification flows through the writer and batched for at most 2 s (`BatchLogRecordProcessor` / `BatchSpanProcessor`, `scheduledDelayMillis` 2000). -Spans export when they end (tools mid-turn, turns at `turn_complete`, root at cleanup). -Flush safety nets: an explicit flush after a terminal error in `signalTaskComplete`, a full shutdown-flush in `cleanupSession` (which SIGTERM reaches via `stop()`), so sandbox teardown cannot eat the tail of a run's telemetry. +Spans export when they end (tools mid-turn, turns at `turn_complete`, root at the run's terminal point). +Sandbox teardown can NOT be a flush point: agent-server is an exec'd process inside the sandbox, so `docker stop` signals only the container's PID 1 and Modal terminate is immediate — the process's SIGTERM handler never runs and anything still queued (or a still-open root span) is lost. Telemetry is therefore ended eagerly at the in-process terminal points: `finalizeRunTelemetry` when a background run's prompt settles, a full shutdown after a terminal failure in `signalTaskComplete`, and `cleanupSession` for interactive `close`. Known limitation: an interactive session ended by hard teardown (e.g. inactivity timeout) loses the root span; its turn/tool spans and logs still assemble under the same trace id. Flush and shutdown are best-effort and per-signal independent (`Promise.allSettled`), and every export is capped at 5 s (`exportTimeoutMillis`, down from the SDK's 30 s default), so a rejecting or hanging traces endpoint can neither starve log delivery nor hold up session cleanup. ## Changes in PostHog/code (companion branch) @@ -86,11 +86,11 @@ Flush and shutdown are best-effort and per-signal independent (`Promise.allSettl - `packages/agent/src/otel-trace-builder.ts` (new): `RunTraceBuilder`, the span state machine described above; `handle(entry)` returns the context each log record should attach to. - `packages/agent/src/otel-attributes.ts` (new): shared pure helpers (`strAttr`, `numAttr`, `usageAttributes`, truncation, caps). - `packages/agent/src/session-log-writer.ts`: optional `sinks: SessionLogSink[]`, teed in `appendRawLine` after the entry is built; a throwing sink warns once and can never break product log persistence; message chunks never reach sinks. `SessionContext` moved here from the otel module. -- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, shuts it down in `cleanupSession`, flushes after terminal errors, and mirrors `enqueueTaskTerminalEvent` payloads into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). Fatal crashes (`reportFatalError`, the uncaught-exception/unhandled-rejection path) also mirror an error record (`error_source=agent_server_crash`) and shut telemetry down, so hard process deaths reach the telemetry project instead of vanishing. +- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, and ends it at the run's in-process terminal points: `finalizeRunTelemetry` (full shutdown when a background run's initial/resume prompt settles — verified necessary because sandbox teardown never delivers SIGTERM to the exec'd process, so waiting for `cleanupSession` left the root span unexported), a shutdown after terminal failures in `signalTaskComplete`, and `cleanupSession` for interactive `close`. `enqueueTaskTerminalEvent` payloads are mirrored into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). Fatal crashes (`reportFatalError`, the uncaught-exception/unhandled-rejection path) also mirror an error record (`error_source=agent_server_crash`) and shut telemetry down, so hard process deaths reach the telemetry project instead of vanishing. - `packages/agent/src/server/bin.ts` + `server/types.ts`: zod-validated env `POSTHOG_AGENT_OTEL_LOGS_URL`, `POSTHOG_AGENT_OTEL_LOGS_TOKEN`, `POSTHOG_AGENT_OTEL_TRACES_URL` → `AgentServerConfig.otelLogsUrl/otelLogsToken/otelTracesUrl`. Telemetry is off unless the logs pair is set; spans additionally require the traces URL (per-signal kill switch). - `packages/agent/src/types.ts`: the February `OtelTransportConfig`/`AgentConfig.otelTransport` remain as `@deprecated`, ignored stubs — `@posthog/agent` is a published package, so removing exported types is an API break reserved for a major. - Dependencies: `@opentelemetry/api`, `@opentelemetry/sdk-trace-base`, `@opentelemetry/exporter-trace-otlp-http`, version-aligned with the existing logs SDK (0.208.x experimental / 2.x stable line). -- Tests (74 passing): parameterized log-mapping matrix, hard privacy tests asserting exported payloads never contain tool args/titles/output or raw error messages, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, and five trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade incl. interrupted tools, orphan export on shutdown, log shutdown isolated from a failing traces endpoint). +- Tests (74 passing in the telemetry suites, plus agent-server terminal-point tests): parameterized log-mapping matrix, hard privacy tests asserting exported payloads never contain tool args/titles/output or raw error messages, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, five trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade incl. interrupted tools, orphan export on shutdown, log shutdown isolated from a failing traces endpoint), and agent-server tests asserting the error mirror lands before the terminal shutdown and that `finalizeRunTelemetry` fires for background runs only. - `packages/agent/README.md`: documents the env vars and behavior. ## Changes in PostHog/posthog (this branch) From 62b1031207e9efa59b0a3668edbce80051f8ffb5 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:17 +0100 Subject: [PATCH 07/26] chore(tasks): fix telemetry report formatting --- REPORT.md | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/REPORT.md b/REPORT.md index f87e086e1e76..32f6c352b33f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -23,7 +23,7 @@ Hard requirements: ## Architecture -``` +```text sandbox (agentsh) └── agent-server (PostHog/code, packages/agent) ├── ACP streams (tapped) ──► SessionLogWriter ──► Django append_log ──► S3 (product log, unchanged) @@ -47,20 +47,20 @@ Resource attributes on every record: `service.name`, `service.version` (agent ve Exported events (allowlist, everything else is dropped): -| Event | Severity | Notable attributes | -| --- | --- | --- | -| `_posthog/run_started` | info | `agent_version`, `session_id` | -| `_posthog/sdk_session` | info | `adapter`, `session_id` | -| `_posthog/usage_update` (and the `session/update` variant) | info | `tokens_input/output/cached_read/cached_write`, `cost_usd` | -| `_posthog/turn_complete` | info | `stop_reason` | -| `_posthog/task_complete` | info | `stop_reason` | -| `_posthog/error` | **error** | `error_source`, `stop_reason`; body is the generic "run error" — the raw message is free text that can embed prompt/repo content, so it stays in the session log and the run's `error_message` | -| `_posthog/progress` | info | `progress_group/step/status` | -| `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | -| `_posthog/mode_change`, `_posthog/compact_boundary` | info | | -| `_posthog/permission_request/response/resolved` | info | `request_id`, `tool_call_id` (identifiers only; tool content excluded) | -| `session/update: tool_call` | info | `tool_call_id`, `tool_kind`, `tool_status` (no title, no rawInput) | -| `session/update: tool_call_update` (terminal only) | info / warn on `failed` | `tool_call_id`, `tool_status` | +| Event | Severity | Notable attributes | +| ---------------------------------------------------------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `_posthog/run_started` | info | `agent_version`, `session_id` | +| `_posthog/sdk_session` | info | `adapter`, `session_id` | +| `_posthog/usage_update` (and the `session/update` variant) | info | `tokens_input/output/cached_read/cached_write`, `cost_usd` | +| `_posthog/turn_complete` | info | `stop_reason` | +| `_posthog/task_complete` | info | `stop_reason` | +| `_posthog/error` | **error** | `error_source`, `stop_reason`; body is the generic "run error" — the raw message is free text that can embed prompt/repo content, so it stays in the session log and the run's `error_message` | +| `_posthog/progress` | info | `progress_group/step/status` | +| `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | +| `_posthog/mode_change`, `_posthog/compact_boundary` | info | | +| `_posthog/permission_request/response/resolved` | info | `request_id`, `tool_call_id` (identifiers only; tool content excluded) | +| `session/update: tool_call` | info | `tool_call_id`, `tool_kind`, `tool_status` (no title, no rawInput) | +| `session/update: tool_call_update` (terminal only) | info / warn on `failed` | `tool_call_id`, `tool_status` | Deliberately dropped: `agent_message`, `agent_message_chunk`, `agent_thought_chunk`, `user_message`, `session/prompt` bodies, in-progress `tool_call_update` snapshots (they re-send the growing tool input/output), `available_commands_update`, `_posthog/console` (free-text agent-server diagnostics interpolate arbitrary data — e.g. the prompt preview logged on user-message handling and stringified extension params — so exporting them would leak content; they stay in the S3 log and event-ingest stream), and any unknown method (fail-closed allowlist). Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `log_attributes` faceting table only indexes key/value pairs under 256 chars). @@ -135,9 +135,9 @@ Flush and shutdown are best-effort and per-signal independent (`Promise.allSettl ## Configuration reference -| Where | Name | Meaning | -| --- | --- | --- | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | +| Where | Name | Meaning | +| ---------------------- | -------------------------------------------------------------------------- | ---------------------------------------------------------------------- | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | | Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | From 637772542424acb0788bc2de0e728391bad150fb Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:19 +0100 Subject: [PATCH 08/26] feat(tasks): mirror scout run logs into posthog logs Incorporates the scout run log mirror from PR #71094 (credit: Andrew Maguire, @andrewm4894) into the agent-run telemetry branch, squashed from the PR's final state so both delivery paths ship together. Persisted task-run log entries are mirrored into the PostHog Logs product, scoped to scout runs. There is no transport of its own: TaskRun.append_log emits one structured stdout line per persisted entry (event=task_run_log) and the per-cluster OTel collector that already tails container stdout ships them into the region's internal project's Logs, parsing JSON keys into queryable attributes and request_id (the run uuid) into a trace id so one run groups as one trace. - run_log_mirror.py translates each ACP JSONL entry: readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, bounded entries per call), severity mapping, and run-identity fields as log attributes - TaskRun.append_log calls the mirror after the S3 write, guarded so any failure is logged and never breaks the run - TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS setting gates which task origins mirror (default signals_scout; empty disables) Complementary to the OTLP run-telemetry export on this branch: the mirror carries full readable bodies for PostHog-authored scout runs into the internal project, while the OTLP path stays metadata-only and covers customer-driven cloud runs. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 19 ++ posthog/settings/temporal.py | 10 + .../backend/logic/services/run_log_mirror.py | 148 ++++++++++++++ products/tasks/backend/models.py | 31 +++ .../backend/tests/test_run_log_mirror.py | 193 ++++++++++++++++++ 5 files changed, 401 insertions(+) create mode 100644 products/tasks/backend/logic/services/run_log_mirror.py create mode 100644 products/tasks/backend/tests/test_run_log_mirror.py diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index a8d3ac381e3e..0a6028a9e994 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -252,6 +252,25 @@ repositories. > **Note:** This only works with `SANDBOX_PROVIDER=docker`. +### Task-run log mirroring to PostHog Logs (dogfooding) + +Task-run log entries (the JSONL appended to object storage via `TaskRun.append_log`) are also mirrored into the PostHog Logs product, +so runs can be browsed and sampled in the Logs UI instead of fetching S3 blobs. + +There is no transport of its own: entries are emitted as structured stdout log lines (`event=task_run_log`), +and the per-cluster OTel collector that already ships all container stdout into the region's internal PostHog project picks them up +(locally, `otel-collector-config.dev.yaml` does the same into your dev logs project). +The collector parses each JSON key into a queryable attribute and turns the emitted `request_id` (the run uuid) into a trace id, +so one run groups as a trace and can be pulled up with an attribute filter on `task_run_id`. + +```bash +# Which task origins to mirror (comma-separated). Defaults to signals scouts only. +# Set empty to disable. +TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=signals_scout +``` + +Mirroring failures are logged and never break the run's log write. + ### How `MODAL_DOCKER` works When both `SANDBOX_PROVIDER=MODAL_DOCKER` and `LOCAL_POSTHOG_CODE_MONOREPO_ROOT` are set: diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 9d8eca0a02fd..1189e8f16362 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -102,6 +102,16 @@ "TASKS_CREDENTIAL_REFRESH_INITIAL_DELAY_SECONDS", 0, type_cast=int ) +# Mirror persisted task-run logs into the PostHog Logs product (dogfooding). +# Entries appended to a run's S3 JSONL log are also emitted as structured stdout log lines; +# the per-cluster OTel collector already ships container stdout into the region's internal +# PostHog project's Logs, so no transport or credentials are needed here. Only runs whose +# task origin_product is in this list are mirrored — scoped to signals scouts for now; +# widen the list to cover more task origins, or set it empty to disable. +TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS: list[str] = get_list( + os.getenv("TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS", "signals_scout") +) + TEMPORAL_LOG_LEVEL_PRODUCE: str = os.getenv("TEMPORAL_LOG_LEVEL_PRODUCE", "DEBUG") TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE: int = get_from_env("TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE", 0, type_cast=int) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py new file mode 100644 index 000000000000..5869d10b7189 --- /dev/null +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -0,0 +1,148 @@ +"""Mirror persisted task-run log entries into the PostHog Logs product via stdout. + +Task-run logs are appended to object storage as one ACP notification envelope per line. +In every PostHog cluster an OTel collector daemonset already tails container stdout and +ships JSON log lines into the region's internal PostHog project's Logs product, parsing +each JSON key into a queryable log attribute, `level` into severity, and `request_id` +into a trace id (see `argocd/otel-collector` in the charts repo; `otel-collector-config.dev.yaml` +does the same for local dev). So dogfooding scout-run logs needs no transport of its own: +emitting one structured stdout line per persisted entry is enough. + +Each mirrored line carries the run's uuid as `request_id`, so a whole run groups as one +trace in the Logs UI and can be pulled up with a `task_run_id` attribute filter. +""" + +import json +from typing import Any + +from django.conf import settings + +import structlog + +logger = structlog.get_logger(__name__) + +# The collector truncates whole log lines at 100 KB (`max_log_size`); cap the body well +# below that so run identity attributes and JSON overhead never push a line over. +MAX_BODY_CHARS = 8_000 + +# Defensive budget per append: origin_product is user-settable on task creation, so a +# hostile append_log request must not be able to flood stdout/the collector with an +# arbitrarily long entry list. Real scout appends are small batches, far below this. +MAX_ENTRIES_PER_CALL = 200 + +_LOG_METHOD_NAMES = {"info": "info", "warn": "warning", "error": "error"} + + +def mirroring_enabled(origin_product: str) -> bool: + return origin_product in settings.TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS + + +def mirror_entries( + entries: list[dict], + *, + team_id: int, + task_id: str, + run_id: str, + origin_product: str, +) -> None: + """Emit one structured stdout log line per persisted entry.""" + if len(entries) > MAX_ENTRIES_PER_CALL: + logger.warning( + "task_run_log_mirror_truncated", + task_run_id=run_id, + dropped=len(entries) - MAX_ENTRIES_PER_CALL, + ) + entries = entries[:MAX_ENTRIES_PER_CALL] + for entry in entries: + if not isinstance(entry, dict): + continue + raw_notification = entry.get("notification") + notification: dict = raw_notification if isinstance(raw_notification, dict) else {} + update = _session_update(notification) + session_update = update.get("sessionUpdate") if isinstance(update.get("sessionUpdate"), str) else None + severity = _severity(notification) + + fields: dict[str, Any] = { + # `request_id` becomes the record's trace id in the collector, grouping the run. + "request_id": run_id, + "task_run_id": run_id, + "task_id": task_id, + "team_id": team_id, + "origin_product": origin_product, + "body": _body(notification, session_update), + } + method = notification.get("method") + if isinstance(method, str): + fields["acp_method"] = method + if session_update: + fields["acp_session_update"] = session_update + entry_timestamp = entry.get("timestamp") + if isinstance(entry_timestamp, str): + fields["entry_timestamp"] = entry_timestamp + + getattr(logger, _LOG_METHOD_NAMES[severity])("task_run_log", **fields) + + +def _session_update(notification: dict) -> dict: + params = notification.get("params") + if not isinstance(params, dict): + return {} + update = params.get("update") + return update if isinstance(update, dict) else {} + + +def _severity(notification: dict) -> str: + if notification.get("method") == "_posthog/error": + return "error" + if notification.get("method") == "_posthog/console": + params = notification.get("params") + level = params.get("level") if isinstance(params, dict) else None + if level in ("warn", "error"): + return level + # No "debug" mapping for thought chunks or debug console lines: the root stdlib log + # level is INFO in production, so a debug line would be filtered before it ever + # reaches stdout and the collector. + return "info" + + +def _body(notification: dict, session_update: str | None) -> str: + raw_params = notification.get("params") + params: dict = raw_params if isinstance(raw_params, dict) else {} + update = _session_update(notification) + + body: str | None = None + if session_update: + text = _extract_text(update.get("content")) + if text is not None: + body = f"[{session_update}] {text}" + elif session_update in ("tool_call", "tool_call_update"): + title = update.get("title") or update.get("toolCallId") or "" + status = update.get("status") + body = f"[{session_update}] {title}" + (f" ({status})" if status else "") + elif notification.get("method") in ("_posthog/console", "_posthog/error"): + message = params.get("message") + if isinstance(message, str): + body = message + elif notification.get("method") == "_posthog/sandbox_output": + stdout = params.get("stdout") or "" + stderr = params.get("stderr") or "" + body = f"[sandbox_output exit={params.get('exitCode')}] {stdout}" + (f"\nstderr: {stderr}" if stderr else "") + elif isinstance(notification.get("result"), dict): + stop_reason = notification["result"].get("stopReason") + if isinstance(stop_reason, str): + body = f"[turn_end] {stop_reason}" + + if body is None: + body = json.dumps(notification) + return body[:MAX_BODY_CHARS] + + +def _extract_text(content: Any) -> str | None: + """Pull plain text out of an ACP content block (single block or list of blocks).""" + if isinstance(content, dict): + text = content.get("text") + return text if isinstance(text, str) else None + if isinstance(content, list): + parts = [t for t in (_extract_text(block) for block in content) if t] + return "\n".join(parts) if parts else None + return None diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py index 57f143634f2a..358c3ad80a66 100644 --- a/products/tasks/backend/models.py +++ b/products/tasks/backend/models.py @@ -1675,6 +1675,8 @@ def append_log(self, entries: list[dict], *, ttl_days: int | None = DEFAULT_LOG_ object_storage.write(self.log_url, content) + self._mirror_logs_to_posthog_logs(entries) + if is_new_file and ttl_days is not None: try: object_storage.tag( @@ -1692,6 +1694,35 @@ def append_log(self, entries: list[dict], *, ttl_days: int | None = DEFAULT_LOG_ error=str(e), ) + def _mirror_logs_to_posthog_logs(self, entries: list[dict]) -> None: + """Mirror persisted entries into the PostHog Logs product via stdout (dogfooding). + + Fire-and-forget: mirroring failures must never break the run's log write. + """ + from products.tasks.backend.logic.services.run_log_mirror import mirror_entries, mirroring_enabled + + if not settings.TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS: + return + + try: + origin_product = self.task.origin_product + if not mirroring_enabled(origin_product): + return + + mirror_entries( + entries, + team_id=self.team_id, + task_id=str(self.task_id), + run_id=str(self.id), + origin_product=origin_product, + ) + except Exception as e: + logger.warning( + "task_run.mirror_logs_to_posthog_logs_failed", + task_run_id=str(self.id), + error=str(e), + ) + def effective_rtk(self) -> bool | None: """rtk posture for analytics: the launch-persisted effective value, falling back to the user's explicit override for runs that never launched.""" diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py new file mode 100644 index 000000000000..c65c9d76ef0e --- /dev/null +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -0,0 +1,193 @@ +import json + +from unittest.mock import MagicMock, patch + +from django.test import SimpleTestCase, TestCase, override_settings + +from parameterized import parameterized + +from posthog.models import Organization, Team + +from products.tasks.backend.logic.services.run_log_mirror import MAX_BODY_CHARS, MAX_ENTRIES_PER_CALL, mirror_entries +from products.tasks.backend.models import Task, TaskRun + +RUN_ID = "0b166f65-9e52-4d1b-b3c4-1a9e3f6d3c21" +TASK_ID = "7d0e9a34-2f1c-4b8a-9c3d-5e6f7a8b9c0d" + + +def _mirror(entries: list[dict]) -> MagicMock: + with patch("products.tasks.backend.logic.services.run_log_mirror.logger") as mock_logger: + mirror_entries(entries, team_id=2, task_id=TASK_ID, run_id=RUN_ID, origin_product="signals_scout") + return mock_logger + + +def _session_update_entry(session_update: str, **update_fields) -> dict: + return { + "type": "notification", + "timestamp": "2026-07-15T10:00:00+00:00", + "notification": { + "method": "session/update", + "params": {"update": {"sessionUpdate": session_update, **update_fields}}, + }, + } + + +class TestMirrorEntries(SimpleTestCase): + @parameterized.expand( + [ + ( + "agent_message", + _session_update_entry("agent_message", content={"type": "text", "text": "hello"}), + "info", + "[agent_message] hello", + ), + ( + "tool_call_without_content", + _session_update_entry("tool_call", title="grep", status="in_progress"), + "info", + "[tool_call] grep (in_progress)", + ), + ( + "posthog_error", + {"notification": {"method": "_posthog/error", "params": {"message": "boom"}}}, + "error", + "boom", + ), + ( + "console_level_passthrough", + {"notification": {"method": "_posthog/console", "params": {"level": "warn", "message": "careful"}}}, + "warning", + "careful", + ), + ( + "sandbox_output", + { + "notification": { + "method": "_posthog/sandbox_output", + "params": {"stdout": "out", "stderr": "err", "exitCode": 1}, + } + }, + "info", + "[sandbox_output exit=1] out\nstderr: err", + ), + ( + "turn_end_result", + {"notification": {"result": {"stopReason": "end_turn"}}}, + "info", + "[turn_end] end_turn", + ), + ] + ) + def test_severity_and_body_mapping(self, _name, entry, expected_log_method, expected_body): + mock_logger = _mirror([entry]) + log_call = getattr(mock_logger, expected_log_method) + log_call.assert_called_once() + self.assertEqual(log_call.call_args.kwargs["body"], expected_body) + + def test_unrecognized_entry_falls_back_to_json_body(self): + notification = {"method": "session/request_permission", "params": {"tool": "bash"}} + mock_logger = _mirror([{"notification": notification}]) + self.assertEqual(json.loads(mock_logger.info.call_args.kwargs["body"]), notification) + + def test_emitted_fields_carry_run_identity(self): + mock_logger = _mirror([_session_update_entry("agent_message", content={"type": "text", "text": "hi"})]) + self.assertEqual(mock_logger.info.call_args.args, ("task_run_log",)) + fields = mock_logger.info.call_args.kwargs + self.assertEqual(fields["request_id"], RUN_ID) + self.assertEqual(fields["task_run_id"], RUN_ID) + self.assertEqual(fields["task_id"], TASK_ID) + self.assertEqual(fields["team_id"], 2) + self.assertEqual(fields["origin_product"], "signals_scout") + self.assertEqual(fields["acp_method"], "session/update") + self.assertEqual(fields["acp_session_update"], "agent_message") + self.assertEqual(fields["entry_timestamp"], "2026-07-15T10:00:00+00:00") + + def test_oversized_body_is_truncated(self): + entry = _session_update_entry("agent_message", content={"type": "text", "text": "x" * (MAX_BODY_CHARS * 2)}) + mock_logger = _mirror([entry]) + self.assertEqual(len(mock_logger.info.call_args.kwargs["body"]), MAX_BODY_CHARS) + + def test_oversized_batch_is_capped(self): + entries = [ + _session_update_entry("agent_message", content={"type": "text", "text": f"line {i}"}) + for i in range(MAX_ENTRIES_PER_CALL + 50) + ] + mock_logger = _mirror(entries) + self.assertEqual(mock_logger.info.call_count, MAX_ENTRIES_PER_CALL) + mock_logger.warning.assert_called_once() + self.assertEqual(mock_logger.warning.call_args.kwargs["dropped"], 50) + + @parameterized.expand([("empty", []), ("non_dict_entries", ["not-a-dict", 42])]) + def test_no_usable_entries_emits_nothing(self, _name, entries): + mock_logger = _mirror(entries) + mock_logger.info.assert_not_called() + mock_logger.warning.assert_not_called() + mock_logger.error.assert_not_called() + + +@override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=["signals_scout"]) +class TestAppendLogMirroring(TestCase): + @classmethod + def setUpTestData(cls): + cls.organization = Organization.objects.create(name="Test Org") + cls.team = Team.objects.create(organization=cls.organization, name="Test Team") + + def _create_run(self, origin_product: str) -> TaskRun: + task = Task.objects.create( + team=self.team, + title="Test Task", + description="Test", + origin_product=origin_product, + ) + return TaskRun.objects.create(team=self.team, task=task) + + @parameterized.expand( + [ + (Task.OriginProduct.SIGNALS_SCOUT, True), + (Task.OriginProduct.USER_CREATED, False), + ] + ) + @patch("products.tasks.backend.logic.services.run_log_mirror.logger") + @patch("products.tasks.backend.models.object_storage") + def test_mirrors_only_allowlisted_origin_products(self, origin_product, expect_mirrored, mock_storage, mock_logger): + mock_storage.read.return_value = None + run = self._create_run(origin_product) + message = _session_update_entry("agent_message", content={"type": "text", "text": "hi"}) + chunk = _session_update_entry("agent_message_chunk", content={"type": "text", "text": "h"}) + + run.append_log([message, chunk]) + + mock_storage.write.assert_called_once() + if expect_mirrored: + # The chunk entry is dropped before persistence, so exactly one line is mirrored. + mock_logger.info.assert_called_once() + self.assertEqual(mock_logger.info.call_args.kwargs["task_run_id"], str(run.id)) + self.assertEqual(mock_logger.info.call_args.kwargs["origin_product"], origin_product) + else: + mock_logger.info.assert_not_called() + + @override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=[]) + @patch("products.tasks.backend.logic.services.run_log_mirror.logger") + @patch("products.tasks.backend.models.object_storage") + def test_no_mirroring_when_disabled(self, mock_storage, mock_logger): + mock_storage.read.return_value = None + run = self._create_run(Task.OriginProduct.SIGNALS_SCOUT) + + run.append_log([_session_update_entry("agent_message", content={"type": "text", "text": "hi"})]) + + mock_storage.write.assert_called_once() + mock_logger.info.assert_not_called() + + @patch( + "products.tasks.backend.logic.services.run_log_mirror.mirror_entries", + side_effect=RuntimeError("kaboom"), + ) + @patch("products.tasks.backend.models.object_storage") + def test_mirror_failure_does_not_break_log_write(self, mock_storage, mock_mirror): + mock_storage.read.return_value = None + run = self._create_run(Task.OriginProduct.SIGNALS_SCOUT) + + run.append_log([_session_update_entry("agent_message", content={"type": "text", "text": "hi"})]) + + mock_storage.write.assert_called_once() + mock_mirror.assert_called_once() From bfaf03516987949962259b52d4838c18df3d1120 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:21 +0100 Subject: [PATCH 09/26] chore(tasks): sync implementation report with rebase and scout log mirror Documents the incorporated scout run log mirror (#71094) alongside the OTLP run-telemetry export: architecture, the two pipelines' differing privacy postures and scopes, changed files, and the new TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS setting in the configuration reference. Also applies ruff formatting to test_utils.py after the rebase conflict resolution. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 29 ++++++++++++++++++- .../temporal/process_task/tests/test_utils.py | 1 + 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index 32f6c352b33f..fc569b473a22 100644 --- a/REPORT.md +++ b/REPORT.md @@ -27,7 +27,9 @@ Hard requirements: sandbox (agentsh) └── agent-server (PostHog/code, packages/agent) ├── ACP streams (tapped) ──► SessionLogWriter ──► Django append_log ──► S3 (product log, unchanged) - │ │ + │ │ │ + │ │ └─ scout runs only: structured stdout line per entry + │ │ └─► cluster OTel collector ──► internal project Logs │ └─ sink ──► OtelRunTelemetry │ ├─ log records ──► POST {POSTHOG_AGENT_OTEL_LOGS_URL} │ └─ RunTraceBuilder spans ──► POST {POSTHOG_AGENT_OTEL_TRACES_URL} @@ -80,6 +82,22 @@ Spans export when they end (tools mid-turn, turns at `turn_complete`, root at th Sandbox teardown can NOT be a flush point: agent-server is an exec'd process inside the sandbox, so `docker stop` signals only the container's PID 1 and Modal terminate is immediate — the process's SIGTERM handler never runs and anything still queued (or a still-open root span) is lost. Telemetry is therefore ended eagerly at the in-process terminal points: `finalizeRunTelemetry` when a background run's prompt settles, a full shutdown after a terminal failure in `signalTaskComplete`, and `cleanupSession` for interactive `close`. Known limitation: an interactive session ended by hard teardown (e.g. inactivity timeout) loses the root span; its turn/tool spans and logs still assemble under the same trace id. Flush and shutdown are best-effort and per-signal independent (`Promise.allSettled`), and every export is capped at 5 s (`exportTimeoutMillis`, down from the SDK's 30 s default), so a rejecting or hanging traces endpoint can neither starve log delivery nor hold up session cleanup. +### Companion pipeline: scout run log mirror (incorporated from PR #71094) + +This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](https://github.com/PostHog/posthog/pull/71094)), a second, complementary delivery path with a different privacy posture and scope: + +| | OTLP export (this work) | Scout run log mirror (#71094) | +| -------------- | ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| Emitter | agent-server inside the sandbox | Django `TaskRun.append_log` (the S3 write choke point) | +| Transport | direct OTLP HTTP to `capture-logs` | structured stdout line per entry (`event=task_run_log`); the per-cluster OTel collector already ships container stdout | +| Destination | chosen telemetry project (via `SANDBOX_AGENT_OTEL_*`) | the region's internal PostHog project's Logs | +| Content | metadata-only allowlist — never prompts, message text, tool args/output, or raw error text | readable bodies (agent messages, tool calls, sandbox output), capped at 8k chars | +| Scope | every cloud task run once the settings are set | runs whose task `origin_product` is in `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default: `signals_scout` only) | +| Traces | one APM trace per run (`task_run`/`turn`/`tool_call:*` spans) | `request_id` = run uuid becomes the record's trace id, grouping one run as one trace | + +The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. +Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. + ## Changes in PostHog/code (companion branch) - `packages/agent/src/otel-telemetry.ts` (renamed from `otel-log-writer.ts`): `OtelRunTelemetry`, the single `SessionLogSink` owning the OTLP log exporter and (when a traces URL is configured) the `RunTraceBuilder`. Contains the pure `mapNotificationToLogRecord()` allowlist mapper. Fixes the dead `/i/v1/agent-logs` default. Never throws into the run; ignores entries for other sessions. @@ -105,6 +123,14 @@ Flush and shutdown are best-effort and per-signal independent (`Promise.allSettl - Tests: parameterized gating matrix on `_build_environment_variables` (5 rows: full config, logs-only, partial configs, traces-without-logs all correctly gated) and a `SimpleTestCase` wiring guard on the snapshot-resume path. - `docs/internal/sandboxes-setup-guide.md`: local-dev setup section for the new settings. +Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout run log mirror, credit Andrew Maguire): + +- `products/tasks/backend/logic/services/run_log_mirror.py` (new): translates each ACP JSONL entry into a `task_run_log` structlog stdout line — readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, entry-count cap per call), severity mapping (`_posthog/error` → error, console level passthrough), run-identity fields (`task_run_id`/`task_id`/`team_id`/`origin_product`/`acp_method`) as log attributes, and `request_id` = run uuid as the trace id. +- `products/tasks/backend/models.py`: `TaskRun.append_log` calls `_mirror_logs_to_posthog_logs` after the S3 write, gated on the origin-product allowlist and wrapped so any failure is logged and never breaks the write. +- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables). +- `products/tasks/backend/tests/test_run_log_mirror.py` (new): severity/body mapping matrix, run-identity fields, truncation/batch caps, and `append_log` gating (allowlist, disabled, mirror-failure isolation). +- `docs/internal/sandboxes-setup-guide.md`: mirroring section. + ## Key design decisions 1. **Emit from the sandbox, not from Django.** Independence from the product log path (see Architecture), matching the streamlit sandbox precedent. The Django-tee alternative would add an outbound call to a hot API path and go dark precisely when `append_log` breaks. @@ -141,3 +167,4 @@ Flush and shutdown are best-effort and per-signal independent (`Promise.allSettl | Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | | Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | | Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | +| Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | diff --git a/products/tasks/backend/temporal/process_task/tests/test_utils.py b/products/tasks/backend/temporal/process_task/tests/test_utils.py index a804ce6de179..9ae562544e27 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_utils.py +++ b/products/tasks/backend/temporal/process_task/tests/test_utils.py @@ -1068,6 +1068,7 @@ def test_collision_detection_is_case_insensitive(self): assert get_relayed_mcp_server_names(task_run, {"grafana"}) == ["Playwright", "internal-cli"] + class TestBuildSandboxEnvironmentVariables(SimpleTestCase): @patch( "products.tasks.backend.logic.services.connection_token.get_sandbox_jwt_public_key", From c937fc1313b23a32d78c852b9ff3c94933ab51ad Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:23 +0100 Subject: [PATCH 10/26] feat(tasks): add direct otlp delivery leg to the run log mirror Local testing showed the mirrored scout-run logs (readable agent message bodies, tool calls, sandbox output) never reach the Logs UI in dev: the mirror emits structured stdout lines for the cluster OTel collector to pick up, but locally append_log runs in the host Django process and otel-collector-config.dev.yaml only tails docker-compose container stdout - so the lines land in the phrocs pane and go nowhere. Only the agent-server's own OTLP metadata records were visible in /logs. Add an optional direct delivery leg: when TASK_RUN_LOGS_MIRROR_OTLP_URL and _TOKEN are set, each mirrored batch is also POSTed as OTLP JSON to that logs endpoint (service.name=task-run-log-mirror, run uuid as the trace id - the same request_id -> trace mapping the production collector applies - and the structlog fields as attributes). Both settings default unset, so production keeps the zero-transport stdout path; delivery failures are logged and never raise into the run. Tests: payload shape (severity mapping, trace id, int attribute encoding) and a parameterized guard that no HTTP call happens unless both settings are set - protecting the hot append_log path from accidental sync network calls in production. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 18 +++- posthog/settings/temporal.py | 8 ++ .../backend/logic/services/run_log_mirror.py | 101 ++++++++++++++++-- .../backend/tests/test_run_log_mirror.py | 44 ++++++++ 4 files changed, 162 insertions(+), 9 deletions(-) diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index 0a6028a9e994..55a1b60857db 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -257,9 +257,8 @@ repositories. Task-run log entries (the JSONL appended to object storage via `TaskRun.append_log`) are also mirrored into the PostHog Logs product, so runs can be browsed and sampled in the Logs UI instead of fetching S3 blobs. -There is no transport of its own: entries are emitted as structured stdout log lines (`event=task_run_log`), -and the per-cluster OTel collector that already ships all container stdout into the region's internal PostHog project picks them up -(locally, `otel-collector-config.dev.yaml` does the same into your dev logs project). +In production there is no transport of its own: entries are emitted as structured stdout log lines (`event=task_run_log`), +and the per-cluster OTel collector that already ships all container stdout into the region's internal PostHog project picks them up. The collector parses each JSON key into a queryable attribute and turns the emitted `request_id` (the run uuid) into a trace id, so one run groups as a trace and can be pulled up with an attribute filter on `task_run_id`. @@ -269,6 +268,19 @@ so one run groups as a trace and can be pulled up with an attribute filter on `t TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=signals_scout ``` +**Local dev needs one extra step**: `append_log` runs in the host Django process, +and the dev collector (`otel-collector-config.dev.yaml`) only tails docker-compose container stdout, +so the mirrored lines never reach your local Logs on their own — they just show up in the Django phrocs pane. +Point the mirror's direct OTLP leg at your local logs ingest to see them in `/logs`: + +```bash +TASK_RUN_LOGS_MIRROR_OTLP_URL=http://localhost:8000/i/v1/logs +TASK_RUN_LOGS_MIRROR_OTLP_TOKEN= +``` + +Records arrive under `service.name=task-run-log-mirror` with the run uuid as the trace id. +Both settings stay unset in production, where the collector delivers. + Mirroring failures are logged and never break the run's log write. ### How `MODAL_DOCKER` works diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 1189e8f16362..5af03da0703c 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -112,6 +112,14 @@ os.getenv("TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS", "signals_scout") ) +# Optional direct OTLP delivery for the mirror above, for hosts whose stdout no collector +# tails. In production the collector daemonset ships pod stdout so these stay unset; in +# local dev `append_log` runs in the host Django process, which the dev collector (docker +# containers only) never sees — point these at the local logs ingest (e.g. +# http://localhost:8000/i/v1/logs with a project API key) to see mirrored runs in /logs. +TASK_RUN_LOGS_MIRROR_OTLP_URL: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_URL", None, optional=True) +TASK_RUN_LOGS_MIRROR_OTLP_TOKEN: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_TOKEN", None, optional=True) + TEMPORAL_LOG_LEVEL_PRODUCE: str = os.getenv("TEMPORAL_LOG_LEVEL_PRODUCE", "DEBUG") TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE: int = get_from_env("TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE", 0, type_cast=int) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 5869d10b7189..1a6c5df44d49 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -4,21 +4,32 @@ In every PostHog cluster an OTel collector daemonset already tails container stdout and ships JSON log lines into the region's internal PostHog project's Logs product, parsing each JSON key into a queryable log attribute, `level` into severity, and `request_id` -into a trace id (see `argocd/otel-collector` in the charts repo; `otel-collector-config.dev.yaml` -does the same for local dev). So dogfooding scout-run logs needs no transport of its own: -emitting one structured stdout line per persisted entry is enough. - -Each mirrored line carries the run's uuid as `request_id`, so a whole run groups as one -trace in the Logs UI and can be pulled up with a `task_run_id` attribute filter. +into a trace id (see `argocd/otel-collector` in the charts repo). So in production +dogfooding scout-run logs needs no transport of its own: emitting one structured stdout +line per persisted entry is enough. + +Local dev is the exception: `otel-collector-config.dev.yaml` only tails docker-compose +container stdout, and `append_log` runs in the host Django process, so the stdout lines +never reach Logs there. Setting `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN` additionally +ships each batch straight to a logs OTLP endpoint — the delivery leg for hosts whose +stdout no collector tails. + +Each mirrored line carries the run's uuid as `request_id` (and as the OTLP trace id on +the direct leg), so a whole run groups as one trace in the Logs UI and can be pulled up +with a `task_run_id` attribute filter. """ import json +import time +from datetime import datetime from typing import Any from django.conf import settings import structlog +from posthog.security.outbound_proxy import internal_requests + logger = structlog.get_logger(__name__) # The collector truncates whole log lines at 100 KB (`max_log_size`); cap the body well @@ -32,6 +43,12 @@ _LOG_METHOD_NAMES = {"info": "info", "warn": "warning", "error": "error"} +_OTLP_SEVERITIES = {"info": ("INFO", 9), "warn": ("WARN", 13), "error": ("ERROR", 17)} + +_OTLP_SERVICE_NAME = "task-run-log-mirror" + +_OTLP_TIMEOUT_SECONDS = 3 + def mirroring_enabled(origin_product: str) -> bool: return origin_product in settings.TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS @@ -53,6 +70,7 @@ def mirror_entries( dropped=len(entries) - MAX_ENTRIES_PER_CALL, ) entries = entries[:MAX_ENTRIES_PER_CALL] + records: list[tuple[str, dict[str, Any]]] = [] for entry in entries: if not isinstance(entry, dict): continue @@ -81,6 +99,77 @@ def mirror_entries( fields["entry_timestamp"] = entry_timestamp getattr(logger, _LOG_METHOD_NAMES[severity])("task_run_log", **fields) + records.append((severity, fields)) + + _post_otlp(records, run_id=run_id) + + +def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> None: + """Ship the batch straight to a logs OTLP endpoint when one is configured. + + Best-effort: a delivery failure is logged and never raises into the run. + """ + url = settings.TASK_RUN_LOGS_MIRROR_OTLP_URL + token = settings.TASK_RUN_LOGS_MIRROR_OTLP_TOKEN + if not url or not token or not records: + return + + # The run uuid without dashes is a valid 16-byte hex trace id, matching the + # request_id -> trace id mapping the production collector applies. + trace_id = run_id.replace("-", "") + log_records = [] + for severity, fields in records: + severity_text, severity_number = _OTLP_SEVERITIES[severity] + log_records.append( + { + "timeUnixNano": str(_time_unix_nano(fields.get("entry_timestamp"))), + "severityText": severity_text, + "severityNumber": severity_number, + "body": {"stringValue": fields["body"]}, + **({"traceId": trace_id} if len(trace_id) == 32 else {}), + "attributes": [ + {"key": key, "value": _otlp_attribute_value(value)} + for key, value in fields.items() + if key != "body" + ], + } + ) + payload = { + "resourceLogs": [ + { + "resource": {"attributes": [{"key": "service.name", "value": {"stringValue": _OTLP_SERVICE_NAME}}]}, + "scopeLogs": [{"scope": {"name": __name__}, "logRecords": log_records}], + } + ] + } + try: + response = internal_requests.post( + url, + json=payload, + headers={"Authorization": f"Bearer {token}"}, + timeout=_OTLP_TIMEOUT_SECONDS, + ) + response.raise_for_status() + except Exception as e: + logger.warning("task_run_log_mirror_otlp_failed", task_run_id=run_id, error=str(e)) + + +def _otlp_attribute_value(value: Any) -> dict[str, Any]: + # OTLP JSON encodes 64-bit ints as strings. + if isinstance(value, bool): + return {"boolValue": value} + if isinstance(value, int): + return {"intValue": str(value)} + return {"stringValue": str(value)} + + +def _time_unix_nano(entry_timestamp: Any) -> int: + if isinstance(entry_timestamp, str): + try: + return int(datetime.fromisoformat(entry_timestamp.replace("Z", "+00:00")).timestamp() * 1_000_000_000) + except (ValueError, OverflowError): + pass + return time.time_ns() def _session_update(notification: dict) -> dict: diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index c65c9d76ef0e..ca0785991759 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -125,6 +125,50 @@ def test_no_usable_entries_emits_nothing(self, _name, entries): mock_logger.error.assert_not_called() +class TestMirrorOtlpDelivery(SimpleTestCase): + @override_settings( + TASK_RUN_LOGS_MIRROR_OTLP_URL="http://localhost:8000/i/v1/logs", + TASK_RUN_LOGS_MIRROR_OTLP_TOKEN="phc_logs", + ) + def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): + entries = [ + _session_update_entry("agent_message", content={"type": "text", "text": "hello"}), + {"notification": {"method": "_posthog/error", "params": {"message": "boom"}}}, + ] + with patch("products.tasks.backend.logic.services.run_log_mirror.internal_requests") as mock_requests: + _mirror(entries) + + mock_requests.post.assert_called_once() + args, kwargs = mock_requests.post.call_args + assert args[0] == "http://localhost:8000/i/v1/logs" + assert kwargs["headers"] == {"Authorization": "Bearer phc_logs"} + records = kwargs["json"]["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + assert [r["severityText"] for r in records] == ["INFO", "ERROR"] + assert records[0]["body"]["stringValue"] == "[agent_message] hello" + assert records[1]["body"]["stringValue"] == "boom" + # The run uuid (dashes stripped) is the trace id, grouping the run. + assert records[0]["traceId"] == RUN_ID.replace("-", "") + attributes = {a["key"]: a["value"] for a in records[0]["attributes"]} + assert attributes["task_run_id"] == {"stringValue": RUN_ID} + # OTLP JSON encodes 64-bit ints as strings. + assert attributes["team_id"] == {"intValue": "2"} + + @parameterized.expand( + [ + ("both_unset", None, None), + ("url_only", "http://localhost:8000/i/v1/logs", None), + ("token_only", None, "phc_logs"), + ] + ) + def test_no_http_delivery_unless_fully_configured(self, _name, url, token): + entry = _session_update_entry("agent_message", content={"type": "text", "text": "hello"}) + with override_settings(TASK_RUN_LOGS_MIRROR_OTLP_URL=url, TASK_RUN_LOGS_MIRROR_OTLP_TOKEN=token): + with patch("products.tasks.backend.logic.services.run_log_mirror.internal_requests") as mock_requests: + _mirror([entry]) + + mock_requests.post.assert_not_called() + + @override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=["signals_scout"]) class TestAppendLogMirroring(TestCase): @classmethod From 74b33dab075be14abb6fefabf2905d1746a168af Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:25 +0100 Subject: [PATCH 11/26] chore(tasks): sync implementation report with mirror otlp delivery leg Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/REPORT.md b/REPORT.md index fc569b473a22..4cda58e2cd6d 100644 --- a/REPORT.md +++ b/REPORT.md @@ -98,6 +98,9 @@ This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](ht The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. +The mirror additionally has a **direct OTLP leg** for hosts whose stdout no collector tails (added on this branch, on top of #71094): in local dev `append_log` runs in the host Django process, and `otel-collector-config.dev.yaml` only tails docker-compose container stdout, so the mirrored lines never reached local Logs — verified during local testing where `/logs` showed only the OTLP metadata records. +Setting `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `TASK_RUN_LOGS_MIRROR_OTLP_TOKEN` ships each batch straight to a logs OTLP endpoint (`service.name=task-run-log-mirror`, run uuid as the trace id, same fields as attributes); both stay unset in production, where the collector daemonset delivers pod stdout. + ## Changes in PostHog/code (companion branch) - `packages/agent/src/otel-telemetry.ts` (renamed from `otel-log-writer.ts`): `OtelRunTelemetry`, the single `SessionLogSink` owning the OTLP log exporter and (when a traces URL is configured) the `RunTraceBuilder`. Contains the pure `mapNotificationToLogRecord()` allowlist mapper. Fixes the dead `/i/v1/agent-logs` default. Never throws into the run; ignores entries for other sessions. @@ -127,7 +130,7 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout - `products/tasks/backend/logic/services/run_log_mirror.py` (new): translates each ACP JSONL entry into a `task_run_log` structlog stdout line — readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, entry-count cap per call), severity mapping (`_posthog/error` → error, console level passthrough), run-identity fields (`task_run_id`/`task_id`/`team_id`/`origin_product`/`acp_method`) as log attributes, and `request_id` = run uuid as the trace id. - `products/tasks/backend/models.py`: `TaskRun.append_log` calls `_mirror_logs_to_posthog_logs` after the S3 write, gated on the origin-product allowlist and wrapped so any failure is logged and never breaks the write. -- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables). +- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables), plus `TASK_RUN_LOGS_MIRROR_OTLP_URL`/`_TOKEN` for the direct dev delivery leg (default unset). - `products/tasks/backend/tests/test_run_log_mirror.py` (new): severity/body mapping matrix, run-identity fields, truncation/batch caps, and `append_log` gating (allowlist, disabled, mirror-failure isolation). - `docs/internal/sandboxes-setup-guide.md`: mirroring section. @@ -168,3 +171,4 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout | Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | | Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | | Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | +| Django settings | `TASK_RUN_LOGS_MIRROR_OTLP_URL` / `_TOKEN` | Direct OTLP delivery for the mirror on hosts no collector tails (local dev); unset in production | From 4faa43b62ee0409a136e69a5c9166e571bec07cd Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:27 +0100 Subject: [PATCH 12/26] refactor(tasks): reuse agent otel settings for the mirror dev delivery leg Drops the mirror-specific TASK_RUN_LOGS_MIRROR_OTLP_URL/_TOKEN settings: in local dev there is one logs destination anyway, and the agent telemetry settings (SANDBOX_AGENT_OTEL_LOGS_URL/_TOKEN) already point at it - so the mirror's direct delivery leg now reuses them, gated on DEBUG. With the agent telemetry configured locally, mirrored scout runs reach /logs with zero extra configuration. The DEBUG gate is what makes the reuse safe: in production those settings point at the customer-facing telemetry project and the collector daemonset already delivers pod stdout to the internal project, so an unconditional reuse would double-deliver scout bodies into the wrong project the moment the telemetry rollout sets them. A test pins that production never posts even with the settings configured. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 15 +++++-------- posthog/settings/temporal.py | 8 ------- .../backend/logic/services/run_log_mirror.py | 22 ++++++++++++------- .../backend/tests/test_run_log_mirror.py | 18 +++++++++------ 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index 55a1b60857db..28b1c9271d78 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -268,18 +268,13 @@ so one run groups as a trace and can be pulled up with an attribute filter on `t TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=signals_scout ``` -**Local dev needs one extra step**: `append_log` runs in the host Django process, +**Local dev delivery**: `append_log` runs in the host Django process, and the dev collector (`otel-collector-config.dev.yaml`) only tails docker-compose container stdout, -so the mirrored lines never reach your local Logs on their own — they just show up in the Django phrocs pane. -Point the mirror's direct OTLP leg at your local logs ingest to see them in `/logs`: - -```bash -TASK_RUN_LOGS_MIRROR_OTLP_URL=http://localhost:8000/i/v1/logs -TASK_RUN_LOGS_MIRROR_OTLP_TOKEN= -``` - +so the mirrored stdout lines never reach your local Logs on their own — they just show up in the Django phrocs pane. +In DEBUG the mirror therefore also ships each batch straight to the logs OTLP endpoint the agent telemetry already uses — +if `SANDBOX_AGENT_OTEL_LOGS_URL` + `SANDBOX_AGENT_OTEL_LOGS_TOKEN` are set (see "Agent run telemetry" above), mirrored scout runs appear in `/logs` with no extra configuration. Records arrive under `service.name=task-run-log-mirror` with the run uuid as the trace id. -Both settings stay unset in production, where the collector delivers. +This direct leg is DEBUG-only: in production those settings point at the customer-facing telemetry project, and the collector already delivers pod stdout to the internal project. Mirroring failures are logged and never break the run's log write. diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 5af03da0703c..1189e8f16362 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -112,14 +112,6 @@ os.getenv("TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS", "signals_scout") ) -# Optional direct OTLP delivery for the mirror above, for hosts whose stdout no collector -# tails. In production the collector daemonset ships pod stdout so these stay unset; in -# local dev `append_log` runs in the host Django process, which the dev collector (docker -# containers only) never sees — point these at the local logs ingest (e.g. -# http://localhost:8000/i/v1/logs with a project API key) to see mirrored runs in /logs. -TASK_RUN_LOGS_MIRROR_OTLP_URL: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_URL", None, optional=True) -TASK_RUN_LOGS_MIRROR_OTLP_TOKEN: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_TOKEN", None, optional=True) - TEMPORAL_LOG_LEVEL_PRODUCE: str = os.getenv("TEMPORAL_LOG_LEVEL_PRODUCE", "DEBUG") TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE: int = get_from_env("TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE", 0, type_cast=int) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 1a6c5df44d49..56b1fc28a952 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -10,9 +10,10 @@ Local dev is the exception: `otel-collector-config.dev.yaml` only tails docker-compose container stdout, and `append_log` runs in the host Django process, so the stdout lines -never reach Logs there. Setting `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN` additionally -ships each batch straight to a logs OTLP endpoint — the delivery leg for hosts whose -stdout no collector tails. +never reach Logs there. In DEBUG each batch is therefore also shipped straight to the +logs OTLP endpoint the agent telemetry already uses (`SANDBOX_AGENT_OTEL_LOGS_URL` + +`_TOKEN`) — DEBUG-only because in production those settings point at the customer-facing +telemetry project and the collector already delivers pod stdout to the internal project. Each mirrored line carries the run's uuid as `request_id` (and as the OTLP trace id on the direct leg), so a whole run groups as one trace in the Logs UI and can be pulled up @@ -105,13 +106,18 @@ def mirror_entries( def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> None: - """Ship the batch straight to a logs OTLP endpoint when one is configured. + """Ship the batch straight to the local logs OTLP endpoint in dev. - Best-effort: a delivery failure is logged and never raises into the run. + DEBUG-only: production delivery is the collector tailing pod stdout, and reusing + `SANDBOX_AGENT_OTEL_LOGS_*` there would double-deliver scout bodies into the + customer-facing telemetry project. Best-effort: a delivery failure is logged and + never raises into the run. """ - url = settings.TASK_RUN_LOGS_MIRROR_OTLP_URL - token = settings.TASK_RUN_LOGS_MIRROR_OTLP_TOKEN - if not url or not token or not records: + if not settings.DEBUG or not records: + return + url = settings.SANDBOX_AGENT_OTEL_LOGS_URL + token = settings.SANDBOX_AGENT_OTEL_LOGS_TOKEN + if not url or not token: return # The run uuid without dashes is a valid 16-byte hex trace id, matching the diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index ca0785991759..b998a2e22805 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -127,8 +127,9 @@ def test_no_usable_entries_emits_nothing(self, _name, entries): class TestMirrorOtlpDelivery(SimpleTestCase): @override_settings( - TASK_RUN_LOGS_MIRROR_OTLP_URL="http://localhost:8000/i/v1/logs", - TASK_RUN_LOGS_MIRROR_OTLP_TOKEN="phc_logs", + DEBUG=True, + SANDBOX_AGENT_OTEL_LOGS_URL="http://localhost:8000/i/v1/logs", + SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_logs", ) def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): entries = [ @@ -155,14 +156,17 @@ def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): @parameterized.expand( [ - ("both_unset", None, None), - ("url_only", "http://localhost:8000/i/v1/logs", None), - ("token_only", None, "phc_logs"), + # Production: never deliver directly, even with the agent telemetry + # settings configured — the collector owns delivery there, and posting + # would put scout bodies in the customer-facing telemetry project. + ("prod_with_settings", False, "http://localhost:8000/i/v1/logs", "phc_logs"), + ("debug_without_url", True, None, "phc_logs"), + ("debug_without_token", True, "http://localhost:8000/i/v1/logs", None), ] ) - def test_no_http_delivery_unless_fully_configured(self, _name, url, token): + def test_no_http_delivery_outside_configured_debug(self, _name, debug, url, token): entry = _session_update_entry("agent_message", content={"type": "text", "text": "hello"}) - with override_settings(TASK_RUN_LOGS_MIRROR_OTLP_URL=url, TASK_RUN_LOGS_MIRROR_OTLP_TOKEN=token): + with override_settings(DEBUG=debug, SANDBOX_AGENT_OTEL_LOGS_URL=url, SANDBOX_AGENT_OTEL_LOGS_TOKEN=token): with patch("products.tasks.backend.logic.services.run_log_mirror.internal_requests") as mock_requests: _mirror([entry]) From 1975380cef068cc163ece15babcb09c6fcc1f9a9 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:29 +0100 Subject: [PATCH 13/26] chore(tasks): sync implementation report with mirror settings reuse Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/REPORT.md b/REPORT.md index 4cda58e2cd6d..b074d8e0dced 100644 --- a/REPORT.md +++ b/REPORT.md @@ -98,8 +98,9 @@ This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](ht The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. -The mirror additionally has a **direct OTLP leg** for hosts whose stdout no collector tails (added on this branch, on top of #71094): in local dev `append_log` runs in the host Django process, and `otel-collector-config.dev.yaml` only tails docker-compose container stdout, so the mirrored lines never reached local Logs — verified during local testing where `/logs` showed only the OTLP metadata records. -Setting `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `TASK_RUN_LOGS_MIRROR_OTLP_TOKEN` ships each batch straight to a logs OTLP endpoint (`service.name=task-run-log-mirror`, run uuid as the trace id, same fields as attributes); both stay unset in production, where the collector daemonset delivers pod stdout. +The mirror additionally has a **direct OTLP leg for local dev** (added on this branch, on top of #71094): `append_log` runs in the host Django process, and `otel-collector-config.dev.yaml` only tails docker-compose container stdout, so the mirrored lines never reached local Logs — verified during local testing where `/logs` showed only the OTLP metadata records. +In DEBUG each batch is also shipped straight to the logs OTLP endpoint the agent telemetry already uses (`SANDBOX_AGENT_OTEL_LOGS_URL` + `_TOKEN` — no new settings), arriving as `service.name=task-run-log-mirror` with the run uuid as the trace id and the structlog fields as attributes. +DEBUG-only by design: in production those settings point at the customer-facing telemetry project, and the collector daemonset already delivers pod stdout to the internal project — reusing them there would double-deliver scout bodies into the wrong project. ## Changes in PostHog/code (companion branch) @@ -130,7 +131,7 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout - `products/tasks/backend/logic/services/run_log_mirror.py` (new): translates each ACP JSONL entry into a `task_run_log` structlog stdout line — readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, entry-count cap per call), severity mapping (`_posthog/error` → error, console level passthrough), run-identity fields (`task_run_id`/`task_id`/`team_id`/`origin_product`/`acp_method`) as log attributes, and `request_id` = run uuid as the trace id. - `products/tasks/backend/models.py`: `TaskRun.append_log` calls `_mirror_logs_to_posthog_logs` after the S3 write, gated on the origin-product allowlist and wrapped so any failure is logged and never breaks the write. -- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables), plus `TASK_RUN_LOGS_MIRROR_OTLP_URL`/`_TOKEN` for the direct dev delivery leg (default unset). +- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables). The dev delivery leg reuses `SANDBOX_AGENT_OTEL_LOGS_URL`/`_TOKEN` under DEBUG — no mirror-specific OTLP settings. - `products/tasks/backend/tests/test_run_log_mirror.py` (new): severity/body mapping matrix, run-identity fields, truncation/batch caps, and `append_log` gating (allowlist, disabled, mirror-failure isolation). - `docs/internal/sandboxes-setup-guide.md`: mirroring section. @@ -171,4 +172,4 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout | Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | | Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | | Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | -| Django settings | `TASK_RUN_LOGS_MIRROR_OTLP_URL` / `_TOKEN` | Direct OTLP delivery for the mirror on hosts no collector tails (local dev); unset in production | +| Django settings (DEBUG) | `SANDBOX_AGENT_OTEL_LOGS_URL` / `_TOKEN` (reused) | Also serve as the mirror's direct dev delivery leg; ignored by the mirror in production | From 33f0fa54ad90ed22f48ae8fee3cd957872de0740 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:31 +0100 Subject: [PATCH 14/26] feat(tasks): deliver the run log mirror otlp leg in production The mirror's direct OTLP leg is meant to run in production, not just dev: it delivers scout run transcripts into PostHog's own internal logs project. The dedicated TASK_RUN_LOGS_MIRROR_OTLP_URL/_TOKEN settings are restored (replacing the short-lived DEBUG-gated reuse of the agent telemetry settings) and the DEBUG gate is dropped. The token is the mechanism that keeps customer projects out of it: scout runs execute for customer teams, but capture-logs routes records by the Bearer key, so a centrally configured internal-project key means the mirrored transcripts only ever land in - and bill - our own project, never the customer's. That destination differs from the agent-telemetry project SANDBOX_AGENT_OTEL_LOGS_TOKEN addresses, which is why the settings stay separate. Unset disables the direct leg; the stdout emission for the collector remains either way. Delivery is one POST per persisted batch, scout runs only, 3s timeout, and never raises into the run. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 19 ++++++++---- posthog/settings/temporal.py | 8 +++++ .../backend/logic/services/run_log_mirror.py | 29 +++++++++---------- .../backend/tests/test_run_log_mirror.py | 24 +++++++-------- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index 28b1c9271d78..01c8cf99f326 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -268,13 +268,20 @@ so one run groups as a trace and can be pulled up with an attribute filter on `t TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=signals_scout ``` -**Local dev delivery**: `append_log` runs in the host Django process, -and the dev collector (`otel-collector-config.dev.yaml`) only tails docker-compose container stdout, -so the mirrored stdout lines never reach your local Logs on their own — they just show up in the Django phrocs pane. -In DEBUG the mirror therefore also ships each batch straight to the logs OTLP endpoint the agent telemetry already uses — -if `SANDBOX_AGENT_OTEL_LOGS_URL` + `SANDBOX_AGENT_OTEL_LOGS_TOKEN` are set (see "Agent run telemetry" above), mirrored scout runs appear in `/logs` with no extra configuration. +The mirror also has a **direct OTLP leg** that ships each batch straight to a logs ingest endpoint: + +```bash +TASK_RUN_LOGS_MIRROR_OTLP_URL=http://localhost:8000/i/v1/logs # prod: https://us.i.posthog.com/i/v1/logs +TASK_RUN_LOGS_MIRROR_OTLP_TOKEN= +``` + +The token pins the destination: scout runs execute for customer teams, +but their mirrored transcripts must only ever land in — and bill — PostHog's own internal logs project, never the customer's. Records arrive under `service.name=task-run-log-mirror` with the run uuid as the trace id. -This direct leg is DEBUG-only: in production those settings point at the customer-facing telemetry project, and the collector already delivers pod stdout to the internal project. + +Locally the direct leg is the only delivery path: `append_log` runs in the host Django process, +and the dev collector (`otel-collector-config.dev.yaml`) only tails docker-compose container stdout, +so without these settings the mirrored lines only show up in the Django phrocs pane. Mirroring failures are logged and never break the run's log write. diff --git a/posthog/settings/temporal.py b/posthog/settings/temporal.py index 1189e8f16362..6fdaa8776611 100644 --- a/posthog/settings/temporal.py +++ b/posthog/settings/temporal.py @@ -112,6 +112,14 @@ os.getenv("TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS", "signals_scout") ) +# Direct OTLP delivery for the mirror above. The token pins the destination: scout runs +# execute for customer teams, but their mirrored logs must only ever land in (and bill) +# PostHog's own internal logs project — so this is the internal project's API key, never +# derived from the run's team. Point locally at the dev logs ingest to see mirrored runs +# in /logs. Unset disables the direct leg (stdout emission for the collector remains). +TASK_RUN_LOGS_MIRROR_OTLP_URL: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_URL", None, optional=True) +TASK_RUN_LOGS_MIRROR_OTLP_TOKEN: str | None = get_from_env("TASK_RUN_LOGS_MIRROR_OTLP_TOKEN", None, optional=True) + TEMPORAL_LOG_LEVEL_PRODUCE: str = os.getenv("TEMPORAL_LOG_LEVEL_PRODUCE", "DEBUG") TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE: int = get_from_env("TEMPORAL_EXTERNAL_LOGS_QUEUE_SIZE", 0, type_cast=int) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 56b1fc28a952..0d752135d3a6 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -8,12 +8,12 @@ dogfooding scout-run logs needs no transport of its own: emitting one structured stdout line per persisted entry is enough. -Local dev is the exception: `otel-collector-config.dev.yaml` only tails docker-compose -container stdout, and `append_log` runs in the host Django process, so the stdout lines -never reach Logs there. In DEBUG each batch is therefore also shipped straight to the -logs OTLP endpoint the agent telemetry already uses (`SANDBOX_AGENT_OTEL_LOGS_URL` + -`_TOKEN`) — DEBUG-only because in production those settings point at the customer-facing -telemetry project and the collector already delivers pod stdout to the internal project. +A direct OTLP leg (`TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN`) additionally ships each +batch straight to a logs ingest endpoint. The token pins the destination to PostHog's own +internal logs project: scout runs execute for customer teams, and their mirrored +transcripts must never land in — or bill — a customer's project. It is also the only +delivery path in local dev, where `append_log` runs in the host Django process whose +stdout the dev collector (docker containers only) never tails. Each mirrored line carries the run's uuid as `request_id` (and as the OTLP trace id on the direct leg), so a whole run groups as one trace in the Logs UI and can be pulled up @@ -106,18 +106,15 @@ def mirror_entries( def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> None: - """Ship the batch straight to the local logs OTLP endpoint in dev. + """Ship the batch straight to the configured logs OTLP endpoint. - DEBUG-only: production delivery is the collector tailing pod stdout, and reusing - `SANDBOX_AGENT_OTEL_LOGS_*` there would double-deliver scout bodies into the - customer-facing telemetry project. Best-effort: a delivery failure is logged and - never raises into the run. + The token routes the records, so the destination is always the internal logs + project the settings point at — never the run's (customer) team. Best-effort: + a delivery failure is logged and never raises into the run. """ - if not settings.DEBUG or not records: - return - url = settings.SANDBOX_AGENT_OTEL_LOGS_URL - token = settings.SANDBOX_AGENT_OTEL_LOGS_TOKEN - if not url or not token: + url = settings.TASK_RUN_LOGS_MIRROR_OTLP_URL + token = settings.TASK_RUN_LOGS_MIRROR_OTLP_TOKEN + if not url or not token or not records: return # The run uuid without dashes is a valid 16-byte hex trace id, matching the diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index b998a2e22805..f1ababb14f75 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -127,9 +127,8 @@ def test_no_usable_entries_emits_nothing(self, _name, entries): class TestMirrorOtlpDelivery(SimpleTestCase): @override_settings( - DEBUG=True, - SANDBOX_AGENT_OTEL_LOGS_URL="http://localhost:8000/i/v1/logs", - SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_logs", + TASK_RUN_LOGS_MIRROR_OTLP_URL="https://us.i.posthog.com/i/v1/logs", + TASK_RUN_LOGS_MIRROR_OTLP_TOKEN="phc_internal", ) def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): entries = [ @@ -141,8 +140,10 @@ def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): mock_requests.post.assert_called_once() args, kwargs = mock_requests.post.call_args - assert args[0] == "http://localhost:8000/i/v1/logs" - assert kwargs["headers"] == {"Authorization": "Bearer phc_logs"} + assert args[0] == "https://us.i.posthog.com/i/v1/logs" + # The configured internal-project token routes the records — never a key + # derived from the run's (customer) team. + assert kwargs["headers"] == {"Authorization": "Bearer phc_internal"} records = kwargs["json"]["resourceLogs"][0]["scopeLogs"][0]["logRecords"] assert [r["severityText"] for r in records] == ["INFO", "ERROR"] assert records[0]["body"]["stringValue"] == "[agent_message] hello" @@ -156,17 +157,14 @@ def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): @parameterized.expand( [ - # Production: never deliver directly, even with the agent telemetry - # settings configured — the collector owns delivery there, and posting - # would put scout bodies in the customer-facing telemetry project. - ("prod_with_settings", False, "http://localhost:8000/i/v1/logs", "phc_logs"), - ("debug_without_url", True, None, "phc_logs"), - ("debug_without_token", True, "http://localhost:8000/i/v1/logs", None), + ("both_unset", None, None), + ("url_only", "https://us.i.posthog.com/i/v1/logs", None), + ("token_only", None, "phc_internal"), ] ) - def test_no_http_delivery_outside_configured_debug(self, _name, debug, url, token): + def test_no_http_delivery_unless_fully_configured(self, _name, url, token): entry = _session_update_entry("agent_message", content={"type": "text", "text": "hello"}) - with override_settings(DEBUG=debug, SANDBOX_AGENT_OTEL_LOGS_URL=url, SANDBOX_AGENT_OTEL_LOGS_TOKEN=token): + with override_settings(TASK_RUN_LOGS_MIRROR_OTLP_URL=url, TASK_RUN_LOGS_MIRROR_OTLP_TOKEN=token): with patch("products.tasks.backend.logic.services.run_log_mirror.internal_requests") as mock_requests: _mirror([entry]) From 5ffecf82498f60750828f6185fc20a2ed8ac304f Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:33 +0100 Subject: [PATCH 15/26] chore(tasks): sync implementation report with prod mirror delivery Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/REPORT.md b/REPORT.md index b074d8e0dced..87793147c295 100644 --- a/REPORT.md +++ b/REPORT.md @@ -98,9 +98,10 @@ This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](ht The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. -The mirror additionally has a **direct OTLP leg for local dev** (added on this branch, on top of #71094): `append_log` runs in the host Django process, and `otel-collector-config.dev.yaml` only tails docker-compose container stdout, so the mirrored lines never reached local Logs — verified during local testing where `/logs` showed only the OTLP metadata records. -In DEBUG each batch is also shipped straight to the logs OTLP endpoint the agent telemetry already uses (`SANDBOX_AGENT_OTEL_LOGS_URL` + `_TOKEN` — no new settings), arriving as `service.name=task-run-log-mirror` with the run uuid as the trace id and the structlog fields as attributes. -DEBUG-only by design: in production those settings point at the customer-facing telemetry project, and the collector daemonset already delivers pod stdout to the internal project — reusing them there would double-deliver scout bodies into the wrong project. +The mirror additionally has a **direct OTLP leg** (added on this branch, on top of #71094): `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN` ship each batch straight to a logs ingest endpoint, arriving as `service.name=task-run-log-mirror` with the run uuid as the trace id and the structlog fields as attributes. +The token pins the destination — scout runs execute for customer teams, and their mirrored transcripts must only ever land in (and bill) PostHog's own internal logs project, never the customer's — so the intended production value is the internal logs project's API key, deliberately not derived from the run's team and not reusing `SANDBOX_AGENT_OTEL_LOGS_TOKEN` (which addresses the separate agent-telemetry project). +It is also the only delivery path in local dev, where `append_log` runs in the host Django process whose stdout the dev collector (docker containers only) never tails — verified during local testing where `/logs` showed only the OTLP metadata records until this leg existed. +Delivery is best-effort (3 s timeout, one POST per persisted batch, scout runs only) and never raises into the run. ## Changes in PostHog/code (companion branch) @@ -131,7 +132,7 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout - `products/tasks/backend/logic/services/run_log_mirror.py` (new): translates each ACP JSONL entry into a `task_run_log` structlog stdout line — readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, entry-count cap per call), severity mapping (`_posthog/error` → error, console level passthrough), run-identity fields (`task_run_id`/`task_id`/`team_id`/`origin_product`/`acp_method`) as log attributes, and `request_id` = run uuid as the trace id. - `products/tasks/backend/models.py`: `TaskRun.append_log` calls `_mirror_logs_to_posthog_logs` after the S3 write, gated on the origin-product allowlist and wrapped so any failure is logged and never breaks the write. -- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables). The dev delivery leg reuses `SANDBOX_AGENT_OTEL_LOGS_URL`/`_TOKEN` under DEBUG — no mirror-specific OTLP settings. +- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables), plus `TASK_RUN_LOGS_MIRROR_OTLP_URL`/`_TOKEN` for the direct delivery leg (default unset; prod value = internal logs project's key). - `products/tasks/backend/tests/test_run_log_mirror.py` (new): severity/body mapping matrix, run-identity fields, truncation/batch caps, and `append_log` gating (allowlist, disabled, mirror-failure isolation). - `docs/internal/sandboxes-setup-guide.md`: mirroring section. @@ -172,4 +173,4 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout | Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | | Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | | Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | -| Django settings (DEBUG) | `SANDBOX_AGENT_OTEL_LOGS_URL` / `_TOKEN` (reused) | Also serve as the mirror's direct dev delivery leg; ignored by the mirror in production | +| Django settings | `TASK_RUN_LOGS_MIRROR_OTLP_URL` / `_TOKEN` | Direct OTLP delivery for the mirror; the token pins records to the internal logs project so customer projects are never billed | From 74fddab76ea8397d79f21a618351e731125fb2c8 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:36 +0100 Subject: [PATCH 16/26] fix(tasks): preserve mirror event in otlp logs --- REPORT.md | 30 +++++++++---------- .../backend/logic/services/run_log_mirror.py | 13 +++++--- .../backend/tests/test_run_log_mirror.py | 1 + 3 files changed, 25 insertions(+), 19 deletions(-) diff --git a/REPORT.md b/REPORT.md index 87793147c295..264b5bbac31f 100644 --- a/REPORT.md +++ b/REPORT.md @@ -86,14 +86,14 @@ Flush and shutdown are best-effort and per-signal independent (`Promise.allSettl This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](https://github.com/PostHog/posthog/pull/71094)), a second, complementary delivery path with a different privacy posture and scope: -| | OTLP export (this work) | Scout run log mirror (#71094) | -| -------------- | ------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | -| Emitter | agent-server inside the sandbox | Django `TaskRun.append_log` (the S3 write choke point) | -| Transport | direct OTLP HTTP to `capture-logs` | structured stdout line per entry (`event=task_run_log`); the per-cluster OTel collector already ships container stdout | -| Destination | chosen telemetry project (via `SANDBOX_AGENT_OTEL_*`) | the region's internal PostHog project's Logs | -| Content | metadata-only allowlist — never prompts, message text, tool args/output, or raw error text | readable bodies (agent messages, tool calls, sandbox output), capped at 8k chars | -| Scope | every cloud task run once the settings are set | runs whose task `origin_product` is in `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default: `signals_scout` only) | -| Traces | one APM trace per run (`task_run`/`turn`/`tool_call:*` spans) | `request_id` = run uuid becomes the record's trace id, grouping one run as one trace | +| | OTLP export (this work) | Scout run log mirror (#71094) | +| ----------- | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | +| Emitter | agent-server inside the sandbox | Django `TaskRun.append_log` (the S3 write choke point) | +| Transport | direct OTLP HTTP to `capture-logs` | structured stdout line per entry (`event=task_run_log`); the per-cluster OTel collector already ships container stdout | +| Destination | chosen telemetry project (via `SANDBOX_AGENT_OTEL_*`) | the region's internal PostHog project's Logs | +| Content | metadata-only allowlist — never prompts, message text, tool args/output, or raw error text | readable bodies (agent messages, tool calls, sandbox output), capped at 8k chars | +| Scope | every cloud task run once the settings are set | runs whose task `origin_product` is in `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default: `signals_scout` only) | +| Traces | one APM trace per run (`task_run`/`turn`/`tool_call:*` spans) | `request_id` = run uuid becomes the record's trace id, grouping one run as one trace | The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. @@ -166,11 +166,11 @@ Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout ## Configuration reference -| Where | Name | Meaning | -| ---------------------- | -------------------------------------------------------------------------- | ---------------------------------------------------------------------- | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | -| Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | -| Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | +| Where | Name | Meaning | +| ---------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | +| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | +| Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | +| Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | | Django settings | `TASK_RUN_LOGS_MIRROR_OTLP_URL` / `_TOKEN` | Direct OTLP delivery for the mirror; the token pins records to the internal logs project so customer projects are never billed | diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 0d752135d3a6..4159f08dfdc3 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -46,6 +46,8 @@ _OTLP_SEVERITIES = {"info": ("INFO", 9), "warn": ("WARN", 13), "error": ("ERROR", 17)} +_LOG_EVENT_NAME = "task_run_log" + _OTLP_SERVICE_NAME = "task-run-log-mirror" _OTLP_TIMEOUT_SECONDS = 3 @@ -99,7 +101,7 @@ def mirror_entries( if isinstance(entry_timestamp, str): fields["entry_timestamp"] = entry_timestamp - getattr(logger, _LOG_METHOD_NAMES[severity])("task_run_log", **fields) + getattr(logger, _LOG_METHOD_NAMES[severity])(_LOG_EVENT_NAME, **fields) records.append((severity, fields)) _post_otlp(records, run_id=run_id) @@ -131,9 +133,12 @@ def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> Non "body": {"stringValue": fields["body"]}, **({"traceId": trace_id} if len(trace_id) == 32 else {}), "attributes": [ - {"key": key, "value": _otlp_attribute_value(value)} - for key, value in fields.items() - if key != "body" + {"key": "event", "value": {"stringValue": _LOG_EVENT_NAME}}, + *[ + {"key": key, "value": _otlp_attribute_value(value)} + for key, value in fields.items() + if key != "body" + ], ], } ) diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index f1ababb14f75..bd012ecb0f97 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -151,6 +151,7 @@ def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): # The run uuid (dashes stripped) is the trace id, grouping the run. assert records[0]["traceId"] == RUN_ID.replace("-", "") attributes = {a["key"]: a["value"] for a in records[0]["attributes"]} + assert attributes["event"] == {"stringValue": "task_run_log"} assert attributes["task_run_id"] == {"stringValue": RUN_ID} # OTLP JSON encodes 64-bit ints as strings. assert attributes["team_id"] == {"intValue": "2"} From f91e18b4acee408f6ff30ddd9a85ef408133d367 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:38 +0100 Subject: [PATCH 17/26] fix(tasks): redact protocol payloads from mirrored logs --- REPORT.md | 1 + .../backend/logic/services/run_log_mirror.py | 27 ++++++++++---- .../backend/tests/test_run_log_mirror.py | 37 ++++++++++++++++--- 3 files changed, 53 insertions(+), 12 deletions(-) diff --git a/REPORT.md b/REPORT.md index 264b5bbac31f..888e48c242e5 100644 --- a/REPORT.md +++ b/REPORT.md @@ -96,6 +96,7 @@ This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](ht | Traces | one APM trace per run (`task_run`/`turn`/`tool_call:*` spans) | `request_id` = run uuid becomes the record's trace id, grouping one run as one trace | The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. +The mirror emits readable message, tool, console, error, and sandbox-output bodies, but reduces protocol setup envelopes and other metadata-only updates to type labels so authentication headers and other configuration payloads never enter Logs. Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. The mirror additionally has a **direct OTLP leg** (added on this branch, on top of #71094): `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN` ship each batch straight to a logs ingest endpoint, arriving as `service.name=task-run-log-mirror` with the run uuid as the trace id and the structlog fields as attributes. diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 4159f08dfdc3..5c573c8e3ab2 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -20,7 +20,6 @@ with a `task_run_id` attribute filter. """ -import json import time from datetime import datetime from typing import Any @@ -46,6 +45,15 @@ _OTLP_SEVERITIES = {"info": ("INFO", 9), "warn": ("WARN", 13), "error": ("ERROR", 17)} +_READABLE_SESSION_UPDATES = { + "agent_message", + "agent_message_chunk", + "agent_thought_chunk", + "user_message_chunk", +} + +_TOOL_SESSION_UPDATES = {"tool_call", "tool_call_update"} + _LOG_EVENT_NAME = "task_run_log" _OTLP_SERVICE_NAME = "task-run-log-mirror" @@ -208,14 +216,18 @@ def _body(notification: dict, session_update: str | None) -> str: update = _session_update(notification) body: str | None = None - if session_update: + if session_update in _READABLE_SESSION_UPDATES: text = _extract_text(update.get("content")) if text is not None: body = f"[{session_update}] {text}" - elif session_update in ("tool_call", "tool_call_update"): - title = update.get("title") or update.get("toolCallId") or "" - status = update.get("status") - body = f"[{session_update}] {title}" + (f" ({status})" if status else "") + else: + body = f"[{session_update}]" + elif session_update in _TOOL_SESSION_UPDATES: + title = update.get("title") or update.get("toolCallId") or "" + status = update.get("status") + body = f"[{session_update}] {title}" + (f" ({status})" if status else "") + elif session_update: + body = f"[{session_update}]" elif notification.get("method") in ("_posthog/console", "_posthog/error"): message = params.get("message") if isinstance(message, str): @@ -230,7 +242,8 @@ def _body(notification: dict, session_update: str | None) -> str: body = f"[turn_end] {stop_reason}" if body is None: - body = json.dumps(notification) + method = notification.get("method") + body = f"[{method}]" if isinstance(method, str) else "[acp_message]" return body[:MAX_BODY_CHARS] diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index bd012ecb0f97..b32c0407aec6 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -1,5 +1,3 @@ -import json - from unittest.mock import MagicMock, patch from django.test import SimpleTestCase, TestCase, override_settings @@ -84,10 +82,39 @@ def test_severity_and_body_mapping(self, _name, entry, expected_log_method, expe log_call.assert_called_once() self.assertEqual(log_call.call_args.kwargs["body"], expected_body) - def test_unrecognized_entry_falls_back_to_json_body(self): - notification = {"method": "session/request_permission", "params": {"tool": "bash"}} + @parameterized.expand( + [ + ( + "protocol_request", + { + "method": "session/new", + "params": { + "mcpServers": [ + { + "headers": [ + {"name": "Authorization", "value": "Bearer protocol-secret"}, + ] + } + ] + }, + }, + "[session/new]", + ), + ( + "metadata_session_update", + _session_update_entry( + "available_commands_update", + availableCommands=[{"name": "example", "description": "metadata-secret"}], + )["notification"], + "[available_commands_update]", + ), + ] + ) + def test_protocol_metadata_omits_payload(self, _name, notification, expected_body): mock_logger = _mirror([{"notification": notification}]) - self.assertEqual(json.loads(mock_logger.info.call_args.kwargs["body"]), notification) + body = mock_logger.info.call_args.kwargs["body"] + self.assertEqual(body, expected_body) + self.assertNotIn("secret", body) def test_emitted_fields_carry_run_identity(self): mock_logger = _mirror([_session_update_entry("agent_message", content={"type": "text", "text": "hi"})]) From d0a1b8d855d9fa1cadf93ae4bfa9083520db315d Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:40 +0100 Subject: [PATCH 18/26] fix(tasks): harden run log mirror against slow endpoints and oversized input Review findings from a subagent pass over the mirror module: - The OTLP POST runs synchronously inside append_log, and a scalar requests timeout applies to connect and read independently - a slow connect followed by a stalled read could block the append for ~6s. Use a (1, 3) connect/read tuple to bound the worst case. - acp_method, acp_session_update, and entry_timestamp were forwarded uncapped from the semi-trusted append_log request; an oversized value could push the stdout line past the collector's 100 KB drop threshold, silently discarding the whole record. Cap them at 200 chars. - _extract_text recursed without a depth bound, so an adversarially nested content list would RecursionError and abort mirroring for the entire batch. Bound it at 10 levels. Tests: identifier-cap and deep-nesting regressions, an OTLP-leg privacy test pinning that protocol payloads never reach the payload and that the attribute set is closed, and a negative gating row for the snapshot-resume env path (logs pair missing -> no OTel keys). Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- .../backend/logic/services/run_log_mirror.py | 28 ++++++++--- .../temporal/process_task/tests/test_utils.py | 18 ++++++++ .../backend/tests/test_run_log_mirror.py | 46 ++++++++++++++++++- 3 files changed, 84 insertions(+), 8 deletions(-) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 5c573c8e3ab2..b8b1642c557c 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -58,7 +58,19 @@ _OTLP_SERVICE_NAME = "task-run-log-mirror" -_OTLP_TIMEOUT_SECONDS = 3 +# (connect, read) — a scalar timeout applies to each stage independently, so a slow +# connect followed by a stalled read would block append_log for the sum. The mirror +# runs synchronously on that hot path; keep the worst case tightly bounded. +_OTLP_TIMEOUT = (1, 3) + +# Identifier-shaped fields (method names, timestamps) come from the same semi-trusted +# request as the entries; cap them so an oversized value can't push the stdout line +# past the collector's 100 KB drop threshold or bloat OTLP attributes. +MAX_IDENTIFIER_CHARS = 200 + +# ACP content blocks nest at most a few levels; an adversarially deep list must not +# RecursionError and abort the whole batch. +_MAX_CONTENT_DEPTH = 10 def mirroring_enabled(origin_product: str) -> bool: @@ -102,12 +114,12 @@ def mirror_entries( } method = notification.get("method") if isinstance(method, str): - fields["acp_method"] = method + fields["acp_method"] = method[:MAX_IDENTIFIER_CHARS] if session_update: - fields["acp_session_update"] = session_update + fields["acp_session_update"] = session_update[:MAX_IDENTIFIER_CHARS] entry_timestamp = entry.get("timestamp") if isinstance(entry_timestamp, str): - fields["entry_timestamp"] = entry_timestamp + fields["entry_timestamp"] = entry_timestamp[:MAX_IDENTIFIER_CHARS] getattr(logger, _LOG_METHOD_NAMES[severity])(_LOG_EVENT_NAME, **fields) records.append((severity, fields)) @@ -163,7 +175,7 @@ def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> Non url, json=payload, headers={"Authorization": f"Bearer {token}"}, - timeout=_OTLP_TIMEOUT_SECONDS, + timeout=_OTLP_TIMEOUT, ) response.raise_for_status() except Exception as e: @@ -247,12 +259,14 @@ def _body(notification: dict, session_update: str | None) -> str: return body[:MAX_BODY_CHARS] -def _extract_text(content: Any) -> str | None: +def _extract_text(content: Any, depth: int = 0) -> str | None: """Pull plain text out of an ACP content block (single block or list of blocks).""" + if depth > _MAX_CONTENT_DEPTH: + return None if isinstance(content, dict): text = content.get("text") return text if isinstance(text, str) else None if isinstance(content, list): - parts = [t for t in (_extract_text(block) for block in content) if t] + parts = [t for t in (_extract_text(block, depth + 1) for block in content) if t] return "\n".join(parts) if parts else None return None diff --git a/products/tasks/backend/temporal/process_task/tests/test_utils.py b/products/tasks/backend/temporal/process_task/tests/test_utils.py index 9ae562544e27..f7fc1a3e7c62 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_utils.py +++ b/products/tasks/backend/temporal/process_task/tests/test_utils.py @@ -1089,3 +1089,21 @@ def test_snapshot_resume_env_includes_otel_config_when_configured(self, _api, _j assert env["POSTHOG_AGENT_OTEL_LOGS_URL"] == "https://us.i.posthog.com/i/v1/logs" assert env["POSTHOG_AGENT_OTEL_LOGS_TOKEN"] == "phc_telemetry" assert env["POSTHOG_AGENT_OTEL_TRACES_URL"] == "https://us.i.posthog.com/i/v1/traces" + + @patch( + "products.tasks.backend.logic.services.connection_token.get_sandbox_jwt_public_key", + return_value="pub", + ) + @patch( + "products.tasks.backend.temporal.process_task.utils.get_sandbox_api_url", + return_value="https://api.example", + ) + def test_snapshot_resume_env_omits_otel_config_without_logs_pair(self, _api, _jwt) -> None: + with override_settings( + SANDBOX_AGENT_OTEL_LOGS_URL="https://us.i.posthog.com/i/v1/logs", + SANDBOX_AGENT_OTEL_LOGS_TOKEN=None, + SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", + ): + env = build_sandbox_environment_variables(None, "access-token", 1) + + assert not any(key.startswith("POSTHOG_AGENT_OTEL_") for key in env) diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index b32c0407aec6..e7e6cd212faa 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -1,3 +1,5 @@ +import json + from unittest.mock import MagicMock, patch from django.test import SimpleTestCase, TestCase, override_settings @@ -6,7 +8,12 @@ from posthog.models import Organization, Team -from products.tasks.backend.logic.services.run_log_mirror import MAX_BODY_CHARS, MAX_ENTRIES_PER_CALL, mirror_entries +from products.tasks.backend.logic.services.run_log_mirror import ( + MAX_BODY_CHARS, + MAX_ENTRIES_PER_CALL, + MAX_IDENTIFIER_CHARS, + mirror_entries, +) from products.tasks.backend.models import Task, TaskRun RUN_ID = "0b166f65-9e52-4d1b-b3c4-1a9e3f6d3c21" @@ -134,6 +141,20 @@ def test_oversized_body_is_truncated(self): mock_logger = _mirror([entry]) self.assertEqual(len(mock_logger.info.call_args.kwargs["body"]), MAX_BODY_CHARS) + def test_oversized_identifier_fields_are_capped(self): + entry = {"timestamp": "9" * 100_000, "notification": {"method": "m" * 100_000}} + mock_logger = _mirror([entry]) + fields = mock_logger.info.call_args.kwargs + self.assertEqual(len(fields["acp_method"]), MAX_IDENTIFIER_CHARS) + self.assertEqual(len(fields["entry_timestamp"]), MAX_IDENTIFIER_CHARS) + + def test_deeply_nested_content_returns_type_only_body(self): + content: list = ["bottom"] + for _ in range(2_000): + content = [content] + mock_logger = _mirror([_session_update_entry("agent_message", content=content)]) + self.assertEqual(mock_logger.info.call_args.kwargs["body"], "[agent_message]") + def test_oversized_batch_is_capped(self): entries = [ _session_update_entry("agent_message", content={"type": "text", "text": f"line {i}"}) @@ -183,6 +204,29 @@ def test_posts_one_otlp_batch_with_severity_trace_and_attributes(self): # OTLP JSON encodes 64-bit ints as strings. assert attributes["team_id"] == {"intValue": "2"} + @override_settings( + TASK_RUN_LOGS_MIRROR_OTLP_URL="https://us.i.posthog.com/i/v1/logs", + TASK_RUN_LOGS_MIRROR_OTLP_TOKEN="phc_internal", + ) + def test_protocol_payloads_never_reach_the_otlp_payload(self): + entry = { + "notification": { + "method": "session/new", + "params": {"mcpServers": [{"headers": [{"name": "Authorization", "value": "Bearer protocol-secret"}]}]}, + } + } + with patch("products.tasks.backend.logic.services.run_log_mirror.internal_requests") as mock_requests: + _mirror([entry]) + + payload = mock_requests.post.call_args.kwargs["json"] + self.assertNotIn("protocol-secret", json.dumps(payload)) + records = payload["resourceLogs"][0]["scopeLogs"][0]["logRecords"] + # The attribute set is closed: run identity plus entry type, never raw params. + self.assertEqual( + {a["key"] for a in records[0]["attributes"]}, + {"event", "request_id", "task_run_id", "task_id", "team_id", "origin_product", "acp_method"}, + ) + @parameterized.expand( [ ("both_unset", None, None), From f9c1a89d222038f654dc2d80ed5295edeb5a642a Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:41 +0100 Subject: [PATCH 19/26] refactor(tasks): simplify run log mirror internals Simplify-pass cleanups, no behavior change: - pass the already-derived session update dict into _body instead of re-parsing the notification per entry - hoist the batch-constant trace fields and event attribute out of the per-record loop in _post_otlp - drop the Z-suffix replace in _time_unix_nano (fromisoformat handles it natively on our Python) Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- .../tasks/backend/logic/services/run_log_mirror.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index b8b1642c557c..17a9ae02d8f6 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -110,7 +110,7 @@ def mirror_entries( "task_id": task_id, "team_id": team_id, "origin_product": origin_product, - "body": _body(notification, session_update), + "body": _body(notification, update, session_update), } method = notification.get("method") if isinstance(method, str): @@ -142,6 +142,8 @@ def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> Non # The run uuid without dashes is a valid 16-byte hex trace id, matching the # request_id -> trace id mapping the production collector applies. trace_id = run_id.replace("-", "") + trace_fields = {"traceId": trace_id} if len(trace_id) == 32 else {} + event_attribute = {"key": "event", "value": {"stringValue": _LOG_EVENT_NAME}} log_records = [] for severity, fields in records: severity_text, severity_number = _OTLP_SEVERITIES[severity] @@ -151,9 +153,9 @@ def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> Non "severityText": severity_text, "severityNumber": severity_number, "body": {"stringValue": fields["body"]}, - **({"traceId": trace_id} if len(trace_id) == 32 else {}), + **trace_fields, "attributes": [ - {"key": "event", "value": {"stringValue": _LOG_EVENT_NAME}}, + event_attribute, *[ {"key": key, "value": _otlp_attribute_value(value)} for key, value in fields.items() @@ -194,7 +196,7 @@ def _otlp_attribute_value(value: Any) -> dict[str, Any]: def _time_unix_nano(entry_timestamp: Any) -> int: if isinstance(entry_timestamp, str): try: - return int(datetime.fromisoformat(entry_timestamp.replace("Z", "+00:00")).timestamp() * 1_000_000_000) + return int(datetime.fromisoformat(entry_timestamp).timestamp() * 1_000_000_000) except (ValueError, OverflowError): pass return time.time_ns() @@ -222,10 +224,9 @@ def _severity(notification: dict) -> str: return "info" -def _body(notification: dict, session_update: str | None) -> str: +def _body(notification: dict, update: dict, session_update: str | None) -> str: raw_params = notification.get("params") params: dict = raw_params if isinstance(raw_params, dict) else {} - update = _session_update(notification) body: str | None = None if session_update in _READABLE_SESSION_UPDATES: From 00553f4acaf742ef54cfc778f8d160356dd2725d Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:43 +0100 Subject: [PATCH 20/26] chore(tasks): drop implementation report REPORT.md was a working artifact for the agent session, not part of the change itself. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- REPORT.md | 177 ------------------------------------------------------ 1 file changed, 177 deletions(-) delete mode 100644 REPORT.md diff --git a/REPORT.md b/REPORT.md deleted file mode 100644 index 888e48c242e5..000000000000 --- a/REPORT.md +++ /dev/null @@ -1,177 +0,0 @@ -# Agent run telemetry: PostHog Code cloud tasks → PostHog Logs + APM - -This branch is one half of a two-repo change; the companion branch is `posthog-code/agent-run-otel-telemetry` in `PostHog/code`. -This repo (`PostHog/posthog`) carries the configuration/injection side; `PostHog/code` carries the telemetry emitter inside the agent. - -## Goal - -Cloud task runs executed by PostHog Code should be observable in PostHog itself: -every run's lifecycle metadata delivered as OTel **logs** into the Logs product, and one OTel **trace** per run (root span, per-turn spans, per-tool-call spans) into APM, with logs and spans cross-linked via `trace_id`/`span_id`. - -Hard requirements: - -- **Filterable per user**: the Logs UI must answer "show me everything cloud runs did for this user" directly, so `user_id`/`distinct_id` (plus `team_id`, `task_id`, `run_id`) are OTel resource attributes, which PostHog Logs facets via `resource_fingerprint`. -- **Cloud tasks only** for now; desktop local runs do not export session telemetry. -- **Metadata only**: the S3 session log remains the source of truth for full transcripts. Telemetry never carries prompts, agent message/thought text, tool arguments, or tool output. - -## Background: what existed before - -- Agent session logs flow from the sandbox `agent-server` through `SessionLogWriter` to the Django endpoint `POST /api/projects/{team}/tasks/{task}/runs/{run}/append_log/`, which appends NDJSON to S3 (`TaskRun.append_log`, 30-day TTL). Two more delivery legs exist: an NDJSON event-ingest stream and SSE to connected clients. -- A February attempt at OTel logs export (`OtelLogWriter`, commits `6abadc79` → `99a3aea8` → `8876c8fb` in `PostHog/code`) was unwired, and its default endpoint `/i/v1/agent-logs` does not exist in the ingest service; it would 404 today. -- The correct ingest is the `capture-logs` Rust service (`rust/capture-logs/`): `POST /i/v1/logs` and `POST /i/v1/traces`, OTLP http/protobuf or http/JSON, auth `Authorization: Bearer `, 2 MB request cap, billed by uncompressed bytes, severity normalized to lowercase, prod host `https://us.i.posthog.com`. -- Working dogfood precedents followed here: the desktop Electron transport (`posthog-code-desktop` → `/i/v1/logs`), the engineering-analytics CI log emitter, the streamlit sandbox proxy (env-injected OTLP config), and the plugin-server metrics exporter (telemetry off unless URL + token are both set). - -## Architecture - -```text -sandbox (agentsh) -└── agent-server (PostHog/code, packages/agent) - ├── ACP streams (tapped) ──► SessionLogWriter ──► Django append_log ──► S3 (product log, unchanged) - │ │ │ - │ │ └─ scout runs only: structured stdout line per entry - │ │ └─► cluster OTel collector ──► internal project Logs - │ └─ sink ──► OtelRunTelemetry - │ ├─ log records ──► POST {POSTHOG_AGENT_OTEL_LOGS_URL} - │ └─ RunTraceBuilder spans ──► POST {POSTHOG_AGENT_OTEL_TRACES_URL} - └── terminal error events ──────────────────────► mirrored into OtelRunTelemetry directly - -capture-logs (Rust) ──► Kafka ──► ClickHouse (logs / trace_spans) ──► Logs + APM UI -``` - -Telemetry is emitted **from inside the sandbox** by the agent-server process, deliberately independent of the Django/S3 product path: if `append_log` is degraded, telemetry still flows, and a broken product log pipeline is exactly the failure telemetry must capture. -Egress works because `*.posthog.com` is in the agentsh `INFRASTRUCTURE_DOMAINS` allowlist (prod) and the new `SANDBOX_AGENT_OTEL_*_URL` hosts join the DEBUG-only firewall list (local dev). - -## What ships per run - -### Log records (service.name=posthog-code-agent) - -Resource attributes on every record: `service.name`, `service.version` (agent version), `run_id`, `task_id`, `team_id`, `user_id`, `distinct_id`, `device_type` (`cloud`), `adapter` (`claude`/`codex`), `run_mode` (`interactive`/`background`). - -Exported events (allowlist, everything else is dropped): - -| Event | Severity | Notable attributes | -| ---------------------------------------------------------- | ----------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `_posthog/run_started` | info | `agent_version`, `session_id` | -| `_posthog/sdk_session` | info | `adapter`, `session_id` | -| `_posthog/usage_update` (and the `session/update` variant) | info | `tokens_input/output/cached_read/cached_write`, `cost_usd` | -| `_posthog/turn_complete` | info | `stop_reason` | -| `_posthog/task_complete` | info | `stop_reason` | -| `_posthog/error` | **error** | `error_source`, `stop_reason`; body is the generic "run error" — the raw message is free text that can embed prompt/repo content, so it stays in the session log and the run's `error_message` | -| `_posthog/progress` | info | `progress_group/step/status` | -| `_posthog/git_checkpoint`, `_posthog/branch_created` | info | `branch` | -| `_posthog/mode_change`, `_posthog/compact_boundary` | info | | -| `_posthog/permission_request/response/resolved` | info | `request_id`, `tool_call_id` (identifiers only; tool content excluded) | -| `session/update: tool_call` | info | `tool_call_id`, `tool_kind`, `tool_status` (no title, no rawInput) | -| `session/update: tool_call_update` (terminal only) | info / warn on `failed` | `tool_call_id`, `tool_status` | - -Deliberately dropped: `agent_message`, `agent_message_chunk`, `agent_thought_chunk`, `user_message`, `session/prompt` bodies, in-progress `tool_call_update` snapshots (they re-send the growing tool input/output), `available_commands_update`, `_posthog/console` (free-text agent-server diagnostics interpolate arbitrary data — e.g. the prompt preview logged on user-message handling and stringified extension params — so exporting them would leak content; they stay in the S3 log and event-ingest stream), and any unknown method (fail-closed allowlist). -Bodies are capped at 2000 chars; free-text attribute values at 200 chars (the `log_attributes` faceting table only indexes key/value pairs under 256 chars). - -### APM trace (one per run) - -- `task_run` root span (kind SERVER): opened at session init, closed at the run's in-process terminal point — a background run's prompt settling (`finalizeRunTelemetry`), a terminal failure (`signalTaskComplete`), or session cleanup (interactive `close`). Status is resolved at shutdown from the latest turn outcome (the sandbox never emits `task_complete` for successful runs — the terminal "completed" status is decided by the workflow outside — so the last turn is the in-sandbox success signal): OK when the last turn ended with `end_turn`, ERROR on `_posthog/error` (an error always wins, later turn completions cannot flip it back; `error_source` lands as a root-span attribute while the raw message is withheld — see the error row above) or when the last turn stopped with `error`, unset otherwise (cancelled / refused / timed out / no completed turns). Resolving at shutdown rather than per-turn means an early clean turn cannot leave a stale OK on a run whose final turn was cancelled. -- `turn` child spans: opened on each ACP `session/prompt` (used purely as a boundary marker; its content is never read), closed on `_posthog/turn_complete`; attributes `turn_index`, `stop_reason`, plus the turn's token counts and `cost_usd` lifted from usage updates, so APM can rank slow or expensive turns directly. -- `tool_call:` grandchild spans (`execute`, `read`, `edit`, ...): opened on `tool_call`, closed on the terminal `tool_call_update`; status ERROR on `failed`; attributes `tool_call_id`, `tool_kind`, `tool_status`. Per-kind span names stay low-cardinality and make APM latency breakdowns by tool kind useful. -- Robustness: orphaned spans are closed (status unset) and exported at shutdown; a new prompt while a turn is open closes the stale turn; duplicate `tool_call` events are idempotent; a run error cascades ERROR status to the open turn and the root, and closes still-open tool spans as ERROR with `tool_status=interrupted` so APM never shows a healthy-looking active tool under a failed run. -- Every log record is emitted under the OTel context of the span it belongs to (tool logs on the tool span, lifecycle logs on the root), so `trace_id`/`span_id` land in the `logs` table columns and the UI links Logs ⇄ trace waterfall. - -### Delivery timing - -Telemetry is near-realtime, not end-of-turn: records are created the moment each notification flows through the writer and batched for at most 2 s (`BatchLogRecordProcessor` / `BatchSpanProcessor`, `scheduledDelayMillis` 2000). -Spans export when they end (tools mid-turn, turns at `turn_complete`, root at the run's terminal point). -Sandbox teardown can NOT be a flush point: agent-server is an exec'd process inside the sandbox, so `docker stop` signals only the container's PID 1 and Modal terminate is immediate — the process's SIGTERM handler never runs and anything still queued (or a still-open root span) is lost. Telemetry is therefore ended eagerly at the in-process terminal points: `finalizeRunTelemetry` when a background run's prompt settles, a full shutdown after a terminal failure in `signalTaskComplete`, and `cleanupSession` for interactive `close`. Known limitation: an interactive session ended by hard teardown (e.g. inactivity timeout) loses the root span; its turn/tool spans and logs still assemble under the same trace id. -Flush and shutdown are best-effort and per-signal independent (`Promise.allSettled`), and every export is capped at 5 s (`exportTimeoutMillis`, down from the SDK's 30 s default), so a rejecting or hanging traces endpoint can neither starve log delivery nor hold up session cleanup. - -### Companion pipeline: scout run log mirror (incorporated from PR #71094) - -This branch also carries the scout-run log mirror by Andrew Maguire ([#71094](https://github.com/PostHog/posthog/pull/71094)), a second, complementary delivery path with a different privacy posture and scope: - -| | OTLP export (this work) | Scout run log mirror (#71094) | -| ----------- | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------------------------------------------------------------------- | -| Emitter | agent-server inside the sandbox | Django `TaskRun.append_log` (the S3 write choke point) | -| Transport | direct OTLP HTTP to `capture-logs` | structured stdout line per entry (`event=task_run_log`); the per-cluster OTel collector already ships container stdout | -| Destination | chosen telemetry project (via `SANDBOX_AGENT_OTEL_*`) | the region's internal PostHog project's Logs | -| Content | metadata-only allowlist — never prompts, message text, tool args/output, or raw error text | readable bodies (agent messages, tool calls, sandbox output), capped at 8k chars | -| Scope | every cloud task run once the settings are set | runs whose task `origin_product` is in `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default: `signals_scout` only) | -| Traces | one APM trace per run (`task_run`/`turn`/`tool_call:*` spans) | `request_id` = run uuid becomes the record's trace id, grouping one run as one trace | - -The privacy postures are compatible because the scopes differ: scout runs are PostHog-authored agents whose transcripts we already own end to end, so full bodies into the internal project are fine there, while the OTLP path covers customer-driven runs and therefore stays metadata-only. -The mirror emits readable message, tool, console, error, and sandbox-output bodies, but reduces protocol setup envelopes and other metadata-only updates to type labels so authentication headers and other configuration payloads never enter Logs. -Mirroring is fire-and-forget — a mirror failure is logged and never breaks the run's S3 log write. - -The mirror additionally has a **direct OTLP leg** (added on this branch, on top of #71094): `TASK_RUN_LOGS_MIRROR_OTLP_URL` + `_TOKEN` ship each batch straight to a logs ingest endpoint, arriving as `service.name=task-run-log-mirror` with the run uuid as the trace id and the structlog fields as attributes. -The token pins the destination — scout runs execute for customer teams, and their mirrored transcripts must only ever land in (and bill) PostHog's own internal logs project, never the customer's — so the intended production value is the internal logs project's API key, deliberately not derived from the run's team and not reusing `SANDBOX_AGENT_OTEL_LOGS_TOKEN` (which addresses the separate agent-telemetry project). -It is also the only delivery path in local dev, where `append_log` runs in the host Django process whose stdout the dev collector (docker containers only) never tails — verified during local testing where `/logs` showed only the OTLP metadata records until this leg existed. -Delivery is best-effort (3 s timeout, one POST per persisted batch, scout runs only) and never raises into the run. - -## Changes in PostHog/code (companion branch) - -- `packages/agent/src/otel-telemetry.ts` (renamed from `otel-log-writer.ts`): `OtelRunTelemetry`, the single `SessionLogSink` owning the OTLP log exporter and (when a traces URL is configured) the `RunTraceBuilder`. Contains the pure `mapNotificationToLogRecord()` allowlist mapper. Fixes the dead `/i/v1/agent-logs` default. Never throws into the run; ignores entries for other sessions. -- `packages/agent/src/otel-trace-builder.ts` (new): `RunTraceBuilder`, the span state machine described above; `handle(entry)` returns the context each log record should attach to. -- `packages/agent/src/otel-attributes.ts` (new): shared pure helpers (`strAttr`, `numAttr`, `usageAttributes`, truncation, caps). -- `packages/agent/src/session-log-writer.ts`: optional `sinks: SessionLogSink[]`, teed in `appendRawLine` after the entry is built; a throwing sink warns once and can never break product log persistence; message chunks never reach sinks. `SessionContext` moved here from the otel module. -- `packages/agent/src/server/agent-server.ts`: builds the telemetry per session from config (`createRunTelemetry`), passes it as the writer sink, stores it on the session, and ends it at the run's in-process terminal points: `finalizeRunTelemetry` (full shutdown when a background run's initial/resume prompt settles — verified necessary because sandbox teardown never delivers SIGTERM to the exec'd process, so waiting for `cleanupSession` left the root span unexported), a shutdown after terminal failures in `signalTaskComplete`, and `cleanupSession` for interactive `close`. `enqueueTaskTerminalEvent` payloads are mirrored into it directly (terminal `_posthog/error` events bypass `SessionLogWriter`, and a failed run is exactly what telemetry must record). Fatal crashes (`reportFatalError`, the uncaught-exception/unhandled-rejection path) also mirror an error record (`error_source=agent_server_crash`) and shut telemetry down, so hard process deaths reach the telemetry project instead of vanishing. -- `packages/agent/src/server/bin.ts` + `server/types.ts`: zod-validated env `POSTHOG_AGENT_OTEL_LOGS_URL`, `POSTHOG_AGENT_OTEL_LOGS_TOKEN`, `POSTHOG_AGENT_OTEL_TRACES_URL` → `AgentServerConfig.otelLogsUrl/otelLogsToken/otelTracesUrl`. Telemetry is off unless the logs pair is set; spans additionally require the traces URL (per-signal kill switch). -- `packages/agent/src/types.ts`: the February `OtelTransportConfig`/`AgentConfig.otelTransport` remain as `@deprecated`, ignored stubs — `@posthog/agent` is a published package, so removing exported types is an API break reserved for a major. -- Dependencies: `@opentelemetry/api`, `@opentelemetry/sdk-trace-base`, `@opentelemetry/exporter-trace-otlp-http`, version-aligned with the existing logs SDK (0.208.x experimental / 2.x stable line). -- Tests (74 passing in the telemetry suites, plus agent-server terminal-point tests): parameterized log-mapping matrix, hard privacy tests asserting exported payloads never contain tool args/titles/output or raw error messages, per-user resource attributes, session-mismatch guard, never-throws guard, sink isolation in `SessionLogWriter`, five trace tests (span tree + statuses + attributes, log⇄span id correlation, error cascade incl. interrupted tools, orphan export on shutdown, log shutdown isolated from a failing traces endpoint), and agent-server tests asserting the error mirror lands before the terminal shutdown and that `finalizeRunTelemetry` fires for background runs only. -- `packages/agent/README.md`: documents the env vars and behavior. - -## Changes in PostHog/posthog (this branch) - -- `posthog/settings/temporal.py`: new optional settings `SANDBOX_AGENT_OTEL_LOGS_URL`, `SANDBOX_AGENT_OTEL_LOGS_TOKEN`, `SANDBOX_AGENT_OTEL_TRACES_URL` (all default unset = telemetry off). -- `products/tasks/backend/temporal/process_task/utils.py`: `get_sandbox_otel_env_vars()` maps those settings to the sandbox env vars, gated on the logs pair; called from **both** env assembly paths so fresh provisioning and snapshot-resume behave identically: - - `activities/provision_sandbox.py` `_build_environment_variables` - - `utils.py` `build_sandbox_environment_variables` (used by `create_sandbox_from_snapshot`) -- `products/tasks/backend/constants.py`: the three env keys added to `RESERVED_SANDBOX_ENVIRONMENT_VARIABLE_KEYS` so user-supplied SandboxEnvironment vars cannot override them. -- `products/tasks/backend/logic/services/agentsh.py`: `SANDBOX_AGENT_OTEL_LOGS_URL`/`SANDBOX_AGENT_OTEL_TRACES_URL` added to `_DEBUG_SANDBOX_URL_SETTINGS` so local-dev hosts pass the agentsh syscall firewall (prod egress already covered by `*.posthog.com`). -- `products/tasks/backend/logic/services/docker_sandbox.py`: `POSTHOG_AGENT_OTEL_LOGS_URL`/`POSTHOG_AGENT_OTEL_TRACES_URL` added to `_DOCKER_URL_ENV_KEYS` so localhost URLs are rewritten to `host.docker.internal` for local Docker sandboxes. -- Tests: parameterized gating matrix on `_build_environment_variables` (5 rows: full config, logs-only, partial configs, traces-without-logs all correctly gated) and a `SimpleTestCase` wiring guard on the snapshot-resume path. -- `docs/internal/sandboxes-setup-guide.md`: local-dev setup section for the new settings. - -Incorporated from [#71094](https://github.com/PostHog/posthog/pull/71094) (scout run log mirror, credit Andrew Maguire): - -- `products/tasks/backend/logic/services/run_log_mirror.py` (new): translates each ACP JSONL entry into a `task_run_log` structlog stdout line — readable bodies for agent messages / tool calls / sandbox output / turn ends (8k char cap, entry-count cap per call), severity mapping (`_posthog/error` → error, console level passthrough), run-identity fields (`task_run_id`/`task_id`/`team_id`/`origin_product`/`acp_method`) as log attributes, and `request_id` = run uuid as the trace id. -- `products/tasks/backend/models.py`: `TaskRun.append_log` calls `_mirror_logs_to_posthog_logs` after the S3 write, gated on the origin-product allowlist and wrapped so any failure is logged and never breaks the write. -- `posthog/settings/temporal.py`: `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` (default `signals_scout`; empty disables), plus `TASK_RUN_LOGS_MIRROR_OTLP_URL`/`_TOKEN` for the direct delivery leg (default unset; prod value = internal logs project's key). -- `products/tasks/backend/tests/test_run_log_mirror.py` (new): severity/body mapping matrix, run-identity fields, truncation/batch caps, and `append_log` gating (allowlist, disabled, mirror-failure isolation). -- `docs/internal/sandboxes-setup-guide.md`: mirroring section. - -## Key design decisions - -1. **Emit from the sandbox, not from Django.** Independence from the product log path (see Architecture), matching the streamlit sandbox precedent. The Django-tee alternative would add an outbound call to a hot API path and go dark precisely when `append_log` breaks. -2. **`POSTHOG_`-prefixed env vars instead of standard `OTEL_*` names.** The sandbox env is inherited by the user's own processes (their tests, their apps). Standard `OTEL_EXPORTER_OTLP_*` vars would make any OTel SDK in user code silently auto-export the user's telemetry into our internal project. Custom names mean only agent-server reads the config. -3. **`service.name=posthog-code-agent`, not `posthog-code`.** `service.name` identifies the emitting process, not the product: the desktop app already ships its process logs as `posthog-code-desktop`, and `service_name` is both the primary Logs UI facet and part of the ClickHouse sort key `(team_id, service_name, timestamp)`, so component-level names keep streams separable and queries narrow. House pattern matches (`posthog-django-*`, `node-*`, `github-ci-logs`). -4. **Fail-closed allowlist for content.** Only known lifecycle events are exported; unknown methods are dropped. This is a privacy boundary (customer prompts/repo content must not reach the telemetry project) and a cost control (logs are billed by bytes; in-progress tool snapshots re-send growing output). -5. **Generic `SessionLogSink` instead of hardcoding OTel into `SessionLogWriter`.** The February attempt was removed partly because of hard coupling; the sink interface keeps the writer single-purpose, is desktop-neutral (no sinks wired there), and isolates sink failures. -6. **Terminal-error mirrors.** Two paths bypass `SessionLogWriter` and are mirrored into telemetry explicitly: `enqueueTaskTerminalEvent` (agent-server-sourced run errors, which feed only the event-ingest stream) and `reportFatalError` (unrecoverable crashes, which mark the run failed via the API with no session log involvement). Without the mirrors the most important records, failed and crashed runs, would be missing from telemetry. -7. **Per-signal kill switch.** Logs and spans have separate URLs; unsetting the traces URL disables spans without touching logs, and unsetting either of the logs pair disables everything. -8. **Token exposure is acceptable by design.** The sandbox receives a project API key of the telemetry project: a write-only, public-by-design key class (the same class that ships in client SDKs), far weaker than the `POSTHOG_PERSONAL_API_KEY` already present in the sandbox. Worst case is junk telemetry writes; `capture-logs` has a token drop list as the kill switch. - -## Verification - -- `PostHog/code`: 74 tests pass in the agent package (including the new telemetry suite), `tsc --noEmit` clean via turbo, biome clean on all touched files (one pre-existing warning untouched). The package's pre-existing test failures in this environment (missing Postgres/git fixtures) were confirmed byte-identical with and without these changes by running the failing files against a stashed tree. -- `PostHog/posthog`: 21 tests pass across `test_provision_sandbox.py` and the new `TestBuildSandboxEnvironmentVariables`; `ruff check`/`format` clean on all touched files. DB-dependent suites in this sandbox fail identically with and without the change (no Postgres available). -- Local-dev routing verified: Caddy serves `/i/v1/logs`/`/i/v1/traces` on `localhost:8000` and proxies to `capture-logs`, and the Docker URL rewrite covers the new vars. - -## Rollout runbook (what remains) - -1. Choose the destination telemetry project and create/locate its project API key. Recommendation: the shared internal project where `posthog-code-desktop` logs and Code analytics already land, so desktop and cloud correlate in one Logs view (separable by `service_name`). -2. Set in prod US: - - `SANDBOX_AGENT_OTEL_LOGS_URL=https://us.i.posthog.com/i/v1/logs` - - `SANDBOX_AGENT_OTEL_LOGS_TOKEN=` - - `SANDBOX_AGENT_OTEL_TRACES_URL=https://us.i.posthog.com/i/v1/traces` -3. Run one cloud task; verify in the destination project: Logs filtered by `service.name=posthog-code-agent` (facet by `distinct_id`/`user_id`/`run_id`), and the APM trace for the run (`task_run` → `turn` → `tool_call:*` waterfall, logs linked from spans). -4. Add a saved Logs view and alerts (error severity on the service; volume anomaly), and watch billed bytes for a week; the event allowlist and body caps are the tuning knobs. -5. Optional follow-ups: unify the desktop transport with the new telemetry module; consider a shared `product` resource attribute across `posthog-code-*` services; sampling if volume warrants. - -## Configuration reference - -| Where | Name | Meaning | -| ---------------------- | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_URL` | Full OTLP logs ingest URL; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_LOGS_TOKEN` | Project API key of the telemetry project; unset = telemetry off | -| Django settings | `SANDBOX_AGENT_OTEL_TRACES_URL` | Full OTLP traces ingest URL; unset = spans off, logs unaffected | -| Sandbox env (injected) | `POSTHOG_AGENT_OTEL_LOGS_URL` / `_TOKEN` / `POSTHOG_AGENT_OTEL_TRACES_URL` | Read by `agent-server` (`bin.ts`); reserved keys, not user-overridable | -| Django settings | `TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS` | Task origins whose run logs mirror to the internal project's Logs (default `signals_scout`; empty disables) | -| Django settings | `TASK_RUN_LOGS_MIRROR_OTLP_URL` / `_TOKEN` | Direct OTLP delivery for the mirror; the token pins records to the internal logs project so customer projects are never billed | From 82042b790594155d65c20d0737cda84964113ac4 Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:06:45 +0100 Subject: [PATCH 21/26] fix(tasks): reject client-supplied signals_scout task origin origin_product was API-settable to signals_scout (only image_builder was rejected), so any task:write caller could forge a scout task whose appended run logs would be mirrored into PostHog's internal Logs project once the mirror ships - flagged by the security review bots on the PR. Scout tasks are created exclusively server-side by the signals scout harness via the tasks facade, so treat signals_scout like image_builder: an internal-only origin the public serializer rejects. With creation server-attested, mirroring_enabled() can keep gating on origin_product. The mirror is new in this PR, so no forged tasks can predate the check where it matters. Test: parameterized serializer-level SimpleTestCase over the internal-only origins (verified to fail without the fix); the existing image_builder endpoint test remains the wiring guard. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- .../tasks/backend/presentation/serializers.py | 5 +++++ .../tests/test_presentation_serializers.py | 15 +++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/products/tasks/backend/presentation/serializers.py b/products/tasks/backend/presentation/serializers.py index e9b7e6be7a56..5a190e064945 100644 --- a/products/tasks/backend/presentation/serializers.py +++ b/products/tasks/backend/presentation/serializers.py @@ -600,6 +600,11 @@ def validate_origin_product(self, value): # would let them expose an arbitrary task to the whole team. The experiments # flow creates its tasks server-side through the facade, never through here. raise serializers.ValidationError("origin_product 'experiments' is reserved for the experiments flow") + if value == tasks_facade.TaskOriginProduct.SIGNALS_SCOUT: + # Scout tasks are created only by the signals scout harness. A forged scout origin + # would route the task's run logs into PostHog's internal Logs project + # (run_log_mirror) and inherit scout visibility semantics. + raise serializers.ValidationError("origin_product 'signals_scout' is reserved for signals scout runs") return value def validate_repository(self, value): diff --git a/products/tasks/backend/tests/test_presentation_serializers.py b/products/tasks/backend/tests/test_presentation_serializers.py index 8ba6c5f48cd8..3cc7d2446dc7 100644 --- a/products/tasks/backend/tests/test_presentation_serializers.py +++ b/products/tasks/backend/tests/test_presentation_serializers.py @@ -9,9 +9,24 @@ from products.tasks.backend.presentation.serializers import ( TaskRunCreateRequestSerializer, TaskRunLivingArtifactCreateRequestSerializer, + TaskWriteSerializer, ) +class TestTaskWriteSerializerOriginProduct(SimpleTestCase): + @parameterized.expand( + [ + ("image_builder", True), + ("signals_scout", True), + ("user_created", False), + ] + ) + def test_internal_only_origins_are_rejected(self, origin_product: str, expected_rejected: bool) -> None: + serializer = TaskWriteSerializer(data={"origin_product": origin_product}) + serializer.is_valid() + assert ("origin_product" in serializer.errors) is expected_rejected + + class TestTaskRunLivingArtifactCreateRequestSerializer(SimpleTestCase): @parameterized.expand( [ From 1642793e3fe573a6cac3241a4668f1434f7182fd Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:25:29 +0100 Subject: [PATCH 22/26] feat(tasks): gate agent run telemetry behind a rollout feature flag Telemetry was env-gated only (SANDBOX_AGENT_OTEL_* / TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS): all-or-nothing per region, with a deploy needed to change anything - no gradual rollout and no runtime kill switch. Add tasks-agent-run-otel-telemetry, evaluated org-targeted once at dispatch and stamped into run state, following the sandbox-event-ingest pattern (_capture_sandbox_event_ingest_flag is generalized to _capture_run_feature_flags, stamping both flags in one fetch + one atomic mutate). The stamp keeps the decision stable across retries and resumes. The flag gates both legs, AND'ed with the existing env settings: - sandbox OTel env injection - fresh provisions read the new TaskProcessingContext.agent_otel_telemetry_enabled field (state override -> DEBUG -> flag), snapshot resumes pass it into build_sandbox_environment_variables - the scout run-log mirror - reads the state stamp directly, so the append_log hot path never evaluates a flag remotely Fail-closed everywhere: missing stamp, missing flag, or evaluation error means telemetry stays off. DEBUG bypasses the flag (the analytics SDK is disabled locally), so local dev remains purely env-gated. Deploys are inert until the flag is created and ramped. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- docs/internal/sandboxes-setup-guide.md | 2 +- products/tasks/backend/constants.py | 4 + products/tasks/backend/feature_flags.py | 31 +++++++ products/tasks/backend/models.py | 6 ++ products/tasks/backend/temporal/client.py | 80 +++++++++++-------- .../create_sandbox_from_snapshot.py | 1 + .../activities/get_task_processing_context.py | 38 +++++++++ .../activities/provision_sandbox.py | 3 +- .../tests/test_provision_sandbox.py | 18 ++++- .../temporal/process_task/tests/test_utils.py | 20 ++++- .../backend/temporal/process_task/utils.py | 4 +- .../backend/tests/test_run_log_mirror.py | 34 +++++++- 12 files changed, 202 insertions(+), 39 deletions(-) diff --git a/docs/internal/sandboxes-setup-guide.md b/docs/internal/sandboxes-setup-guide.md index 01c8cf99f326..602c5c7b2a42 100644 --- a/docs/internal/sandboxes-setup-guide.md +++ b/docs/internal/sandboxes-setup-guide.md @@ -187,7 +187,7 @@ SANDBOX_AGENT_OTEL_LOGS_TOKEN= SANDBOX_AGENT_OTEL_TRACES_URL=http://localhost:8000/i/v1/traces # optional, enables APM spans ``` -They're injected into the sandbox as `POSTHOG_AGENT_OTEL_LOGS_URL`/`_TOKEN`/`POSTHOG_AGENT_OTEL_TRACES_URL` (deliberately not standard `OTEL_*` names, so OTel SDKs in user code don't auto-export into the telemetry project). +In cloud, emission is additionally gated per run by the `tasks-agent-run-otel-telemetry` feature flag (org-targeted, stamped into run state at dispatch; it also gates the scout run-log mirror). `DEBUG` bypasses the flag, so locally these settings are the only switch. They're injected into the sandbox as `POSTHOG_AGENT_OTEL_LOGS_URL`/`_TOKEN`/`POSTHOG_AGENT_OTEL_TRACES_URL` (deliberately not standard `OTEL_*` names, so OTel SDKs in user code don't auto-export into the telemetry project). The agent-server exports run/turn/tool lifecycle metadata (never message content or tool arguments), tagged with `run_id`/`task_id`/`team_id`/`user_id`/`distinct_id` resource attributes and `service.name=posthog-code-agent`. Telemetry stays off when either of the first two vars is unset. For local Docker sandboxes the localhost URLs are rewritten to `host.docker.internal` automatically; local ingestion requires the `capture-logs` service to be running. diff --git a/products/tasks/backend/constants.py b/products/tasks/backend/constants.py index bb6469737018..b329cc792fed 100644 --- a/products/tasks/backend/constants.py +++ b/products/tasks/backend/constants.py @@ -7,6 +7,10 @@ AGENT_PROXY_KEEP_STREAM_OPEN_FEATURE_FLAG = "tasks-agent-proxy-keep-stream-open" MODAL_VM_SANDBOX_FEATURE_FLAG = "tasks-modal-vm-sandbox" MODAL_NETWORK_ALLOWLIST_FEATURE_FLAG = "tasks-modal-network-allowlist" +AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG = "tasks-agent-run-otel-telemetry" +# Run-state key the telemetry flag decision is stamped under at dispatch (temporal/client.py). +# Consumers read the stamp, so the decision stays stable for the run's whole lifetime. +AGENT_OTEL_TELEMETRY_STATE_KEY = "agent_otel_telemetry_enabled" def vm_sandbox_allowed_origin_products(payload: object) -> set[str]: diff --git a/products/tasks/backend/feature_flags.py b/products/tasks/backend/feature_flags.py index dc6f8ba68abb..894d8f0a3c0f 100644 --- a/products/tasks/backend/feature_flags.py +++ b/products/tasks/backend/feature_flags.py @@ -4,6 +4,8 @@ import posthoganalytics +from products.tasks.backend.constants import AGENT_OTEL_TELEMETRY_STATE_KEY, AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG + logger = logging.getLogger(__name__) NATIVE_STEERING_SIGNALS_FEATURE_FLAG = "tasks-native-steering-signals" @@ -26,3 +28,32 @@ def is_native_steering_signals_enabled() -> bool: except Exception: logger.exception("native_steering_signals_feature_flag_check_failed") return False + + +def is_agent_otel_telemetry_enabled(*, distinct_id: str, organization_id: str) -> bool: + """Org-gated rollout of agent-run OTel telemetry; fail-closed when evaluation fails.""" + try: + return bool( + posthoganalytics.feature_enabled( + AGENT_RUN_OTEL_TELEMETRY_FEATURE_FLAG, + distinct_id=distinct_id, + groups={"organization": organization_id}, + group_properties={"organization": {"id": organization_id}}, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) + ) + except Exception: + logger.exception("agent_otel_telemetry_flag_check_failed") + return False + + +def agent_otel_telemetry_enabled_for_state(state: dict | None) -> bool: + """Per-run telemetry decision, read from the flag value stamped into run state at dispatch. + + DEBUG bypasses the flag: the analytics SDK is disabled in local dev, where the + telemetry env settings / mirror settings are themselves the opt-in. + """ + if settings.DEBUG: + return True + return (state or {}).get(AGENT_OTEL_TELEMETRY_STATE_KEY) is True diff --git a/products/tasks/backend/models.py b/products/tasks/backend/models.py index 358c3ad80a66..5b190c61f3dc 100644 --- a/products/tasks/backend/models.py +++ b/products/tasks/backend/models.py @@ -1699,11 +1699,17 @@ def _mirror_logs_to_posthog_logs(self, entries: list[dict]) -> None: Fire-and-forget: mirroring failures must never break the run's log write. """ + from products.tasks.backend.feature_flags import agent_otel_telemetry_enabled_for_state from products.tasks.backend.logic.services.run_log_mirror import mirror_entries, mirroring_enabled if not settings.TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS: return + # Per-run rollout decision (tasks-agent-run-otel-telemetry), stamped into run + # state at dispatch; fail closed while the stamp is absent. + if not agent_otel_telemetry_enabled_for_state(self.state if isinstance(self.state, dict) else None): + return + try: origin_product = self.task.origin_product if not mirroring_enabled(origin_product): diff --git a/products/tasks/backend/temporal/client.py b/products/tasks/backend/temporal/client.py index bc5c3bff73d5..ee3d44f8d5cb 100644 --- a/products/tasks/backend/temporal/client.py +++ b/products/tasks/backend/temporal/client.py @@ -15,9 +15,9 @@ from posthog.temporal.common.client import async_connect, sync_connect from posthog.temporal.oauth import PosthogMcpScopes -from products.tasks.backend.constants import SANDBOX_EVENT_INGEST_FEATURE_FLAG +from products.tasks.backend.constants import AGENT_OTEL_TELEMETRY_STATE_KEY, SANDBOX_EVENT_INGEST_FEATURE_FLAG from products.tasks.backend.error_telemetry import truncate_error_message -from products.tasks.backend.feature_flags import is_native_steering_signals_enabled +from products.tasks.backend.feature_flags import is_agent_otel_telemetry_enabled, is_native_steering_signals_enabled from products.tasks.backend.metrics import observe_task_run_workflow_start from products.tasks.backend.models import Task, TaskRun from products.tasks.backend.temporal.build_image.workflow import BuildSandboxImageInput @@ -113,15 +113,21 @@ def _get_task_run_for_metrics(run_id: str) -> TaskRun | None: return None -def _capture_sandbox_event_ingest_flag(run_id: str) -> None: +def _capture_run_feature_flags(run_id: str) -> None: + """Evaluate per-run rollout flags once at dispatch and stamp them into run state. + + Idempotent per key, so retries and resumes keep the decision the run started with. + """ try: task_run = TaskRun.objects.select_related("task__created_by", "task__team").get(id=run_id) except Exception: - logger.exception("sandbox_event_ingest_capture_run_missing", extra={"run_id": run_id}) + logger.exception("run_feature_flag_capture_run_missing", extra={"run_id": run_id}) return state = task_run.state or {} - if isinstance(state.get("sandbox_event_ingest_enabled"), bool): + need_event_ingest = not isinstance(state.get("sandbox_event_ingest_enabled"), bool) + need_otel_telemetry = not isinstance(state.get(AGENT_OTEL_TELEMETRY_STATE_KEY), bool) + if not need_event_ingest and not need_otel_telemetry: return task = task_run.task @@ -130,33 +136,43 @@ def _capture_sandbox_event_ingest_flag(run_id: str) -> None: task.created_by.distinct_id if task.created_by and task.created_by.distinct_id else "process_task_workflow" ) - try: - enabled = bool( - posthoganalytics.feature_enabled( - SANDBOX_EVENT_INGEST_FEATURE_FLAG, - distinct_id=distinct_id, - groups={"organization": organization_id}, - group_properties={"organization": {"id": organization_id}}, - only_evaluate_locally=False, - send_feature_flag_events=False, + event_ingest_enabled = False + if need_event_ingest: + try: + event_ingest_enabled = bool( + posthoganalytics.feature_enabled( + SANDBOX_EVENT_INGEST_FEATURE_FLAG, + distinct_id=distinct_id, + groups={"organization": organization_id}, + group_properties={"organization": {"id": organization_id}}, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) ) - ) - except Exception as e: - logger.warning( - "sandbox_event_ingest_capture_flag_failed", - extra={"run_id": run_id, "task_id": str(task.id), "error": str(e)}, - ) - enabled = False + except Exception as e: + logger.warning( + "sandbox_event_ingest_capture_flag_failed", + extra={"run_id": run_id, "task_id": str(task.id), "error": str(e)}, + ) + otel_telemetry_enabled = need_otel_telemetry and is_agent_otel_telemetry_enabled( + distinct_id=distinct_id, organization_id=organization_id + ) - def _set_sandbox_event_ingest_flag(latest_state: dict[str, Any]) -> None: - if not isinstance(latest_state.get("sandbox_event_ingest_enabled"), bool): - latest_state["sandbox_event_ingest_enabled"] = enabled + def _stamp_flags(latest_state: dict[str, Any]) -> None: + if need_event_ingest and not isinstance(latest_state.get("sandbox_event_ingest_enabled"), bool): + latest_state["sandbox_event_ingest_enabled"] = event_ingest_enabled + if need_otel_telemetry and not isinstance(latest_state.get(AGENT_OTEL_TELEMETRY_STATE_KEY), bool): + latest_state[AGENT_OTEL_TELEMETRY_STATE_KEY] = otel_telemetry_enabled - captured_state = TaskRun.mutate_state_atomic(task_run.id, _set_sandbox_event_ingest_flag) - captured_enabled = captured_state.get("sandbox_event_ingest_enabled", enabled) + captured_state = TaskRun.mutate_state_atomic(task_run.id, _stamp_flags) logger.info( - "sandbox_event_ingest_captured", - extra={"run_id": run_id, "task_id": str(task.id), "sandbox_event_ingest_enabled": captured_enabled}, + "run_feature_flags_captured", + extra={ + "run_id": run_id, + "task_id": str(task.id), + "sandbox_event_ingest_enabled": captured_state.get("sandbox_event_ingest_enabled"), + "agent_otel_telemetry_enabled": captured_state.get(AGENT_OTEL_TELEMETRY_STATE_KEY), + }, ) @@ -196,7 +212,7 @@ async def execute_task_processing_workflow_async( task_run_for_metrics = await _aget_task_run_for_metrics(run_id) observe_task_run_workflow_start(task_run_for_metrics, outcome="attempted", reason="requested") await Team.objects.select_related("organization").aget(id=team_id) - await sync_to_async(_capture_sandbox_event_ingest_flag)(run_id) + await sync_to_async(_capture_run_feature_flags)(run_id) workflow_id = TaskRun.get_workflow_id(task_id, run_id, workflow_id_prefix) if workflow_id_prefix: @@ -279,7 +295,7 @@ def execute_task_processing_workflow( ) Team.objects.get(id=team_id) - _capture_sandbox_event_ingest_flag(run_id) + _capture_run_feature_flags(run_id) workflow_id = TaskRun.get_workflow_id(task_id, run_id, workflow_id_prefix) if workflow_id_prefix: @@ -431,7 +447,7 @@ def redispatch_orphaned_task_run(run_id: str) -> str: ) observe_task_run_workflow_start(task_run, outcome="attempted", reason="reconcile") - _capture_sandbox_event_ingest_flag(run_id) + _capture_run_feature_flags(run_id) try: client = sync_connect() asyncio.run( @@ -461,7 +477,7 @@ def redispatch_orphaned_task_run(run_id: str) -> str: def resume_task_in_cloud_workflow(run_id: str, workflow_id: str) -> None: - _capture_sandbox_event_ingest_flag(run_id) + _capture_run_feature_flags(run_id) client = sync_connect() asyncio.run( client.start_workflow( diff --git a/products/tasks/backend/temporal/process_task/activities/create_sandbox_from_snapshot.py b/products/tasks/backend/temporal/process_task/activities/create_sandbox_from_snapshot.py index 7a37cc291c89..2ee3266a2681 100644 --- a/products/tasks/backend/temporal/process_task/activities/create_sandbox_from_snapshot.py +++ b/products/tasks/backend/temporal/process_task/activities/create_sandbox_from_snapshot.py @@ -126,6 +126,7 @@ def create_sandbox_from_snapshot(input: CreateSandboxFromSnapshotInput) -> Creat access_token=access_token, team_id=ctx.team_id, sandbox_environment=sandbox_env, + otel_telemetry_enabled=ctx.agent_otel_telemetry_enabled, ) environment_variables.update(get_git_identity_env_vars(task, ctx.state)) diff --git a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py index b38f4dd2111e..200663954da8 100644 --- a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py @@ -11,6 +11,7 @@ from posthog.temporal.common.utils import asyncify, close_db_connections from products.tasks.backend.constants import ( + AGENT_OTEL_TELEMETRY_STATE_KEY, AGENT_PROXY_KEEP_STREAM_OPEN_FEATURE_FLAG, CONTINUE_AS_NEW_FEATURE_FLAG, MODAL_DIRECTORY_RESUME_SNAPSHOTS_FEATURE_FLAG, @@ -23,6 +24,7 @@ vm_sandbox_default_base_origin_products, ) from products.tasks.backend.exceptions import TaskInvalidStateError, TaskRunNotReadyError +from products.tasks.backend.feature_flags import is_agent_otel_telemetry_enabled from products.tasks.backend.logic.services.sandbox_config import ( MAX_SANDBOX_CPU_CORES, MAX_SANDBOX_MEMORY_GB, @@ -82,6 +84,9 @@ class TaskProcessingContext: # Captured at workflow start so the sandbox event transport branch is # deterministic for the full run. sandbox_event_ingest_enabled: bool = False + # Captured at workflow start so telemetry env injection (and the run-log mirror, + # which reads the same state stamp) is deterministic for the full run. + agent_otel_telemetry_enabled: bool = False use_modal_vm_sandbox: bool = False use_modal_network_allowlist: bool = False # Burstable by default; the per-run state can opt out to pin a fixed-size box @@ -252,6 +257,7 @@ def _is_agent_proxy_keep_stream_open_enabled( try: enabled = bool( posthoganalytics.feature_enabled( + AGENT_OTEL_TELEMETRY_STATE_KEY, AGENT_PROXY_KEEP_STREAM_OPEN_FEATURE_FLAG, distinct_id=distinct_id, groups={"organization": organization_id}, @@ -362,6 +368,31 @@ def _is_sandbox_event_ingest_enabled( return enabled +def _is_agent_otel_telemetry_enabled( + *, + distinct_id: str, + organization_id: str, + run_id: str, + state: dict | None = None, +) -> bool: + state_override = (state or {}).get(AGENT_OTEL_TELEMETRY_STATE_KEY) + if isinstance(state_override, bool): + return state_override + + # Local dev disables the analytics SDK, so the flag below would always read False; + # the SANDBOX_AGENT_OTEL_* settings are the local opt-in and gate emission themselves. + if settings.DEBUG: + return True + + enabled = is_agent_otel_telemetry_enabled(distinct_id=distinct_id, organization_id=organization_id) + log_with_activity_context( + "agent_otel_telemetry_flag_checked", + run_id=run_id, + agent_otel_telemetry_enabled=enabled, + ) + return enabled + + def _is_modal_vm_sandbox_enabled( *, distinct_id: str, @@ -760,6 +791,12 @@ def get_task_processing_context(input: GetTaskProcessingContextInput) -> TaskPro "debug", f"sandbox_event_ingest_enabled: {sandbox_event_ingest_enabled} for this task run", ) + agent_otel_telemetry_enabled = _is_agent_otel_telemetry_enabled( + distinct_id=distinct_id, + organization_id=organization_id, + run_id=run_id, + state=state, + ) use_modal_vm_sandbox = _is_modal_vm_sandbox_enabled( distinct_id=distinct_id, organization_id=organization_id, @@ -886,6 +923,7 @@ def get_task_processing_context(input: GetTaskProcessingContextInput) -> TaskPro use_modal_resume_snapshots=settings.TASKS_USE_MODAL_RESUME_SNAPSHOTS or use_modal_directory_resume_snapshots, use_modal_directory_resume_snapshots=use_modal_directory_resume_snapshots, sandbox_event_ingest_enabled=sandbox_event_ingest_enabled, + agent_otel_telemetry_enabled=agent_otel_telemetry_enabled, use_modal_vm_sandbox=use_modal_vm_sandbox, use_modal_network_allowlist=use_modal_network_allowlist, burstable_sandbox_resources_enabled=burstable_sandbox_resources_enabled, diff --git a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py index 1cc2ee477533..2c4ec913182f 100644 --- a/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py +++ b/products/tasks/backend/temporal/process_task/activities/provision_sandbox.py @@ -345,7 +345,8 @@ def _build_environment_variables( # the cache key) and cost attribution. Rely on Temporal retries instead. environment_variables["POSTHOG_DISABLE_MODEL_FALLBACK"] = "1" - environment_variables.update(get_sandbox_otel_env_vars()) + if ctx.agent_otel_telemetry_enabled: + environment_variables.update(get_sandbox_otel_env_vars()) if ctx.allowed_domains is not None: environment_variables.update(NETWORK_RESTRICTED_AGENT_ENV) diff --git a/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py b/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py index 5125b5a21920..0806f9d08e1a 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py +++ b/products/tasks/backend/temporal/process_task/tests/test_provision_sandbox.py @@ -226,7 +226,7 @@ def test_build_environment_variables_disables_telemetry_when_restricted( def test_build_environment_variables_injects_otel_env_only_when_fully_configured( _api, _jwt, _git, url, token, traces_url, expected_keys ): - ctx = _context() + ctx = _context(agent_otel_telemetry_enabled=True) with override_settings( SANDBOX_AGENT_OTEL_LOGS_URL=url, @@ -236,3 +236,19 @@ def test_build_environment_variables_injects_otel_env_only_when_fully_configured env = _build_environment_variables(ctx, MagicMock(), "", "access-token") assert {key for key in env if key.startswith("POSTHOG_AGENT_OTEL_")} == expected_keys + + +@patch(f"{_PROVISION}.get_git_identity_env_vars", return_value={}) +@patch(f"{_PROVISION}.get_sandbox_jwt_public_key", return_value="pub") +@patch(f"{_PROVISION}.get_sandbox_api_url", return_value="https://api.example") +def test_build_environment_variables_omits_otel_env_when_flag_disabled(_api, _jwt, _git): + ctx = _context() + + with override_settings( + SANDBOX_AGENT_OTEL_LOGS_URL="https://us.i.posthog.com/i/v1/logs", + SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_telemetry", + SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", + ): + env = _build_environment_variables(ctx, MagicMock(), "", "access-token") + + assert not any(key.startswith("POSTHOG_AGENT_OTEL_") for key in env) diff --git a/products/tasks/backend/temporal/process_task/tests/test_utils.py b/products/tasks/backend/temporal/process_task/tests/test_utils.py index f7fc1a3e7c62..bfe800dbac81 100644 --- a/products/tasks/backend/temporal/process_task/tests/test_utils.py +++ b/products/tasks/backend/temporal/process_task/tests/test_utils.py @@ -1084,7 +1084,7 @@ def test_snapshot_resume_env_includes_otel_config_when_configured(self, _api, _j SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_telemetry", SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", ): - env = build_sandbox_environment_variables(None, "access-token", 1) + env = build_sandbox_environment_variables(None, "access-token", 1, otel_telemetry_enabled=True) assert env["POSTHOG_AGENT_OTEL_LOGS_URL"] == "https://us.i.posthog.com/i/v1/logs" assert env["POSTHOG_AGENT_OTEL_LOGS_TOKEN"] == "phc_telemetry" @@ -1103,6 +1103,24 @@ def test_snapshot_resume_env_omits_otel_config_without_logs_pair(self, _api, _jw SANDBOX_AGENT_OTEL_LOGS_URL="https://us.i.posthog.com/i/v1/logs", SANDBOX_AGENT_OTEL_LOGS_TOKEN=None, SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", + ): + env = build_sandbox_environment_variables(None, "access-token", 1, otel_telemetry_enabled=True) + + assert not any(key.startswith("POSTHOG_AGENT_OTEL_") for key in env) + + @patch( + "products.tasks.backend.logic.services.connection_token.get_sandbox_jwt_public_key", + return_value="pub", + ) + @patch( + "products.tasks.backend.temporal.process_task.utils.get_sandbox_api_url", + return_value="https://api.example", + ) + def test_snapshot_resume_env_omits_otel_config_when_flag_disabled(self, _api, _jwt) -> None: + with override_settings( + SANDBOX_AGENT_OTEL_LOGS_URL="https://us.i.posthog.com/i/v1/logs", + SANDBOX_AGENT_OTEL_LOGS_TOKEN="phc_telemetry", + SANDBOX_AGENT_OTEL_TRACES_URL="https://us.i.posthog.com/i/v1/traces", ): env = build_sandbox_environment_variables(None, "access-token", 1) diff --git a/products/tasks/backend/temporal/process_task/utils.py b/products/tasks/backend/temporal/process_task/utils.py index 788d8db0318d..e7ebb32083d5 100644 --- a/products/tasks/backend/temporal/process_task/utils.py +++ b/products/tasks/backend/temporal/process_task/utils.py @@ -1118,6 +1118,7 @@ def build_sandbox_environment_variables( access_token: str, team_id: int, sandbox_environment: Optional[Any] = None, + otel_telemetry_enabled: bool = False, ) -> dict[str, str]: """Build the environment variables dict for a sandbox, merging user env vars from SandboxEnvironment. @@ -1148,7 +1149,8 @@ def build_sandbox_environment_variables( if settings.SANDBOX_LLM_GATEWAY_URL: env_vars["LLM_GATEWAY_URL"] = settings.SANDBOX_LLM_GATEWAY_URL - env_vars.update(get_sandbox_otel_env_vars()) + if otel_telemetry_enabled: + env_vars.update(get_sandbox_otel_env_vars()) return env_vars diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index e7e6cd212faa..84c55f301b04 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -8,6 +8,7 @@ from posthog.models import Organization, Team +from products.tasks.backend.feature_flags import agent_otel_telemetry_enabled_for_state from products.tasks.backend.logic.services.run_log_mirror import ( MAX_BODY_CHARS, MAX_ENTRIES_PER_CALL, @@ -243,6 +244,23 @@ def test_no_http_delivery_unless_fully_configured(self, _name, url, token): mock_requests.post.assert_not_called() +class TestAgentOtelTelemetryStateGate(SimpleTestCase): + @parameterized.expand( + [ + ("stamped_true", {"agent_otel_telemetry_enabled": True}, True), + ("stamped_false", {"agent_otel_telemetry_enabled": False}, False), + ("missing", {}, False), + ("none_state", None, False), + ] + ) + def test_fails_closed_outside_debug(self, _name, state, expected): + assert agent_otel_telemetry_enabled_for_state(state) is expected + + @override_settings(DEBUG=True) + def test_debug_bypasses_the_flag(self): + assert agent_otel_telemetry_enabled_for_state(None) is True + + @override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=["signals_scout"]) class TestAppendLogMirroring(TestCase): @classmethod @@ -250,14 +268,15 @@ def setUpTestData(cls): cls.organization = Organization.objects.create(name="Test Org") cls.team = Team.objects.create(organization=cls.organization, name="Test Team") - def _create_run(self, origin_product: str) -> TaskRun: + def _create_run(self, origin_product: str, telemetry_enabled: bool | None = True) -> TaskRun: task = Task.objects.create( team=self.team, title="Test Task", description="Test", origin_product=origin_product, ) - return TaskRun.objects.create(team=self.team, task=task) + state = {"agent_otel_telemetry_enabled": telemetry_enabled} if telemetry_enabled is not None else {} + return TaskRun.objects.create(team=self.team, task=task, state=state) @parameterized.expand( [ @@ -284,6 +303,17 @@ def test_mirrors_only_allowlisted_origin_products(self, origin_product, expect_m else: mock_logger.info.assert_not_called() + @patch("products.tasks.backend.logic.services.run_log_mirror.logger") + @patch("products.tasks.backend.models.object_storage") + def test_no_mirroring_without_telemetry_flag_stamp(self, mock_storage, mock_logger): + mock_storage.read.return_value = None + run = self._create_run(Task.OriginProduct.SIGNALS_SCOUT, telemetry_enabled=None) + + run.append_log([_session_update_entry("agent_message", content={"type": "text", "text": "hi"})]) + + mock_storage.write.assert_called_once() + mock_logger.info.assert_not_called() + @override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=[]) @patch("products.tasks.backend.logic.services.run_log_mirror.logger") @patch("products.tasks.backend.models.object_storage") From ed1836954d50db63b260cabcd6f977fc8d5177ac Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:31:16 +0100 Subject: [PATCH 23/26] fix(tasks): satisfy mypy on the telemetry gating changes Two Python code quality (mypy) failures on the branch: - run_log_mirror.mirror_entries typed entries as list[dict], making the defensive non-dict guard "unreachable" under warn_unreachable. The guard is load-bearing (entries arrive from a semi-trusted request payload), so type the parameter list[Any] to match reality. - the flag-gating commit accidentally injected AGENT_OTEL_TELEMETRY_STATE_KEY as the first positional argument of the keep-stream-open feature_enabled call (an over-broad substring replacement), which mypy flagged as a duplicate distinct_id. Restore the call. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- products/tasks/backend/logic/services/run_log_mirror.py | 4 +++- .../process_task/activities/get_task_processing_context.py | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 17a9ae02d8f6..02443c4f039f 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -78,7 +78,9 @@ def mirroring_enabled(origin_product: str) -> bool: def mirror_entries( - entries: list[dict], + # list[Any], not list[dict]: entries arrive from a semi-trusted request payload, + # so the non-dict guard below is load-bearing. + entries: list[Any], *, team_id: int, task_id: str, diff --git a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py index 200663954da8..ab1dfcd12c34 100644 --- a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py @@ -257,7 +257,6 @@ def _is_agent_proxy_keep_stream_open_enabled( try: enabled = bool( posthoganalytics.feature_enabled( - AGENT_OTEL_TELEMETRY_STATE_KEY, AGENT_PROXY_KEEP_STREAM_OPEN_FEATURE_FLAG, distinct_id=distinct_id, groups={"organization": organization_id}, From ed105b9be5147827ae47e0fcc2037a669a4e92de Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:39:13 +0100 Subject: [PATCH 24/26] fix(tasks): protect rollout flag stamps from run-state PATCHes Review bots flagged that agent_otel_telemetry_enabled was not in _PROTECTED_RUN_STATE_KEYS, so a same-team task:write caller could PATCH a run's state to force the stamp true and bypass the org rollout flag - injecting the internal OTLP capture token into their sandbox on resume and re-enabling the run-log mirror with the rollout off. Add the shared AGENT_OTEL_TELEMETRY_STATE_KEY constant to the protected set, and sandbox_event_ingest_enabled alongside it - the sibling rollout stamp had the same gap (a caller could flip the event transport branch mid-run). Dispatch (_capture_run_feature_flags) remains the sole writer. Extends the existing protected-state endpoint test: both stamps in the attack payload and assertions, and the telemetry stamp in the state_remove_keys leg. Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- products/tasks/backend/facade/api.py | 7 +++++++ products/tasks/backend/tests/test_api.py | 8 ++++++++ 2 files changed, 15 insertions(+) diff --git a/products/tasks/backend/facade/api.py b/products/tasks/backend/facade/api.py index a427210a1cb2..4cf8974890c3 100644 --- a/products/tasks/backend/facade/api.py +++ b/products/tasks/backend/facade/api.py @@ -37,6 +37,7 @@ from posthog.models.integration import Integration from products.tasks.backend.constants import ( + AGENT_OTEL_TELEMETRY_STATE_KEY, MAX_CUSTOM_IMAGES_PER_TEAM, MAX_CUSTOM_IMAGES_PER_USER, RESERVED_SANDBOX_ENVIRONMENT_VARIABLE_KEYS, @@ -1687,6 +1688,12 @@ def _sync_automation_schedule(automation: TaskAutomation) -> None: "wizard_head_branch", "use_modal_directory_resume_snapshots", "use_modal_vm_sandbox", + # Rollout stamps written once at dispatch by _capture_run_feature_flags; a PATCHable + # value would let a task controller bypass the org feature flags (for telemetry, that + # means injecting the internal OTLP capture token into their sandbox and re-enabling + # the run-log mirror with the rollout off). + AGENT_OTEL_TELEMETRY_STATE_KEY, + "sandbox_event_ingest_enabled", "snapshot_external_id", "snapshot_kind", "snapshot_mount_path", diff --git a/products/tasks/backend/tests/test_api.py b/products/tasks/backend/tests/test_api.py index 26f05f5b4888..2bb29f7e3bb4 100644 --- a/products/tasks/backend/tests/test_api.py +++ b/products/tasks/backend/tests/test_api.py @@ -4317,6 +4317,8 @@ def test_patch_cannot_mutate_protected_credential_state_keys(self, _mock_publish "inactivity_timeout_seconds": 600, "use_modal_directory_resume_snapshots": True, "use_modal_vm_sandbox": False, + "agent_otel_telemetry_enabled": False, + "sandbox_event_ingest_enabled": False, "snapshot_external_id": "im-real", "snapshot_kind": "directory", "snapshot_mount_path": "/tmp", @@ -4348,6 +4350,8 @@ def test_patch_cannot_mutate_protected_credential_state_keys(self, _mock_publish "wizard_config": {}, "use_modal_directory_resume_snapshots": False, "use_modal_vm_sandbox": True, + "agent_otel_telemetry_enabled": True, + "sandbox_event_ingest_enabled": True, "snapshot_external_id": "im-attacker", "snapshot_kind": "directory", "snapshot_mount_path": "/tmp/workspace", @@ -4378,6 +4382,8 @@ def test_patch_cannot_mutate_protected_credential_state_keys(self, _mock_publish assert "wizard_config" not in run.state # caller cannot mark a run as a wizard run assert run.state["use_modal_directory_resume_snapshots"] is True assert run.state["use_modal_vm_sandbox"] is False + assert run.state["agent_otel_telemetry_enabled"] is False + assert run.state["sandbox_event_ingest_enabled"] is False assert run.state["snapshot_external_id"] == "im-real" assert run.state["snapshot_kind"] == "directory" assert run.state["snapshot_mount_path"] == "/tmp" @@ -4394,6 +4400,7 @@ def test_patch_cannot_mutate_protected_credential_state_keys(self, _mock_publish "state": {}, "state_remove_keys": [ "github_credential_source", + "agent_otel_telemetry_enabled", "sandbox_id", "use_modal_directory_resume_snapshots", "use_modal_vm_sandbox", @@ -4412,6 +4419,7 @@ def test_patch_cannot_mutate_protected_credential_state_keys(self, _mock_publish self.assertEqual(response.status_code, status.HTTP_200_OK) run.refresh_from_db() assert run.state["github_credential_source"] == "caller_token" # protected key survives removal + assert run.state["agent_otel_telemetry_enabled"] is False # protected key survives removal assert run.state["sandbox_id"] == "sb-real" # protected key survives removal assert run.state["use_modal_directory_resume_snapshots"] is True # protected key survives removal assert run.state["use_modal_vm_sandbox"] is False # protected key survives removal From 664beaf17ebd6801b754c96b9e5fb5554ae2bfbf Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 17:51:42 +0100 Subject: [PATCH 25/26] feat(tasks): add prometheus counters for telemetry rollout observability Three counters so the cloud-background-agents-runs Grafana dashboard can verify the telemetry rollout: - posthog_tasks_agent_otel_telemetry_stamped_total{enabled} - first-time rollout stamps at dispatch; the enabled=true share tracks the tasks-agent-run-otel-telemetry flag ramp on new runs - posthog_tasks_run_log_mirror_entries_total{origin_product} - entries the run-log mirror actually emitted - posthog_tasks_run_log_mirror_otlp_batches_total{outcome} - direct-OTLP mirror delivery outcomes (local-dev leg; stays flat in cloud) Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- .../backend/logic/services/run_log_mirror.py | 6 ++++++ products/tasks/backend/metrics.py | 19 +++++++++++++++++++ products/tasks/backend/temporal/client.py | 6 +++++- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/products/tasks/backend/logic/services/run_log_mirror.py b/products/tasks/backend/logic/services/run_log_mirror.py index 02443c4f039f..167d8adbe78b 100644 --- a/products/tasks/backend/logic/services/run_log_mirror.py +++ b/products/tasks/backend/logic/services/run_log_mirror.py @@ -30,6 +30,8 @@ from posthog.security.outbound_proxy import internal_requests +from products.tasks.backend.metrics import RUN_LOG_MIRROR_ENTRIES_TOTAL, RUN_LOG_MIRROR_OTLP_BATCHES_TOTAL + logger = structlog.get_logger(__name__) # The collector truncates whole log lines at 100 KB (`max_log_size`); cap the body well @@ -126,6 +128,8 @@ def mirror_entries( getattr(logger, _LOG_METHOD_NAMES[severity])(_LOG_EVENT_NAME, **fields) records.append((severity, fields)) + if records: + RUN_LOG_MIRROR_ENTRIES_TOTAL.labels(origin_product=origin_product).inc(len(records)) _post_otlp(records, run_id=run_id) @@ -182,7 +186,9 @@ def _post_otlp(records: list[tuple[str, dict[str, Any]]], *, run_id: str) -> Non timeout=_OTLP_TIMEOUT, ) response.raise_for_status() + RUN_LOG_MIRROR_OTLP_BATCHES_TOTAL.labels(outcome="sent").inc() except Exception as e: + RUN_LOG_MIRROR_OTLP_BATCHES_TOTAL.labels(outcome="failed").inc() logger.warning("task_run_log_mirror_otlp_failed", task_run_id=run_id, error=str(e)) diff --git a/products/tasks/backend/metrics.py b/products/tasks/backend/metrics.py index 3f6415c8701f..39a82319a3fe 100644 --- a/products/tasks/backend/metrics.py +++ b/products/tasks/backend/metrics.py @@ -59,6 +59,25 @@ ], ) +AGENT_OTEL_TELEMETRY_STAMPED_TOTAL = Counter( + "posthog_tasks_agent_otel_telemetry_stamped_total", + "Agent-run OTel telemetry rollout decisions stamped into run state at dispatch " + "(tasks-agent-run-otel-telemetry flag). First-time stamps only; resumes reuse the stamp.", + labelnames=["enabled"], +) + +RUN_LOG_MIRROR_ENTRIES_TOTAL = Counter( + "posthog_tasks_run_log_mirror_entries_total", + "Task-run log entries mirrored to stdout for the internal Logs project (run_log_mirror).", + labelnames=["origin_product"], +) + +RUN_LOG_MIRROR_OTLP_BATCHES_TOTAL = Counter( + "posthog_tasks_run_log_mirror_otlp_batches_total", + "Direct-OTLP mirror batch deliveries by outcome (the local-dev leg; unset in cloud).", + labelnames=["outcome"], +) + PREWARMED_ACTIVATED_TOTAL = Counter( "posthog_tasks_prewarmed_activated_total", "Pre-warmed Runs that received their first user message (the warm sandbox got used, not reaped)", diff --git a/products/tasks/backend/temporal/client.py b/products/tasks/backend/temporal/client.py index ee3d44f8d5cb..4f5a80c7971d 100644 --- a/products/tasks/backend/temporal/client.py +++ b/products/tasks/backend/temporal/client.py @@ -18,7 +18,7 @@ from products.tasks.backend.constants import AGENT_OTEL_TELEMETRY_STATE_KEY, SANDBOX_EVENT_INGEST_FEATURE_FLAG from products.tasks.backend.error_telemetry import truncate_error_message from products.tasks.backend.feature_flags import is_agent_otel_telemetry_enabled, is_native_steering_signals_enabled -from products.tasks.backend.metrics import observe_task_run_workflow_start +from products.tasks.backend.metrics import AGENT_OTEL_TELEMETRY_STAMPED_TOTAL, observe_task_run_workflow_start from products.tasks.backend.models import Task, TaskRun from products.tasks.backend.temporal.build_image.workflow import BuildSandboxImageInput from products.tasks.backend.temporal.constants import ( @@ -165,6 +165,10 @@ def _stamp_flags(latest_state: dict[str, Any]) -> None: latest_state[AGENT_OTEL_TELEMETRY_STATE_KEY] = otel_telemetry_enabled captured_state = TaskRun.mutate_state_atomic(task_run.id, _stamp_flags) + if need_otel_telemetry: + AGENT_OTEL_TELEMETRY_STAMPED_TOTAL.labels( + enabled=str(bool(captured_state.get(AGENT_OTEL_TELEMETRY_STATE_KEY))).lower() + ).inc() logger.info( "run_feature_flags_captured", extra={ From 541d3e72fc32b2eca168000bd949ad4bd48e0a7b Mon Sep 17 00:00:00 2001 From: Alessandro Pogliaghi Date: Thu, 23 Jul 2026 18:09:33 +0100 Subject: [PATCH 26/26] fix(tasks): debug-first telemetry gate, dual-flag capture test, mypy annotations Three fixes for the review comment + red CI on the branch: - _is_agent_otel_telemetry_enabled checked the state stamp before DEBUG, but local dev always stamps False (the analytics SDK is disabled there), so the DEBUG bypass was dead and local sandboxes would never get the OTel env even with SANDBOX_AGENT_OTEL_* set - flagged by graphite-app, and inconsistent with the mirror's agent_otel_telemetry_enabled_for_state. DEBUG now wins first at both gates; pinned by a parametrized precedence test. - test_captures_sandbox_event_ingest_flag_before_starting_workflow patched the shared posthoganalytics module attribute and asserted a single feature_enabled call; the dispatch capture now evaluates two flags, so the test asserts both stamps and both calls. - TestAppendLogMirroring needed class-level organization/team annotations for the setUpTestData attributes (mypy attr-defined, the "Python code quality" failure). Generated-By: PostHog Code Task-Id: 0c511836-2180-455a-9b58-45df2a0661ec --- .../activities/get_task_processing_context.py | 12 ++++++---- .../tests/test_get_task_processing_context.py | 21 +++++++++++++++++ .../backend/tests/test_run_log_mirror.py | 3 +++ .../backend/tests/test_temporal_client.py | 23 +++++++++++-------- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py index ab1dfcd12c34..bde97d9b82c5 100644 --- a/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/get_task_processing_context.py @@ -374,15 +374,17 @@ def _is_agent_otel_telemetry_enabled( run_id: str, state: dict | None = None, ) -> bool: + # DEBUG first: local dev disables the analytics SDK, so the dispatch capture always + # stamps False there - honoring the stamp would disable telemetry locally even with + # the SANDBOX_AGENT_OTEL_* settings (the local opt-in, which gate emission themselves) + # set. Matches agent_otel_telemetry_enabled_for_state, which the mirror reads. + if settings.DEBUG: + return True + state_override = (state or {}).get(AGENT_OTEL_TELEMETRY_STATE_KEY) if isinstance(state_override, bool): return state_override - # Local dev disables the analytics SDK, so the flag below would always read False; - # the SANDBOX_AGENT_OTEL_* settings are the local opt-in and gate emission themselves. - if settings.DEBUG: - return True - enabled = is_agent_otel_telemetry_enabled(distinct_id=distinct_id, organization_id=organization_id) log_with_activity_context( "agent_otel_telemetry_flag_checked", diff --git a/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py b/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py index 6424664bb0ee..cab7cff82544 100644 --- a/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py +++ b/products/tasks/backend/temporal/process_task/activities/tests/test_get_task_processing_context.py @@ -24,6 +24,7 @@ from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import ( GetTaskProcessingContextInput, TaskProcessingContext, + _is_agent_otel_telemetry_enabled, _is_agent_proxy_keep_stream_open_enabled, _is_burstable_sandbox_resources_enabled, _is_continue_as_new_enabled, @@ -38,6 +39,26 @@ @pytest.mark.requires_secrets +class TestIsAgentOtelTelemetryEnabled: + @pytest.mark.parametrize( + "debug,state,expected", + [ + # DEBUG must win over the stamp: local dev always stamps False (SDK disabled), + # and the SANDBOX_AGENT_OTEL_* settings are the local opt-in. + (True, {"agent_otel_telemetry_enabled": False}, True), + (True, {}, True), + (False, {"agent_otel_telemetry_enabled": True}, True), + (False, {"agent_otel_telemetry_enabled": False}, False), + ], + ) + def test_debug_wins_then_stamp(self, debug, state, expected): + with override_settings(DEBUG=debug): + assert ( + _is_agent_otel_telemetry_enabled(distinct_id="d", organization_id="o", run_id="r", state=state) + is expected + ) + + class TestGetTaskProcessingContextActivity: def _create_task_with_repo(self, team, user, github_integration, repo_config): return Task.objects.create( diff --git a/products/tasks/backend/tests/test_run_log_mirror.py b/products/tasks/backend/tests/test_run_log_mirror.py index 84c55f301b04..7efdde6dc8e4 100644 --- a/products/tasks/backend/tests/test_run_log_mirror.py +++ b/products/tasks/backend/tests/test_run_log_mirror.py @@ -263,6 +263,9 @@ def test_debug_bypasses_the_flag(self): @override_settings(TASK_RUN_LOGS_MIRROR_ORIGIN_PRODUCTS=["signals_scout"]) class TestAppendLogMirroring(TestCase): + organization: Organization + team: Team + @classmethod def setUpTestData(cls): cls.organization = Organization.objects.create(name="Test Org") diff --git a/products/tasks/backend/tests/test_temporal_client.py b/products/tasks/backend/tests/test_temporal_client.py index f57c38d75e93..3f3805d3afec 100644 --- a/products/tasks/backend/tests/test_temporal_client.py +++ b/products/tasks/backend/tests/test_temporal_client.py @@ -209,7 +209,7 @@ def test_does_not_overwrite_run_that_already_started(self, executor: str) -> Non self.assertIsNone(run.completed_at) @parameterized.expand([("sync",), ("async",)]) - def test_captures_sandbox_event_ingest_flag_before_starting_workflow(self, executor: str) -> None: + def test_captures_run_feature_flags_before_starting_workflow(self, executor: str) -> None: run = self._create_run() client = Mock() client.start_workflow = AsyncMock() @@ -229,14 +229,19 @@ def test_captures_sandbox_event_ingest_flag_before_starting_workflow(self, execu run.refresh_from_db() self.assertEqual(run.state["sandbox_event_ingest_enabled"], True) - flag.assert_called_once_with( - "tasks-cloud-runs-sandbox-event-ingest", - distinct_id="process_task_workflow", - groups={"organization": str(self.organization.id)}, - group_properties={"organization": {"id": str(self.organization.id)}}, - only_evaluate_locally=False, - send_feature_flag_events=False, - ) + self.assertEqual(run.state["agent_otel_telemetry_enabled"], True) + # Patching the shared posthoganalytics module attribute covers both evaluation + # sites (event ingest in client.py, telemetry in feature_flags.py). + self.assertEqual(flag.call_count, 2) + for flag_key in ("tasks-cloud-runs-sandbox-event-ingest", "tasks-agent-run-otel-telemetry"): + flag.assert_any_call( + flag_key, + distinct_id="process_task_workflow", + groups={"organization": str(self.organization.id)}, + group_properties={"organization": {"id": str(self.organization.id)}}, + only_evaluate_locally=False, + send_feature_flag_events=False, + ) def test_captures_sandbox_event_ingest_flag_before_resuming_workflow(self) -> None: run = self._create_run()