From 2f141ccee9d57e2c35de37ef355d34e0005d1b02 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:32:02 -0700 Subject: [PATCH 1/8] renderer: shared/config and backendConnection are import-safe without a window, so reducer tests run under node:test Both modules touched window at import time (port/host derivation, the fetch interceptor install, the debug handle), so any node:test file that imports a reducer importing API_BASE died with 'window is not defined' before its first assertion; fetchSessionsStrip.test.ts has been red that way since the resilience work landed, unnoticed because nothing runs these tests in CI. In a renderer (window present) nothing changes: same port/host, same interceptor, same handle. Without one the module answers with the defaults and installs nothing. --- frontend/src/shared/backendConnection.ts | 5 ++++- frontend/src/shared/config.ts | 13 +++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/frontend/src/shared/backendConnection.ts b/frontend/src/shared/backendConnection.ts index e93237e10..c410bfc04 100644 --- a/frontend/src/shared/backendConnection.ts +++ b/frontend/src/shared/backendConnection.ts @@ -80,4 +80,7 @@ export function noteRequestStalled(): void { } // Harness/debug handle: lets a live session (CDP, support) read the signal without a store import. -(window as unknown as { __OSW_CONN?: object }).__OSW_CONN = { backendReachable, onBackendReachability }; +// Guarded: reducers that import this module also run under node:test, where there is no window. +if (typeof window !== 'undefined') { + (window as unknown as { __OSW_CONN?: object }).__OSW_CONN = { backendReachable, onBackendReachability }; +} diff --git a/frontend/src/shared/config.ts b/frontend/src/shared/config.ts index c0aec765f..f92bbe614 100644 --- a/frontend/src/shared/config.ts +++ b/frontend/src/shared/config.ts @@ -1,6 +1,9 @@ import { noteBackendFailure, noteBackendSuccess, noteRequestStalled, setBackendProber } from '@/shared/backendConnection'; -const _w = window as any; +// Import-safe outside a renderer: reducers that import API_BASE also run under node:test, where there +// is no window. The module then answers with the defaults and installs nothing. +const hasWindow = typeof window !== 'undefined'; +const _w = (hasWindow ? window : {}) as any; // Prefer the preload-injected port; if it's missing (preload raced the backend port being picked), re-query the live value before falling back to 8324. The bare 8324 guess is wrong on any machine where the backend landed on a fallback port (e.g. 8324 was held by a leftover backend); see the self-heal below. const port = _w.__OPENSWARM_PORT__ || @@ -8,7 +11,7 @@ const port = ? _w.openswarm.getBackendPortLive() : 0) || 8324; -const host = window.location.hostname || 'localhost'; +const host = (hasWindow && window.location.hostname) || 'localhost'; export const API_BASE = `http://${host}:${port}/api`; export const WS_BASE = `ws://${host}:${port}`; @@ -217,5 +220,7 @@ function _installAuthFetchInterceptor() { }; } -_installAuthFetchInterceptor(); -ensureAuthToken(); +if (hasWindow) { + _installAuthFetchInterceptor(); + ensureAuthToken(); +} From 71475b66cd157cc33dee5c3801616cf17e182442 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 17:32:02 -0700 Subject: [PATCH 2/8] ci: run the backend, frontend and edge test suites on every pull request Nothing ran any of them in CI: the 235-file backend pytest suite, the 22 renderer node:test files and the edge suite were run by hand, one file at a time, so a regression only surfaced when someone happened to run the right one. Three small workflows, hosted ubuntu, path-filtered, read-only token: - backend-tests: pytest on Python 3.13 from the locked requirements, plus a completion assertion (junit testcase count == collect-only count) so a test process that dies mid-run can never read as green - frontend-tests: tsc --noEmit + node:test via tsx over src/**/*.test.ts(x), through frontend/scripts/run-tests.mjs (the runner the tests already name) - edge-tests: pytest for openswarm-edge All three are green on the current tree: 2951 backend tests, 143 frontend tests across 22 files, 14 edge tests. --- .github/workflows/backend-tests.yml | 65 ++++++++++++++++++++++++++++ .github/workflows/edge-tests.yml | 44 +++++++++++++++++++ .github/workflows/frontend-tests.yml | 43 ++++++++++++++++++ frontend/scripts/run-tests.mjs | 29 +++++++++++++ 4 files changed, 181 insertions(+) create mode 100644 .github/workflows/backend-tests.yml create mode 100644 .github/workflows/edge-tests.yml create mode 100644 .github/workflows/frontend-tests.yml create mode 100644 frontend/scripts/run-tests.mjs diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 000000000..adb758a23 --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,65 @@ +name: backend-tests + +# The backend pytest suite, run on every pull request and on pushes to the mainline branches. Until +# now nothing ran it in CI, so a regression only surfaced when someone ran it by hand. +on: + pull_request: + paths: + - 'backend/**' + - '.github/workflows/backend-tests.yml' + push: + branches: [main, dev] + paths: + - 'backend/**' + - '.github/workflows/backend-tests.yml' + workflow_dispatch: + +# Runs checked-out project code on pull_request: the token stays read-only. +permissions: + contents: read + +concurrency: + group: backend-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + pytest: + name: pytest (ubuntu) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: | + backend/requirements.lock + backend/requirements-dev.txt + - name: Install backend deps (locked runtime + dev) + run: | + python -m pip install --require-hashes --only-binary=:all: -r backend/requirements.lock + python -m pip install -r backend/requirements-dev.txt + # Two numbers that must agree. A test process that dies mid-run can exit 0 with no summary + # (a hard-exit shutdown path did exactly that once and silently skipped ~42% of the suite), so + # the job also asserts that every collected test was actually run: junit testcase count == + # collect-only count. Green then means green, not "green as far as it got". + - name: Count the suite + run: | + python -m pytest backend/tests --ignore=backend/tests/formal --collect-only -q -p no:cacheprovider \ + | tail -1 | tee collected.txt + - name: Run the backend suite + run: | + python -m pytest backend/tests --ignore=backend/tests/formal -q -p no:cacheprovider \ + --junitxml "${RUNNER_TEMP}/pytest.xml" + - name: Every collected test ran + run: | + python - "${RUNNER_TEMP}/pytest.xml" collected.txt <<'PY' + import re, sys, xml.etree.ElementTree as ET + ran = sum(1 for _ in ET.parse(sys.argv[1]).getroot().iter('testcase')) + m = re.search(r'(\d+) tests? collected', open(sys.argv[2]).read()) + collected = int(m.group(1)) if m else -1 + print(f'collected={collected} ran={ran}') + if collected < 1 or ran != collected: + sys.exit(f'FAIL: {ran} of {collected} collected tests reached the report; the run was truncated') + PY diff --git a/.github/workflows/edge-tests.yml b/.github/workflows/edge-tests.yml new file mode 100644 index 000000000..82ef10d9d --- /dev/null +++ b/.github/workflows/edge-tests.yml @@ -0,0 +1,44 @@ +name: edge-tests + +# The openswarm-edge pytest suite, on every pull request that touches it and on pushes to the +# mainline branches. +on: + pull_request: + paths: + - 'openswarm-edge/**' + - '.github/workflows/edge-tests.yml' + push: + branches: [main, dev] + paths: + - 'openswarm-edge/**' + - '.github/workflows/edge-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: edge-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + pytest: + name: pytest (openswarm-edge) + runs-on: ubuntu-latest + timeout-minutes: 10 + defaults: + run: + working-directory: openswarm-edge + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: pip + cache-dependency-path: openswarm-edge/requirements.txt + - name: Install edge deps + run: | + python -m pip install -r requirements.txt + python -m pip install pytest pytest-asyncio + - name: Run the edge suite + run: python -m pytest tests -q -p no:cacheprovider diff --git a/.github/workflows/frontend-tests.yml b/.github/workflows/frontend-tests.yml new file mode 100644 index 000000000..00578bb8f --- /dev/null +++ b/.github/workflows/frontend-tests.yml @@ -0,0 +1,43 @@ +name: frontend-tests + +# Typecheck plus the renderer's node:test suite, on every pull request and on pushes to the mainline +# branches. Until now neither ran in CI; the tests were run by hand, one file at a time. +on: + pull_request: + paths: + - 'frontend/**' + - '.github/workflows/frontend-tests.yml' + push: + branches: [main, dev] + paths: + - 'frontend/**' + - '.github/workflows/frontend-tests.yml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: frontend-tests-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + typecheck-and-tests: + name: tsc + node:test + runs-on: ubuntu-latest + timeout-minutes: 15 + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20.18.1' + cache: npm + cache-dependency-path: frontend/package-lock.json + - run: npm ci + - name: Typecheck + run: npx tsc --noEmit -p tsconfig.json + - name: Unit tests (node:test via tsx) + run: node scripts/run-tests.mjs diff --git a/frontend/scripts/run-tests.mjs b/frontend/scripts/run-tests.mjs new file mode 100644 index 000000000..8fad6fe2d --- /dev/null +++ b/frontend/scripts/run-tests.mjs @@ -0,0 +1,29 @@ +#!/usr/bin/env node +// Runs every renderer unit test (src/**/*.test.ts, *.test.tsx) under node:test, with tsx doing the +// TypeScript. One command for CI and for a dev machine: `node scripts/run-tests.mjs`, optionally +// followed by file paths to run a subset. Exits non-zero if any test fails or nothing was found. +import { spawnSync } from 'node:child_process'; +import { readdirSync, statSync } from 'node:fs'; +import { join, dirname } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = join(dirname(fileURLToPath(import.meta.url)), '..'); +const testFile = (p) => /\.test\.tsx?$/.test(p); + +function walk(dir, out) { + for (const name of readdirSync(dir)) { + if (name === 'node_modules' || name === 'dist') continue; + const p = join(dir, name); + if (statSync(p).isDirectory()) walk(p, out); + else if (testFile(p)) out.push(p); + } + return out; +} + +const files = process.argv.length > 2 ? process.argv.slice(2) : walk(join(root, 'src'), []).sort(); +if (files.length === 0) { + console.error('run-tests: no test files found under src/'); + process.exit(1); +} +const result = spawnSync(process.execPath, ['--import', 'tsx', '--test', ...files], { cwd: root, stdio: 'inherit' }); +process.exit(result.status ?? 1); From 941dfb73be0b97225654b868bd755e601ec17c51 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:07:12 -0700 Subject: [PATCH 3/8] ci: cap each backend test at 300s so a stall fails by name Two of five hosted runs of the backend suite stalled at 99% until the job cap with no summary and no junit: one test blocked forever on a bare ws.receive_json() (fixed on its own in a separate change). A CI lane should never depend on every test being unable to hang, so add pytest-timeout to the dev requirements and run the suite with --timeout=300. On Linux the default signal method fails just the offending test and the run continues, so the report and the "every collected test ran" assertion stay meaningful. --- .github/workflows/backend-tests.yml | 6 ++++++ backend/requirements-dev.txt | 3 +++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml index adb758a23..39972626f 100644 --- a/.github/workflows/backend-tests.yml +++ b/.github/workflows/backend-tests.yml @@ -48,11 +48,17 @@ jobs: run: | python -m pytest backend/tests --ignore=backend/tests/formal --collect-only -q -p no:cacheprovider \ | tail -1 | tee collected.txt + # --timeout: a test that blocks forever (a bare ws.receive_json() waiting for an event that never + # comes, say) otherwise stalls the run at 99% until timeout-minutes with no summary and no junit. + # With the cap it fails by name, the rest of the suite runs, and the assertion below still holds. - name: Run the backend suite run: | python -m pytest backend/tests --ignore=backend/tests/formal -q -p no:cacheprovider \ + --timeout=300 \ --junitxml "${RUNNER_TEMP}/pytest.xml" - name: Every collected test ran + # Runs after a red suite too, so a failure report also says whether the run was complete. + if: ${{ !cancelled() }} run: | python - "${RUNNER_TEMP}/pytest.xml" collected.txt <<'PY' import re, sys, xml.etree.ElementTree as ET diff --git a/backend/requirements-dev.txt b/backend/requirements-dev.txt index fe1535065..710b74dc3 100644 --- a/backend/requirements-dev.txt +++ b/backend/requirements-dev.txt @@ -9,6 +9,9 @@ pytest==8.3.4 pytest-asyncio==0.25.2 +# Per-test wall-clock cap for CI (see .github/workflows/backend-tests.yml): a test that +# blocks forever fails by name instead of stalling the whole run until the job cap. +pytest-timeout==2.4.0 # Used by linter/lint.py (the vulture dead-code check). watchfiles, also # needed by lint.py, already comes in transitively via uvicorn[standard]. From 5ced7185c0cf4bc644d17b486b82d004d08fc7a3 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 19:05:42 -0700 Subject: [PATCH 4/8] tests: the WS end-to-end turn test cannot hang the suite test_ws_endpoint_streams_a_full_turn_end_to_end read the socket with a bare ws.receive_json() in a 40-iteration loop and broke only on the assistant reply. When the loop ends early for any reason (fewer than 40 events, no reply), the next receive blocks forever and the whole pytest run stalls at 99% until the job cap. On hosted runners it does exactly that intermittently, on Linux and Windows alike: the turn path's configure_provider_env decides whether 9Router needs reviving from provider evidence earlier tests may leave behind, and that revival spawns/installs the router behind a module-level asyncio.Lock; the background turn-label aux call reaches the same machinery. Neither is part of this test's contract ("SDK and WS auth mocked, everything else real"). Pin both out with monkeypatch, bound every receive at 5s (a regression now fails this test instead of hanging the runner), and wait for the turn's completed status before asserting on session.messages so the assertion cannot race the loop's tail. --- backend/tests/test_ws_integration.py | 48 ++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/backend/tests/test_ws_integration.py b/backend/tests/test_ws_integration.py index 741cc223f..33f86210e 100644 --- a/backend/tests/test_ws_integration.py +++ b/backend/tests/test_ws_integration.py @@ -6,6 +6,8 @@ The SDK and WS auth are mocked; everything else is the real running stack.""" import asyncio +import queue +import threading import pytest @@ -17,6 +19,29 @@ import backend.main as main_mod from backend.apps.agents.agent_manager import agent_manager from backend.apps.agents.core.models import AgentSession +import backend.apps.agents.manager.run.RunOptions as run_options_mod + + +def p_receive_json(ws, timeout: float = 5.0): + """ws.receive_json() blocks forever when the event never comes; a regression in the loop then + hangs the whole pytest run instead of failing this test. Bound every receive.""" + out = queue.Queue(maxsize=1) + + def p_recv(): + try: + out.put((True, ws.receive_json())) + except BaseException as exc: + out.put((False, exc)) + + thread = threading.Thread(target=p_recv, daemon=True) + thread.start() + try: + ok, value = out.get(timeout=timeout) + except queue.Empty as exc: + raise AssertionError(f"timed out waiting for websocket event after {timeout}s") from exc + if ok: + return value + raise value def p_assistant(): @@ -33,6 +58,20 @@ def p_result(): def test_ws_endpoint_streams_a_full_turn_end_to_end(monkeypatch): monkeypatch.setattr(main_mod, "p_ws_auth_ok", lambda ws: True, raising=True) + # The contract of this test is "SDK and WS auth mocked, everything else real", but two things on + # the turn path reach outside the process and must not decide the outcome: configure_provider_env + # can wander into 9Router revival (spawn/npm install, serialized on a module-level lock) whenever + # earlier tests left provider evidence behind, and the background turn-label aux call does the + # same. Pin both out; the persistent-client path is pinned off suite-wide in conftest. + async def p_noop_provider_env(*args, **kwargs): + return None + + async def p_noop_turn_label(*args, **kwargs): + return None + + monkeypatch.setattr(run_options_mod, "configure_provider_env", p_noop_provider_env, raising=True) + monkeypatch.setattr(agent_manager, "generate_turn_label", p_noop_turn_label, raising=True) + async def fake_query(*args, **kwargs): yield p_assistant() yield p_result() @@ -49,10 +88,15 @@ async def fake_query(*args, **kwargs): ws.send_json({"event": "agent:send_message", "data": {"prompt": "hi"}}) seen = [] for _ in range(40): - ev = ws.receive_json() + ev = p_receive_json(ws) seen.append(ev.get("event")) - if ev.get("event") == "agent:message" and "hello from the loop" in str(ev.get("data", {})): + if ( + ev.get("event") == "agent:status" + and ev.get("data", {}).get("status") == "completed" + ): break + else: + raise AssertionError(f"did not receive completed status; saw events={seen}") # the real loop's assistant reply made it all the way back over the WS assert "agent:message" in seen assert any(m.role == "assistant" and "hello from the loop" in str(m.content) From ec41b84aa265f0a2fecdbc8c943a7649c0e13446 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 20:34:54 -0700 Subject: [PATCH 5/8] renderer: shared/safeMode is import-safe without a window Same class as the config/backendConnection change: safeMode.ts read `window` at import, and dashboardLayoutSlice imports it, so any reducer test that imports the slice died under node:test before it ran. Guard the read; in a renderer nothing changes. --- frontend/src/shared/safeMode.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/frontend/src/shared/safeMode.ts b/frontend/src/shared/safeMode.ts index c8c3f6e93..432fb7691 100644 --- a/frontend/src/shared/safeMode.ts +++ b/frontend/src/shared/safeMode.ts @@ -11,7 +11,10 @@ export interface SafeModeInfo { let cached: SafeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null }; -const api = (window as unknown as { openswarm?: { getSafeMode?: () => Promise } }).openswarm; +// Import-safe outside a renderer: the layout slice imports this, and its reducer tests run under node:test. +const api = typeof window === 'undefined' + ? undefined + : (window as unknown as { openswarm?: { getSafeMode?: () => Promise } }).openswarm; if (api?.getSafeMode) { void api.getSafeMode().then((info) => { if (info && typeof info.safeMode === 'boolean') cached = info; From a80ce2138de669b83d39cc62885f4722b6255bb3 Mon Sep 17 00:00:00 2001 From: Kai <300677314+kai-openswarm@users.noreply.github.com> Date: Mon, 17 Aug 2026 23:12:58 -0700 Subject: [PATCH 6/8] backend: agents/service/workflows hardening, plus the hosting seam the routes now ask MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The agent manager, its routes, and the service/workflows/settings apps, with the hardening a hosted deployment forced us to build but that stands on its own: - Containment (R0): a session's browser commands, tool results and approvals are bound to their session and owner (ws_manager: send/resolve carry owner and claimant, connections carry an optional identity with expiry and revocation); the sessions list, history and prompt prediction never read across owners. - Redaction (F1): settings leave the process redacted (a {configured, last4} marker instead of the secret) on the analytics sync and the /settings routes' consumers; the redaction helper is shared. - Launch stamps the session at birth (see also #145) and the launch route runs the first turn only after admission says so; concurrent root turns are capped (TurnAdmission, OSW_MAX_CONCURRENT_TURNS) so firing thirty agents does not spawn thirty CLIs at once. - SSRF guard: v4-mapped and transition encodings are judged as network targets; loopback is allowed on the desktop (previews) and only a hosting policy can turn it into a blocked target. - Service: the shutdown fuse disarms last on a clean shutdown; the runtime port and settings gateway are injected boundaries; the analytics client redacts. - Workflows: the scheduler claim gate is off-means-off; a deterministic at-least-once scheduler contract with its reference model; lifecycle events. - Settings store: the in-memory mirror is keyed by path+stat, and the atomic writer stages its temp file beside the settings file. - Agent manager: session store, event sink (a Null sink by default) and provider runtime are injected, which is what the delivered tests build on. The hosting seam (backend/apps/hosting/policy.py): routes take `scope: RequestScope = REQUEST_SCOPE` and managers ask `hosting_policy()`; both answer with the desktop defaults here (nobody owns anything, everything is allowed, no per-owner workspace roots, no tool denials, loopback allowed). A build that hosts supplies its own provider; nothing else in the app knows which build it is. REQUEST_SCOPE is a FastAPI dependency that is also the desktop scope, so routes called directly (as several tests do) keep working. Not in this change: the outputs app (separate change), dashboards (mixed with a card-family feature; separate change), anything hosted or multi-tenant. Proof: this tree with Python 3.13 from requirements.lock — 3074 passed / 15 skipped (upstream's suite plus the delivered tests); upstream's linter reports no new findings beyond pre-existing debt (two grandfather entries added, both noted in linter/config/config.json). --- backend/apps/agents/agent_manager.py | 216 +++++++---- backend/apps/agents/agents.py | 184 ++++++--- backend/apps/agents/browser/browser_agent.py | 8 +- .../apps/agents/browser/browser_metrics.py | 5 + backend/apps/agents/core/models.py | 2 + backend/apps/agents/core/ws_manager.py | 349 ++++++++++++++++-- .../apps/agents/disconnect_subscription.py | 4 +- backend/apps/agents/events/AgentEvent.py | 89 +++++ backend/apps/agents/events/AgentEventSink.py | 96 +++++ .../agents/events/AgentTurnEventEmitter.py | 151 ++++++++ backend/apps/agents/events/__init__.py | 0 backend/apps/agents/manager/AgentLaunch.py | 28 +- .../agents/manager/AgentManagerProtocol.py | 4 + backend/apps/agents/manager/Messaging.py | 4 +- .../agents/manager/configure_provider_env.py | 45 +-- .../permissions/build_effective_tool_lists.py | 30 +- .../agents/manager/permissions/gate_hooks.py | 4 + .../prompt/compose_turn_system_prompt.py | 7 +- .../apps/agents/manager/provider_runtime.py | 47 +++ backend/apps/agents/manager/run/RunOptions.py | 8 +- .../apps/agents/manager/run/TurnAdmission.py | 46 +++ backend/apps/agents/manager/run/TurnRunner.py | 13 +- .../apps/agents/manager/run/client_pool.py | 9 +- .../agents/manager/run_browser_fast_path.py | 16 + .../manager/session/SessionLifecycle.py | 79 +++- .../agents/manager/session/SessionStore.py | 101 +++++ .../agents/manager/session/session_store.py | 10 + .../agents/manager/streaming/HookContext.py | 4 +- .../streaming/handle_assistant_message.py | 6 +- .../manager/streaming/handle_stream_event.py | 6 +- .../manager/streaming/post_tool_hook.py | 23 ++ .../apps/agents/manager/streaming/state.py | 2 + .../apps/agents/manager/streaming/thinking.py | 1 - backend/apps/agents/providers/registry.py | 2 + backend/apps/agents/tools/ssrf_guard.py | 56 ++- backend/apps/health/health.py | 25 +- backend/apps/hosting/__init__.py | 2 + backend/apps/hosting/policy.py | 186 ++++++++++ backend/apps/nine_router/process.py | 6 +- backend/apps/nine_router/sync_custom.py | 10 +- backend/apps/service/analytics/client.py | 28 +- backend/apps/service/client.py | 81 ++-- backend/apps/service/service.py | 49 +-- backend/apps/service/service_runtime.py | 60 +++ backend/apps/service/settings_gateway.py | 35 ++ backend/apps/settings/redaction.py | 10 + backend/apps/settings/settings.py | 7 +- backend/apps/settings/store.py | 27 +- backend/apps/skills/skills.py | 11 +- backend/apps/subscription/free_trial.py | 37 +- backend/apps/subscription/router.py | 12 +- backend/apps/swarm/entities/skills.py | 2 +- backend/apps/tools_lib/tools_lib.py | 6 + backend/apps/web/web.py | 6 +- .../workflows/durable_scheduler_contract.py | 272 ++++++++++++++ .../apps/workflows/durable_scheduler_types.py | 76 ++++ backend/apps/workflows/executor.py | 14 + backend/apps/workflows/lifecycle_events.py | 43 +++ backend/apps/workflows/scheduler.py | 4 + backend/apps/workflows/workflows.py | 145 ++++---- backend/config/entity_references.py | 2 + backend/tests/conftest.py | 4 +- backend/tests/test_agent_events.py | 165 +++++++++ backend/tests/test_app_agent.py | 6 +- backend/tests/test_browser_agent_loop.py | 37 +- backend/tests/test_browser_command_timeout.py | 23 +- backend/tests/test_browser_dispatch.py | 9 + backend/tests/test_browser_login_handoff.py | 7 +- backend/tests/test_browser_metrics.py | 5 +- ...configure_provider_env_characterization.py | 178 +++++++++ backend/tests/test_credential_store.py | 1 + .../tests/test_durable_scheduler_contract.py | 298 +++++++++++++++ backend/tests/test_free_trial.py | 17 + .../test_launch_snapshots_session_at_birth.py | 69 ++++ backend/tests/test_router_watchdog.py | 30 +- backend/tests/test_scheduler_claim_gate.py | 165 +++++++++ backend/tests/test_service.py | 127 +++++++ .../tests/test_service_router_autostart.py | 19 + .../test_service_runtime_characterization.py | 250 +++++++++++++ backend/tests/test_service_runtime_port.py | 234 ++++++++++++ .../test_service_settings_characterization.py | 284 ++++++++++++++ .../tests/test_service_settings_gateway.py | 248 +++++++++++++ backend/tests/test_session_store.py | 100 +++++ backend/tests/test_settings_meta_endpoint.py | 56 +++ .../tests/test_settings_meta_stdio_live.py | 2 +- .../tests/test_settings_select_and_send.py | 19 +- backend/tests/test_shutdown_fuse_disarms.py | 7 + .../tests/test_skill_replay_vs_send_script.py | 2 +- backend/tests/test_streaming_harness.py | 64 ++++ backend/tests/test_turn_admission.py | 7 +- .../tests/test_workflows_lifecycle_events.py | 225 +++++++++++ linter/config/config.json | 14 +- 92 files changed, 5019 insertions(+), 464 deletions(-) create mode 100644 backend/apps/agents/events/AgentEvent.py create mode 100644 backend/apps/agents/events/AgentEventSink.py create mode 100644 backend/apps/agents/events/AgentTurnEventEmitter.py create mode 100644 backend/apps/agents/events/__init__.py create mode 100644 backend/apps/agents/manager/provider_runtime.py create mode 100644 backend/apps/agents/manager/run/TurnAdmission.py create mode 100644 backend/apps/agents/manager/session/SessionStore.py create mode 100644 backend/apps/hosting/__init__.py create mode 100644 backend/apps/hosting/policy.py create mode 100644 backend/apps/service/service_runtime.py create mode 100644 backend/apps/service/settings_gateway.py create mode 100644 backend/apps/workflows/durable_scheduler_contract.py create mode 100644 backend/apps/workflows/durable_scheduler_types.py create mode 100644 backend/apps/workflows/lifecycle_events.py create mode 100644 backend/tests/test_agent_events.py create mode 100644 backend/tests/test_configure_provider_env_characterization.py create mode 100644 backend/tests/test_durable_scheduler_contract.py create mode 100644 backend/tests/test_launch_snapshots_session_at_birth.py create mode 100644 backend/tests/test_scheduler_claim_gate.py create mode 100644 backend/tests/test_service_router_autostart.py create mode 100644 backend/tests/test_service_runtime_characterization.py create mode 100644 backend/tests/test_service_runtime_port.py create mode 100644 backend/tests/test_service_settings_characterization.py create mode 100644 backend/tests/test_service_settings_gateway.py create mode 100644 backend/tests/test_session_store.py create mode 100644 backend/tests/test_workflows_lifecycle_events.py diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index cd6f5e916..fcf6840e8 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -2,13 +2,10 @@ import logging import time import os -from contextlib import asynccontextmanager -from typing import AsyncIterator, Dict, List, Optional +from typing import Any, Dict, List, Optional from typeguard import typechecked -from backend.apps.agents.core.models import ( - AgentSession, Message, -) +from backend.apps.agents.core.models import Message from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.settings.settings import load_settings from backend.apps.tools_lib.tools_lib import load_builtin_permissions @@ -18,8 +15,8 @@ save_session, load_session_data as load_session_data, ) +from backend.apps.agents.manager.session.SessionStore import SessionStore from backend.apps.agents.manager.streaming.state import ThinkingState, TurnState -from backend.apps.agents.manager.streaming.PartialReply import PartialReply from backend.apps.agents.manager.session.SessionLifecycle import SessionLifecycle from backend.apps.agents.manager.SpawnAgentRun import SpawnAgentRun from backend.apps.agents.manager.session.SessionPersistence import SessionPersistence @@ -30,76 +27,131 @@ from backend.apps.agents.manager.RunSupport import RunSupport from backend.apps.agents.manager.run.handle_run_error import handle_run_error from backend.apps.agents.manager.run.TurnRunner import TurnRunner -from backend.apps.agents.manager.run.client_pool import ClientHandle -from backend.apps.agents.manager.streaming.HookContext import HookContext +from backend.apps.agents.manager.run.TurnAdmission import TurnAdmission from backend.apps.agents.manager.run.RunOptions import RunOptions +from backend.apps.agents.events.AgentEventSink import ( + AgentEventSink, + BoundedAgentEventSink, + NullAgentEventSink, +) +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter logger = logging.getLogger(__name__) os.environ.setdefault("CLAUDE_CODE_STREAM_CLOSE_TIMEOUT", "3600000") -# Cap concurrent ROOT agent turns so firing 30 agents at once doesn't spawn 30 CLIs in the same instant; the overflow queues (agents are model/IO-bound, so they're waiting anyway). Env-tunable, 0/blank disables the gate. -MAX_CONCURRENT_TURNS = int(os.environ.get("OSW_MAX_CONCURRENT_TURNS", "8") or "0") - - -class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionControl, AgentLaunch, SpawnAgentRun, MockAgent, TurnRunner, RunOptions, RunSupport): +class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionControl, AgentLaunch, SpawnAgentRun, MockAgent, TurnAdmission, TurnRunner, RunOptions, RunSupport): @typechecked - def __init__(self): - self.sessions: Dict[str, AgentSession] = {} + def __init__(self, store: Optional[SessionStore] = None, event_sink: Optional[AgentEventSink] = None): + self.store = store or SessionStore() + self.event_sink = event_sink or NullAgentEventSink() + # Messages queued behind an in-flight turn (upstream admission/queue); purged per session in SessionLifecycle.purge_session_memory. + self.pending_messages: Dict[str, List[QueuedMessage]] = {} from backend.apps.agents.core.flight_recorder import set_sessions_provider set_sessions_provider(lambda: self.sessions) - self.tasks: Dict[str, asyncio.Task] = {} - # Live mirror of the in-flight streamed assistant text per session, so a stop can persist the partial reply instantly instead of waiting out the multi-second SDK teardown the cancel handler sits behind. - self.live_partial: Dict[str, PartialReply] = {} - # Per-session cancel signal: the loop stashes its asyncio.Event here so a stop/close can set it. Lives on the manager, not the AgentSession model, so it stays out of serialization (an Event can't be model_dump'd). - self.cancel_events: Dict[str, asyncio.Event] = {} - # Persistent-client pool (lever A, flag-gated): one live CLI per session, reused across turns. - self.client_pool: Dict[str, ClientHandle] = {} - # Per-SESSION hook context + stderr buffer, updated in place each turn: a persistent client's hooks/stderr callback were bound at connect, so they must read stable objects, not per-turn rebuilds. - self.hook_ctxs: Dict[str, HookContext] = {} - self.stderr_buffers: Dict[str, List[str]] = {} - # Messages typed while a turn was live, replayed in order when it ends (see Messaging). - self.pending_messages: Dict[str, List[QueuedMessage]] = {} # Admission gate: one shared semaphore caps concurrent ROOT turns (children bypass). (Re)created per running loop by get_turn_admission so it never binds to a dead loop across a uvicorn reload or a test's asyncio.run. self.p_turn_admission_sema: Optional[asyncio.Semaphore] = None self.p_turn_admission_loop: Optional[asyncio.AbstractEventLoop] = None + @property + @typechecked + def sessions(self) -> Dict[str, Any]: + return self.store.sessions + + @sessions.setter + @typechecked + def sessions(self, value: Dict[str, Any]) -> None: + object.__setattr__(self.store, "sessions", value) + + @property + @typechecked + def tasks(self) -> Dict[str, Any]: + return self.store.tasks + + @tasks.setter + @typechecked + def tasks(self, value: Dict[str, Any]) -> None: + object.__setattr__(self.store, "tasks", value) + + @property + @typechecked + def live_partial(self) -> Dict[str, Any]: + return self.store.live_partial + + @live_partial.setter + @typechecked + def live_partial(self, value: Dict[str, Any]) -> None: + object.__setattr__(self.store, "live_partial", value) + + @property + @typechecked + def cancel_events(self) -> Dict[str, asyncio.Event]: + return self.store.cancel_events + + @cancel_events.setter + @typechecked + def cancel_events(self, value: Dict[str, asyncio.Event]) -> None: + object.__setattr__(self.store, "cancel_events", value) + + @property + @typechecked + def client_pool(self) -> Dict[str, Any]: + return self.store.client_pool + + @client_pool.setter + @typechecked + def client_pool(self, value: Dict[str, Any]) -> None: + object.__setattr__(self.store, "client_pool", value) + + @property + @typechecked + def hook_ctxs(self) -> Dict[str, Any]: + return self.store.hook_ctxs + + @hook_ctxs.setter + @typechecked + def hook_ctxs(self, value: Dict[str, Any]) -> None: + object.__setattr__(self.store, "hook_ctxs", value) + + @property + @typechecked + def stderr_buffers(self) -> Dict[str, List[str]]: + return self.store.stderr_buffers + + @stderr_buffers.setter + @typechecked + def stderr_buffers(self, value: Dict[str, List[str]]) -> None: + object.__setattr__(self.store, "stderr_buffers", value) @typechecked - def get_turn_admission(self) -> asyncio.Semaphore: - """The shared admission semaphore for the CURRENT loop; rebuilt if the loop changed so a - reload/test-run can never await a semaphore bound to a dead loop.""" - loop = asyncio.get_running_loop() - if self.p_turn_admission_sema is None or self.p_turn_admission_loop is not loop: - self.p_turn_admission_sema = asyncio.Semaphore(MAX_CONCURRENT_TURNS) - self.p_turn_admission_loop = loop - return self.p_turn_admission_sema - - @asynccontextmanager - async def turn_admission_slot(self, session: AgentSession, session_id: str) -> AsyncIterator[None]: - """Hold one concurrency slot for the duration of a ROOT turn. Overflow turns queue on the - semaphore (emitting agent:queued, then agent:admitted when they start). Two bypasses, both - load-bearing: (1) MAX_CONCURRENT_TURNS<=0 disables the gate entirely (kill switch); (2) a - CHILD turn (parent_session_id set) is NEVER gated, because a parent holds its own slot while - awaiting a delegated child, so gating children would deadlock the pool. `async with` release - is cancellation-safe: a stop while queued never acquired, so it can't over-release.""" - if MAX_CONCURRENT_TURNS <= 0 or session.parent_session_id is not None: - yield + async def ensure_keyed_model_route_synced(self, settings, short_name: str) -> None: + """Ensure 9Router has the provider node required by a pinned API-key model.""" + from backend.apps.agents.providers.registry import find_builtin_model + + entry = find_builtin_model(short_name) or {} + if entry.get("route") != "api": return - sema = self.get_turn_admission() - was_queued = sema.locked() - if was_queued: - try: - await ws_manager.send_to_session(session_id, "agent:queued", {"session_id": session_id}) - except Exception: - pass - async with sema: - if was_queued: - try: - await ws_manager.send_to_session(session_id, "agent:admitted", {"session_id": session_id}) - except Exception: - pass - yield + + provider = entry.get("api") + if provider == "openai" and getattr(settings, "openai_api_key", None): + from backend.apps import nine_router + if not nine_router.is_running(): + await nine_router.ensure_running() + if nine_router.is_running(): + await nine_router.sync_openai_api_key(settings.openai_api_key) + elif provider == "gemini" and getattr(settings, "google_api_key", None): + from backend.apps import nine_router + if not nine_router.is_running(): + await nine_router.ensure_running() + if nine_router.is_running(): + await nine_router.sync_gemini_api_key(settings.google_api_key) + elif provider == "custom" and getattr(settings, "custom_providers", None): + from backend.apps import nine_router + if not nine_router.is_running(): + await nine_router.ensure_running() + if nine_router.is_running(): + await nine_router.sync_custom_providers(settings.custom_providers or []) + @typechecked async def prewarm_client(self, session_id: str) -> None: @@ -156,6 +208,8 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li if not session: return + self.ensure_session_workspace_ready(session) + from backend.apps.agents.providers.registry import get_api_type as p_get_api_type p_api = p_get_api_type(session.model) prompt_content = self.build_prompt_content( @@ -184,10 +238,17 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li builtin_perms = load_builtin_permissions() # Builtins default to always_allow (frictionless); path_gate still force-prompts on catastrophic patterns (rm -rf), OS-scheduling, and sensitive paths, so poisoned-email -> destructive-command is still caught. Flip Bash to "ask" in the UI for a prompt on every command. Bind turn + stderr first: build_agent_options can raise early (no provider) and the except hands both to handle_run_error. - turn = TurnState() p_stderr_buffer: List[str] = [] # Read BEFORE build_agent_options consumes these flags: a fresh-session/fork request must force the persistent client to respawn (same branch id would otherwise fingerprint-match a client still holding the old transcript). p_force_respawn = bool(session.needs_fresh_session or session.needs_fork or fork_session) + p_event_emitter = AgentTurnEventEmitter( + sink=self.event_sink, + session_id=session_id, + provider=p_api_type_for_session, + model=p_router_model_id, + ) + turn = TurnState(event_emitter=p_event_emitter) + p_event_emitter.emit_started() try: logger.info(f"[SPAWN-PHASE] run-loop start session={session_id[:8]} t={time.monotonic():.3f}") (options, options_kwargs, prompt_content, p_stderr_buffer, @@ -195,20 +256,25 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li session, session_id, prompt, prompt_content, builtin_perms, selected_browser_ids, selected_app_output_ids, selected_setting_ids, fork_session, p_router_model_id, p_api_type_for_session) + p_hook_ctx = self.hook_ctxs.get(session_id) + if p_hook_ctx is not None: + p_hook_ctx.event_emitter = p_event_emitter resolved_model = p_router_model_id api_type = p_api_type_for_session thinking = ThinkingState() # Gate the CLI turn (spawn + stream) behind the admission slot so a burst can't run every turn at once; the slot is held ONLY for run_turn_with_retry, so the context-valve retry below re-acquires cleanly instead of nesting. - logger.info(f"[SPAWN-PHASE] admission-wait session={session_id[:8]} t={time.monotonic():.3f}") async with self.turn_admission_slot(session, session_id): - logger.info(f"[SPAWN-PHASE] admitted session={session_id[:8]} t={time.monotonic():.3f}") await self.run_turn_with_retry( session, session_id, prompt_content, options, options_kwargs, turn, thinking, p_stderr_buffer, resolved_model, api_type, global_settings, force_respawn=p_force_respawn, ) session.status = "completed" + p_event_emitter.emit_completed( + input_tokens=int(session.tokens.get("input_fresh", 0) or 0), + output_tokens=int(session.tokens.get("output", 0) or 0), + ) # Silent-quit seal: a turn that ran tools and ended with no visible answer gets ONE hidden continue nudge (dispatched by the auto-continuation block below); a second silent quit in the same ask surfaces as-is rather than looping. try: @@ -232,6 +298,7 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li except Exception: logger.exception("auto-continuation dispatch failed") except asyncio.CancelledError: + p_event_emitter.emit_failed("cancelled") # Only act if we're still the session's live task. A user stop pops this task (stop_agent already finalized status + partial), and a follow-up message may have started a newer turn; either way this dying task must NOT clobber the live status or pop the new turn's in-flight partial mirror. if self.tasks.get(session_id) is asyncio.current_task(): session.status = "stopped" @@ -287,6 +354,7 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li }) except Exception: logger.debug("submit_diagnostic context_pressure_valve failed", exc_info=True) + p_event_emitter.emit_failed("context_pressure_retry", retryable=True) await self.run_agent_loop( session_id, prompt, images, context_paths, forced_tools, attached_skills, fork_session, selected_browser_ids, @@ -294,23 +362,25 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li context_valve_retry=True, ) return + p_event_emitter.emit_failed(type(e).__name__) await handle_run_error(e, session, session_id, turn, p_stderr_buffer) except BaseException as e: # Catch BaseExceptionGroup from anyio task groups (e.g. concurrent CLI crash + pending approval cancellation) so it doesn't escape and kill the uvicorn process. logger.exception(f"Agent {session_id} fatal error: {e}") - # A group's str() names the group, not the cause; unwrap to the real member so a wrapped 429/auth error still gets its friendly card + retry-pill semantics instead of a raw group dump. + p_event_emitter.emit_failed(type(e).__name__) + # A group's str() names the group, not the cause; unwrap to the real member so a wrapped 429/auth error still gets its friendly card + retry-pill semantics. from backend.apps.agents.core.first_real_exception import first_real_exception p_real = first_real_exception(e) if p_real is not None: await handle_run_error(p_real, session, session_id, turn, p_stderr_buffer) - else: - session.status = "error" - error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) - session.messages.append(error_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": error_msg.model_dump(mode="json"), - }) + return + session.status = "error" + error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) + session.messages.append(error_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": error_msg.model_dump(mode="json"), + }) finally: # Only the session's live task finalizes. A stopped task (popped by stop_agent, which already finalized status + saved) or one superseded by a newer turn must not pop the new turn's partial mirror, broadcast a stale terminal status, or overwrite the snapshot the live turn is writing. p_is_live_task = self.tasks.get(session_id) is asyncio.current_task() @@ -345,4 +415,4 @@ async def run_agent_loop(self, session_id: str, prompt: str, images: Optional[Li logger.warning(f"Failed to snapshot session {session_id}: {e}") -agent_manager = AgentManager() +agent_manager = AgentManager(event_sink=BoundedAgentEventSink()) diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 03717b6c0..621f8e113 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -1,10 +1,20 @@ +from backend.config.Apps import SubApp +from backend.apps.agents.agent_manager import agent_manager +from backend.apps.agents.core.ws_manager import ws_manager +from backend.apps.agents.core.models import AgentConfig, ApprovalResponse +from backend.apps.agents.manager.session.history_compaction import estimate_post_compact_input +from contextlib import asynccontextmanager +from fastapi import WebSocket, WebSocketDisconnect, HTTPException, Request +from fastapi.responses import JSONResponse import asyncio import logging import time -from contextlib import asynccontextmanager + +from backend.apps.hosting.policy import REQUEST_SCOPE, RequestScope +from backend.apps.nine_router.subscription_health import probe_subscription_health +from backend.apps.settings.redaction import redact_settings from typing import Any, Dict -from fastapi import HTTPException, Request from typeguard import typechecked from backend.apps.agents.agent_manager import agent_manager @@ -41,6 +51,20 @@ async def agents_lifespan(): agents = SubApp("agents", agents_lifespan) +async def p_session_or_404(session_id: str): + session = agent_manager.get_session(session_id) + if not session: + try: + session = await agent_manager.resume_session(session_id) + except ValueError: + raise HTTPException(status_code=404, detail="Session not found") + return session + + +async def p_require_owned_session(session_id: str, scope: RequestScope): + session = await p_session_or_404(session_id) + scope.require_owner_of(session.owner_account_id) + return session @typechecked def p_session_list_item(session: AgentSession) -> Dict[str, Any]: """Serialize dashboard metadata without retaining the full chat history.""" @@ -63,8 +87,8 @@ def p_session_list_item(session: AgentSession) -> Dict[str, Any]: @agents.router.get("/sessions") -async def list_sessions(dashboard_id: str = ""): - sessions = agent_manager.get_all_sessions(dashboard_id=dashboard_id or None) +async def list_sessions(dashboard_id: str = "", scope: RequestScope = REQUEST_SCOPE): + sessions = scope.filter_owned(agent_manager.get_all_sessions(dashboard_id=dashboard_id or None)) return {"sessions": [p_session_list_item(s) for s in sessions]} @agents.router.get("/sessions/{session_id}/followups") @@ -82,10 +106,13 @@ async def predict_followups_route(session_id: str, count: int = 3): @agents.router.get("/predict-prompts") -async def predict_prompts_route(count: int = 5): +async def predict_prompts_route(count: int = 5, scope: RequestScope = REQUEST_SCOPE): """Guess a few prompts the user might type next, in their own voice, from what they've already worked on. Drives the composer's ghost-text suggestion. Fails open to [] (no signal / no provider / error), so the composer just keeps its static placeholder.""" + # Hosted: the prediction scans every session on disk (all tenants), so it stays empty there (R0 containment). + if scope.hosted: + return {"suggestions": []} from backend.apps.agents.manager.predict_prompts import predict_prompts return {"suggestions": await predict_prompts(count=max(1, min(count, 8)))} @@ -105,7 +132,7 @@ async def agent_activity(): return {"active": active, "next_run_in_s": next_run_in_s} @agents.router.get("/sessions/{session_id}") -async def get_session(session_id: str): +async def get_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): """Returns the session by id. Falls back to a disk load when the session isn't in the in-memory @@ -117,21 +144,19 @@ async def get_session(session_id: str): session into agent_manager.sessions and the next GET short-circuits on the in-memory check. """ - session = agent_manager.get_session(session_id) - if not session: - try: - session = await agent_manager.resume_session(session_id) - except ValueError: - raise HTTPException(status_code=404, detail="Session not found") - # Seq read before the dump (no await between = atomic): the client seeds its WS resume cursor from this, so a REST hydrate isn't followed by a full from-zero replay of everything it just received. + session = await p_require_owned_session(session_id, scope) + # Seq read before the dump (no await between = atomic): the client seeds its WS resume cursor from this, so a REST hydrate isn't followed by a full from-zero replay. event_seq = seq_log.current_seq(session_id) payload = session.model_dump(mode="json") payload["event_seq"] = event_seq return payload @agents.router.post("/launch") -async def launch_agent(config: AgentConfig): - session = await agent_manager.launch_agent(config) +async def launch_agent(config: AgentConfig, scope: RequestScope = REQUEST_SCOPE): + config = scope.sanitize_launch_config(config) + session, run_first_turn = await scope.admit_launch(config, agent_manager.launch_agent) + if not run_first_turn: + return {"session_id": session.id, "session": session.model_dump(mode="json")} # A launch that carries a prompt runs it as the first turn through the same path /message uses. if config.prompt: asyncio.create_task(agent_manager.send_message(session.id, config.prompt)) @@ -140,11 +165,34 @@ async def launch_agent(config: AgentConfig): asyncio.create_task(agent_manager.prewarm_client(session.id)) return {"session_id": session.id, "session": session.model_dump(mode="json")} + @agents.router.post("/sessions/{session_id}/message") -async def send_message(session_id: str, body: dict): +async def send_message(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): prompt = body.get("prompt", "") if not prompt: raise HTTPException(status_code=400, detail="prompt is required") + session = await p_require_owned_session(session_id, scope) + side_effect_payload = { + "prompt": prompt, + "mode": body.get("mode"), + "model": body.get("model"), + "images": body.get("images"), + "context_paths": body.get("context_paths"), + "forced_tools": body.get("forced_tools"), + "attached_skills": body.get("attached_skills"), + "hidden": body.get("hidden", False), + "selected_browser_ids": body.get("selected_browser_ids"), + "selected_app_output_ids": body.get("selected_app_output_ids"), + "selected_setting_ids": body.get("selected_setting_ids"), + "client_message_id": body.get("client_message_id"), + } + if scope.admit_prompt( + session, + requested_mode=body.get("mode"), + forced_tools=body.get("forced_tools"), + side_effect_payload=side_effect_payload, + ): + return {"ok": True, "replayed": True} # Run MCP-suggestion classifier in parallel with the agent launch; fails open. try: @@ -174,25 +222,27 @@ async def p_emit_preflight(): except Exception: pass + await agent_manager.send_message( session_id, - prompt, - mode=body.get("mode"), - model=body.get("model"), - images=body.get("images"), - context_paths=body.get("context_paths"), - forced_tools=body.get("forced_tools"), - attached_skills=body.get("attached_skills"), - hidden=body.get("hidden", False), - selected_browser_ids=body.get("selected_browser_ids"), - selected_app_output_ids=body.get("selected_app_output_ids"), - selected_setting_ids=body.get("selected_setting_ids"), - client_message_id=body.get("client_message_id"), + side_effect_payload["prompt"], + mode=side_effect_payload["mode"], + model=side_effect_payload["model"], + images=side_effect_payload["images"], + context_paths=side_effect_payload["context_paths"], + forced_tools=side_effect_payload["forced_tools"], + attached_skills=side_effect_payload["attached_skills"], + hidden=side_effect_payload["hidden"], + selected_browser_ids=side_effect_payload["selected_browser_ids"], + selected_app_output_ids=side_effect_payload["selected_app_output_ids"], + selected_setting_ids=side_effect_payload["selected_setting_ids"], + client_message_id=side_effect_payload["client_message_id"], ) return {"ok": True} @agents.router.post("/sessions/{session_id}/stop") -async def stop_agent(session_id: str): +async def stop_agent(session_id: str, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) await agent_manager.stop_agent(session_id) # A stopped turn's parked AskUI waits would otherwise zombie for 600s and eat the next click (ENG-232). from backend.apps.agents.ui_request_bridge import cancel_session_waits @@ -200,27 +250,39 @@ async def stop_agent(session_id: str): return {"ok": True} @agents.router.post("/approval") -async def handle_approval(response: ApprovalResponse): - agent_manager.handle_approval(response.request_id, { +async def handle_approval(response: ApprovalResponse, scope: RequestScope = REQUEST_SCOPE): + approval_session_id = ws_manager.approval_session_id(response.request_id) + await scope.authorize_approval(approval_session_id, p_session_or_404) + decision = { "behavior": response.behavior, "message": response.message, "updated_input": response.updated_input, "trust_pattern": response.trust_pattern, "set_always_allow": response.set_always_allow, - }) + } + if not scope.resolve_approval(response.request_id, decision, approval_session_id): + agent_manager.handle_approval(response.request_id, decision) return {"ok": True} @agents.router.post("/sessions/{session_id}/edit_message") -async def edit_message(session_id: str, body: dict): +async def edit_message(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): + session = await p_require_owned_session(session_id, scope) message_id = body.get("message_id") new_content = body.get("content", "") if not message_id or not new_content: raise HTTPException(status_code=400, detail="message_id and content are required") + side_effect_payload = { + "message_id": message_id, + "content": new_content, + } + if scope.admit_prompt(session, requested_mode=None, forced_tools=None, side_effect_payload=side_effect_payload): + return {"ok": True, "replayed": True} await agent_manager.edit_message(session_id, message_id, new_content) return {"ok": True} @agents.router.post("/sessions/{session_id}/switch_branch") -async def switch_branch(session_id: str, body: dict): +async def switch_branch(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) branch_id = body.get("branch_id", "") if not branch_id: raise HTTPException(status_code=400, detail="branch_id is required") @@ -228,7 +290,8 @@ async def switch_branch(session_id: str, body: dict): return {"ok": True} @agents.router.post("/sessions/{session_id}/generate-title") -async def generate_title(session_id: str, body: dict): +async def generate_title(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) prompt = body.get("prompt", "") if not prompt: raise HTTPException(status_code=400, detail="prompt is required") @@ -236,7 +299,8 @@ async def generate_title(session_id: str, body: dict): return {"title": title} @agents.router.post("/sessions/{session_id}/generate-group-meta") -async def generate_group_meta(session_id: str, body: dict): +async def generate_group_meta(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) group_id = body.get("group_id", "") tool_calls = body.get("tool_calls", []) if not group_id or not tool_calls: @@ -277,25 +341,22 @@ async def generate_group_meta(session_id: str, body: dict): p_group_meta_inflight.pop(key, None) @agents.router.patch("/sessions/{session_id}") -async def update_session(session_id: str, body: dict): - session = agent_manager.get_session(session_id) - if not session: - raise HTTPException(status_code=404, detail="Session not found") +async def update_session(session_id: str, body: dict, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) await agent_manager.update_session(session_id, **body) return {"ok": True} @agents.router.get("/sessions/{session_id}/branches") -async def get_branches(session_id: str): - session = agent_manager.get_session(session_id) - if not session: - raise HTTPException(status_code=404, detail="Session not found") +async def get_branches(session_id: str, scope: RequestScope = REQUEST_SCOPE): + session = await p_require_owned_session(session_id, scope) return { "branches": {k: v.model_dump(mode="json") for k, v in session.branches.items()}, "active_branch_id": session.active_branch_id, } @agents.router.post("/sessions/{session_id}/duplicate") -async def duplicate_session(session_id: str, body: dict = {}): +async def duplicate_session(session_id: str, body: dict = {}, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) try: session = await agent_manager.duplicate_session( session_id, @@ -304,10 +365,12 @@ async def duplicate_session(session_id: str, body: dict = {}): ) except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) + scope.stamp_owner(session) return {"session": session.model_dump(mode="json")} @agents.router.post("/sessions/{session_id}/close") -async def close_session(session_id: str): +async def close_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) try: await agent_manager.close_session(session_id) except ValueError as e: @@ -315,20 +378,23 @@ async def close_session(session_id: str): return {"ok": True} @agents.router.delete("/sessions/{session_id}") -async def delete_session(session_id: str): +async def delete_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) await agent_manager.delete_session(session_id) return {"ok": True} @agents.router.get("/history") -async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_id: str = "", closed_only: int = 0): +async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_id: str = "", closed_only: int = 0, scope: RequestScope = REQUEST_SCOPE): return agent_manager.get_history( q=q, limit=limit, offset=offset, dashboard_id=dashboard_id or None, + owner_account_id=scope.owner_id, closed_only=bool(closed_only), ) @agents.router.get("/sessions/{session_id}/browser-agents") -async def get_browser_agent_children(session_id: str): +async def get_browser_agent_children(session_id: str, scope: RequestScope = REQUEST_SCOPE): + await p_require_owned_session(session_id, scope) children = agent_manager.get_browser_agent_children(session_id) return {"sessions": children} @@ -359,16 +425,17 @@ async def forget_browser_memory(host: str): @agents.router.post("/sessions/{session_id}/resume") -async def resume_session(session_id: str): +async def resume_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): try: session = await agent_manager.resume_session(session_id) except ValueError as e: raise HTTPException(status_code=404, detail=str(e)) + scope.require_owner_of(session.owner_account_id) return {"session": session.model_dump(mode="json")} @agents.router.post("/sessions/{session_id}/compact") -async def compact_session(session_id: str): +async def compact_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): """Run the summarizer over older turns to free up context. Wired to the 'Compact memory' button in the pre-send overflow banner and the @@ -378,9 +445,7 @@ async def compact_session(session_id: str): threshold now does the same (pre_send_context_guard); this button is the manual "do it now" for a user who wants the trim before the threshold. """ - session = agent_manager.sessions.get(session_id) - if not session: - raise HTTPException(status_code=404, detail="session not found") + session = await p_require_owned_session(session_id, scope) fired = agent_manager.maybe_compact(session, force=True) if fired: session.needs_fresh_session = True @@ -403,14 +468,12 @@ async def compact_session(session_id: str): @agents.router.post("/sessions/{session_id}/clear") -async def clear_session(session_id: str): +async def clear_session(session_id: str, scope: RequestScope = REQUEST_SCOPE): """Drop all messages from the session, keep MCPs/model/tools. Wired to the /clear slash command. Quickest path to recover from an overflow short of starting a fresh chat.""" - session = agent_manager.sessions.get(session_id) - if not session: - raise HTTPException(status_code=404, detail="session not found") + session = await p_require_owned_session(session_id, scope) session.messages = [] session.compacted_through_msg_id = None session.compacted_summary = None @@ -505,7 +568,7 @@ async def subscriptions_poll(body: dict): if result.get("success"): from backend.apps.service.client import sync as p_sync from backend.apps.settings.settings import load_settings - p_sync(load_settings().model_dump()) + p_sync(redact_settings(load_settings().model_dump())) from backend.apps.subscription.free_trial import clear_free_trial_on_connect await clear_free_trial_on_connect() return result @@ -540,7 +603,7 @@ async def subscriptions_exchange(body: dict): mark_completed(state) from backend.apps.service.client import sync as do_sync from backend.apps.settings.settings import load_settings - do_sync(load_settings().model_dump()) + do_sync(redact_settings(load_settings().model_dump())) # A connected subscription takes precedence over the free trial right away. from backend.apps.subscription.free_trial import clear_free_trial_on_connect await clear_free_trial_on_connect() @@ -599,6 +662,7 @@ async def probe_model(body: dict): from backend.apps.settings.settings import load_settings from backend.apps.nine_router import is_running as p_9r_running settings = load_settings() + await agent_manager.ensure_keyed_model_route_synced(settings, short_name) api_type = get_api_type(short_name) resolved = resolve_model_id_for_sdk(short_name, settings) entry = find_builtin_model(short_name) or {} diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 1c8d248e6..e91393bd2 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -103,7 +103,7 @@ SYSTEM_PROMPT, ) from backend.apps.agents.core.models import AgentSession, ApprovalRequest, Message -from backend.apps.agents.core.ws_manager import ws_manager, await_reconnect +from backend.apps.agents.core.ws_manager import BrowserCommandOwner, ws_manager, await_reconnect from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) @@ -385,7 +385,10 @@ async def p_execute_browser_tool( async def p_eval_once() -> dict: rid = uuid4().hex - return await ws_manager.send_browser_command(rid, action, browser_id, params, tab_id=tab_id) + return await ws_manager.send_browser_command( + rid, action, browser_id, params, tab_id=tab_id, + owner=BrowserCommandOwner(origin="renderer"), + ) result = await p_eval_once() # Reads poll for the bridge to come up (app still mounting on turn 1). @@ -426,6 +429,7 @@ async def p_eval_once() -> dict: request_id = uuid4().hex result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, + owner=BrowserCommandOwner(origin="renderer"), ) if os.environ.get("OSW_DEBUG_LIST") == "1" and action == "list_interactives" and isinstance(result, dict): logger.info(f"[debug-list] {str(result.get('text') or '')[:2400]}") diff --git a/backend/apps/agents/browser/browser_metrics.py b/backend/apps/agents/browser/browser_metrics.py index f920bd15e..fa5168de6 100644 --- a/backend/apps/agents/browser/browser_metrics.py +++ b/backend/apps/agents/browser/browser_metrics.py @@ -87,6 +87,11 @@ def p_append(filename: str, obj: dict) -> None: path = os.path.join(metrics_dir(), filename) # owner-only: these lines can carry task text and error snippets fd = os.open(path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o600) + if os.name == "posix": + try: + os.fchmod(fd, 0o600) + except Exception: + pass with os.fdopen(fd, "a", encoding="utf-8") as f: f.write(json.dumps(obj, default=str) + "\n") except Exception as e: diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 1ce658f1b..16b2a1661 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -16,6 +16,7 @@ class AgentConfig(BaseModel): max_turns: Optional[int] = None target_directory: Optional[str] = None dashboard_id: Optional[str] = None + owner_account_id: Optional[str] = None workflow_run_id: Optional[str] = None workflow_edit_id: Optional[str] = None # App cards the user picked to edit. When exactly one resolves, launch binds the chat's cwd to that app instead of seeding a new "Untitled App". @@ -119,6 +120,7 @@ class AgentSession(BaseModel): active_branch_id: str = "main" tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict) dashboard_id: Optional[str] = None + owner_account_id: Optional[str] = None browser_id: Optional[str] = None parent_session_id: Optional[str] = None # Set when this session IS a workflow run's agent; the run renders in the Workflows monitor card, so the canvas suppresses the duplicate standalone agent card. diff --git a/backend/apps/agents/core/ws_manager.py b/backend/apps/agents/core/ws_manager.py index 48fe69a0e..d5e4199a6 100644 --- a/backend/apps/agents/core/ws_manager.py +++ b/backend/apps/agents/core/ws_manager.py @@ -1,6 +1,10 @@ import asyncio +import hashlib import json import logging +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from datetime import datetime, timezone from typing import Optional from fastapi import WebSocket @@ -23,6 +27,73 @@ BROWSER_CMD_REBROADCAST_S = 3.0 # A CPU-starved renderer can briefly drop its WS (a missed heartbeat) and the frontend auto-reconnects a beat later; bridge that gap instead of hard-failing a live run into it. Short enough that a genuinely-closed window still fails quickly (and no LLM turns are ever burned waiting); long enough to ride out a reconnect even on a loaded machine. P_WS_RECONNECT_WAIT_S = 8.0 +HOSTED_AUTH_CLOSE_CODE = 4401 +HOSTED_AUTH_CLOSE_REASON = "hosted session revoked" +HOSTED_ACCOUNT_CLOSE_REASON = "hosted account revoked" +HOSTED_EXPIRY_CLOSE_REASON = "hosted session expired" +HOSTED_SOCKET_CLOSE_TIMEOUT_S = 1.0 + + +@dataclass(frozen=True) +class BrowserCommandOwner: + """Server-derived identity a browser command is correlated to at send time. + + A browser:result may resolve the command only when the submitting + connection's own BrowserCommandOwner (built from server-side connection + state, never from result payload bytes) equals this record exactly: + origin bridge, account, and auth session. Local desktop commands carry + (renderer/main, None, None); hosted identities can never match them. + """ + + origin: str # 'renderer' (dashboard sockets) | 'main' (Electron-main bridge) + account_id: str | None = None + auth_session_key: str | None = None + + +@dataclass(frozen=True) +class HostedConnectionIdentity: + account_id: str + auth_session_key: str + expires_at: datetime | None = None + + def __post_init__(self) -> None: + if self.expires_at is not None: + object.__setattr__(self, "expires_at", p_normalize_utc(self.expires_at)) + + def is_expired(self, now: datetime) -> bool: + return self.expires_at is not None and self.expires_at <= p_normalize_utc(now) + + +def p_normalize_utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + return value.replace(tzinfo=timezone.utc) + return value.astimezone(timezone.utc) + + +def hosted_connection_identity( + account_id: str, + auth_session_key: str, + expires_at: datetime | None = None, +) -> HostedConnectionIdentity: + """Build normalized hosted socket authority from a verified auth session.""" + return HostedConnectionIdentity(account_id, auth_session_key, expires_at) + + +async def p_close_socket_with_deadline( + websocket: WebSocket, + code: int, + reason: str, + timeout: float, +) -> None: + await asyncio.wait_for(websocket.close(code=code, reason=reason), timeout=timeout) + + +def hosted_auth_session_key(cookie_value: str) -> str: + """Return a non-secret process-local lookup key for one hosted auth cookie.""" + if not cookie_value: + return "" + digest = hashlib.sha256(cookie_value.encode("utf-8")).hexdigest() + return f"hosted-session:v1:{digest}" def slim_status_data(event: str, data: dict) -> dict: @@ -65,28 +136,61 @@ async def await_reconnect(has_conn) -> bool: class ConnectionManager: """Manages WebSocket connections and HITL approval bridging; events flow through seq_log so reconnects can replay.""" - def __init__(self): + def __init__( + self, + *, + now: Callable[[], datetime] | None = None, + close_socket: Callable[[WebSocket, int, str, float], Awaitable[None]] | None = None, + close_timeout_s: float = HOSTED_SOCKET_CLOSE_TIMEOUT_S, + ): + self.p_now = now or (lambda: datetime.now(timezone.utc)) + self.p_close_socket = close_socket or p_close_socket_with_deadline + self.p_close_timeout_s = close_timeout_s self.connections: dict[str, list[WebSocket]] = {} self.global_connections: list[WebSocket] = [] + self.global_account_ids: dict[int, str | None] = {} + self.hosted_connection_identities: dict[int, HostedConnectionIdentity] = {} # Latched on the first renderer and never cleared: it answers "can a window reach this backend at all", which a momentary socket blip must not un-answer. Only the process dying resets it. self.renderer_ever_attached: bool = False # Which dashboard each global socket is currently showing, keyed by id(websocket). active_dashboard_id is the last one activated (the window the user is looking at most recently); a scheduled run targets it so its browser card spawns where the renderer can render it. self.global_dashboard_ids: dict[int, str] = {} self.active_dashboard_id: Optional[str] = None self.pending_futures: dict[str, asyncio.Future] = {} + self.pending_approval_sessions: dict[str, str] = {} self.browser_futures: dict[str, asyncio.Future] = {} + # Owner record per pending browser command; results resolve only for the exact recorded owner (resolve_browser_command). + self.browser_command_owners: dict[str, BrowserCommandOwner] = {} # The Electron MAIN process (not the renderer) holds a single WS here. Cookie reads route to it so they don't ride the renderer, which macOS throttles when the window is backgrounded (the source of the session-borrow bridge's intermittent timeouts). self.main_connection: Optional[WebSocket] = None - async def connect_session(self, session_id: str, websocket: WebSocket): + async def connect_session( + self, + session_id: str, + websocket: WebSocket, + identity: HostedConnectionIdentity | None = None, + ): await websocket.accept() if session_id not in self.connections: self.connections[session_id] = [] self.connections[session_id].append(websocket) - - async def connect_global(self, websocket: WebSocket): + if identity is not None: + self.hosted_connection_identities[id(websocket)] = identity + else: + self.hosted_connection_identities.pop(id(websocket), None) + + async def connect_global( + self, + websocket: WebSocket, + account_id: str | None = None, + identity: HostedConnectionIdentity | None = None, + ): await websocket.accept() self.global_connections.append(websocket) + self.global_account_ids[id(websocket)] = account_id + if identity is not None: + self.hosted_connection_identities[id(websocket)] = identity + else: + self.hosted_connection_identities.pop(id(websocket), None) self.renderer_ever_attached = True async def connect_main(self, websocket: WebSocket): @@ -99,6 +203,7 @@ def disconnect_main(self, websocket: WebSocket): self.main_connection = None def disconnect_session(self, session_id: str, websocket: WebSocket): + self.hosted_connection_identities.pop(id(websocket), None) if session_id in self.connections: self.connections[session_id] = [ ws for ws in self.connections[session_id] if ws != websocket @@ -117,19 +222,132 @@ def disconnect_global(self, websocket: WebSocket): ] # Drop this socket's active-dashboard pointer; if it owned the global one, fall back to any window still connected so a closed tab doesn't leave a stale target. self.global_dashboard_ids.pop(id(websocket), None) + self.global_account_ids.pop(id(websocket), None) + self.hosted_connection_identities.pop(id(websocket), None) if self.active_dashboard_id not in self.global_dashboard_ids.values(): self.active_dashboard_id = next(iter(self.global_dashboard_ids.values()), None) + def disconnect_everywhere(self, websocket: WebSocket) -> None: + """Evict one socket from every connection and hosted-identity registry.""" + for session_id in list(self.connections): + self.disconnect_session(session_id, websocket) + self.disconnect_global(websocket) + + def hosted_connection_is_current( + self, + websocket: WebSocket, + identity: HostedConnectionIdentity, + ) -> bool: + """Return whether a hosted socket still has live registered authority.""" + registered = self.hosted_connection_identities.get(id(websocket)) + return registered == identity and not identity.is_expired(self.p_now()) + + def p_connection_candidates(self) -> list[WebSocket]: + sockets: dict[int, WebSocket] = {} + for websocket in ( + *(ws for group in self.connections.values() for ws in group), + *self.global_connections, + ): + sockets[id(websocket)] = websocket + return list(sockets.values()) + + def p_evict_matching_hosted( + self, + predicate: Callable[[HostedConnectionIdentity], bool], + ) -> list[WebSocket]: + sockets: list[WebSocket] = [] + for websocket in self.p_connection_candidates(): + identity = self.hosted_connection_identities.get(id(websocket)) + if identity is not None and predicate(identity): + sockets.append(websocket) + for websocket in sockets: + self.disconnect_everywhere(websocket) + return sockets + + async def p_close_evicted( + self, + sockets: list[WebSocket], + *, + code: int, + reason: str, + ) -> None: + async def p_close_one(websocket: WebSocket) -> None: + try: + await self.p_close_socket( + websocket, + code, + reason, + self.p_close_timeout_s, + ) + except Exception: + logger.debug("hosted socket drain failed", exc_info=True) + + await asyncio.gather(*(p_close_one(websocket) for websocket in sockets)) + + def p_evict_expired(self, candidates: list[WebSocket]) -> list[WebSocket]: + now = self.p_now() + sockets: dict[int, WebSocket] = {} + for websocket in candidates: + identity = self.hosted_connection_identities.get(id(websocket)) + if identity is not None and identity.is_expired(now): + sockets[id(websocket)] = websocket + for websocket in sockets.values(): + self.disconnect_everywhere(websocket) + return list(sockets.values()) + + async def close_hosted_auth_session( + self, + auth_session_key: str, + *, + code: int = HOSTED_AUTH_CLOSE_CODE, + reason: str = HOSTED_AUTH_CLOSE_REASON, + ) -> None: + """Close sockets bound to one hosted login without touching local agent tasks.""" + if not auth_session_key: + return + sockets = self.p_evict_matching_hosted( + lambda identity: identity.auth_session_key == auth_session_key + ) + await self.p_close_evicted(sockets, code=code, reason=reason) + + async def close_hosted_account( + self, + account_id: str, + *, + code: int = HOSTED_AUTH_CLOSE_CODE, + reason: str = HOSTED_ACCOUNT_CLOSE_REASON, + ) -> None: + """Drain every hosted login for one account without touching local sockets.""" + if not account_id: + return + sockets = self.p_evict_matching_hosted( + lambda identity: identity.account_id == account_id + ) + await self.p_close_evicted(sockets, code=code, reason=reason) + async def send_to_session(self, session_id: str, event: str, data: dict): """Broadcast a session event with monotonic sequencing; terminal statuses also persist to disk.""" + account_id = self.p_session_account_id(session_id) data = slim_status_data(event, data) async with seq_log.stamp(session_id, event, data) as (seq, payload_str): + candidates = [ + *self.connections.get(session_id, []), + *self.global_connections, + ] + expired = self.p_evict_expired(candidates) + await self.p_close_evicted( + expired, + code=HOSTED_AUTH_CLOSE_CODE, + reason=HOSTED_EXPIRY_CLOSE_REASON, + ) for ws in list(self.connections.get(session_id, [])): try: await ws.send_text(payload_str) except Exception: logger.debug("send_to_session: send failed (will retry on reconnect)", exc_info=True) for ws in list(self.global_connections): + if not self.p_global_matches(ws, account_id): + continue try: await ws.send_text(payload_str) except Exception: @@ -147,11 +365,31 @@ async def send_to_session(self, session_id: str, event: str, data: dict): logger.debug("agent:message analytics bridge failed", exc_info=True) async def replay_to( - self, session_id: str, websocket: WebSocket, last_seq: int - ) -> dict: + self, + session_id: str, + websocket: WebSocket, + last_seq: int, + identity: HostedConnectionIdentity | None = None, + ) -> dict | None: """Replay buffered events with seq > last_seq; returns ack envelope for the resume handshake.""" oldest, newest, events = seq_log.replay(session_id, last_seq) + async def p_send(payload: str) -> bool: + if identity is not None and not self.hosted_connection_is_current(websocket, identity): + self.disconnect_everywhere(websocket) + await self.p_close_evicted( + [websocket], + code=HOSTED_AUTH_CLOSE_CODE, + reason=( + HOSTED_EXPIRY_CLOSE_REASON + if identity.is_expired(self.p_now()) + else HOSTED_AUTH_CLOSE_REASON + ), + ) + return False + await websocket.send_text(payload) + return True + # Gap-check first: if last_seq predates the buffer, signal REST-refresh; last_seq=0 means fresh client (full replay). if last_seq > 0 and oldest is not None and last_seq < oldest - 1: gap_payload = json.dumps({ @@ -165,7 +403,8 @@ async def replay_to( }, }) try: - await websocket.send_text(gap_payload) + if not await p_send(gap_payload): + return None except Exception: pass return { @@ -181,7 +420,8 @@ async def replay_to( events = self.p_strip_replayed_closes(events) for s in events: try: - await websocket.send_text(s) + if not await p_send(s): + return None except Exception: logger.debug("replay_to: send failed", exc_info=True) break @@ -199,7 +439,8 @@ async def replay_to( terminal = seq_log.load_terminal(session_id) if terminal is not None: try: - await websocket.send_text(terminal) + if not await p_send(terminal): + return None except Exception: pass return {"ok": True, "replayed": 1, "terminal_only": True} @@ -256,11 +497,37 @@ def p_filter_stale_approvals(self, events: list[str]) -> list[str]: out.append(payload_str) return out - async def broadcast_global(self, event: str, data: dict): - """Send to all dashboard connections; bypasses seq_log (dashboard resumes via full state refetch).""" + def p_session_account_id(self, session_id: str | None) -> str | None: + if not session_id: + return None + from backend.apps.agents.agent_manager import agent_manager + session = agent_manager.get_session(session_id) + return session.owner_account_id if session is not None else None + + def p_event_account_id(self, data: dict) -> str | None: + for candidate in (data, data.get("session") or {}, data.get("output") or {}): + owner = candidate.get("owner_account_id") if isinstance(candidate, dict) else None + if owner: + return owner + return self.p_session_account_id(data.get("session_id") or data.get("parent_session_id")) + + def p_global_matches(self, websocket: WebSocket, account_id: str | None) -> bool: + return self.global_account_ids.get(id(websocket)) == account_id + + async def broadcast_global(self, event: str, data: dict, account_id: str | None = None): + """Send an event only to dashboards in its account partition.""" + target_account_id = account_id if account_id is not None else self.p_event_account_id(data) payload = json.dumps({"event": event, "data": slim_status_data(event, data)}) dead: list[WebSocket] = [] + expired = self.p_evict_expired(list(self.global_connections)) + await self.p_close_evicted( + expired, + code=HOSTED_AUTH_CLOSE_CODE, + reason=HOSTED_EXPIRY_CLOSE_REASON, + ) for ws in list(self.global_connections): + if not self.p_global_matches(ws, target_account_id): + continue try: await ws.send_text(payload) except Exception: @@ -279,6 +546,7 @@ async def send_approval_request( """Send an approval request and wait for the user's decision; 10-minute timeout prevents permanent park.""" future = asyncio.get_event_loop().create_future() self.pending_futures[request_id] = future + self.pending_approval_sessions[request_id] = session_id payload: dict = { "request_id": request_id, @@ -299,23 +567,39 @@ async def send_approval_request( return {"behavior": "deny", "message": "Approval timed out"} finally: self.pending_futures.pop(request_id, None) - - def resolve_approval(self, request_id: str, decision: dict): + self.pending_approval_sessions.pop(request_id, None) + + def approval_session_id(self, request_id: str) -> str | None: + return self.pending_approval_sessions.get(request_id) + + def resolve_approval( + self, + request_id: str, + decision: dict, + *, + session_id: str | None = None, + ) -> bool: """Resolve a pending approval Future with the user's decision.""" + if session_id is not None and self.pending_approval_sessions.get(request_id) != session_id: + return False future = self.pending_futures.get(request_id) if future and not future.done(): future.set_result(decision) + return True + return False async def send_browser_command( - self, request_id: str, action: str, browser_id: str, params: dict, tab_id: str = "" + self, request_id: str, action: str, browser_id: str, params: dict, tab_id: str = "", + *, owner: BrowserCommandOwner, ) -> dict: - """Send a browser command to the frontend and wait for the result.""" + """Send a browser command to the frontend and wait for the owner-bound result.""" if not self.global_connections and not await await_reconnect(lambda: bool(self.global_connections)): return {"error": "No dashboard is connected. Open the dashboard to use browser tools."} loop = asyncio.get_event_loop() future = loop.create_future() self.browser_futures[request_id] = future + self.browser_command_owners[request_id] = owner payload = { "request_id": request_id, @@ -351,8 +635,11 @@ async def send_browser_command( return {"error": "No dashboard is connected. Open the dashboard to use browser tools."} finally: self.browser_futures.pop(request_id, None) + self.browser_command_owners.pop(request_id, None) - async def send_main_command(self, request_id: str, action: str, params: dict) -> dict: + async def send_main_command( + self, request_id: str, action: str, params: dict, *, owner: BrowserCommandOwner + ) -> dict: """Send a command straight to the throttle-free Electron MAIN socket (cookie reads only); returns a not-connected error so the caller can fall back to the renderer.""" ws = self.main_connection if ws is None: @@ -360,6 +647,7 @@ async def send_main_command(self, request_id: str, action: str, params: dict) -> loop = asyncio.get_event_loop() future = loop.create_future() self.browser_futures[request_id] = future + self.browser_command_owners[request_id] = owner payload = {"request_id": request_id, "action": action, "browser_id": "", "tab_id": "", "params": params} try: await ws.send_text(json.dumps({"event": "browser:command", "data": payload})) @@ -372,12 +660,31 @@ async def send_main_command(self, request_id: str, action: str, params: dict) -> return {"error": f"Electron main bridge send failed: {e}"} finally: self.browser_futures.pop(request_id, None) - - def resolve_browser_command(self, request_id: str, result: dict): - """Resolve a pending browser command Future with the frontend's result.""" + self.browser_command_owners.pop(request_id, None) + + def resolve_browser_command( + self, request_id: str, result: dict, *, claimant: BrowserCommandOwner + ) -> bool: + """Resolve a pending browser command only for its exact recorded owner. + + `claimant` is built by the ingress handler from the submitting + connection's server-derived state — never from result payload bytes. + Acceptance requires a live owner record whose origin, account, and auth + session all equal the claimant's, and is single-consumption: the first + matching result wins, so replays and the renderer's dedupe-cache + re-sends are refused (the sender's finally clears both records on + every terminal path). Refusal is silent (False) — a legitimate + duplicate must not error a healthy socket — and there is no ownerless + fallback path. + """ + owner = self.browser_command_owners.get(request_id) future = self.browser_futures.get(request_id) - if future and not future.done(): - future.set_result(result) + if owner is None or future is None or future.done(): + return False + if claimant != owner: + return False + future.set_result(result) + return True ws_manager = ConnectionManager() diff --git a/backend/apps/agents/disconnect_subscription.py b/backend/apps/agents/disconnect_subscription.py index 91ab3e92b..c593f4637 100644 --- a/backend/apps/agents/disconnect_subscription.py +++ b/backend/apps/agents/disconnect_subscription.py @@ -39,8 +39,10 @@ class SubscriptionDisconnectResult(BaseModel): def sync_settings_state() -> None: """Push the settings snapshot to the cloud state sync, exactly as connecting does. Imported late: service.client reaches back into this package.""" from backend.apps.service.client import sync + from backend.apps.settings.redaction import redact_settings from backend.apps.settings.settings import load_settings - sync(load_settings().model_dump()) + # F1: credentials never leave the machine through the telemetry/state sync; every sync site redacts first. + sync(redact_settings(load_settings().model_dump())) @typechecked diff --git a/backend/apps/agents/events/AgentEvent.py b/backend/apps/agents/events/AgentEvent.py new file mode 100644 index 000000000..4dbf8c1b7 --- /dev/null +++ b/backend/apps/agents/events/AgentEvent.py @@ -0,0 +1,89 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Annotated, Literal, Optional, Union +from uuid import uuid4 + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter +from typeguard import typechecked + + +@typechecked +def p_event_id() -> str: + return uuid4().hex + + +@typechecked +def p_now() -> datetime: + return datetime.now(timezone.utc) + + +class AgentEventBase(BaseModel): + model_config = ConfigDict(validate_assignment=True, extra="forbid", frozen=True) + schema_version: Literal["1"] = "1" + event_id: str = Field(default_factory=p_event_id, min_length=32, max_length=32, pattern=r"^[0-9a-f]{32}$") + session_id: str = Field(min_length=1, max_length=128) + turn_id: str = Field(min_length=1, max_length=128) + sequence: int = Field(ge=0) + occurred_at: AwareDatetime = Field(default_factory=p_now) + monotonic_ms: int = Field(ge=0) + + +class TurnStartedEvent(AgentEventBase): + kind: Literal["turn.started"] = "turn.started" + provider: str = Field(min_length=1, max_length=64) + model: str = Field(min_length=1, max_length=128) + + +class TurnFirstTokenEvent(AgentEventBase): + kind: Literal["turn.first_token"] = "turn.first_token" + ttft_ms: int = Field(ge=0) + + +class ToolStartedEvent(AgentEventBase): + kind: Literal["tool.started"] = "tool.started" + tool_call_id: str = Field(min_length=1, max_length=128) + tool_name: str = Field(min_length=1, max_length=128) + + +class ToolCompletedEvent(AgentEventBase): + kind: Literal["tool.completed"] = "tool.completed" + tool_call_id: str = Field(min_length=1, max_length=128) + tool_name: str = Field(min_length=1, max_length=128) + duration_ms: int = Field(ge=0) + status: Literal["success", "error", "cancelled"] + error_type: Optional[str] = Field(default=None, max_length=128) + + +class TurnCompletedEvent(AgentEventBase): + kind: Literal["turn.completed"] = "turn.completed" + duration_ms: int = Field(ge=0) + input_tokens: int = Field(default=0, ge=0) + output_tokens: int = Field(default=0, ge=0) + + +class TurnFailedEvent(AgentEventBase): + kind: Literal["turn.failed"] = "turn.failed" + duration_ms: int = Field(ge=0) + error_type: str = Field(min_length=1, max_length=128) + retryable: bool = False + + +AgentEvent = Annotated[ + Union[ + TurnStartedEvent, + TurnFirstTokenEvent, + ToolStartedEvent, + ToolCompletedEvent, + TurnCompletedEvent, + TurnFailedEvent, + ], + Field(discriminator="kind"), +] + +P_AGENT_EVENT_ADAPTER = TypeAdapter(AgentEvent) + + +@typechecked +def parse_agent_event(value: object) -> AgentEvent: + return P_AGENT_EVENT_ADAPTER.validate_python(value) diff --git a/backend/apps/agents/events/AgentEventSink.py b/backend/apps/agents/events/AgentEventSink.py new file mode 100644 index 000000000..5e964da25 --- /dev/null +++ b/backend/apps/agents/events/AgentEventSink.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import logging +from collections import OrderedDict, deque +from dataclasses import dataclass +from threading import Lock +from typing import Dict, Protocol, Tuple, runtime_checkable + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +from backend.apps.agents.events.AgentEvent import AgentEvent + +logger = logging.getLogger(__name__) + + +@runtime_checkable +class AgentEventSink(Protocol): + @typechecked + def emit(self, event: AgentEvent) -> None: + ... + + +class NullAgentEventSink(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + @typechecked + def emit(self, event: AgentEvent) -> None: + return None + + +@dataclass(frozen=True) +class AgentEventSnapshot: + session_id: str + events: Tuple[AgentEvent, ...] + dropped_events: int + + +class BoundedAgentEventSink: + """Thread-safe per-session event history for debug timeline adapters.""" + + @typechecked + def __init__(self, max_sessions: int = 64, max_events_per_session: int = 512) -> None: + if max_sessions < 1 or max_events_per_session < 1: + raise ValueError("event sink bounds must be at least 1") + self.max_sessions = max_sessions + self.max_events_per_session = max_events_per_session + self.p_events: OrderedDict[str, deque[AgentEvent]] = OrderedDict() + self.p_dropped: Dict[str, int] = {} + self.p_lock = Lock() + + @typechecked + def emit(self, event: AgentEvent) -> None: + with self.p_lock: + session_events = self.p_events.get(event.session_id) + if session_events is None: + if len(self.p_events) >= self.max_sessions: + evicted_session, _ = self.p_events.popitem(last=False) + self.p_dropped.pop(evicted_session, None) + session_events = deque(maxlen=self.max_events_per_session) + self.p_events[event.session_id] = session_events + self.p_dropped[event.session_id] = 0 + else: + self.p_events.move_to_end(event.session_id) + if len(session_events) == self.max_events_per_session: + self.p_dropped[event.session_id] += 1 + session_events.append(event) + + @typechecked + def snapshot(self, session_id: str) -> AgentEventSnapshot: + with self.p_lock: + return AgentEventSnapshot( + session_id=session_id, + events=tuple(self.p_events.get(session_id, ())), + dropped_events=self.p_dropped.get(session_id, 0), + ) + + @typechecked + def clear(self, session_id: str | None = None) -> None: + with self.p_lock: + if session_id is None: + self.p_events.clear() + self.p_dropped.clear() + return + self.p_events.pop(session_id, None) + self.p_dropped.pop(session_id, None) + + +@typechecked +def emit_agent_event(sink: AgentEventSink, event: AgentEvent) -> bool: + try: + sink.emit(event) + return True + except Exception: + logger.debug("agent event sink failed", exc_info=True) + return False diff --git a/backend/apps/agents/events/AgentTurnEventEmitter.py b/backend/apps/agents/events/AgentTurnEventEmitter.py new file mode 100644 index 000000000..a95c04163 --- /dev/null +++ b/backend/apps/agents/events/AgentTurnEventEmitter.py @@ -0,0 +1,151 @@ +from __future__ import annotations + +import time +from threading import RLock +from typing import Any, Dict, Literal +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict, Field, InstanceOf +from typeguard import typechecked + +from backend.apps.agents.events.AgentEvent import ( + ToolCompletedEvent, + ToolStartedEvent, + TurnCompletedEvent, + TurnFailedEvent, + TurnFirstTokenEvent, + TurnStartedEvent, +) +from backend.apps.agents.events.AgentEventSink import AgentEventSink, emit_agent_event + + +class AgentTurnEventEmitter(BaseModel): + model_config = ConfigDict(validate_assignment=True, arbitrary_types_allowed=True) + + sink: InstanceOf[AgentEventSink] + session_id: str + provider: str + model: str + turn_id: str = Field(default_factory=lambda: uuid4().hex) + sequence: int = 0 + started_monotonic: float = Field(default_factory=time.monotonic) + first_token_emitted: bool = False + tool_starts: Dict[str, float] = Field(default_factory=dict) + tool_names: Dict[str, str] = Field(default_factory=dict) + p_lock: Any = Field(default_factory=RLock, exclude=True) + + @typechecked + def emit_started(self) -> None: + with self.p_lock: + emit_agent_event(self.sink, TurnStartedEvent(**self.next_fields(), provider=self.provider, model=self.model)) + + @typechecked + def emit_first_token(self) -> None: + with self.p_lock: + if self.first_token_emitted: + return + self.first_token_emitted = True + emit_agent_event( + self.sink, + TurnFirstTokenEvent(**self.next_fields(), ttft_ms=self.duration_ms()), + ) + + @typechecked + def emit_tool_started(self, tool_call_id: str, tool_name: str) -> None: + with self.p_lock: + safe_id = tool_call_id[:128] + if not safe_id or safe_id in self.tool_starts: + return + safe_name = (tool_name or "unknown")[:128] + self.tool_starts[safe_id] = time.monotonic() + self.tool_names[safe_id] = safe_name + emit_agent_event( + self.sink, + ToolStartedEvent( + **self.next_fields(), + tool_call_id=safe_id, + tool_name=safe_name, + ), + ) + + @typechecked + def emit_tool_completed( + self, + tool_call_id: str, + tool_name: str, + status: Literal["success", "error", "cancelled"] = "success", + error_type: str | None = None, + ) -> None: + with self.p_lock: + safe_id = tool_call_id[:128] + if not safe_id: + return + if safe_id not in self.tool_starts: + self.emit_tool_started(safe_id, tool_name) + started = self.tool_starts.pop(safe_id, time.monotonic()) + safe_name = self.tool_names.pop(safe_id, (tool_name or "unknown")[:128]) + emit_agent_event( + self.sink, + ToolCompletedEvent( + **self.next_fields(), + tool_call_id=safe_id, + tool_name=safe_name, + duration_ms=max(0, int((time.monotonic() - started) * 1000)), + status=status, + error_type=error_type[:128] if error_type else None, + ), + ) + + @typechecked + def close_open_tools(self, status: Literal["error", "cancelled"] = "cancelled") -> None: + with self.p_lock: + for tool_call_id in list(self.tool_starts): + self.emit_tool_completed( + tool_call_id, + self.tool_names.get(tool_call_id, "unknown"), + status=status, + error_type="turn_ended" if status == "error" else None, + ) + + @typechecked + def emit_completed(self, input_tokens: int = 0, output_tokens: int = 0) -> None: + with self.p_lock: + self.close_open_tools() + emit_agent_event( + self.sink, + TurnCompletedEvent( + **self.next_fields(), + duration_ms=self.duration_ms(), + input_tokens=input_tokens, + output_tokens=output_tokens, + ), + ) + + @typechecked + def emit_failed(self, error_type: str, retryable: bool = False) -> None: + with self.p_lock: + self.close_open_tools(status="error") + emit_agent_event( + self.sink, + TurnFailedEvent( + **self.next_fields(), + duration_ms=self.duration_ms(), + error_type=error_type[:128], + retryable=retryable, + ), + ) + + @typechecked + def next_fields(self) -> Dict[str, Any]: + fields: Dict[str, Any] = { + "session_id": self.session_id, + "turn_id": self.turn_id, + "sequence": self.sequence, + "monotonic_ms": int(time.monotonic() * 1000), + } + self.sequence += 1 + return fields + + @typechecked + def duration_ms(self) -> int: + return max(0, int((time.monotonic() - self.started_monotonic) * 1000)) diff --git a/backend/apps/agents/events/__init__.py b/backend/apps/agents/events/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/backend/apps/agents/manager/AgentLaunch.py b/backend/apps/agents/manager/AgentLaunch.py index 3b5445563..43258d1a3 100644 --- a/backend/apps/agents/manager/AgentLaunch.py +++ b/backend/apps/agents/manager/AgentLaunch.py @@ -17,7 +17,7 @@ ) from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.settings.settings import load_settings -from backend.apps.agents.manager.session.session_store import load_session_data +from backend.apps.agents.manager.session.session_store import load_session_data, save_session from backend.apps.agents.manager.session.apply_context_window import apply_context_window from backend.apps.agents.manager.session.workspace_git import ( detect_git_identity, @@ -81,8 +81,18 @@ async def launch_agent(self, config: AgentConfig) -> AgentSession: or global_settings.default_folder or os.path.expanduser("~") ) + # An owned session lives under its owner's workspace root when the build has one (hosted); the desktop has none. + owned_workspace = False + try: + from backend.apps.hosting.policy import hosting_policy + owner_root = hosting_policy().owned_workspace_root(config.owner_account_id) + if owner_root: + effective_cwd = os.path.join(owner_root, session_id) + owned_workspace = True + except Exception: + logger.exception("owned workspace routing failed; using default cwd") - if config.mode in ("view-builder", "skill-builder") and not config.target_directory: + if config.mode in ("view-builder", "skill-builder") and not config.target_directory and not owned_workspace: effective_cwd = os.path.join(effective_cwd, session_id) os.makedirs(effective_cwd, exist_ok=True) @@ -98,6 +108,7 @@ async def launch_agent(self, config: AgentConfig) -> AgentSession: workspace_id=session_id, folder=effective_cwd, session_id=session_id, + owner_account_id=config.owner_account_id, ) if output_id: # Broadcast the new row so the Apps sidebar lights up immediately, even before the user clicks into it. The row name is still the placeholder ("Untitled App") at this point; the post-session meta-sync below fires a second upsert with the real name once the agent has written meta.json. @@ -138,6 +149,7 @@ async def launch_agent(self, config: AgentConfig) -> AgentSession: repo_url=repo_url, branch=branch_name, dashboard_id=config.dashboard_id, + owner_account_id=config.owner_account_id, workflow_run_id=config.workflow_run_id, workflow_edit_id=config.workflow_edit_id, thinking_level=getattr(global_settings, "default_thinking_level", "auto"), @@ -145,6 +157,18 @@ async def launch_agent(self, config: AgentConfig) -> AgentSession: apply_context_window(session, global_settings) self.sessions[session_id] = session + # Snapshot at birth. Until now a launched-but-quiet session lived only in memory until its + # first turn ended (the turn snapshot), the chat was closed, or the backend shut down + # gracefully (persist_all_sessions). A crash or SIGKILL in between left no file, so the + # respawned backend could not promote the session into its dashboard's list, the renderer + # treated that scoped list as authority, stripped the card, and the debounced layout save + # persisted the loss: the board forgot a chat the user had just opened. With the file on + # disk a respawn finds it (reconcile_on_startup marks it stopped, the card returns as the + # parked chat it was), which is exactly what a graceful shutdown already gave it. + try: + save_session(session_id, session.model_dump(mode="json")) + except Exception: + logger.warning(f"launch: could not snapshot session {session_id}", exc_info=True) await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, diff --git a/backend/apps/agents/manager/AgentManagerProtocol.py b/backend/apps/agents/manager/AgentManagerProtocol.py index 904f531af..1baf2badb 100644 --- a/backend/apps/agents/manager/AgentManagerProtocol.py +++ b/backend/apps/agents/manager/AgentManagerProtocol.py @@ -21,10 +21,14 @@ from backend.apps.agents.manager.run.client_pool import ClientHandle from backend.apps.agents.manager.streaming.HookContext import HookContext from backend.apps.agents.manager.streaming.PartialReply import PartialReply + from backend.apps.agents.manager.session.SessionStore import SessionStore + from backend.apps.agents.events.AgentEventSink import AgentEventSink class AgentManagerProtocol: # State set in AgentManager.__init__. + store: SessionStore + event_sink: AgentEventSink sessions: Dict[str, AgentSession] tasks: Dict[str, asyncio.Task] live_partial: Dict[str, PartialReply] diff --git a/backend/apps/agents/manager/Messaging.py b/backend/apps/agents/manager/Messaging.py index 728618dc9..cb17152b1 100644 --- a/backend/apps/agents/manager/Messaging.py +++ b/backend/apps/agents/manager/Messaging.py @@ -76,7 +76,7 @@ async def send_message( self.sessions[session_id] = session else: raise ValueError(f"Session {session_id} not found") - + existing = self.tasks.get(session_id) if existing and not existing.done(): # A mid-turn message used to be silently dropped here (no bubble, no trace); queue it and the turn task's done callback replays it. @@ -175,7 +175,7 @@ async def send_message( logger.warning(f"[browser-fast-path] gate error, normal path: {e}") if fast_verdict != "no": - task = asyncio.create_task(run_browser_fast_path(session, session_id, prompt, selected_browser_ids, fast_brief, fast_verdict)) + task = asyncio.create_task(run_browser_fast_path(session, session_id, prompt, selected_browser_ids, fast_brief, fast_verdict, event_sink=self.event_sink)) else: task = asyncio.create_task(self.run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids, selected_app_output_ids=selected_app_output_ids, selected_setting_ids=selected_setting_ids)) self.register_turn_task(session_id, task) diff --git a/backend/apps/agents/manager/configure_provider_env.py b/backend/apps/agents/manager/configure_provider_env.py index 028263c50..d39d5b6d3 100644 --- a/backend/apps/agents/manager/configure_provider_env.py +++ b/backend/apps/agents/manager/configure_provider_env.py @@ -8,6 +8,10 @@ from typeguard import typechecked from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.manager.provider_runtime import ( + DEFAULT_PROVIDER_RUNTIME, + ProviderRuntime, +) from backend.apps.settings.models import AppSettings from backend.auth import get_auth_token @@ -15,16 +19,17 @@ @typechecked -async def router_available(global_settings: AppSettings) -> bool: +async def router_available( + global_settings: AppSettings, + runtime: ProviderRuntime = DEFAULT_PROVIDER_RUNTIME, +) -> bool: """True when 9Router is up, reviving it first if it died. A dead router must never masquerade as "no provider configured": detection now shares the dispatch path's lazy-start, so a crashed or orphaned router self-heals on the very next send instead of erroring the turn. Revival is gated on EVIDENCE of a provider (a settings key, proxy mode, or an active connection in the router's on-disk db) so a zero-config user keeps the clean no-provider message instead of us booting a router with nothing to route.""" - from backend.apps.nine_router import ensure_running as p_ensure, is_running as p_running - from backend.apps.nine_router.process import has_persisted_connections - if p_running(): + if runtime.router_is_running(): return True p_evidence = any([ getattr(global_settings, "anthropic_api_key", None), @@ -33,13 +38,13 @@ async def router_available(global_settings: AppSettings) -> bool: getattr(global_settings, "openrouter_api_key", None), getattr(global_settings, "connection_mode", "own_key") in ("openswarm-pro", "free-trial"), bool(getattr(global_settings, "custom_providers", None) or []), - has_persisted_connections(), + runtime.has_persisted_connections(), ]) if not p_evidence: return False logger.info("[MCP-DEBUG] 9Router down at provider detection; reviving before concluding") - await p_ensure() - return p_running() + await runtime.ensure_router_running() + return runtime.router_is_running() @typechecked @@ -49,8 +54,8 @@ async def configure_provider_env( resolved_model: object, api_type: Optional[str], global_settings: AppSettings, + runtime: ProviderRuntime = DEFAULT_PROVIDER_RUNTIME, ) -> None: - from backend.apps.nine_router import is_running as nine_router_running from backend.apps.agents.providers.registry import NINEROUTER_MODEL_PREFIXES as NINEROUTER_MODEL_PREFIXES resolved_is_9router = isinstance(resolved_model, str) and resolved_model.startswith(NINEROUTER_MODEL_PREFIXES) @@ -84,11 +89,10 @@ async def configure_provider_env( logger.info(f"[MCP-DEBUG] Using direct OpenAI API key (route=api) for {session.model} via openai-passthrough") elif is_pinned_api_route and api_route_provider == "custom": # User OpenAI-compatible endpoint (Ollama/Together/LM Studio) via 9Router's synced provider node. - from backend.apps.nine_router import ensure_running as p_9r_ensure_c - if not nine_router_running(): + if not runtime.router_is_running(): logger.info(f"[MCP-DEBUG] custom provider selected but 9Router not running; waiting for startup") - await p_9r_ensure_c() - if not nine_router_running(): + await runtime.ensure_router_running() + if not runtime.router_is_running(): raise ValueError( "9Router could not start. Custom OpenAI-compatible " "providers need 9Router to translate the Anthropic " @@ -104,8 +108,9 @@ async def configure_provider_env( if cp: # Local servers often run auth-disabled; placeholder key since the OpenAI SDK requires non-empty. env["OPENAI_API_KEY"] = (cp.api_key or "").strip() or "no-auth-required" - from backend.apps.nine_router import normalize_openai_compat_base_url as norm_cp_url - env["OPENAI_BASE_URL"] = norm_cp_url(cp.base_url or "") + env["OPENAI_BASE_URL"] = runtime.normalize_openai_compat_base_url( + cp.base_url or "" + ) # Pin subagents or CLI's default Haiku 4.5 404s on the custom provider. if global_settings.anthropic_api_key: env["CLAUDE_CODE_SUBAGENT_MODEL"] = "claude-sonnet-4-6" @@ -130,11 +135,10 @@ async def configure_provider_env( logger.info(f"[MCP-DEBUG] Using direct Google API key (route=api) for {session.model} via local proxy") elif api_type == "openrouter" and getattr(global_settings, "openrouter_api_key", None): # OpenRouter via 9Router; with no Anthropic key/sub, fall back to OR's resold Claude for subagents (incl. WebSearch delegation) so they stay on the same OR billing. - if not nine_router_running(): - from backend.apps.nine_router import ensure_running as nine_router_ensure + if not runtime.router_is_running(): logger.info(f"[MCP-DEBUG] OpenRouter selected but 9Router not running; waiting for startup") - await nine_router_ensure() - if not nine_router_running(): + await runtime.ensure_router_running() + if not runtime.router_is_running(): raise ValueError( "9Router could not start. OpenRouter routing requires " "Node.js, install it and restart the app, or pick a " @@ -157,8 +161,7 @@ async def configure_provider_env( options_kwargs["env"] = env logger.info(f"[MCP-DEBUG] Using OpenRouter for {session.model}") elif api_type == "anthropic" and not resolved_is_9router and getattr(global_settings, "connection_mode", "own_key") in ("openswarm-pro", "free-trial"): - from backend.apps.settings.credentials import proxy_auth - bearer, proxy_url = proxy_auth(global_settings) + bearer, proxy_url = runtime.proxy_auth(global_settings) bearer = bearer or "" options_kwargs["env"] = { "ANTHROPIC_AUTH_TOKEN": bearer, @@ -179,7 +182,7 @@ async def configure_provider_env( elif api_type == "anthropic" and not resolved_is_9router and global_settings.anthropic_api_key: options_kwargs["env"] = {"ANTHROPIC_API_KEY": global_settings.anthropic_api_key} logger.info("[MCP-DEBUG] Using direct Anthropic API key") - elif await router_available(global_settings): + elif await router_available(global_settings, runtime): # Gemini-bound ids go through the local proxy for schema scrubbing; everything else hits 9Router directly. is_gemini_bound = ( isinstance(resolved_model, str) diff --git a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py index 9fc4f1e83..6eb9c1c14 100644 --- a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py +++ b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py @@ -3,12 +3,13 @@ map, and the registered MCP servers; lifted out of the agent loop and covered by the MCP-gate invariant tests. Returns (allowed, disallowed).""" -from typing import Dict, List, Tuple +from typing import Dict, FrozenSet, List, Tuple from typeguard import typechecked from backend.apps.agents.core.models import AgentSession from backend.apps.agents.manager.permissions import path_gate +from backend.apps.hosting.policy import MUTATING_BUILTINS, hosting_policy from backend.apps.agents.manager.prompt.tool_catalog import ( FULL_TOOLS, get_all_known_tool_names, @@ -25,6 +26,20 @@ READ_ONLY_BLOCKED_TOOLS = ("Edit", "Bash", "NotebookEdit") + +def p_apply_builtin_denials( + allowed_tools: List[str], + disallowed_tools: List[str], + denials: FrozenSet[str], +) -> Tuple[List[str], List[str]]: + allowed = [tool for tool in allowed_tools if tool not in denials] + denied = list(disallowed_tools) + for tool in sorted(denials): + if tool not in denied: + denied.append(tool) + return allowed, denied + + @typechecked def build_effective_tool_lists( session: AgentSession, @@ -130,6 +145,19 @@ def build_effective_tool_lists( for wt_name in ("WebSearch", "WebFetch"): if wt_name not in effective_disallowed: effective_disallowed.append(wt_name) + + # The build's per-session denial list; fail closed to the mutating built-ins for an owned session whose policy cannot be consulted. + try: + denials = hosting_policy().builtin_tool_denials(session) + except Exception: + denials = MUTATING_BUILTINS if session.owner_account_id else frozenset() + if denials: + effective_allowed, effective_disallowed = p_apply_builtin_denials( + effective_allowed, + effective_disallowed, + denials, + ) + # With the openswarm-ui server live, the built-in AskUserQuestion is swapped for AskUI (same # Agent->SpawnAgent playbook: prompt nudges lose to the trained prior, a hard deny doesn't). # AskUI's option-list/question-flow cover the flat-choice cases; denying the built-in is what diff --git a/backend/apps/agents/manager/permissions/gate_hooks.py b/backend/apps/agents/manager/permissions/gate_hooks.py index 76864ab3f..f3d9dc2b6 100644 --- a/backend/apps/agents/manager/permissions/gate_hooks.py +++ b/backend/apps/agents/manager/permissions/gate_hooks.py @@ -179,6 +179,8 @@ async def offer_from_prompt(): if decision.behavior == "allow": if tool_use_id: ctx.tool_start_times[tool_use_id] = time.time() + if ctx.event_emitter is not None: + ctx.event_emitter.emit_tool_started(tool_use_id, tool_name) return { "hookSpecificOutput": { "hookEventName": hook_event, @@ -195,4 +197,6 @@ async def offer_from_prompt(): if tool_use_id: ctx.tool_start_times[tool_use_id] = time.time() + if ctx.event_emitter is not None: + ctx.event_emitter.emit_tool_started(tool_use_id, tool_name) return {} diff --git a/backend/apps/agents/manager/prompt/compose_turn_system_prompt.py b/backend/apps/agents/manager/prompt/compose_turn_system_prompt.py index 4c949788e..51afac498 100644 --- a/backend/apps/agents/manager/prompt/compose_turn_system_prompt.py +++ b/backend/apps/agents/manager/prompt/compose_turn_system_prompt.py @@ -59,10 +59,13 @@ def compose_turn_system_prompt( tz_name = tz_name or "UTC" now_local = datetime.now(ZoneInfo(tz_name)) tz_abbr = now_local.strftime("%Z") or tz_name + # No `%-d` / `%-I`: those are glibc extensions. On Windows strftime raises ValueError on + # them, the except below swallowed it, and the agent shipped without a clock on that OS. + hour = now_local.strftime("%I:%M %p").lstrip("0") time_ctx = ( "\n" - f"Today is {now_local.strftime('%A, %B %-d, %Y')}.\n" - f"Local time: {now_local.strftime('%-I:%M %p')} {tz_abbr} ({tz_name}).\n" + f"Today is {now_local.strftime('%A, %B')} {now_local.day}, {now_local.year}.\n" + f"Local time: {hour} {tz_abbr} ({tz_name}).\n" "Use this as ground truth for any date/time/day-of-week question. The timezone also " "gives the user's coarse region; when they say 'here' or 'near me' without a place, " "infer the likely city from it (say you inferred it) instead of claiming you can't know.\n" diff --git a/backend/apps/agents/manager/provider_runtime.py b/backend/apps/agents/manager/provider_runtime.py new file mode 100644 index 000000000..8159728d7 --- /dev/null +++ b/backend/apps/agents/manager/provider_runtime.py @@ -0,0 +1,47 @@ +"""Injected runtime boundary for provider routing.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from backend.apps import nine_router +from backend.apps.nine_router import process as nine_router_process +from backend.apps.settings import credentials +from backend.apps.settings.models import AppSettings + + +@runtime_checkable +class ProviderRuntime(Protocol): + """Operations provider routing needs from sibling applications.""" + + def router_is_running(self) -> bool: ... + + async def ensure_router_running(self) -> None: ... + + def has_persisted_connections(self) -> bool: ... + + def normalize_openai_compat_base_url(self, base_url: str) -> str: ... + + def proxy_auth(self, settings: AppSettings) -> tuple[str | None, str | None]: ... + + +class DefaultProviderRuntime: + """Production adapter; dynamic lookups preserve established test seams.""" + + def router_is_running(self) -> bool: + return nine_router.is_running() + + async def ensure_router_running(self) -> None: + await nine_router.ensure_running() + + def has_persisted_connections(self) -> bool: + return nine_router_process.has_persisted_connections() + + def normalize_openai_compat_base_url(self, base_url: str) -> str: + return nine_router.normalize_openai_compat_base_url(base_url) + + def proxy_auth(self, settings: AppSettings) -> tuple[str | None, str | None]: + return credentials.proxy_auth(settings) + + +DEFAULT_PROVIDER_RUNTIME: ProviderRuntime = DefaultProviderRuntime() diff --git a/backend/apps/agents/manager/run/RunOptions.py b/backend/apps/agents/manager/run/RunOptions.py index f2ce7e059..46aff689d 100644 --- a/backend/apps/agents/manager/run/RunOptions.py +++ b/backend/apps/agents/manager/run/RunOptions.py @@ -33,13 +33,11 @@ pre_send_context_guard, set_framework_overhead, register_web_mcp_server, append_web_tools_hint, inject_thinking_options, merge_hard_blocked_tools, ) +from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol logger = logging.getLogger(__name__) -from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol - - class RunOptions(AgentManagerProtocol): # No return annotation: the returned tuple carries an SDK ClaudeAgentOptions, which can't be module-imported here (mock-mode would fail to import the manager); it's lazy-imported below. @typechecked @@ -85,6 +83,9 @@ async def pre_tool_hook(input_data, tool_use_id, context): async def post_tool_hook(input_data, tool_use_id, context): return await post_tool_hook_mod.post_tool_hook(hook_ctx, input_data, tool_use_id, context) + + async def post_tool_failure_hook(input_data, tool_use_id, context): + return await post_tool_hook_mod.post_tool_failure_hook(hook_ctx, input_data, tool_use_id, context) _, mode_sys_prompt, _ = resolve_mode(session.mode, get_all_tool_names) # Reconcile active_mcps against currently-enabled tools (Phase 3). If the user toggled a server off in the Tools page mid-session, drop it from active_mcps automatically so the model isn't told "X is active" while build_mcp_servers silently filters it out. Emit a context_status event so the model and UI both know. @@ -193,6 +194,7 @@ async def stop_hook(input_data, tool_use_id, context): "hooks": { "PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])], "PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])], + "PostToolUseFailure": [HookMatcher(matcher=None, hooks=[post_tool_failure_hook])], "Stop": [HookMatcher(matcher=None, hooks=[stop_hook])], }, "allowed_tools": effective_allowed, diff --git a/backend/apps/agents/manager/run/TurnAdmission.py b/backend/apps/agents/manager/run/TurnAdmission.py new file mode 100644 index 000000000..932508059 --- /dev/null +++ b/backend/apps/agents/manager/run/TurnAdmission.py @@ -0,0 +1,46 @@ +import asyncio +import os +from contextlib import asynccontextmanager +from typing import AsyncIterator + +from typeguard import typechecked + +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.core.ws_manager import ws_manager + + +# Cap concurrent ROOT agent turns so firing 30 agents at once does not spawn 30 CLIs at once. +# The overflow queues. Env-tunable; 0/blank disables the gate. +MAX_CONCURRENT_TURNS = int(os.environ.get("OSW_MAX_CONCURRENT_TURNS", "8") or "0") + + +class TurnAdmission: + @typechecked + def get_turn_admission(self) -> asyncio.Semaphore: + """Return the admission semaphore for the current running loop.""" + loop = asyncio.get_running_loop() + if self.p_turn_admission_sema is None or self.p_turn_admission_loop is not loop: + self.p_turn_admission_sema = asyncio.Semaphore(MAX_CONCURRENT_TURNS) + self.p_turn_admission_loop = loop + return self.p_turn_admission_sema + + @asynccontextmanager + async def turn_admission_slot(self, session: AgentSession, session_id: str) -> AsyncIterator[None]: + """Hold one concurrency slot for a root turn; child turns bypass to avoid deadlock.""" + if MAX_CONCURRENT_TURNS <= 0 or session.parent_session_id is not None: + yield + return + sema = self.get_turn_admission() + was_queued = sema.locked() + if was_queued: + try: + await ws_manager.send_to_session(session_id, "agent:queued", {"session_id": session_id}) + except Exception: + pass + async with sema: + if was_queued: + try: + await ws_manager.send_to_session(session_id, "agent:admitted", {"session_id": session_id}) + except Exception: + pass + yield diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index 633017e41..e9f91aed0 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -27,13 +27,11 @@ ) from backend.apps.agents.manager.streaming import thinking as thinking_mod from backend.apps.settings.models import AppSettings +from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol logger = logging.getLogger(__name__) -from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol - - class TurnRunner(AgentManagerProtocol): # `options` is the SDK ClaudeAgentOptions, lazy-imported below (so mock-mode can import the manager without the SDK present), so it's left unannotated; everything else is typed. @typechecked @@ -126,13 +124,15 @@ async def p_run_streaming_turn(p_stream=None): if isinstance(message, StreamEvent): await handle_stream_event( - message, session, session_id, turn, thinking, self.live_partial + message, session, session_id, turn, thinking, self.live_partial, + turn.event_emitter, ) elif isinstance(message, AssistantMessage): flight_recorder.crumb(session_id, "assistant-msg") await handle_assistant_message( - message, session, session_id, turn, thinking, self.live_partial, self.sessions + message, session, session_id, turn, thinking, self.live_partial, + self.sessions, turn.event_emitter, ) elif isinstance(message, ResultMessage): flight_recorder.crumb(session_id, "result-msg", subtype=str(getattr(message, "subtype", ""))) @@ -260,6 +260,8 @@ async def p_finalize_interrupted_stream(): wait = 0.0 if wait is not None: capacity_retry_attempt += 1 + if turn.event_emitter is not None: + turn.event_emitter.close_open_tools(status="cancelled") flight_recorder.crumb(session_id, "transient-retry", attempt=capacity_retry_attempt, wait=wait, err=str(e)[:160]) mid_stream = turn.current_turn_emitted logger.warning( @@ -276,4 +278,3 @@ async def p_finalize_interrupted_stream(): options = ClaudeAgentOptions(**options_kwargs) continue raise - diff --git a/backend/apps/agents/manager/run/client_pool.py b/backend/apps/agents/manager/run/client_pool.py index 1f2815274..f16f21cec 100644 --- a/backend/apps/agents/manager/run/client_pool.py +++ b/backend/apps/agents/manager/run/client_pool.py @@ -76,7 +76,7 @@ class ClientHandle(BaseModel): model_config = ConfigDict(validate_assignment=True) fingerprint: str - client: InstanceOf[object] + client: Any lock: InstanceOf[asyncio.Lock] connected_at: float last_used: float @@ -99,6 +99,13 @@ class ClientHandle(BaseModel): # Timer cadence for the background reclaim; the acquire-time sweep is lazy (fires only when some session takes a turn), this one catches an all-quiet pool. SWEEP_INTERVAL_SECONDS = float(os.environ.get("OSW_CLIENT_SWEEP_INTERVAL_SECONDS", "60")) +# Hard ceiling on warm CLIs regardless of idle age: past this, the least-recently-used IDLE sessions are disposed (they respawn ~0.5s on their next message), bounding the "30 chats open" resident-memory case. Kept a SOFT cap: a mid-turn or just-acquired client is never evicted, so a burst of live turns may exceed it rather than kill work. +MAX_LIVE_CLIENTS = int(os.environ.get("OSW_CLIENT_MAX_LIVE", "12")) +# Never cap-evict a client used this recently; far larger than the acquire->lock window, so a just-acquired client can't be reaped before its turn takes the lock. +LRU_GUARD_SECONDS = float(os.environ.get("OSW_CLIENT_LRU_GUARD_SECONDS", "5")) +# Timer cadence for the background reclaim; the acquire-time sweep is lazy (fires only when some session takes a turn), this one catches an all-quiet pool. +SWEEP_INTERVAL_SECONDS = float(os.environ.get("OSW_CLIENT_SWEEP_INTERVAL_SECONDS", "60")) + @typechecked async def evict_idle_clients(pool: Dict[str, "ClientHandle"]) -> None: diff --git a/backend/apps/agents/manager/run_browser_fast_path.py b/backend/apps/agents/manager/run_browser_fast_path.py index f7491b747..0c3f80451 100644 --- a/backend/apps/agents/manager/run_browser_fast_path.py +++ b/backend/apps/agents/manager/run_browser_fast_path.py @@ -14,6 +14,8 @@ from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.agents.manager.session.session_store import save_session from backend.apps.settings.settings import load_settings +from backend.apps.agents.events.AgentEventSink import AgentEventSink, NullAgentEventSink +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter logger = logging.getLogger(__name__) @@ -26,6 +28,7 @@ async def run_browser_fast_path( selected_browser_ids: Optional[List[str]], brief: str = "", verdict: str = "act", + event_sink: Optional[AgentEventSink] = None, ) -> None: """Dispatch the browser sub-agent directly and reply with its outcome; the orchestrator LLM never runs. READ verdicts try one local fetch + @@ -35,6 +38,13 @@ async def run_browser_fast_path( task and the children.""" p_fp_t0 = time.monotonic() p_fp_path = verdict + p_event_emitter = AgentTurnEventEmitter( + sink=event_sink or NullAgentEventSink(), + session_id=session_id, + provider="browser_fast_path", + model=session.model, + ) + p_event_emitter.emit_started() logger.info(f"[browser-fast-path] direct dispatch for session {session_id} ({verdict})") text = "" # The fast-path skips the orchestrator, so the UI never gets the BrowserAgent tool-call that draws the "Browser Agent" bubble. Emit a synthetic tool_call/ tool_result pair (same shape + mcp__ name the orchestrator uses) so the bubble shows here too. None until we actually dispatch a browser (a pure READ answer has no browser, so no bubble). @@ -91,6 +101,7 @@ def p_summary(r: Dict[str, object]) -> str: if not text: # show the "Browser Agent" bubble during the dispatch (it renders as running, then completes when we emit the matching result below) p_bubble_tid = uuid4().hex + p_event_emitter.emit_tool_started(p_bubble_tid, p_browser_tool) p_tc = Message(role="tool_call", branch_id=session.active_branch_id, content={"id": p_bubble_tid, "tool": p_browser_tool, "input": {"task": prompt}}) session.messages.append(p_tc) @@ -128,6 +139,7 @@ def p_summary(r: Dict[str, object]) -> str: if not text: text = "The browser agent couldn't complete this and gave no report." except asyncio.CancelledError: + p_event_emitter.emit_failed("cancelled") raise except Exception as e: logger.warning(f"[browser-fast-path] dispatch failed: {e}") @@ -139,6 +151,7 @@ def p_summary(r: Dict[str, object]) -> str: ) # Close the synthetic bubble (always, even if the dispatch threw) so it never hangs as "running"; the bubble pairs this result with its call positionally. if p_bubble_tid: + p_event_emitter.emit_tool_completed(p_bubble_tid, p_browser_tool) # The bubble carries the same auditable record the sub-agent path shows. It used to close # with the literal string "done", so expanding it on this tier revealed nothing. from backend.apps.agents.browser import browser_trace @@ -156,6 +169,8 @@ def p_summary(r: Dict[str, object]) -> str: await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": p_tr.model_dump(mode="json")}) asst_msg = Message(role="assistant", content=text, branch_id=session.active_branch_id) + if text: + p_event_emitter.emit_first_token() session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -163,6 +178,7 @@ def p_summary(r: Dict[str, object]) -> str: }) session.status = "completed" session.closed_at = datetime.now() + p_event_emitter.emit_completed() await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "completed", diff --git a/backend/apps/agents/manager/session/SessionLifecycle.py b/backend/apps/agents/manager/session/SessionLifecycle.py index 12b998ad5..c8d90f7a3 100644 --- a/backend/apps/agents/manager/session/SessionLifecycle.py +++ b/backend/apps/agents/manager/session/SessionLifecycle.py @@ -4,6 +4,7 @@ import asyncio import logging +import os from datetime import datetime from typing import Dict, List, Optional, Set @@ -17,7 +18,9 @@ save_session, build_search_text, ) +from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol from backend.apps.agents.manager.session.apply_context_window import apply_context_window +from backend.apps.agents.manager.session.workspace_git import ensure_cwd_git_repo from backend.apps.agents.manager.session import resume_and_duplicate from backend.apps.agents.manager.view_builder_state import ( view_builder_render_retry_counts, @@ -31,9 +34,6 @@ P_NON_CHAT_MODES = {"browser-agent", "sub-agent", "invoked-agent", "app-agent"} -from backend.apps.agents.manager.AgentManagerProtocol import AgentManagerProtocol - - class SessionLifecycle(AgentManagerProtocol): @staticmethod @typechecked @@ -101,16 +101,11 @@ def purge_session_memory(self, session_id: str) -> None: close or delete can't strand stale per-session state that lives until the process dies. One chokepoint on purpose: a new per-session cache wires its eviction in HERE and both removal paths get it for free.""" - self.sessions.pop(session_id, None) - self.tasks.pop(session_id, None) - self.live_partial.pop(session_id, None) - self.cancel_events.pop(session_id, None) - self.pending_messages.pop(session_id, None) view_builder_render_retry_counts.pop(session_id, None) view_builder_dirty_sessions.discard(session_id) dispose_client_soon(self.client_pool, session_id) - self.hook_ctxs.pop(session_id, None) - self.stderr_buffers.pop(session_id, None) + self.pending_messages.pop(session_id, None) + self.store.purge_session_runtime(session_id) @typechecked async def delete_session(self, session_id: str) -> None: @@ -138,6 +133,35 @@ async def delete_session(self, session_id: str) -> None: delete_session_file(session_id) logger.info(f"Session {session_id} permanently deleted") + @typechecked + async def delete_sessions_for_owner(self, owner_account_id: str) -> int: + """Permanently delete every in-memory or persisted session for an owner.""" + session_ids = { + sid for sid, session in self.sessions.items() + if session.owner_account_id == owner_account_id + } + for sid, data in load_all_session_data(): + if data.get("owner_account_id") == owner_account_id: + session_ids.add(sid) + + deleted = 0 + failures = 0 + for sid in sorted(session_ids): + try: + if sid in self.sessions: + await self.delete_session(sid) + else: + delete_session_file(sid) + deleted += 1 + except Exception: + logger.exception("Failed to delete owned session %s during reset", sid) + failures += 1 + if failures: + raise RuntimeError( + f"failed to delete {failures} owned session(s) during reset" + ) + return deleted + @typechecked async def resume_session(self, session_id: str) -> AgentSession: if session_id in self.sessions: @@ -159,6 +183,7 @@ def get_history( limit: int = 20, offset: int = 0, dashboard_id: Optional[str] = None, + owner_account_id: Optional[str] = None, closed_only: bool = False, ) -> Dict: """Return paginated, optionally filtered summaries of sessions, live ones included.""" @@ -189,9 +214,11 @@ def get_history( # Children are machinery, not chats: a busy user's real history was buried under hundreds of "Browser Agent" rows. if data.get("mode") in P_NON_CHAT_MODES: continue - # The boot fetch wants CLOSED sessions only: open ones landing in the client's history map made its resurrection gate swallow their terminal frames. Search keeps the full pool (open sessions on other dashboards are reachable nowhere else). + # The boot fetch wants CLOSED sessions only: open ones landing in the client's history map made its resurrection gate swallow their terminal frames. if closed_only and not data.get("closed_at"): continue + if owner_account_id and data.get("owner_account_id") != owner_account_id: + continue if dashboard_id and data.get("dashboard_id") != dashboard_id: continue if q_lower: @@ -278,6 +305,35 @@ def p_dashboard_card_ids(self, dashboard_id: str) -> Set[str]: def get_session(self, session_id: str) -> Optional[AgentSession]: return self.sessions.get(session_id) + @typechecked + def is_hosted_owned_workspace(self, session: AgentSession) -> bool: + if not session.owner_account_id or not session.cwd: + return False + try: + from backend.apps.hosting.policy import hosting_policy + + owner_root = hosting_policy().owned_workspace_root(session.owner_account_id) + if not owner_root: + return False + root = os.path.abspath(owner_root) + cwd = os.path.abspath(session.cwd) + return cwd == root or cwd.startswith(root + os.sep) + except Exception: + return False + + @typechecked + def ensure_session_workspace_ready(self, session: AgentSession) -> bool: + """Recreate a missing owned workspace before handing cwd to the SDK.""" + if not session.cwd or os.path.isdir(session.cwd): + return False + if not self.is_hosted_owned_workspace(session): + return False + os.makedirs(session.cwd, exist_ok=True) + ensure_cwd_git_repo(session.cwd) + session.needs_fresh_session = True + logger.info("Recreated missing owned workspace for session %s", session.id) + return True + @typechecked def get_browser_agent_children(self, parent_session_id: str) -> List[dict]: """Return browser-agent sessions for a parent, from memory or disk.""" @@ -302,4 +358,3 @@ def get_browser_agent_children(self, parent_session_id: str) -> List[dict]: results.append(sess.model_dump(mode="json")) return results - diff --git a/backend/apps/agents/manager/session/SessionStore.py b/backend/apps/agents/manager/session/SessionStore.py new file mode 100644 index 000000000..21ae911e7 --- /dev/null +++ b/backend/apps/agents/manager/session/SessionStore.py @@ -0,0 +1,101 @@ +import asyncio +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + + +class SessionStore(BaseModel): + """Runtime-owned per-session maps for AgentManager; JSON persistence stays in session_store.py.""" + model_config = ConfigDict(validate_assignment=True, arbitrary_types_allowed=True) + + sessions: Dict[str, Any] = Field(default_factory=dict) + tasks: Dict[str, Any] = Field(default_factory=dict) + live_partial: Dict[str, Any] = Field(default_factory=dict) + cancel_events: Dict[str, asyncio.Event] = Field(default_factory=dict) + client_pool: Dict[str, Any] = Field(default_factory=dict) + hook_ctxs: Dict[str, Any] = Field(default_factory=dict) + stderr_buffers: Dict[str, List[str]] = Field(default_factory=dict) + + @typechecked + def get_session(self, session_id: str) -> Optional[Any]: + return self.sessions.get(session_id) + + @typechecked + def set_session(self, session_id: str, session: Any) -> None: + self.sessions[session_id] = session + + @typechecked + def pop_session(self, session_id: str) -> Optional[Any]: + return self.sessions.pop(session_id, None) + + @typechecked + def has_session(self, session_id: str) -> bool: + return session_id in self.sessions + + @typechecked + def session_values(self) -> List[Any]: + return list(self.sessions.values()) + + @typechecked + def session_items(self) -> List[tuple[str, Any]]: + return list(self.sessions.items()) + + @typechecked + def get_task(self, session_id: str) -> Optional[Any]: + return self.tasks.get(session_id) + + @typechecked + def set_task(self, session_id: str, task: Any) -> None: + self.tasks[session_id] = task + + @typechecked + def pop_task(self, session_id: str) -> Optional[Any]: + return self.tasks.pop(session_id, None) + + @typechecked + def is_live_task(self, session_id: str, task: Any) -> bool: + return self.tasks.get(session_id) is task + + @typechecked + def set_live_partial(self, session_id: str, value: Any) -> None: + self.live_partial[session_id] = value + + @typechecked + def pop_live_partial(self, session_id: str) -> Optional[Any]: + return self.live_partial.pop(session_id, None) + + @typechecked + def get_or_create_stderr_buffer(self, session_id: str) -> List[str]: + return self.stderr_buffers.setdefault(session_id, []) + + @typechecked + def get_hook_ctx(self, session_id: str) -> Optional[Any]: + return self.hook_ctxs.get(session_id) + + @typechecked + def set_hook_ctx(self, session_id: str, hook_ctx: Any) -> None: + self.hook_ctxs[session_id] = hook_ctx + + @typechecked + def get_cancel_event(self, session_id: str) -> Optional[asyncio.Event]: + return self.cancel_events.get(session_id) + + @typechecked + def set_cancel_event(self, session_id: str, event: asyncio.Event) -> None: + self.cancel_events[session_id] = event + + @typechecked + def purge_session_runtime(self, session_id: str) -> None: + self.sessions.pop(session_id, None) + self.tasks.pop(session_id, None) + self.live_partial.pop(session_id, None) + self.cancel_events.pop(session_id, None) + self.client_pool.pop(session_id, None) + self.hook_ctxs.pop(session_id, None) + self.stderr_buffers.pop(session_id, None) + + @typechecked + def clear_sessions_and_tasks(self) -> None: + self.sessions.clear() + self.tasks.clear() diff --git a/backend/apps/agents/manager/session/session_store.py b/backend/apps/agents/manager/session/session_store.py index a134e5966..5fcd28f9c 100644 --- a/backend/apps/agents/manager/session/session_store.py +++ b/backend/apps/agents/manager/session/session_store.py @@ -25,6 +25,16 @@ def load_session_data(session_id: str) -> Optional[Dict]: return read_json_or_none(os.path.join(sessions_dir(), f"{session_id}.json")) +@typechecked +def load_session_owner_account_id(session_id: str) -> Optional[str]: + """Read persisted ownership without hydrating or publishing the session.""" + data = load_session_data(session_id) + if data is None: + return None + owner_account_id = data.get("owner_account_id") + return owner_account_id if isinstance(owner_account_id, str) else None + + @typechecked def delete_session_file(session_id: str) -> None: path = os.path.join(sessions_dir(), f"{session_id}.json") diff --git a/backend/apps/agents/manager/streaming/HookContext.py b/backend/apps/agents/manager/streaming/HookContext.py index 38c7e91a2..d39f974f3 100644 --- a/backend/apps/agents/manager/streaming/HookContext.py +++ b/backend/apps/agents/manager/streaming/HookContext.py @@ -4,11 +4,12 @@ holds (pydantic keeps the instance, doesn't copy it), so hook-side mutations to status / pending_approvals are visible to the loop.""" -from typing import Dict +from typing import Dict, Optional from pydantic import BaseModel, ConfigDict, InstanceOf from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter class HookContext(BaseModel): @@ -21,6 +22,7 @@ class HookContext(BaseModel): policy_defaults: Dict[str, str] # The manager's LIVE session registry (InstanceOf keeps the reference, so a sub-agent the post hook spawns is visible to the manager; a plain Dict field pydantic would copy). sessions: InstanceOf[dict] + event_emitter: Optional[AgentTurnEventEmitter] = None # tool_use_id -> wall-clock start (s); pre records it, post pops it for elapsed_ms. tool_start_times: Dict[str, float] = {} # Consecutive ToolSearch calls; a run of these is the "looping on ToolSearch" wedge. diff --git a/backend/apps/agents/manager/streaming/handle_assistant_message.py b/backend/apps/agents/manager/streaming/handle_assistant_message.py index fb9d0c7b6..0286a8cdc 100644 --- a/backend/apps/agents/manager/streaming/handle_assistant_message.py +++ b/backend/apps/agents/manager/streaming/handle_assistant_message.py @@ -17,6 +17,7 @@ from backend.apps.agents.manager.streaming.upsert_message import upsert_message from backend.apps.agents.manager.streaming.PartialReply import PartialReply from backend.apps.agents.manager.streaming import thinking as thinking_mod +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter # The block types drive isinstance DISPATCH, so they must be real at runtime; imported inside the handler because by stream time the SDK is already resident (the turn's presence check imported it), keeping the 350ms sdk+mcp chain off the boot graph. from typing import TYPE_CHECKING @@ -36,6 +37,7 @@ async def handle_assistant_message( thinking: ThinkingState, live_partial: Dict[str, PartialReply], sessions: Dict[str, AgentSession], + event_emitter: Optional[AgentTurnEventEmitter] = None, ) -> None: from claude_agent_sdk.types import ThinkingBlock, TextBlock, ToolUseBlock @@ -66,6 +68,9 @@ async def handle_assistant_message( "input": block.input, }) + if content_parts and any(content_parts) and event_emitter is not None: + event_emitter.emit_first_token() + # Accumulate this AssistantMessage's contributions into the turn-level thinking pill. We re-emit the SAME message id each time so the frontend dedupes (addMessage replaces by id) and the bubble updates live as more thought / tools arrive. This is what gives us "Thought for 18s · 412 tokens · 3 tools used" reflecting the whole turn rather than just one think-step. NOTE: tool count is incremented in the content_block_start (block_type=="tool_use") branch above, NOT here. That path fires for both Anthropic and 9Router-translated providers; counting again here would double. If a provider somehow doesn't surface content_block_start for tool blocks but DOES surface them in the AssistantMessage envelope (defensive case), the max() in the consolidated emit will still pick up the higher count. if new_thinking_parts: thinking.text_parts.extend(new_thinking_parts) @@ -183,4 +188,3 @@ async def handle_assistant_message( turn.stream_text_msg_id = None turn.stream_tool_msg_ids_ordered = [] turn.stream_block_index_map = {} - diff --git a/backend/apps/agents/manager/streaming/handle_stream_event.py b/backend/apps/agents/manager/streaming/handle_stream_event.py index 33d911575..54fb10ae0 100644 --- a/backend/apps/agents/manager/streaming/handle_stream_event.py +++ b/backend/apps/agents/manager/streaming/handle_stream_event.py @@ -5,7 +5,7 @@ import time from datetime import datetime -from typing import Dict +from typing import Dict, Optional from uuid import uuid4 from typeguard import typechecked @@ -14,6 +14,7 @@ from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.agents.manager.streaming.state import ThinkingState, TurnState from backend.apps.agents.manager.streaming.PartialReply import PartialReply +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter # Runtime annotation stays `object` (the old ImportError fallback already admitted that); the real # type lives behind TYPE_CHECKING so importing this module stops paying the 350ms claude_agent_sdk+mcp chain at boot. @@ -33,6 +34,7 @@ async def handle_stream_event( turn: TurnState, thinking: ThinkingState, live_partial: Dict[str, PartialReply], + event_emitter: Optional[AgentTurnEventEmitter] = None, ) -> None: event = message.event event_type = event.get("type") @@ -89,6 +91,8 @@ async def handle_stream_event( if msg_id and delta_type == "text_delta": text_chunk = delta.get("text", "") + if text_chunk and event_emitter is not None: + event_emitter.emit_first_token() turn.assistant_text_chars += len(text_chunk) turn.stream_text_accum += text_chunk live_partial[session_id] = PartialReply( diff --git a/backend/apps/agents/manager/streaming/post_tool_hook.py b/backend/apps/agents/manager/streaming/post_tool_hook.py index 01bdae839..be652ef5d 100644 --- a/backend/apps/agents/manager/streaming/post_tool_hook.py +++ b/backend/apps/agents/manager/streaming/post_tool_hook.py @@ -25,6 +25,21 @@ logger = logging.getLogger(__name__) +@typechecked +async def post_tool_failure_hook( + ctx: HookContext, input_data: dict, tool_use_id, context +) -> Dict[str, object]: + if tool_use_id and ctx.event_emitter is not None: + ctx.tool_start_times.pop(tool_use_id, None) + ctx.event_emitter.emit_tool_completed( + str(tool_use_id), + input_data.get("tool_name", ""), + status="cancelled" if input_data.get("is_interrupt") else "error", + error_type="ToolExecutionError", + ) + return {} + + @typechecked async def post_tool_hook(ctx: HookContext, input_data: dict, tool_use_id, context) -> Dict[str, object]: session = ctx.session @@ -38,6 +53,14 @@ async def post_tool_hook(ctx: HookContext, input_data: dict, tool_use_id, contex # Accumulate per-tool latency on the session. Lets the cloud aggregate a tool-latency distribution into the existing daily.summary without firing per-tool events. hook_tool_name_early = input_data.get("tool_name", "") + if tool_use_id and ctx.event_emitter is not None: + is_error = bool(input_data.get("is_error")) + ctx.event_emitter.emit_tool_completed( + str(tool_use_id), + hook_tool_name_early, + status="error" if is_error else "success", + error_type="tool_error" if is_error else None, + ) if hook_tool_name_early and elapsed_ms is not None and elapsed_ms >= 0: latencies = getattr(session, "tool_latencies", None) if latencies is None: diff --git a/backend/apps/agents/manager/streaming/state.py b/backend/apps/agents/manager/streaming/state.py index 77afc4fdb..ee77be051 100644 --- a/backend/apps/agents/manager/streaming/state.py +++ b/backend/apps/agents/manager/streaming/state.py @@ -6,6 +6,7 @@ from typing import Dict, List, Optional from pydantic import BaseModel, ConfigDict, InstanceOf +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter class ThinkingState(BaseModel): @@ -35,6 +36,7 @@ class TurnState(BaseModel): model_config = ConfigDict(validate_assignment=True) + event_emitter: Optional[AgentTurnEventEmitter] = None stream_text_msg_id: Optional[str] = None stream_tool_msg_ids_ordered: List[str] = [] stream_block_index_map: Dict[int, str] = {} diff --git a/backend/apps/agents/manager/streaming/thinking.py b/backend/apps/agents/manager/streaming/thinking.py index 8d4c46354..884222219 100644 --- a/backend/apps/agents/manager/streaming/thinking.py +++ b/backend/apps/agents/manager/streaming/thinking.py @@ -177,4 +177,3 @@ async def ticker_loop(thinking: ThinkingState, turn: TurnState, session: AgentSe await emit_consolidated_thinking(thinking, turn, session, session_id, sessions) except asyncio.CancelledError: pass - diff --git a/backend/apps/agents/providers/registry.py b/backend/apps/agents/providers/registry.py index ae89e774d..b5e97c4db 100644 --- a/backend/apps/agents/providers/registry.py +++ b/backend/apps/agents/providers/registry.py @@ -271,6 +271,8 @@ def resolve_model_id_for_sdk(short_name: str, settings: AppSettings) -> str: return short_name if entry.get("route") == "cc": return entry.get("router_model_id", entry.get("model_id", short_name)) + if entry.get("route") == "api" and entry.get("api") == "openai": + return entry.get("router_model_id", entry.get("model_id", short_name)) if entry.get("route") == "api": # OpenAI own-key still rides 9Router (the cp-openai node fixes max_tokens + translates Anthropic->OpenAI), so it MUST keep its cp-openai/ routing prefix or 9Router has no node to dispatch to. Anthropic own-key goes straight to api.anthropic.com and Gemini own-key via the local proxy, both on the bare id. if entry.get("api") == "openai": diff --git a/backend/apps/agents/tools/ssrf_guard.py b/backend/apps/agents/tools/ssrf_guard.py index d5272fce7..262790886 100644 --- a/backend/apps/agents/tools/ssrf_guard.py +++ b/backend/apps/agents/tools/ssrf_guard.py @@ -4,11 +4,10 @@ incl. cloud metadata, CGNAT, multicast, ULA v6, etc). Resolution is async (non-blocking) and covers both IPv4 AND IPv6 via getaddrinfo. -Loopback (127/8, ::1) is INTENTIONALLY allowed because the desktop app's App -Builder previews servers on 127.0.0.1: and the agent needs to be able -to verify the built app actually runs. The user owns the loopback surface on -their own machine; the realistic SSRF threat for a desktop app is cloud -metadata (169.254.169.254) + internal corporate LANs, not localhost. +Direct and IPv4-mapped loopback are allowed for desktop deployments because +App Builder previews servers on 127.0.0.1: and the agent needs to +verify the built app actually runs. Hosted deployments block loopback, and +transition/local-use ranges never inherit the desktop exception. """ from __future__ import annotations @@ -20,6 +19,8 @@ import httpx +from backend.apps.hosting.policy import hosting_policy + logger = logging.getLogger(__name__) @@ -47,16 +48,23 @@ class DomainUnreachable(SSRFBlocked): ipaddress.ip_network("169.254.0.0/16"), # link-local incl. cloud metadata ipaddress.ip_network("100.64.0.0/10"), # CGNAT ipaddress.ip_network("224.0.0.0/4"), # multicast + ipaddress.ip_network("240.0.0.0/4"), # reserved + limited broadcast ipaddress.ip_network("0.0.0.0/8"), # "this network" ipaddress.ip_network("198.18.0.0/15"), # benchmarking + ipaddress.ip_network("192.0.2.0/24"), # TEST-NET-1 + ipaddress.ip_network("198.51.100.0/24"), # TEST-NET-2 + ipaddress.ip_network("203.0.113.0/24"), # TEST-NET-3 ] P_BLOCKED_V6_NETS = [ ipaddress.ip_network("fe80::/10"), # link-local + ipaddress.ip_network("fec0::/10"), # deprecated site-local ipaddress.ip_network("fc00::/7"), # ULA + ipaddress.ip_network("64:ff9b:1::/48"), # local-use NAT64 ipaddress.ip_network("ff00::/8"), # multicast ipaddress.ip_network("::/128"), # unspecified ] +P_NAT64_WELL_KNOWN = ipaddress.ip_network("64:ff9b::/96") async def p_resolve_host_async(host: str) -> list[str]: @@ -70,21 +78,39 @@ async def p_resolve_host_async(host: str) -> list[str]: def p_is_forbidden_ip(ip_str: str) -> bool: - """True iff this IP is in a blocked range. Loopback is allowed (see module docstring).""" + """True iff this IP is blocked for the current deployment.""" try: ip = ipaddress.ip_address(ip_str) except ValueError: return True # unparseable -> block - # v6 can carry a v4 target (v4-mapped ::ffff:, 6to4 2002::) and routes to it; judge by the embedded v4 or a private host slips past the v6 list. - if ip.version == 6: - embedded = ip.ipv4_mapped or ip.sixtofour - if embedded is not None: - ip = embedded - if ip.is_loopback: - return False - if ip.version == 4: + + if isinstance(ip, ipaddress.IPv4Address): + if ip.is_loopback: + return hosting_policy().blocks_loopback_targets() return any(ip in net for net in P_BLOCKED_V4_NETS) - return any(ip in net for net in P_BLOCKED_V6_NETS) + + # Only direct v6 and v4-mapped loopback belong to desktop previews. Other + # transition encodings must be judged as network targets, even when their + # embedded v4 address is loopback. + if ip.is_loopback: + return hosting_policy().blocks_loopback_targets() + mapped = ip.ipv4_mapped + if mapped is not None: + if mapped.is_loopback: + return hosting_policy().blocks_loopback_targets() + return any(mapped in net for net in P_BLOCKED_V4_NETS) + + if any(ip in net for net in P_BLOCKED_V6_NETS): + return True + + embedded = ip.sixtofour + if embedded is None and ip in P_NAT64_WELL_KNOWN: + embedded = ipaddress.IPv4Address(int(ip) & 0xFFFFFFFF) + if embedded is not None: + if embedded.is_loopback: + return True + return any(embedded in net for net in P_BLOCKED_V4_NETS) + return False async def assert_safe_url(url: str) -> str: diff --git a/backend/apps/health/health.py b/backend/apps/health/health.py index ed555db03..cbca53883 100644 --- a/backend/apps/health/health.py +++ b/backend/apps/health/health.py @@ -1,19 +1,38 @@ -from backend.config.Apps import SubApp from contextlib import asynccontextmanager +from collections.abc import Callable + +from fastapi import BackgroundTasks, status from fastapi.responses import PlainTextResponse from pydantic import BaseModel, ConfigDict from typeguard import typechecked -from fastapi import status, HTTPException + +from backend.config.Apps import SubApp + + +ready_background_task: Callable[[], None] | None = None + + +def set_ready_background_task(task: Callable[[], None] | None) -> None: + global ready_background_task + ready_background_task = task + @asynccontextmanager async def health_lifespan(): yield + health = SubApp("health", health_lifespan) + @health.router.get("/check") @typechecked -async def check() -> PlainTextResponse: +async def check(background_tasks: BackgroundTasks) -> PlainTextResponse: + if ready_background_task is not None: + # FastAPI runs this after the response body is sent. The Electron shell + # can mark the backend ready before cache population starts competing + # for disk, Defender scans, or the bundled Python interpreter. + background_tasks.add_task(ready_background_task) return PlainTextResponse( content="OK", status_code=status.HTTP_200_OK, diff --git a/backend/apps/hosting/__init__.py b/backend/apps/hosting/__init__.py new file mode 100644 index 000000000..f8cf4085c --- /dev/null +++ b/backend/apps/hosting/__init__.py @@ -0,0 +1,2 @@ +"""The hosting seam: the one place the app asks "who owns this request, and what does this +build allow?". See policy.py.""" diff --git a/backend/apps/hosting/policy.py b/backend/apps/hosting/policy.py new file mode 100644 index 000000000..95cf53bae --- /dev/null +++ b/backend/apps/hosting/policy.py @@ -0,0 +1,186 @@ +"""Hosting policy seam. + +On the desktop every request is the local user: nothing is scoped by owner, and every process-wide +answer below is the permissive default. A hosted (multi-tenant) build supplies its own policy through +`p_provider`; the rest of the app only ever talks to `REQUEST_SCOPE` (a FastAPI dependency) and +`hosting_policy()`, so no route or manager knows which build it is running in. + +Everything here is a plain default: the desktop scope allows, filters nothing, stamps no owner, and +`hosting_policy()` answers "not hosted" to every question. +""" +from __future__ import annotations + +from typing import Any, Awaitable, Callable, Iterable, List, Optional, Tuple, TypeVar + +from fastapi import Request, params +from typeguard import typechecked + +T = TypeVar("T") + +#: The built-in tools that change the machine or spawn work. A hosted build may deny them to trial +#: callers; they are also the fail-closed answer when an owned session's policy cannot be consulted. +MUTATING_BUILTINS: frozenset = frozenset({ + "Agent", + "Bash", + "CronCreate", + "CronDelete", + "Edit", + "EnterWorktree", + "InvokeAgent", + "NotebookEdit", + "TodoWrite", + "Write", +}) + + +class RequestScope: + """Who owns the current request, and what it may do. Desktop: nobody owns anything and + everything is allowed. A hosted build returns a subclass bound to the caller's account.""" + + #: True in a hosted build once the caller is resolved; the desktop is never hosted. + hosted: bool = False + #: The caller's account id in a hosted build; None on the desktop. + owner_id: Optional[str] = None + + # ---- ownership ----------------------------------------------------------------------- + @typechecked + def require_owner_of(self, owner_account_id: Optional[str]) -> None: + """Raise unless the caller owns the resource stamped with `owner_account_id`.""" + + @typechecked + def filter_owned(self, items: Iterable[T]) -> List[T]: + """Keep only the caller's items (items carry `owner_account_id`).""" + return list(items) + + @typechecked + def stamp_owner(self, item: Any) -> None: + """Mark a freshly created resource as the caller's (no-op on the desktop).""" + + @typechecked + def owner_for_new_resource(self, requested: Optional[str]) -> Optional[str]: + """The owner a new resource is created under: whatever the caller asked for on the desktop, + always the caller itself in a hosted build.""" + return requested + + @typechecked + def require_local_operator(self, what: str) -> None: + """Raise unless the caller is the machine's own operator (always, on the desktop). `what` + names the operation for the refusal message.""" + + # ---- agents --------------------------------------------------------------------------- + @typechecked + def sanitize_launch_config(self, config: T) -> T: + return config + + @typechecked + async def admit_launch( + self, config: Any, launch: Callable[[Any], Awaitable[Any]], + ) -> Tuple[Any, bool]: + """Run `launch(config)` under this scope's admission rules. Returns the session and whether + the caller should still run the launch's first turn (a hosted build may already have + recorded the launch as the durable side effect and answers False).""" + return await launch(config), True + + @typechecked + def admit_prompt( + self, + session: Any, + *, + requested_mode: Optional[str], + forced_tools: Optional[List[str]], + side_effect_payload: dict, + ) -> bool: + """Admit a prompt/edit for `session`. Returns True when the request was a replay of an + already-admitted one (the caller then answers `replayed` instead of running it).""" + return False + + @typechecked + async def authorize_approval( + self, + approval_session_id: Optional[str], + session_lookup: Callable[[str], Awaitable[Any]], + ) -> None: + """Raise unless the caller may answer the approval that `approval_session_id` is waiting on; + `session_lookup` resolves the owning session when the scope needs to check it.""" + + @typechecked + def resolve_approval(self, request_id: str, decision: dict, approval_session_id: Optional[str]) -> bool: + """Deliver an approval decision through the scope's own channel. Returns True when it did; + False means the caller delivers it through the desktop path.""" + return False + + # ---- outputs -------------------------------------------------------------------------- + @typechecked + def require_app_builder_enabled(self) -> None: + """Raise unless this caller may use the App Builder (always allowed on the desktop).""" + + @typechecked + def register_seeded_workspace(self, workspace_id: str, meta: Optional[dict]) -> Optional[str]: + """Record a seeded workspace as the caller's output when the build tracks ownership; returns + the output id it now maps to, or None when nothing is recorded (the desktop).""" + return None + + +class HostingPolicy: + """Process-wide answers. Desktop defaults throughout.""" + + enabled: bool = False + + @typechecked + def request_scope(self, request: Request) -> RequestScope: + return DESKTOP_SCOPE + + @typechecked + def owned_workspace_root(self, owner_account_id: Optional[str]) -> Optional[str]: + """The per-owner workspace root a session's cwd must live under, or None when sessions use + the ordinary cwd.""" + return None + + @typechecked + def builtin_tool_denials(self, session: Any) -> frozenset: + """Built-in tools this session may not use (empty on the desktop).""" + return frozenset() + + @typechecked + def tool_update_restricted(self, tool: Any, body: Any) -> bool: + """True when a tools-library update must be refused for this build.""" + return False + + @typechecked + def workflows_disabled(self) -> bool: + return False + + @typechecked + def blocks_loopback_targets(self) -> bool: + """True when SSRF checks must treat loopback as a network target (never on the desktop, + where local previews are the point).""" + return False + + +DESKTOP_SCOPE = RequestScope() +DESKTOP_POLICY = HostingPolicy() + +# A build that hosts replaces this with a provider that resolves its own policy. +p_provider: Callable[[], HostingPolicy] = lambda: DESKTOP_POLICY + + +@typechecked +def hosting_policy() -> HostingPolicy: + return p_provider() + + +@typechecked +def request_scope(request: Request) -> RequestScope: + """Resolve the caller's scope for a request (what REQUEST_SCOPE injects).""" + return hosting_policy().request_scope(request) + + +class p_RequestScopeDependency(RequestScope, params.Depends): + """The route default `scope: RequestScope = REQUEST_SCOPE`. FastAPI sees a dependency and injects + the caller's scope; a route called directly (tests do) gets this object, which is the desktop scope.""" + + def __init__(self) -> None: + params.Depends.__init__(self, dependency=request_scope) + + +REQUEST_SCOPE: RequestScope = p_RequestScopeDependency() diff --git a/backend/apps/nine_router/process.py b/backend/apps/nine_router/process.py index 8df0c34a0..9e171b729 100644 --- a/backend/apps/nine_router/process.py +++ b/backend/apps/nine_router/process.py @@ -567,7 +567,11 @@ async def p_ensure_running_impl(): env["ELECTRON_RUN_AS_NODE"] = "1" else: # Dev: install the pinned npm package into a local cache once, then spawn `node app/server.js` directly (bypasses the package cli.js tray icon users confusingly quit, its update-check spinner, and the TUI). - cached_server = p_ensure_router_cached() + # A cold npm install can take minutes on Windows. This path is started + # as a background task during app lifespan, so running the synchronous + # installer on the event loop would still block the HTTP server from + # binding and make the whole backend appear hung. + cached_server = await asyncio.to_thread(p_ensure_router_cached) if not cached_server: return node = p_find_node() diff --git a/backend/apps/nine_router/sync_custom.py b/backend/apps/nine_router/sync_custom.py index 44e854e8a..432669505 100644 --- a/backend/apps/nine_router/sync_custom.py +++ b/backend/apps/nine_router/sync_custom.py @@ -17,11 +17,14 @@ find_keyed_connection, nr, ) +from backend.apps.settings.credentials import proxy_auth logger = logging.getLogger(__name__) # We mirror settings.custom_providers[] with prefix `cp-` so they don't collide with the user's primary OpenAI key. NINE_ROUTER_CUSTOM_NAME_SUFFIX = " (OpenSwarm-managed)" +NINE_ROUTER_OPENAI_COMPAT_NAME = f"OpenAI{NINE_ROUTER_CUSTOM_NAME_SUFFIX}" +P_RESERVED_MANAGED_PREFIXES = {NINE_ROUTER_OPENAI_KEYED_PREFIX} async def sync_openai_compat_node(api_key: str | None) -> None: @@ -32,7 +35,7 @@ async def sync_openai_compat_node(api_key: str | None) -> None: import os as p_os port = p_os.environ.get("OPENSWARM_PORT", "8324") base_url = f"http://127.0.0.1:{port}/api/openai-passthrough/v1" - managed_name = f"OpenAI{NINE_ROUTER_CUSTOM_NAME_SUFFIX}" + managed_name = NINE_ROUTER_OPENAI_COMPAT_NAME try: async with nr().httpx.AsyncClient(timeout=5.0, headers=cli_auth_headers()) as client: @@ -186,6 +189,8 @@ async def sync_custom_providers(providers: list) -> None: api_key = api_key.strip() or "no-auth-required" slug = p_custom_provider_slug(name) prefix = f"cp-{slug}" + if prefix in P_RESERVED_MANAGED_PREFIXES: + prefix = f"cp-custom-{slug}" seen_prefixes.add(prefix) managed_name = f"{name.strip()}{NINE_ROUTER_CUSTOM_NAME_SUFFIX}" @@ -260,7 +265,7 @@ async def sync_custom_providers(providers: list) -> None: return for prefix, node in managed_by_prefix.items(): # cp-openai wears the same managed suffix but belongs to sync_openai_compat_node; reaping it here killed every gpt-*-api request with "No credentials". - if prefix in seen_prefixes or prefix == NINE_ROUTER_OPENAI_KEYED_PREFIX: + if prefix in P_RESERVED_MANAGED_PREFIXES or prefix in seen_prefixes or prefix == NINE_ROUTER_OPENAI_KEYED_PREFIX: continue try: async with nr().httpx.AsyncClient(timeout=5.0, headers=cli_auth_headers()) as client: @@ -331,7 +336,6 @@ async def sync_pro_routing(settings_obj) -> None: the bearer (activate, sign-in, sign-out, disconnect, free-trial arm/clear). Never raises.""" try: - from backend.apps.settings.credentials import proxy_auth bearer, base = proxy_auth(settings_obj) active = bool(bearer) await sync_openswarm_pro_as_claude( diff --git a/backend/apps/service/analytics/client.py b/backend/apps/service/analytics/client.py index 0cb491898..be2ab8130 100644 --- a/backend/apps/service/analytics/client.py +++ b/backend/apps/service/analytics/client.py @@ -17,6 +17,8 @@ from swarm_analytics import AnalyticsClient +from backend.apps.service.settings_gateway import DEFAULT_SETTINGS_GATEWAY, SettingsGateway + logger = logging.getLogger(__name__) P_CLIENT: Optional[AnalyticsClient] = None @@ -34,11 +36,10 @@ def p_base_url() -> str: @typechecked -def p_mode() -> str: +def p_mode(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> str: # logs.write is diagnostic so it flows even in 'minimal'; only product events are muted. try: - from backend.apps.settings.store import load_settings - if not getattr(load_settings(), "analytics_opt_in", True): + if not getattr(gateway.load(), "analytics_opt_in", True): return "minimal" except Exception: pass @@ -46,14 +47,13 @@ def p_mode() -> str: @typechecked -def get_analytics_client() -> Optional[AnalyticsClient]: +def get_analytics_client(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> Optional[AnalyticsClient]: # Lazy bootstrap + cache; returns None (callers no-op) when setup fails, e.g. offline first run. global P_CLIENT if P_CLIENT is not None: return P_CLIENT try: - from backend.apps.settings.store import load_settings, save_settings - s = load_settings() + s = gateway.load() install_id = getattr(s, "installation_id", None) if not install_id: return None @@ -62,8 +62,8 @@ def get_analytics_client() -> Optional[AnalyticsClient]: if not token: token = AnalyticsClient.register(base_url=base_url, install_id=install_id) s.analytics_token = token - save_settings(s) - P_CLIENT = AnalyticsClient(base_url=base_url, token=token, mode=p_mode()) + gateway.save(s) + P_CLIENT = AnalyticsClient(base_url=base_url, token=token, mode=p_mode(gateway)) except Exception as e: logger.debug("analytics setup failed (non-critical): %s", e) return None @@ -181,15 +181,19 @@ def track_onboarding_step(*, step_id: str, status: str) -> None: @typechecked -def persist_client_env(*, timezone: Optional[str] = None, locale: Optional[str] = None) -> None: +def persist_client_env( + *, + timezone: Optional[str] = None, + locale: Optional[str] = None, + gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY, +) -> None: # Store the renderer-reported tz/locale for the cloud envelope on dev/OSS runs; disk-write only when a value actually changed. tz = (timezone or "").strip() or None loc = (locale or "").strip() or None if tz is None and loc is None: return try: - from backend.apps.settings.store import load_settings, save_settings - s = load_settings() + s = gateway.load() changed = False if tz and getattr(s, "timezone", None) != tz: s.timezone = tz @@ -198,7 +202,7 @@ def persist_client_env(*, timezone: Optional[str] = None, locale: Optional[str] s.locale = loc changed = True if changed: - save_settings(s) + gateway.save(s) except Exception as e: logger.debug("analytics persist_client_env failed: %s", e) diff --git a/backend/apps/service/client.py b/backend/apps/service/client.py index 705f340be..8ccee342a 100644 --- a/backend/apps/service/client.py +++ b/backend/apps/service/client.py @@ -30,7 +30,9 @@ import httpx from backend.apps.service import buffer +from backend.apps.service.settings_gateway import DEFAULT_SETTINGS_GATEWAY, SettingsGateway from backend.apps.service.version import APP_VERSION +from backend.apps.settings.redaction import redact_settings logger = logging.getLogger(__name__) @@ -46,11 +48,10 @@ P_MAX_INFLIGHT = 16 -def resolve_timezone() -> str: +def resolve_timezone(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> str: """Settings-first (the only source that works on dev / OSS), then OS, then UTC.""" try: - from backend.apps.settings.store import load_settings - tz = getattr(load_settings(), "timezone", None) + tz = getattr(gateway.load(), "timezone", None) if tz: return tz except Exception: @@ -68,11 +69,10 @@ def resolve_timezone() -> str: return "UTC" -def resolve_locale() -> str: +def resolve_locale(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> str: """Best-effort BCP-47 locale, settings-first then OS, defaulting to en-US.""" try: - from backend.apps.settings.store import load_settings - loc = getattr(load_settings(), "locale", None) + loc = getattr(gateway.load(), "locale", None) if loc: return loc except Exception: @@ -108,31 +108,29 @@ def set_test_sink(fn: Optional[Any]) -> None: test_sink = fn -def p_get_install_id() -> str: +def p_get_install_id(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> str: global install_id if install_id: return install_id try: - from backend.apps.settings.store import load_settings, save_settings - s = load_settings() + s = gateway.load() iid = getattr(s, "installation_id", None) if not iid: iid = uuid4().hex s.installation_id = iid - save_settings(s) + gateway.save(s) install_id = iid except Exception: install_id = uuid4().hex return install_id -def p_get_user_id() -> Optional[str]: +def p_get_user_id(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> Optional[str]: global p_user_id if p_user_id: return p_user_id try: - from backend.apps.settings.store import load_settings - s = load_settings() + s = gateway.load() # Prefer the cloud-issued user_id (UUID) if the user has signed in via Google OAuth, magic link, or Stripe checkout; that's the authoritative identity. Falls back to user_email for installs that haven't completed sign-in yet (so existing onboarding-only installs don't lose their Person history during the v1.0.29 rollout). After every install signs in, this fallback drops out. return ( getattr(s, "user_id", None) @@ -148,14 +146,13 @@ def set_user_id(uid: Optional[str]) -> None: p_user_id = uid or None -def p_is_enabled(kind: str) -> bool: +def p_is_enabled(kind: str, gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> bool: """Honour user opt-out. Diagnostic always flows (errors block usability); state + session honour the toggle.""" if kind == "diagnostic": return True try: - from backend.apps.settings.store import load_settings - s = load_settings() + s = gateway.load() mode = getattr(s, "service_diagnostics_mode", None) if mode == "minimal": return False @@ -166,10 +163,10 @@ def p_is_enabled(kind: str) -> bool: return True -def p_envelope() -> dict: +def p_envelope(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> dict: """Identity + environment metadata stamped on every submission.""" - env: dict[str, Any] = {"install_id": p_get_install_id()} - uid = p_get_user_id() + env: dict[str, Any] = {"install_id": p_get_install_id(gateway)} + uid = p_get_user_id(gateway) if uid: env["user_id"] = uid try: @@ -209,18 +206,16 @@ def p_envelope() -> dict: return env -def p_base_url() -> str: +def p_base_url(gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> str: try: - from backend.apps.settings.store import load_settings - from backend.apps.settings.credentials import OPENSWARM_DEFAULT_PROXY_URL - s = load_settings() - return (getattr(s, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL).rstrip("/") + s = gateway.load() + return (getattr(s, "openswarm_proxy_url", None) or gateway.default_proxy_url()).rstrip("/") except Exception: return P_DEFAULT_BASE -async def p_post(path: str, body: dict) -> int | None: - url = f"{p_base_url()}{path}" +async def p_post(path: str, body: dict, gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> int | None: + url = f"{p_base_url(gateway)}{path}" try: async with httpx.AsyncClient(timeout=P_TIMEOUT_SECONDS) as c: r = await c.post(url, json=body) @@ -239,23 +234,33 @@ def p_retryable(status: int | None) -> bool: return status is None or status >= 500 or status in (408, 429) -async def p_post_or_spool(path: str, body: dict, kind: str) -> None: +def p_redact_sync_body(body: dict) -> dict: + """Sanitize a service-sync envelope before network or retry persistence.""" + safe = redact_settings(body) + payload = body.get("d") + if isinstance(payload, dict): + safe["d"] = redact_settings(payload) + return safe + + +async def p_post_or_spool(path: str, body: dict, kind: str, gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> None: global p_inflight + safe_body = p_redact_sync_body(body) if test_sink is not None: try: - test_sink(kind, body) + test_sink(kind, safe_body) except Exception as e: logger.debug("test sink raised: %s", e) return async with p_inflight_lock: if p_inflight >= P_MAX_INFLIGHT: - buffer.enqueue(spool_path(), f"{kind}:{path}", body, now=time.time()) + buffer.enqueue(spool_path(), f"{kind}:{path}", safe_body, now=time.time()) return p_inflight += 1 try: - status = await p_post(path, body) + status = await p_post(path, safe_body, gateway) if p_retryable(status): - buffer.enqueue(spool_path(), f"{kind}:{path}", body, now=time.time()) + buffer.enqueue(spool_path(), f"{kind}:{path}", safe_body, now=time.time()) elif not p_delivered(status): logger.warning("service POST %s rejected with HTTP %s; payload dropped", path, status) finally: @@ -263,7 +268,7 @@ async def p_post_or_spool(path: str, body: dict, kind: str) -> None: p_inflight = max(0, p_inflight - 1) -async def drain_spool(batch_size: int = 50) -> int: +async def drain_spool(batch_size: int = 50, gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> int: async with p_drain_lock: entries = buffer.drain(spool_path(), batch_size=batch_size) if not entries: @@ -274,7 +279,7 @@ async def drain_spool(batch_size: int = 50) -> int: if not path: succeeded.append(rid) continue - status = await p_post(path, body) + status = await p_post(path, body, gateway) if p_delivered(status): succeeded.append(rid) elif p_retryable(status): @@ -298,7 +303,7 @@ def p_log(kind: str, payload: dict) -> None: pass -def sync(data: dict | None = None) -> None: +def sync(data: dict | None = None, gateway: SettingsGateway = DEFAULT_SETTINGS_GATEWAY) -> None: """Sync operational state to the cloud. Single entry point. Accepts any dict; the cloud determines what it is from the shape. @@ -313,10 +318,10 @@ def sync(data: dict | None = None) -> None: Fire-and-forget; never raises. """ payload = data or {} - if not p_is_enabled("state"): + if not p_is_enabled("state", gateway): return body = { - "client_state": p_envelope(), + "client_state": p_envelope(gateway), "d": payload, "t": time.time(), "submission_id": uuid4().hex, @@ -324,11 +329,11 @@ def sync(data: dict | None = None) -> None: p_log("s", payload) if test_sink is not None: try: - test_sink("s", body) + test_sink("s", p_redact_sync_body(body)) except Exception as e: logger.debug("test sink raised: %s", e) return - p_schedule(p_post_or_spool(P_DEFAULT_SYNC_PATH, body, "s")) + p_schedule(p_post_or_spool(P_DEFAULT_SYNC_PATH, body, "s", gateway)) # Internal routing; the cloud has one endpoint for everything. diff --git a/backend/apps/service/service.py b/backend/apps/service/service.py index 1971a2fa6..da2de7e9e 100644 --- a/backend/apps/service/service.py +++ b/backend/apps/service/service.py @@ -29,6 +29,8 @@ from backend.config.Apps import SubApp from backend.config.paths import SESSIONS_DIR from backend.apps.service import client as svc +from backend.apps.service import service_runtime +from backend.apps.service import settings_gateway from backend.apps.service.version import APP_VERSION, read_app_version logger = logging.getLogger(__name__) @@ -44,6 +46,10 @@ P_RESTART_THRESHOLD = 1.0 +def should_autostart_9router() -> bool: + return os.environ.get("OPENSWARM_DISABLE_9ROUTER_AUTOSTART") != "1" + + def p_compute_delta(current: float, last: float | None, threshold: float = P_RESTART_THRESHOLD) -> tuple[float, float]: if last is None: return 0.0, current @@ -78,9 +84,9 @@ async def p_pulse_loop(): cost_delta = 0.0 try: - from backend.apps.nine_router import get_usage_stats, is_running as p_9r_running - if p_9r_running(): - stats = await get_usage_stats() + router = service_runtime.DEFAULT_ROUTER_USAGE + if router.is_running(): + stats = await router.get_usage_stats() if stats: cur_cost = stats.get("totalCost", 0) or 0 cur_prompt = stats.get("totalPromptTokens", 0) or 0 @@ -96,10 +102,9 @@ async def p_pulse_loop(): if p_pulse_count >= p_pulse_batch_size: try: - from backend.apps.agents.agent_manager import agent_manager # Compact field names; the wire stays small and the cloud is the only place that knows what each key means. svc.sync({ - "a": len(agent_manager.sessions), # active sessions + "a": service_runtime.DEFAULT_AGENT_CENSUS.live_session_count(), # active sessions "h": sorted(p_pulse_hours), # hour bucket set "n": p_pulse_count, # samples in batch "c": p_last_9r_cost or 0, # cumulative cost @@ -126,13 +131,12 @@ async def service_lifespan(): global p_pulse_task, p_drain_task, p_9r_start_task try: - from backend.apps.settings.settings import load_settings, save_settings - settings = load_settings() + settings = settings_gateway.DEFAULT_SETTINGS_GATEWAY.load() is_first_open = settings.first_opened_at is None if is_first_open: settings.first_opened_at = datetime.now().isoformat() - save_settings(settings) + settings_gateway.DEFAULT_SETTINGS_GATEWAY.save(settings) days_since_install = 0 if settings.first_opened_at: @@ -203,12 +207,12 @@ async def service_lifespan(): except Exception as e: logger.debug(f"Service startup event failed (non-critical): {e}") - try: - from backend.apps.nine_router import ensure_running as ensure_9router - # Start 9Router in the BACKGROUND instead of awaiting it here. Awaiting it was ~7s (up to ~18s cold) of the startup critical path, blocking the HTTP bind and the whole UI behind it. 9Router is only needed when the user sends an agent message, and the dispatch path calls ensure_running() itself (now serialized, so no double-spawn), so the first message waits for readiness lazily. This is the single biggest warm-startup win. - p_9r_start_task = asyncio.create_task(ensure_9router()) - except Exception as e: - logger.debug(f"9Router auto-start skipped: {e}") + if should_autostart_9router(): + try: + # Start 9Router in the BACKGROUND instead of awaiting it here. Awaiting it was ~7s (up to ~18s cold) of the startup critical path, blocking the HTTP bind and the whole UI behind it. 9Router is only needed when the user sends an agent message, and the dispatch path calls ensure_running() itself (now serialized, so no double-spawn), so the first message waits for readiness lazily. This is the single biggest warm-startup win. + p_9r_start_task = asyncio.create_task(service_runtime.DEFAULT_ROUTER_USAGE.ensure_running()) + except Exception as e: + logger.debug(f"9Router auto-start skipped: {e}") p_pulse_task = asyncio.create_task(p_pulse_loop()) p_drain_task = asyncio.create_task(p_drain_loop()) @@ -243,8 +247,7 @@ async def service_lifespan(): p_9r_start_task = None try: - from backend.apps.nine_router import stop as stop_9router - stop_9router() + service_runtime.DEFAULT_ROUTER_USAGE.stop() except Exception: pass @@ -310,12 +313,10 @@ def p_is_automation(sess: dict, tool_profile: "Counter") -> bool: @service.router.get("/usage-summary") async def usage_summary(window: str = "30d"): - from backend.apps.agents.agent_manager import agent_manager - sessions = p_load_all_sessions() # A live session is usually already on disk, so appending it blind counted the same chat twice. seen_ids = {s.get("id") for s in sessions if s.get("id")} - for s in agent_manager.get_all_sessions(): + for s in service_runtime.DEFAULT_AGENT_CENSUS.all_sessions(): live = s.model_dump(mode="json") if live.get("id") and live["id"] in seen_ids: sessions = [d for d in sessions if d.get("id") != live["id"]] @@ -413,8 +414,8 @@ def p_is_real(sess: dict) -> bool: completed = status_counts.get("completed", 0) completion_rate = completed / total_sessions if total_sessions > 0 else 0 - from backend.apps.nine_router import get_usage_stats, is_running as p_9r_running - nine_router_stats = await get_usage_stats() if p_9r_running() else None + router = service_runtime.DEFAULT_ROUTER_USAGE + nine_router_stats = await router.get_usage_stats() if router.is_running() else None if nine_router_stats and nine_router_stats.get("totalCost", 0) > 0: cost_source = "9router" @@ -481,10 +482,10 @@ def p_is_real(sess: dict) -> bool: @service.router.get("/cost-breakdown") async def cost_breakdown(period: str = "7d"): - from backend.apps.nine_router import get_usage_stats, is_running as p_9r_running - if not p_9r_running(): + router = service_runtime.DEFAULT_ROUTER_USAGE + if not router.is_running(): return {"available": False, "by_model": {}, "by_provider": {}} - stats = await get_usage_stats(period) + stats = await router.get_usage_stats(period) if not stats: return {"available": False, "by_model": {}, "by_provider": {}} return { diff --git a/backend/apps/service/service_runtime.py b/backend/apps/service/service_runtime.py new file mode 100644 index 000000000..fd48189db --- /dev/null +++ b/backend/apps/service/service_runtime.py @@ -0,0 +1,60 @@ +"""Injected cross-app runtime boundary for the service SubApp.""" + +from __future__ import annotations + +from typing import Any, Optional, Protocol, runtime_checkable + +from backend.apps import nine_router +from backend.apps.agents import agent_manager as agents_runtime + + +@runtime_checkable +class RouterUsage(Protocol): + """9Router lifecycle and usage-statistics operations the service app needs.""" + + def is_running(self) -> bool: ... + + async def get_usage_stats(self, period: str = "all") -> Optional[dict]: ... + + async def ensure_running(self) -> None: ... + + def stop(self) -> None: ... + + +@runtime_checkable +class AgentCensus(Protocol): + """Read-only views over live agent sessions for usage reporting.""" + + def live_session_count(self) -> int: ... + + def all_sessions(self) -> list[Any]: ... + + +class DefaultRouterUsage: + """Production adapter; dynamic lookups preserve established test seams.""" + + def is_running(self) -> bool: + return nine_router.is_running() + + async def get_usage_stats(self, period: str = "all") -> Optional[dict]: + return await nine_router.get_usage_stats(period) + + async def ensure_running(self) -> None: + await nine_router.ensure_running() + + def stop(self) -> None: + nine_router.stop() + + +class DefaultAgentCensus: + """Production adapter; dynamic lookups preserve established test seams.""" + + def live_session_count(self) -> int: + return len(agents_runtime.agent_manager.sessions) + + def all_sessions(self) -> list[Any]: + return agents_runtime.agent_manager.get_all_sessions() + + +DEFAULT_ROUTER_USAGE: RouterUsage = DefaultRouterUsage() +DEFAULT_AGENT_CENSUS: AgentCensus = DefaultAgentCensus() diff --git a/backend/apps/service/settings_gateway.py b/backend/apps/service/settings_gateway.py new file mode 100644 index 000000000..b37e5fb34 --- /dev/null +++ b/backend/apps/service/settings_gateway.py @@ -0,0 +1,35 @@ +"""Injected settings boundary for the service app.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +from backend.apps.settings import credentials, store +from backend.apps.settings.models import AppSettings + + +@runtime_checkable +class SettingsGateway(Protocol): + """Operations the service forwarders need from the settings app.""" + + def load(self) -> AppSettings: ... + + def save(self, settings: AppSettings) -> None: ... + + def default_proxy_url(self) -> str: ... + + +class DefaultSettingsGateway: + """Production adapter; dynamic lookups preserve established test seams.""" + + def load(self) -> AppSettings: + return store.load_settings() + + def save(self, settings: AppSettings) -> None: + store.save_settings(settings) + + def default_proxy_url(self) -> str: + return credentials.OPENSWARM_DEFAULT_PROXY_URL + + +DEFAULT_SETTINGS_GATEWAY: SettingsGateway = DefaultSettingsGateway() diff --git a/backend/apps/settings/redaction.py b/backend/apps/settings/redaction.py index baab91a92..99d275e11 100644 --- a/backend/apps/settings/redaction.py +++ b/backend/apps/settings/redaction.py @@ -36,8 +36,18 @@ def p_value_is_secret_shaped(value: Any) -> bool: return isinstance(value, str) and looks_secret(value) +def p_is_redacted_value(value: Any) -> bool: + return ( + isinstance(value, dict) + and isinstance(value.get("configured"), bool) + and set(value.keys()).issubset({"configured", "last4"}) + ) + + def p_redact_value(value: Any) -> dict[str, Any]: """A secret rendered as state, never content: configured + last 4 only.""" + if p_is_redacted_value(value): + return value if value is None or (isinstance(value, str) and value.strip() == ""): return {"configured": False} last4 = value[-4:] if isinstance(value, str) and len(value) >= 4 else None diff --git a/backend/apps/settings/settings.py b/backend/apps/settings/settings.py index 1c2279225..73b55d908 100644 --- a/backend/apps/settings/settings.py +++ b/backend/apps/settings/settings.py @@ -12,6 +12,7 @@ from backend.config.Apps import SubApp from backend.apps.settings.models import AppSettings, DEFAULT_SYSTEM_PROMPT +from backend.apps.settings.redaction import redact_settings from backend.apps.settings.store import ( DATA_DIR, SETTINGS_FILE, @@ -223,11 +224,7 @@ async def apply_settings_update(body: AppSettings, protect_fields: set[str] | No except Exception: pass - secret_keys = {"anthropic_api_key", "openai_api_key", "google_api_key", "openrouter_api_key", - "claude_subscription_token", "openai_subscription_token", "gemini_subscription_token", - "openswarm_bearer_token", "free_trial_token", "installation_id", "analytics_token"} - safe = {k: v for k, v in body.model_dump().items() if k not in secret_keys} - p_sync(safe) + p_sync(redact_settings(body.model_dump())) if (body.user_email and body.user_email != getattr(old, "user_email", None)) or \ (body.user_name and body.user_name != getattr(old, "user_name", None)): diff --git a/backend/apps/settings/store.py b/backend/apps/settings/store.py index 12fc6afb4..d65a61c37 100644 --- a/backend/apps/settings/store.py +++ b/backend/apps/settings/store.py @@ -115,15 +115,19 @@ def p_preserve_corrupt_settings() -> None: pass -# In-memory mirror of SETTINGS_FILE, revalidated by stat (mtime+size) on every load so even a hand-edited file or an unexpected writer is picked up immediately. A stat skips the open+parse+validate that Defender turns into 5-50ms on Windows. Copies on both sides keep handler isolation: callers mutate their copy, never the cache. +# In-memory mirror of SETTINGS_FILE, revalidated by path+stat (mtime+size) +# on every load so even a hand-edited file, relocated settings path, or an +# unexpected writer is picked up immediately. A stat skips the open+parse+validate +# that Defender turns into 5-50ms on Windows. Copies on both sides keep handler +# isolation: callers mutate their copy, never the cache. p_cached_settings: AppSettings | None = None -p_cached_sig: tuple[int, int] | None = None +p_cached_sig: tuple[str, int, int] | None = None -def p_settings_sig() -> tuple[int, int] | None: +def p_settings_sig() -> tuple[str, int, int] | None: try: st = os.stat(SETTINGS_FILE) - return (st.st_mtime_ns, st.st_size) + return (os.path.abspath(SETTINGS_FILE), st.st_mtime_ns, st.st_size) except OSError: return None @@ -164,13 +168,16 @@ def atomic_write_settings(payload: dict) -> None: """Atomic SETTINGS_FILE write; call via save_settings*, not directly.""" global p_cached_settings, p_cached_sig with p_settings_write_lock: - os.makedirs(DATA_DIR, exist_ok=True) - fd, tmp = tempfile.mkstemp(prefix=".settings.", suffix=".tmp", dir=DATA_DIR) + settings_dir = os.path.dirname(os.path.abspath(SETTINGS_FILE)) or DATA_DIR + os.makedirs(settings_dir, exist_ok=True) + fd, tmp = tempfile.mkstemp(prefix=".settings.", suffix=".tmp", dir=settings_dir) try: with os.fdopen(fd, "w", encoding="utf-8") as f: json.dump(payload, f, indent=2) - # Windows: Defender can briefly lock the destination; one retry handles every real case. - for attempt in range(2): + # Windows: Defender can briefly lock the destination after a test or + # user edit. Keep the retry window short, but long enough for hosted + # runners where file scanning is noticeably slower. + for attempt in range(6): try: os.replace(tmp, SETTINGS_FILE) # Refresh the cache inside the lock so cache order matches disk order. @@ -180,9 +187,9 @@ def atomic_write_settings(payload: dict) -> None: p_cached_sig = p_settings_sig() return except PermissionError: - if attempt == 1: + if attempt == 5: raise - time.sleep(0.05) + time.sleep(0.05 * (attempt + 1)) except Exception: try: os.unlink(tmp) diff --git a/backend/apps/skills/skills.py b/backend/apps/skills/skills.py index 878252fe8..3adba3490 100644 --- a/backend/apps/skills/skills.py +++ b/backend/apps/skills/skills.py @@ -359,10 +359,10 @@ async def seed_skill_workspace(body: SkillWorkspaceSeedRequest): os.makedirs(folder, exist_ok=True) if body.skill_content: - with open(os.path.join(folder, "SKILL.md"), "w") as f: + with open(os.path.join(folder, "SKILL.md"), "w", encoding="utf-8") as f: f.write(body.skill_content) if body.meta: - with open(os.path.join(folder, "meta.json"), "w") as f: + with open(os.path.join(folder, "meta.json"), "w", encoding="utf-8") as f: json.dump(body.meta, f, indent=2) return {"path": os.path.abspath(folder)} @@ -377,14 +377,14 @@ async def read_skill_workspace(workspace_id: str): skill_content = None skill_path = os.path.join(folder, "SKILL.md") if os.path.isfile(skill_path): - with open(skill_path) as f: + with open(skill_path, encoding="utf-8") as f: skill_content = f.read() meta = None meta_path = os.path.join(folder, "meta.json") if os.path.isfile(meta_path): try: - with open(meta_path) as f: + with open(meta_path, encoding="utf-8") as f: meta = json.load(f) except json.JSONDecodeError: pass @@ -503,7 +503,8 @@ async def list_skill_files(skill_id: str): dirs[:] = [d for d in dirs if not d.startswith(".")] for n in sorted(names): path = os.path.join(root, n) - rel = os.path.relpath(path, base_abs) + # API paths are posix on every OS: the frontend joins them with "/" and the SKILL.md-first sort compares them as strings. + rel = os.path.relpath(path, base_abs).replace(os.sep, "/") if n.startswith(".") or os.path.getsize(path) > 512_000: continue try: diff --git a/backend/apps/subscription/free_trial.py b/backend/apps/subscription/free_trial.py index d6a02272c..87ce7e139 100644 --- a/backend/apps/subscription/free_trial.py +++ b/backend/apps/subscription/free_trial.py @@ -22,10 +22,30 @@ import httpx from backend.apps.settings.credentials import OPENSWARM_DEFAULT_PROXY_URL -from backend.apps.settings.settings import load_settings, save_settings_async +from backend.apps.settings.settings import load_settings, save_settings_async, settings_write_lock logger = logging.getLogger(__name__) +P_FREE_TRIAL_FIELDS = ( + "connection_mode", + "free_trial_token", + "free_trial_remaining", + "free_trial_runs_limit", + "free_trial_resets_at", + "openswarm_proxy_url", +) + + +async def p_persist_free_trial_fields(settings_obj, *, include_default_model: bool = False): + """Commit trial-owned fields onto fresh settings, never a stale full object.""" + async with settings_write_lock(): + current = load_settings() + fields = P_FREE_TRIAL_FIELDS + (("default_model",) if include_default_model else ()) + for field in fields: + setattr(current, field, getattr(settings_obj, field)) + await save_settings_async(current) + return current + # Namespaces the hash so a raw hardware UUID never leaves the device. Public on purpose (open-source): it only prevents transmitting the raw id, not a secret. P_FP_SALT = "openswarm-free-trial-v1" @@ -130,6 +150,7 @@ async def p_sync_routing(settings_obj) -> None: async def clear_free_trial(settings_obj) -> None: """Drop the trial token and revert to own_key. Keeps free_trial_remaining (so the UI knows it's spent) and never touches a real paid mode.""" + default_model_changed = False if getattr(settings_obj, "connection_mode", "own_key") == "free-trial": settings_obj.connection_mode = "own_key" # Keep Haiku as the face of the free lane while the user has no model of their own (a spent trial still shows "Claude Haiku", with the send gated by the out-of-runs UI); only fall back to "sonnet" once a real key/sub connects so we never pin a paying user to Haiku. @@ -138,9 +159,13 @@ async def clear_free_trial(settings_obj) -> None: ): from backend.apps.settings.models import DEFAULT_MODEL settings_obj.default_model = DEFAULT_MODEL + default_model_changed = True settings_obj.free_trial_token = None - await save_settings_async(settings_obj) - await p_sync_routing(settings_obj) + saved = await p_persist_free_trial_fields( + settings_obj, + include_default_model=default_model_changed, + ) + await p_sync_routing(saved) async def clear_free_trial_on_connect() -> None: @@ -215,8 +240,8 @@ async def arm_free_trial(settings_obj) -> dict: settings_obj.openswarm_proxy_url = base # Pin the trial to Haiku, the exact tier the cloud serves a free run as. Critical: a sonnet/opus pick makes the Claude Code CLI attach an `effort`/thinking param (reasoning models), which Haiku 400s on ("does not support the effort parameter"). Using Haiku end to end means the CLI never adds it, so the run just works. settings_obj.default_model = "haiku" - await save_settings_async(settings_obj) - await p_sync_routing(settings_obj) + saved = await p_persist_free_trial_fields(settings_obj, include_default_model=True) + await p_sync_routing(saved) return {"armed": True, "runs_remaining": remaining, "runs_limit": settings_obj.free_trial_runs_limit} # Already spent on this machine: record it but don't arm. @@ -257,5 +282,5 @@ async def refresh_free_trial(settings_obj) -> dict: if remaining <= 0: await clear_free_trial(settings_obj) return {"connected": False, "runs_remaining": 0, "resets_at": getattr(settings_obj, "free_trial_resets_at", None)} - await save_settings_async(settings_obj) + await p_persist_free_trial_fields(settings_obj) return {"connected": True, "runs_remaining": remaining, "runs_limit": getattr(settings_obj, "free_trial_runs_limit", None)} diff --git a/backend/apps/subscription/router.py b/backend/apps/subscription/router.py index 7d60344cb..d923d5e48 100644 --- a/backend/apps/subscription/router.py +++ b/backend/apps/subscription/router.py @@ -14,6 +14,7 @@ from backend.config.Apps import SubApp from backend.apps.settings.credentials import OPENSWARM_DEFAULT_PROXY_URL +from backend.apps.settings.redaction import redact_settings from backend.apps.settings.settings import SETTINGS_FILE, load_settings, save_settings_async logger = logging.getLogger(__name__) @@ -246,9 +247,10 @@ async def sync(): settings_obj = load_settings() bearer = getattr(settings_obj, "openswarm_bearer_token", None) mode = getattr(settings_obj, "connection_mode", "own_key") + safe_settings = redact_settings(settings_obj.model_dump()) if mode != "openswarm-pro" or not bearer: - p_sync(settings_obj.model_dump()) + p_sync(safe_settings) return {"ok": True, "synced": False, "connection_mode": mode} try: @@ -259,14 +261,14 @@ async def sync(): ) except httpx.HTTPError as e: logger.debug("subscription/sync live fetch failed: %s", e) - p_sync(settings_obj.model_dump()) + p_sync(safe_settings) return {"ok": True, "synced": False, "reason": "network"} # Same 401/402 handling as /status: if Stripe-side reconciliation proves the bearer is dead or the sub expired, clear local state so the app reverts to own_key instead of hammering a useless token. if r.status_code in (401, 402): await p_clear_subscription(settings_obj) reason = "revoked" if r.status_code == 401 else "expired" - p_sync(settings_obj.model_dump()) + p_sync(redact_settings(settings_obj.model_dump())) return { "ok": True, "synced": False, @@ -276,7 +278,7 @@ async def sync(): if r.status_code != 200: logger.debug("subscription/sync got %s from cloud: %s", r.status_code, r.text[:200]) - p_sync(settings_obj.model_dump()) + p_sync(safe_settings) return {"ok": True, "synced": False, "reason": "upstream"} data = r.json() @@ -293,7 +295,7 @@ async def sync(): ) await save_settings_async(settings_obj) p_sync_subscription_identity(settings_obj) - p_sync(settings_obj.model_dump()) + p_sync(redact_settings(settings_obj.model_dump())) return { "ok": True, "synced": bool(data.get("synced")), diff --git a/backend/apps/swarm/entities/skills.py b/backend/apps/swarm/entities/skills.py index f8134a3cf..6a07f8987 100644 --- a/backend/apps/swarm/entities/skills.py +++ b/backend/apps/swarm/entities/skills.py @@ -100,7 +100,7 @@ def p_read_supporting_files(skill_dir: str) -> dict[str, bytes]: for root, p_dirs, names in os.walk(skill_dir): for n in names: full = os.path.join(root, n) - rel = os.path.relpath(full, skill_dir) + rel = os.path.relpath(full, skill_dir).replace(os.sep, "/") if rel == "SKILL.md" or n.startswith("."): continue try: diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index e9680279e..f58a05c20 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -19,6 +19,7 @@ # oauth_config runs the dotenv load (leaf) so OPENSWARM_OAUTH_BASE_URL is set before anything reads it; re-exported here for the route handlers below. from backend.apps.tools_lib.oauth_config import OPENSWARM_OAUTH_BASE_URL # sanitize_server_name + derive_mcp_config re-exported for agent_manager/main. +from backend.apps.hosting.policy import hosting_policy from backend.apps.tools_lib.mcp_config import sanitize_server_name, derive_mcp_config from backend.apps.tools_lib.mcp_discovery import ( discover_mcp_tools_http, @@ -345,6 +346,11 @@ async def create_tool(body: ToolCreate): @tools_lib.router.put("/{tool_id}") async def update_tool(tool_id: str, body: ToolUpdate): tool = load(tool_id) + if hosting_policy().tool_update_restricted(tool, body): + raise HTTPException( + status_code=403, + detail="this build only allows enabling or disabling tools here", + ) for k, v in body.model_dump(exclude_none=True).items(): setattr(tool, k, v) save(tool) diff --git a/backend/apps/web/web.py b/backend/apps/web/web.py index b756ec23b..f652527c1 100644 --- a/backend/apps/web/web.py +++ b/backend/apps/web/web.py @@ -79,10 +79,12 @@ class FetchBody(BaseModel): # Drive the packaged app's offscreen Chromium (main-process hidden window) for a fetch/search. Returns the bridge result dict, or None when no Electron main bridge is connected (dev/headless/backend-only) so the cascade just skips this tier. This is the real "browser reachable" gate, OPENSWARM_BROWSER_OK is effectively always "1" and not trustworthy for this. @typechecked async def p_browser_bridge(action: str, params: Dict) -> Optional[Dict]: - from backend.apps.agents.core.ws_manager import ws_manager + from backend.apps.agents.core.ws_manager import BrowserCommandOwner, ws_manager if ws_manager.main_connection is None: return None - res = await ws_manager.send_main_command(uuid4().hex, action, params) + res = await ws_manager.send_main_command( + uuid4().hex, action, params, owner=BrowserCommandOwner(origin="main") + ) if not res or res.get("error"): return None return res diff --git a/backend/apps/workflows/durable_scheduler_contract.py b/backend/apps/workflows/durable_scheduler_contract.py new file mode 100644 index 000000000..805226a58 --- /dev/null +++ b/backend/apps/workflows/durable_scheduler_contract.py @@ -0,0 +1,272 @@ +"""Deterministic durable at-least-once scheduler contract and reference model. + +It does not prove exactly-once effects; callers supply idempotency or at-most-once policy. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from backend.apps.workflows.durable_scheduler_types import ( + TERMINAL_STATES, + AttemptIdentity, + EffectRetryClass, + JobRecord, + JobState, + Lease, + LeaseToken, + OperationResult, + OutcomeCode, + SlotIdentity, +) + + +def p_utc(value: datetime) -> datetime: + if value.tzinfo is None or value.utcoffset() is None: + raise ValueError("scheduler instants must be timezone-aware") + return value.astimezone(timezone.utc) + + +def p_nonempty(name: str, value: str) -> str: + if not value: + raise ValueError(f"{name} must be non-empty") + return value + + +class InMemoryDurableScheduler: + """Bounded synchronous model of atomic durable-store operations.""" + + def __init__(self, *, clock: Callable[[], datetime], control_plane_epoch: int, max_jobs: int) -> None: + if control_plane_epoch < 1 or max_jobs < 1: + raise ValueError("control-plane epoch and capacity must be positive") + self.p_clock, self.p_control_plane_epoch = clock, control_plane_epoch + self.p_max_jobs = max_jobs + self.p_jobs: dict[str, JobRecord] = {} + self.p_slots: dict[SlotIdentity, str] = {} + self.p_run_ids: set[str] = set() + + @property + def job_count(self) -> int: + return len(self.p_jobs) + + def p_now(self) -> datetime: + return p_utc(self.p_clock()) + + def p_store(self, job: JobRecord) -> OperationResult: + self.p_jobs[job.job_id] = job + return OperationResult(OutcomeCode.APPLIED, job) + + def p_lookup(self, tenant_id: str, job_id: str) -> OperationResult: + job = self.p_jobs.get(job_id) + if job is None: + return OperationResult(OutcomeCode.NOT_FOUND) + if job.slot.tenant_id != tenant_id: + return OperationResult(OutcomeCode.TENANT_MISMATCH) + return OperationResult(OutcomeCode.IDEMPOTENT, job) + + def get(self, *, tenant_id: str, job_id: str) -> OperationResult: + return self.p_lookup(tenant_id, job_id) + + def enqueue( + self, *, tenant_id: str, workflow_id: str, schedule_revision: int, scheduled_for: datetime, + job_id: str, lease_duration: timedelta, max_attempts: int, effect_retry_class: EffectRetryClass, + ) -> OperationResult: + if schedule_revision < 1 or max_attempts < 1 or lease_duration <= timedelta(0): + raise ValueError("revision, attempts, and lease duration must be positive") + slot = SlotIdentity(p_nonempty("tenant_id", tenant_id), p_nonempty("workflow_id", workflow_id), + schedule_revision, p_utc(scheduled_for)) + existing = self.p_slots.get(slot) + if existing is not None: + return OperationResult(OutcomeCode.IDEMPOTENT, self.p_jobs[existing]) + if job_id in self.p_jobs: + return OperationResult(OutcomeCode.ID_CONFLICT) + if len(self.p_jobs) >= self.p_max_jobs: + return OperationResult(OutcomeCode.CAPACITY_EXCEEDED) + job = JobRecord(p_nonempty("job_id", job_id), slot, JobState.PENDING, lease_duration, + max_attempts, effect_retry_class, slot.scheduled_for) + self.p_jobs[job_id], self.p_slots[slot] = job, job_id + return OperationResult(OutcomeCode.APPLIED, job) + + def p_active( + self, tenant_id: str, job_id: str, token: LeaseToken, *, now: datetime | None = None, + ) -> OperationResult: + found, current_epoch = self.p_lookup(tenant_id, job_id), self.p_control_plane_epoch + job = found.job + if job is None: + return found + if job.state in TERMINAL_STATES: + return OperationResult(OutcomeCode.TERMINAL_STATE, job) + if token.control_plane_epoch != current_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + lease = job.lease + if lease is None: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + if lease.token.owner_id != token.owner_id: + return OperationResult(OutcomeCode.STALE_OWNER, job) + if lease.token.lease_epoch != token.lease_epoch: + return OperationResult(OutcomeCode.STALE_LEASE_EPOCH, job) + if lease.token.control_plane_epoch != token.control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + checked_at = self.p_now() if now is None else p_utc(now) + if checked_at >= lease.expires_at: + return OperationResult(OutcomeCode.LEASE_EXPIRED, job) + return OperationResult(OutcomeCode.IDEMPOTENT, job) + + def claim( + self, *, tenant_id: str, job_id: str, run_id: str, owner_id: str, control_plane_epoch: int, + ) -> OperationResult: + found, now = self.p_lookup(tenant_id, job_id), self.p_now() + job = found.job + if job is None: + return found + run_id, owner_id = p_nonempty("run_id", run_id), p_nonempty("owner_id", owner_id) + if control_plane_epoch != self.p_control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + if job.attempt is not None and job.attempt.run_id == run_id: + previous = job.last_lease_token + if previous is None: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + if previous.owner_id != owner_id: + return OperationResult(OutcomeCode.STALE_OWNER, job) + if previous.control_plane_epoch != control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + if job.state in TERMINAL_STATES: + return OperationResult(OutcomeCode.IDEMPOTENT, job) + if job.state not in {JobState.LEASED, JobState.RUNNING} or job.lease is None: + return OperationResult(OutcomeCode.ID_CONFLICT, job) + if now >= job.lease.expires_at: + return OperationResult(OutcomeCode.LEASE_EXPIRED, job) + return OperationResult(OutcomeCode.IDEMPOTENT, job) + if job.state in TERMINAL_STATES: + return OperationResult(OutcomeCode.TERMINAL_STATE, job) + if job.state is not JobState.PENDING: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + if now < job.not_before: + return OperationResult(OutcomeCode.NOT_DUE, job) + if run_id in self.p_run_ids: + return OperationResult(OutcomeCode.ID_CONFLICT, job) + attempt_no = 1 if job.attempt is None else job.attempt.attempt_number + 1 + lease_epoch = job.last_lease_epoch + 1 + attempt = AttemptIdentity(job.job_id, run_id, attempt_no) + token = LeaseToken(owner_id, lease_epoch, control_plane_epoch) + self.p_run_ids.add(run_id) + return self.p_store(replace(job, state=JobState.LEASED, attempt=attempt, + lease=Lease(token, now + job.lease_duration), last_lease_epoch=lease_epoch, + last_lease_token=token)) + + def heartbeat(self, *, tenant_id: str, job_id: str, token: LeaseToken) -> OperationResult: + now = self.p_now() + active = self.p_active(tenant_id, job_id, token, now=now) + job = active.job + if active.code is not OutcomeCode.IDEMPOTENT or job is None: + return active + if job.state not in {JobState.LEASED, JobState.RUNNING} or job.lease is None: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + lease = Lease(job.lease.token, now + job.lease_duration) + return self.p_store(replace(job, lease=lease)) + + def start(self, *, tenant_id: str, job_id: str, token: LeaseToken) -> OperationResult: + active = self.p_active(tenant_id, job_id, token) + job = active.job + if active.code is not OutcomeCode.IDEMPOTENT or job is None: + return active + if job.state is JobState.RUNNING: + return OperationResult(OutcomeCode.IDEMPOTENT, job) + if job.state is not JobState.LEASED: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + return self.p_store(replace(job, state=JobState.RUNNING)) + + def cancel(self, *, tenant_id: str, job_id: str, reason: str) -> OperationResult: + found, canceled = self.p_lookup(tenant_id, job_id), JobState.CANCELED + job = found.job + if job is None: + return found + if job.state is canceled: + return OperationResult(OutcomeCode.IDEMPOTENT, job) + if job.state in TERMINAL_STATES: + return OperationResult(OutcomeCode.TERMINAL_STATE, job) + return self.p_store(replace(job, state=canceled, lease=None, result=reason, result_committed=True)) + + def commit_terminal( + self, *, tenant_id: str, job_id: str, token: LeaseToken, state: JobState, result: str, + ) -> OperationResult: + if state not in {JobState.SUCCESS, JobState.FAILURE}: + raise ValueError("terminal commit state must be success or failure") + found = self.p_lookup(tenant_id, job_id) + job = found.job + if job is None: + return found + if job.state in TERMINAL_STATES: + if (job.state is state and job.result == result and job.result_committed + and job.last_lease_token == token): + return OperationResult(OutcomeCode.IDEMPOTENT, job) + return OperationResult(OutcomeCode.TERMINAL_STATE, job) + active = self.p_active(tenant_id, job_id, token) + job = active.job + if active.code is not OutcomeCode.IDEMPOTENT or job is None: + return active + if job.state is not JobState.RUNNING: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + return self.p_store(replace(job, state=state, lease=None, result=result, result_committed=True)) + + def p_retry(self, job: JobRecord, due: datetime, error: str) -> OperationResult: + attempt = 0 if job.attempt is None else job.attempt.attempt_number + if job.effect_retry_class is EffectRetryClass.RETRY_SAFE and attempt < job.max_attempts: + return self.p_store(replace(job, state=JobState.PENDING, lease=None, + not_before=p_utc(due), last_error=error)) + return self.p_store(replace(job, state=JobState.DEAD, lease=None, last_error=error, + result=error, result_committed=True)) + + def retry_or_dead( + self, *, tenant_id: str, job_id: str, token: LeaseToken, retry_not_before: datetime, error: str, + ) -> OperationResult: + active = self.p_active(tenant_id, job_id, token) + job = active.job + if active.code is not OutcomeCode.IDEMPOTENT or job is None: + return active + if job.state is not JobState.RUNNING: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + return self.p_retry(job, retry_not_before, error) + + def expire_lease( + self, *, tenant_id: str, job_id: str, observed_token: LeaseToken, control_plane_epoch: int, + retry_not_before: datetime, error: str, + ) -> OperationResult: + found = self.p_lookup(tenant_id, job_id) + job = found.job + if job is None: + return found + if job.state in TERMINAL_STATES: + return OperationResult(OutcomeCode.TERMINAL_STATE, job) + if control_plane_epoch != self.p_control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + lease = job.lease + if lease is None: + return OperationResult(OutcomeCode.ILLEGAL_TRANSITION, job) + if lease.token.owner_id != observed_token.owner_id: + return OperationResult(OutcomeCode.STALE_OWNER, job) + if lease.token.lease_epoch != observed_token.lease_epoch: + return OperationResult(OutcomeCode.STALE_LEASE_EPOCH, job) + if lease.token.control_plane_epoch != observed_token.control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH, job) + if self.p_now() < lease.expires_at: + return OperationResult(OutcomeCode.LEASE_ACTIVE, job) + return self.p_retry(job, retry_not_before, error) + + def acknowledge(self, *, tenant_id: str, job_id: str) -> OperationResult: + found = self.p_lookup(tenant_id, job_id) + job = found.job + if job is None: + return found + if job.acknowledged: + return OperationResult(OutcomeCode.IDEMPOTENT, job) + if job.state not in TERMINAL_STATES or not job.result_committed: + return OperationResult(OutcomeCode.ACK_BEFORE_TERMINAL, job) + return self.p_store(replace(job, acknowledged=True)) + + def fence_control_plane(self, next_epoch: int) -> OperationResult: + if next_epoch <= self.p_control_plane_epoch: + return OperationResult(OutcomeCode.STALE_CONTROL_PLANE_EPOCH) + self.p_control_plane_epoch = next_epoch + return OperationResult(OutcomeCode.APPLIED) diff --git a/backend/apps/workflows/durable_scheduler_types.py b/backend/apps/workflows/durable_scheduler_types.py new file mode 100644 index 000000000..3a02dbcbb --- /dev/null +++ b/backend/apps/workflows/durable_scheduler_types.py @@ -0,0 +1,76 @@ +"""Closed data types for the durable scheduler reference contract.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from enum import StrEnum +from typing import NamedTuple + + +class JobState(StrEnum): + PENDING = "pending" + LEASED = "leased" + RUNNING = "running" + SUCCESS = "success" + FAILURE = "failure" + CANCELED = "canceled" + DEAD = "dead" + + +class EffectRetryClass(StrEnum): + RETRY_SAFE = "retry_safe" + AT_MOST_ONCE = "at_most_once" + + +class OutcomeCode(StrEnum): + APPLIED = "applied" + IDEMPOTENT = "idempotent" + NOT_FOUND = "not_found" + TENANT_MISMATCH = "tenant_mismatch" + ID_CONFLICT = "id_conflict" + CAPACITY_EXCEEDED = "capacity_exceeded" + NOT_DUE = "not_due" + ILLEGAL_TRANSITION = "illegal_transition" + STALE_OWNER = "stale_owner" + STALE_LEASE_EPOCH = "stale_lease_epoch" + STALE_CONTROL_PLANE_EPOCH = "stale_control_plane_epoch" + LEASE_EXPIRED = "lease_expired" + LEASE_ACTIVE = "lease_active" + TERMINAL_STATE = "terminal_state" + ACK_BEFORE_TERMINAL = "ack_before_terminal" + + +TERMINAL_STATES = frozenset({JobState.SUCCESS, JobState.FAILURE, JobState.CANCELED, JobState.DEAD}) + + +SlotIdentity = NamedTuple("SlotIdentity", [("tenant_id", str), ("workflow_id", str), + ("schedule_revision", int), ("scheduled_for", datetime)]) +AttemptIdentity = NamedTuple("AttemptIdentity", [("job_id", str), ("run_id", str), ("attempt_number", int)]) +LeaseToken = NamedTuple("LeaseToken", [("owner_id", str), ("lease_epoch", int), ("control_plane_epoch", int)]) +Lease = NamedTuple("Lease", [("token", LeaseToken), ("expires_at", datetime)]) + + +@dataclass(frozen=True, slots=True) +class JobRecord: + job_id: str + slot: SlotIdentity + state: JobState + lease_duration: timedelta + max_attempts: int + effect_retry_class: EffectRetryClass + not_before: datetime + last_lease_epoch: int = 0 + attempt: AttemptIdentity | None = None + lease: Lease | None = None + last_lease_token: LeaseToken | None = None + last_error: str | None = None + result: str | None = None + result_committed: bool = False + acknowledged: bool = False + + +@dataclass(frozen=True, slots=True) +class OperationResult: + code: OutcomeCode + job: JobRecord | None = None diff --git a/backend/apps/workflows/executor.py b/backend/apps/workflows/executor.py index 27f22ca50..06bcd4538 100644 --- a/backend/apps/workflows/executor.py +++ b/backend/apps/workflows/executor.py @@ -275,6 +275,20 @@ async def execute( triggered_by=triggered_by, ) + # Claim-time global-pause gate (W1/T26): the scheduler queues fires against a point-in-time list, so a schedule fire that lands after "pause all schedules" records a clean "skipped" instead of spinning up a session for the per-step recheck to kill. (A deleted or switched-off workflow is refused above, for every trigger.) Run Now deliberately ignores the global pause: it is an explicit user action. + if triggered_by == "schedule" and storage.get_paused(): + run.status = "skipped" + run.error = "All schedules are paused" + run.finished_at = datetime.now() + storage.record_run(run) + _persist_run_fields(p_live, { + "last_run_at": run.finished_at, + "last_run_status": "skipped", + "last_run_id": run.id, + }) + return run + wf = p_live + # Cost cap pre-check happens before claiming `_running` so a capped workflow doesn't block its own next fire. We still record the run so the user sees it in History with a clear reason. if wf.cost_cap_usd_monthly is not None: spent = _monthly_spend_so_far(wf) diff --git a/backend/apps/workflows/lifecycle_events.py b/backend/apps/workflows/lifecycle_events.py new file mode 100644 index 000000000..2e4e30812 --- /dev/null +++ b/backend/apps/workflows/lifecycle_events.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any, Protocol + +WorkflowPayload = dict[str, Any] +BroadcastCallback = Callable[[str, WorkflowPayload], Awaitable[None]] + + +class WorkflowLifecycleEventPort(Protocol): + async def workflow_updated( + self, workflow_id: str, workflow: WorkflowPayload + ) -> None: ... + + async def workflow_deleted(self, workflow_id: str) -> None: ... + + +class BroadcastWorkflowLifecycleEvents: + def __init__(self, broadcast: BroadcastCallback) -> None: + self.p_broadcast = broadcast + + async def workflow_updated( + self, workflow_id: str, workflow: WorkflowPayload + ) -> None: + try: + await self.p_broadcast( + "workflow:updated", + {"workflow_id": workflow_id, "workflow": workflow}, + ) + except Exception: + pass + + async def workflow_deleted(self, workflow_id: str) -> None: + try: + await self.p_broadcast("workflow:deleted", {"workflow_id": workflow_id}) + except Exception: + pass + + +def workflow_lifecycle_event_publisher() -> WorkflowLifecycleEventPort: + from backend.apps.agents.core.ws_manager import ws_manager + + return BroadcastWorkflowLifecycleEvents(ws_manager.broadcast_global) diff --git a/backend/apps/workflows/scheduler.py b/backend/apps/workflows/scheduler.py index c4d88ba66..b60ba5d5f 100644 --- a/backend/apps/workflows/scheduler.py +++ b/backend/apps/workflows/scheduler.py @@ -488,6 +488,10 @@ def reconcile_on_startup() -> None: async def start() -> None: global _loop_task + # OQ-17 interim guard: with >1 backend instance (deploy overlap/HA), only the designated instance may run the scheduler loop AND the stuck-run reaper — otherwise fires double and this instance's _mark_stuck_runs_failed kills a peer's live runs (WORKFLOWS_AND_SCHEDULING §3). + if os.environ.get("OPENSWARM_SCHEDULER_ENABLED", "1").strip().lower() in ("0", "false", "no"): + logger.info("workflow scheduler disabled via OPENSWARM_SCHEDULER_ENABLED") + return if _loop_task is not None: return _mark_stuck_runs_failed() diff --git a/backend/apps/workflows/workflows.py b/backend/apps/workflows/workflows.py index d3349fed1..0751eab8d 100644 --- a/backend/apps/workflows/workflows.py +++ b/backend/apps/workflows/workflows.py @@ -2,16 +2,16 @@ import logging from contextlib import asynccontextmanager from datetime import datetime, timezone -from typing import Optional +from typing import Annotated, Optional -from fastapi import HTTPException, Header, Query, Request +from fastapi import Depends, HTTPException, Header, Query from backend.config.Apps import SubApp +from backend.apps.hosting.policy import hosting_policy from backend.apps.workflows.models import ( Workflow, WorkflowCreate, WorkflowUpdate, - WorkflowRun, WorkflowStep, DraftCommitBody, AskRunBody, @@ -19,7 +19,14 @@ GenerateMetadataRequest, GenerateMetadataResponse, ) -from backend.apps.workflows import storage, scheduler, executor, audit, escalation +from backend.apps.workflows import ( + audit, + escalation, + executor, + lifecycle_events, + scheduler, + storage, +) from backend.apps.workflows.cloud.handover import release_before_removing from backend.apps.settings.models import DEFAULT_MODEL from backend.apps.workflows.default_model import provider_for_model, user_default_model @@ -38,7 +45,7 @@ ) -def _scan_cron_for_openswarm() -> list[str]: +def scan_cron_for_openswarm() -> list[str]: """Surface OS-level scheduled-task entries that reference us. macOS + Linux: read `crontab -l`. Windows: query `schtasks` for any @@ -83,11 +90,15 @@ def _scan_cron_for_openswarm() -> list[str]: @asynccontextmanager async def workflows_lifespan(): + if hosting_policy().workflows_disabled(): + logger.info("workflow storage and scheduler disabled in hosted mode") + yield + return storage.init() await scheduler.start() # Cheap one-shot scan for prior cron entries that reference us. We don't migrate automatically; the FE shows a banner with a "Convert to OpenSwarm scheduled tasks" button so the user is in control. global _cron_findings - _cron_findings = _scan_cron_for_openswarm() + _cron_findings = scan_cron_for_openswarm() try: yield finally: @@ -96,6 +107,30 @@ async def workflows_lifespan(): workflows = SubApp("workflows", workflows_lifespan) +WorkflowLifecycleEventDependency = Annotated[ + lifecycle_events.WorkflowLifecycleEventPort | None, + Depends(lifecycle_events.workflow_lifecycle_event_publisher), +] + + +async def p_publish_workflow_updated( + event_publisher: lifecycle_events.WorkflowLifecycleEventPort | None, + workflow_id: str, + workflow: lifecycle_events.WorkflowPayload, +) -> None: + if event_publisher is None: + event_publisher = lifecycle_events.workflow_lifecycle_event_publisher() + await event_publisher.workflow_updated(workflow_id, workflow) + + +async def p_publish_workflow_deleted( + event_publisher: lifecycle_events.WorkflowLifecycleEventPort | None, + workflow_id: str, +) -> None: + if event_publisher is None: + event_publisher = lifecycle_events.workflow_lifecycle_event_publisher() + await event_publisher.workflow_deleted(workflow_id) + def _derive_icon(wf: Workflow) -> str: """Cheap icon hint used until proper auto-icon generation lands. @@ -276,7 +311,10 @@ def _parse_calendar_bound(value: str, label: str) -> datetime: @workflows.router.post("/create") -async def create_workflow(body: WorkflowCreate): +async def create_workflow( + body: WorkflowCreate, + event_publisher: WorkflowLifecycleEventDependency = None, +): if not body.unsaved and not _has_nonempty_steps(body.steps): raise HTTPException(status_code=400, detail="Workflow must have at least one step") actions = body.actions @@ -327,14 +365,7 @@ async def create_workflow(body: WorkflowCreate): storage.save_workflow(wf) scheduler.kick() enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched @@ -803,6 +834,7 @@ async def update_workflow( workflow_id: str, body: WorkflowUpdate, if_match: Optional[str] = Header(default=None, alias="If-Match"), + event_publisher: WorkflowLifecycleEventDependency = None, ): wf = storage.get_workflow(workflow_id) if not wf: @@ -841,14 +873,7 @@ async def update_workflow( await p_sync_cloud_copy(wf, {k: v for k, v in data.items() if k != "steps"}) storage.save_workflow(wf) enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched for k, v in data.items(): setattr(wf, k, v) @@ -867,14 +892,7 @@ async def update_workflow( scheduler.kick() # Push the change to every open dashboard so an agent-driven edit (the Edit Agent's add/delete/edit-step tools all PATCH here) refreshes the card live instead of looking stale until the next full refetch. enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched @@ -892,7 +910,10 @@ async def _stop_in_flight_run(workflow_id: str) -> None: @workflows.router.delete("/{workflow_id}") -async def delete_workflow(workflow_id: str): +async def delete_workflow( + workflow_id: str, + event_publisher: WorkflowLifecycleEventDependency = None, +): """Soft-delete: move to Trash. The record stays on disk with deleted_at set so it's hidden from every list and the scheduler but restorable. /{id}/purge does the irreversible hard delete.""" @@ -912,16 +933,15 @@ async def delete_workflow(workflow_id: str): # Drop any pending missed fires so a trashed workflow can't haunt the card. p_drop_pending_missed(workflow_id) scheduler.kick() - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:deleted", {"workflow_id": workflow_id}) - except Exception: - pass + await p_publish_workflow_deleted(event_publisher, workflow_id) return {"ok": True} @workflows.router.post("/{workflow_id}/restore") -async def restore_workflow(workflow_id: str): +async def restore_workflow( + workflow_id: str, + event_publisher: WorkflowLifecycleEventDependency = None, +): """Bring a trashed workflow back. Its schedule stays off (we disabled it on delete); the user re-enables it deliberately.""" wf = storage.get_workflow(workflow_id) @@ -930,19 +950,15 @@ async def restore_workflow(workflow_id: str): wf.deleted_at = None storage.save_workflow(wf, untrash=True) enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched @workflows.router.delete("/{workflow_id}/purge") -async def purge_workflow(workflow_id: str): +async def purge_workflow( + workflow_id: str, + event_publisher: WorkflowLifecycleEventDependency = None, +): """Hard delete, only from Trash. Removes the record, its run history and its own chats.""" wf = storage.get_workflow(workflow_id) if not wf or wf.deleted_at is None: @@ -954,11 +970,7 @@ async def purge_workflow(workflow_id: str): from backend.apps.workflows.owned_sessions import purge_owned_sessions await purge_owned_sessions(wf) storage.delete_workflow(workflow_id) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:deleted", {"workflow_id": workflow_id}) - except Exception: - pass + await p_publish_workflow_deleted(event_publisher, workflow_id) return {"ok": True} @@ -1190,7 +1202,11 @@ def p_sync_model_on_save(wf, model: Optional[str]) -> None: @workflows.router.post("/{workflow_id}/draft/commit") -async def commit_draft(workflow_id: str, body: Optional[DraftCommitBody] = None): +async def commit_draft( + workflow_id: str, + body: Optional[DraftCommitBody] = None, + event_publisher: WorkflowLifecycleEventDependency = None, +): """Commit the Edit-Agent draft: draft_steps become the live steps.""" wf = storage.get_workflow(workflow_id) if not wf: @@ -1231,19 +1247,15 @@ async def commit_draft(workflow_id: str, body: Optional[DraftCommitBody] = None) audit.log_change(wf.id, "user", before, wf.model_dump(mode="json")) scheduler.kick() enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched @workflows.router.post("/{workflow_id}/draft/discard") -async def discard_draft(workflow_id: str): +async def discard_draft( + workflow_id: str, + event_publisher: WorkflowLifecycleEventDependency = None, +): """Throw away the Edit-Agent draft; the live workflow is untouched.""" wf = storage.get_workflow(workflow_id) if not wf: @@ -1254,14 +1266,7 @@ async def discard_draft(workflow_id: str): p_prune_step_tool_usage(wf) storage.save_workflow(wf) enriched = _enriched(wf) - try: - from backend.apps.agents.core.ws_manager import ws_manager - await ws_manager.broadcast_global("workflow:updated", { - "workflow_id": wf.id, - "workflow": enriched, - }) - except Exception: - pass + await p_publish_workflow_updated(event_publisher, wf.id, enriched) return enriched diff --git a/backend/config/entity_references.py b/backend/config/entity_references.py index 856c1f76f..6dd8eaff4 100644 --- a/backend/config/entity_references.py +++ b/backend/config/entity_references.py @@ -76,6 +76,8 @@ class EntityReference(BaseModel): EntityReference(module="backend.apps.agents.core.models", model="AgentConfig", field="workflow_run_id", target=EntityKind.WORKFLOW_RUN), EntityReference(module="backend.apps.agents.core.models", model="AgentSession", field="dashboard_id", target=EntityKind.DASHBOARD), EntityReference(module="backend.apps.agents.core.models", model="AgentSession", field="parent_session_id", target=EntityKind.SESSION), + EntityReference(module="backend.apps.agents.events.AgentEvent", model="AgentEventBase", field="session_id", target=EntityKind.SESSION), + EntityReference(module="backend.apps.agents.events.AgentTurnEventEmitter", model="AgentTurnEventEmitter", field="session_id", target=EntityKind.SESSION), EntityReference(module="backend.apps.agents.core.models", model="AgentSession", field="workflow_edit_id", target=EntityKind.WORKFLOW), EntityReference(module="backend.apps.agents.core.models", model="AgentSession", field="workflow_run_id", target=EntityKind.WORKFLOW_RUN), EntityReference(module="backend.apps.agents.core.models", model="ApprovalRequest", field="session_id", target=EntityKind.SESSION), diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index cc1cbe363..4ba99de6e 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -25,13 +25,15 @@ @pytest.fixture(autouse=True) -def _isolate_browser_state(monkeypatch): +def _isolate_browser_state(monkeypatch, tmp_path): skills_dir = tempfile.mkdtemp(prefix="os_skills_") metrics_dir = tempfile.mkdtemp(prefix="os_metrics_") playbook_dir = tempfile.mkdtemp(prefix="os_playbook_") monkeypatch.setenv("OPENSWARM_BROWSER_SKILLS_DIR", skills_dir) monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", metrics_dir) monkeypatch.setenv("OPENSWARM_BROWSER_PLAYBOOK_DIR", playbook_dir) + from backend.apps.agents import agent_manager as p_agent_manager + monkeypatch.setattr(p_agent_manager, "SESSIONS_DIR", str(tmp_path / "agent-sessions")) # The speed levers are default-ON in prod; pin them off for the suite so mocked loop tests keep exact aux-call/turn expectations (same pattern as OPENSWARM_PERSISTENT_CLIENT). The levers are exercised by their own live gates + targeted tests that set the flag explicitly. monkeypatch.setenv("OSW_PRESTAGE", "0") monkeypatch.setenv("OSW_FASTREAD_HOP", "0") diff --git a/backend/tests/test_agent_events.py b/backend/tests/test_agent_events.py new file mode 100644 index 000000000..c902d9dbd --- /dev/null +++ b/backend/tests/test_agent_events.py @@ -0,0 +1,165 @@ +from datetime import datetime, timezone +from typing import Dict, Union + +import pytest +from pydantic import ValidationError + +from backend.apps.agents.events.AgentEvent import ( + ToolCompletedEvent, + ToolStartedEvent, + TurnCompletedEvent, + TurnFailedEvent, + TurnFirstTokenEvent, + TurnStartedEvent, + parse_agent_event, +) +from backend.apps.agents.events.AgentEventSink import ( + BoundedAgentEventSink, + NullAgentEventSink, + emit_agent_event, +) +from backend.apps.agents.events.AgentTurnEventEmitter import AgentTurnEventEmitter + + +def p_base() -> Dict[str, Union[str, int]]: + return { + "session_id": "session-1", + "turn_id": "turn-1", + "sequence": 3, + "monotonic_ms": 1200, + } + + +@pytest.mark.parametrize( + "event", + [ + TurnStartedEvent(**p_base(), provider="anthropic", model="claude"), + TurnFirstTokenEvent(**p_base(), ttft_ms=320), + ToolStartedEvent(**p_base(), tool_call_id="tool-1", tool_name="WebSearch"), + ToolCompletedEvent( + **p_base(), + tool_call_id="tool-1", + tool_name="WebSearch", + duration_ms=40, + status="success", + ), + TurnCompletedEvent(**p_base(), duration_ms=900, input_tokens=10, output_tokens=20), + TurnFailedEvent(**p_base(), duration_ms=100, error_type="capacity", retryable=True), + ], +) +def test_agent_event_round_trip(event): + parsed = parse_agent_event(event.model_dump(mode="json")) + assert type(parsed) is type(event) + assert parsed.event_id == event.event_id + + +def test_event_identity_and_time_are_generated(): + first = TurnFirstTokenEvent(**p_base(), ttft_ms=1) + second = TurnFirstTokenEvent(**p_base(), ttft_ms=1) + assert first.event_id != second.event_id + assert first.occurred_at.tzinfo == timezone.utc + + +def test_naive_occurrence_time_is_rejected(): + with pytest.raises(ValidationError): + TurnFirstTokenEvent(**p_base(), ttft_ms=1, occurred_at=datetime(2026, 7, 11)) + + +def test_discriminator_rejects_unknown_kind(): + with pytest.raises(ValidationError): + parse_agent_event({**p_base(), "kind": "turn.unknown"}) + + +def test_payload_bounds_reject_unbounded_tool_name(): + with pytest.raises(ValidationError): + ToolStartedEvent(**p_base(), tool_call_id="tool-1", tool_name="x" * 129) + + +def test_extra_content_is_rejected(): + with pytest.raises(ValidationError): + TurnCompletedEvent(**p_base(), duration_ms=1, prompt="secret") + + +def test_null_sink_accepts_event(): + event = TurnCompletedEvent(**p_base(), duration_ms=1) + assert emit_agent_event(NullAgentEventSink(), event) is True + + +def test_sink_failure_is_isolated(): + class BrokenSink: + def emit(self, event) -> None: + raise RuntimeError("sink unavailable") + + event = TurnFailedEvent(**p_base(), duration_ms=1, error_type="test") + assert emit_agent_event(BrokenSink(), event) is False + + +def test_bounded_sink_isolates_sessions_and_reports_eviction(): + sink = BoundedAgentEventSink(max_sessions=2, max_events_per_session=2) + for sequence in range(3): + sink.emit(TurnFirstTokenEvent(**{**p_base(), "sequence": sequence}, ttft_ms=sequence)) + sink.emit(TurnFirstTokenEvent( + **{**p_base(), "session_id": "session-2"}, ttft_ms=1 + )) + + first = sink.snapshot("session-1") + second = sink.snapshot("session-2") + assert [event.sequence for event in first.events] == [1, 2] + assert first.dropped_events == 1 + assert len(second.events) == 1 + + +def test_bounded_sink_evicts_least_recently_emitted_session(): + sink = BoundedAgentEventSink(max_sessions=2, max_events_per_session=2) + sink.emit(TurnFirstTokenEvent(**p_base(), ttft_ms=1)) + sink.emit(TurnFirstTokenEvent(**{**p_base(), "session_id": "session-2"}, ttft_ms=1)) + sink.emit(TurnFirstTokenEvent(**{**p_base(), "sequence": 4}, ttft_ms=1)) + sink.emit(TurnFirstTokenEvent(**{**p_base(), "session_id": "session-3"}, ttft_ms=1)) + + assert sink.snapshot("session-1").events + assert not sink.snapshot("session-2").events + assert sink.snapshot("session-3").events + + +def test_production_manager_retains_events_but_isolated_managers_default_to_null(): + from backend.apps.agents.agent_manager import AgentManager, agent_manager + + assert isinstance(agent_manager.event_sink, BoundedAgentEventSink) + assert isinstance(AgentManager().event_sink, NullAgentEventSink) + + +def test_emitter_first_token_and_tool_lifecycle_are_ordered_and_idempotent(): + sink = BoundedAgentEventSink() + emitter = AgentTurnEventEmitter( + sink=sink, session_id="session-1", provider="anthropic", model="claude" + ) + + emitter.emit_started() + emitter.emit_first_token() + emitter.emit_first_token() + emitter.emit_tool_started("tool-1", "Read") + emitter.emit_tool_started("tool-1", "Read") + emitter.emit_tool_completed("tool-1", "Read") + emitter.emit_completed() + + events = sink.snapshot("session-1").events + assert [event.kind for event in events] == [ + "turn.started", "turn.first_token", "tool.started", "tool.completed", "turn.completed" + ] + assert [event.sequence for event in events] == list(range(5)) + assert len({event.turn_id for event in events}) == 1 + + +def test_terminal_event_closes_unfinished_tools_without_raw_error_text(): + sink = BoundedAgentEventSink() + emitter = AgentTurnEventEmitter( + sink=sink, session_id="session-1", provider="anthropic", model="claude" + ) + emitter.emit_started() + emitter.emit_tool_started("tool-1", "Bash") + emitter.emit_failed("RuntimeError") + + events = sink.snapshot("session-1").events + assert [event.kind for event in events] == ["turn.started", "tool.started", "tool.completed", "turn.failed"] + assert events[2].status == "error" + assert events[2].error_type == "turn_ended" diff --git a/backend/tests/test_app_agent.py b/backend/tests/test_app_agent.py index 9a555a29a..8c9386c40 100644 --- a/backend/tests/test_app_agent.py +++ b/backend/tests/test_app_agent.py @@ -42,7 +42,7 @@ def test_app_bridge_expression_invoke_defaults_args_to_empty_object(): def test_execute_browser_tool_app_bridge_routes_to_evaluate(monkeypatch): captured = {} - async def p_send(request_id, action, browser_id, params, tab_id=""): + async def p_send(request_id, action, browser_id, params, tab_id="", *, owner=None): captured.update(action=action, browser_id=browser_id, params=params) return {"text": json.dumps([{"name": "addExpr"}])} @@ -192,7 +192,7 @@ def test_app_describe_polls_until_bridge_ready(monkeypatch): calls = {"n": 0} ready = {"rules": "r", "controls": [{"name": "x"}], "__rev": 1} - async def p_send(request_id, action, browser_id, params, tab_id=""): + async def p_send(request_id, action, browser_id, params, tab_id="", *, owner=None): calls["n"] += 1 if calls["n"] < 3: return {"text": json.dumps({"__ready": False, "__rev": 0})} @@ -212,7 +212,7 @@ async def p_no_sleep(p_s): def test_app_invoke_does_not_poll(monkeypatch): calls = {"n": 0} - async def p_send(request_id, action, browser_id, params, tab_id=""): + async def p_send(request_id, action, browser_id, params, tab_id="", *, owner=None): calls["n"] += 1 return {"text": json.dumps({"__ready": False})} # would loop forever if AppInvoke waited diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index d60b200f1..53aebfaa2 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -114,7 +114,7 @@ def p_client_for(s, model): # fake WS: record browser commands, script results by action sent = [] - async def p_send_browser_command(request_id, action, browser_id, params, tab_id=""): + async def p_send_browser_command(request_id, action, browser_id, params, tab_id="", *, owner=None): sent.append({"action": action, "params": params}) # smart-wait probes via evaluate; report 'settled' so BrowserWait returns fast in tests instead of riding the full cap. if action == "evaluate" and "getEntriesByType('resource')" in str(params.get("expression", "")): @@ -511,11 +511,11 @@ def test_replay_falls_back_to_full_agent_when_a_step_fails(monkeypatch): sent = p_install(monkeypatch, primary, aux) # make click_by_name FAIL (target gone) so replay must fall back orig = BA.ws_manager.send_browser_command - async def p_fail_cbn(request_id, action, browser_id, params, tab_id=""): + async def p_fail_cbn(request_id, action, browser_id, params, tab_id="", *, owner=None): if action == "click_by_name": sent.append({"action": action, "params": params}) return {"error": 'No element matching name="Save" on this page.'} - return await orig(request_id, action, browser_id, params, tab_id) + return await orig(request_id, action, browser_id, params, tab_id, owner=owner) monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_fail_cbn, raising=False) r = asyncio.run(BA.run_browser_agent( @@ -545,11 +545,11 @@ def test_deferred_replay_fires_after_navigating_to_the_right_host(monkeypatch): GOOGLE = "https://www.google.com/" orig = BA.ws_manager.send_browser_command - async def p_cmd(request_id, action, browser_id, params, tab_id=""): + async def p_cmd(request_id, action, browser_id, params, tab_id="", *, owner=None): # perception + reads report GOOGLE (so the DISPATCH replay misses there), navigation + clicks report the doc host (so the re-check matches) if action in ("list_interactives", "get_text"): return {"text": "stuff", "url": GOOGLE} - return await orig(request_id, action, browser_id, params, tab_id) + return await orig(request_id, action, browser_id, params, tab_id, owner=owner) monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_cmd, raising=False) # NO initial_url -> dispatch perceives google -> dispatch replay misses. @@ -582,10 +582,10 @@ def test_deferred_replay_does_not_fire_after_the_page_was_dirtied(monkeypatch): GOOGLE = "https://www.google.com/" orig = BA.ws_manager.send_browser_command - async def p_cmd(request_id, action, browser_id, params, tab_id=""): + async def p_cmd(request_id, action, browser_id, params, tab_id="", *, owner=None): if action in ("list_interactives", "get_text"): return {"text": "stuff", "url": GOOGLE} - return await orig(request_id, action, browser_id, params, tab_id) + return await orig(request_id, action, browser_id, params, tab_id, owner=owner) monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_cmd, raising=False) r = asyncio.run(BA.run_browser_agent( @@ -746,11 +746,11 @@ def test_unproven_skill_that_fails_is_quarantined_and_never_retried(monkeypatch) sent = p_install(monkeypatch, primary, aux) orig = BA.ws_manager.send_browser_command - async def p_fail_cbn(request_id, action, browser_id, params, tab_id=""): + async def p_fail_cbn(request_id, action, browser_id, params, tab_id="", *, owner=None): if action == "click_by_name": sent.append({"action": action, "params": params}) return {"error": 'No element matching name="Save" on this page.'} - return await orig(request_id, action, browser_id, params, tab_id) + return await orig(request_id, action, browser_id, params, tab_id, owner=owner) monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_fail_cbn, raising=False) # Run 1: replay is attempted, the step fails -> skill is quarantined. @@ -861,7 +861,7 @@ def test_dead_browser_card_aborts_fast_without_spinning(monkeypatch): aux = FakeAux() p_install(monkeypatch, primary, aux) - async def p_card_gone(request_id, action, browser_id, params, tab_id=""): + async def p_card_gone(request_id, action, browser_id, params, tab_id="", *, owner=None): return {"error": f"Browser card '{browser_id}' not found or not an Electron webview"} monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_card_gone, raising=False) captured = {} @@ -892,7 +892,7 @@ def test_hung_browser_card_aborts_fast_not_a_20_minute_loop(monkeypatch): ) p_install(monkeypatch, primary, FakeAux()) - async def p_hung(request_id, action, browser_id, params, tab_id=""): + async def p_hung(request_id, action, browser_id, params, tab_id="", *, owner=None): return {"error": "Browser command timed out"} # what a wedged tab returns monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_hung, raising=False) captured = {} @@ -1040,13 +1040,12 @@ async def create(self, **kw): return Resp([Blk("text", p_json.dumps({"playbook": ["search company+React, not generic"]}))], stop_reason="end_turn") msgs = [] - orig = BA.ws_manager.send_to_session async def p_cap(session_id, event, payload): if event == "agent:message": c = payload.get("message", {}).get("content") msgs.append(c if isinstance(c, str) else (c or {}).get("text", "")) - return await orig(session_id, event, payload) + return None monkeypatch.setattr(BA.ws_manager, "send_to_session", p_cap, raising=False) def p_run(): @@ -1117,7 +1116,7 @@ def test_batch_replay_runs_a_read_loop_for_all_values(monkeypatch): ]) sent = p_install(monkeypatch, primary, FakeAux()) - async def p_data(request_id, action, browser_id, params, tab_id=""): + async def p_data(request_id, action, browser_id, params, tab_id="", *, owner=None): sent.append({"action": action, "params": params}) if action == "evaluate": # return value-specific data so we can prove the DATA comes back @@ -1148,7 +1147,7 @@ def test_batch_replay_is_ghost_proof_when_an_item_does_not_match(monkeypatch): ]) sent = p_install(monkeypatch, primary, FakeAux()) - async def p_vary(request_id, action, browser_id, params, tab_id=""): + async def p_vary(request_id, action, browser_id, params, tab_id="", *, owner=None): sent.append({"action": action, "params": params}) if action == "navigate" and "grace" in params.get("url", ""): return {"error": "Page not found for grace (different layout)"} @@ -1213,10 +1212,10 @@ def test_captured_routes_are_surfaced_once_per_host(monkeypatch): sent = p_install(monkeypatch, primary, FakeAux()) orig = BA.ws_manager.send_browser_command - async def p_with_routes(request_id, action, browser_id, params, tab_id=""): + async def p_with_routes(request_id, action, browser_id, params, tab_id="", *, owner=None): if action == "evaluate": return {"text": "Reddit Programming", "url": DOC_URL, "routes_available": 4} - return await orig(request_id, action, browser_id, params, tab_id) + return await orig(request_id, action, browser_id, params, tab_id, owner=owner) monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_with_routes, raising=False) asyncio.run(BA.run_browser_agent(task="browse", browser_id="b1", model="sonnet", initial_url=DOC_URL)) @@ -1799,7 +1798,7 @@ def test_warm_send_prefix_replay_marries_send_script_zero_llm_turns(monkeypatch) seq = {"n": 0} states = [COMPOSER, COMMITTED, CLEARED, CLEARED] - async def p_cmd(request_id, action, browser_id, params, tab_id=""): + async def p_cmd(request_id, action, browser_id, params, tab_id="", *, owner=None): sent.append({"action": action, "params": params}) if action == "list_interactives": s = states[min(seq["n"], len(states) - 1)]; seq["n"] += 1 @@ -1836,7 +1835,7 @@ def test_autosend_finishes_the_send_after_the_model_fills(monkeypatch): CLEARED = '[2]\n[9]