From 70bd0a0aae73465719716ad425d9c6c3cf2271c2 Mon Sep 17 00:00:00 2001 From: Jim Fan Date: Sun, 15 Nov 2026 17:00:20 -0700 Subject: [PATCH 1/5] Upload safe host stages for opted-in CloudEx sessions --- sima_cli/cloudex/client.py | 47 +++++++++++++++++++++++++++++++ tests/unit/test_cloudex_client.py | 23 +++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/sima_cli/cloudex/client.py b/sima_cli/cloudex/client.py index 51a0220e..be75bfb4 100644 --- a/sima_cli/cloudex/client.py +++ b/sima_cli/cloudex/client.py @@ -6,6 +6,7 @@ import json import os import shutil +import stat import subprocess import sys import time @@ -40,6 +41,15 @@ "signaling session conflict", "signaling rate limited", } +UPLOAD_FORWARDER_STAGES = SAFE_FORWARDER_STAGES | { + "ICE candidate gathering complete", + "ICE connection state Checking", + "ICE connection state Connected", + "ICE connection state Completed", + "ICE connection state Disconnected", + "ICE connection state Failed", + "ICE connection state Closed", +} class CloudExError(RuntimeError): @@ -322,6 +332,37 @@ def _safe_last_stage(log_path): return stages[-1] if stages else "" +def _submit_diagnostics(api, session, store, phase, outcome): + sid = session.get("remote_session_id") + if session.get("diagnostics") is not True or not _valid_session_id(sid): + return + log_path = store.runtime_paths(session["session_id"])[3] + stages = [] + try: + limit = 64 * 1024 + fd = os.open(str(log_path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as log: + info = os.fstat(log.fileno()) + if stat.S_ISREG(info.st_mode): + log.seek(max(0, info.st_size - limit)) + if log.tell(): + log.readline(limit) + for line in log.read(limit).decode("utf-8", errors="replace").splitlines(): + prefix = "kerrigan-p2p-forwarder:" + event = line[len(prefix):].strip() if line.startswith(prefix) else line.strip() + if event in UPLOAD_FORWARDER_STAGES and (not stages or stages[-1] != event): + stages.append(event) + except OSError: + pass + try: + api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", + {"phase": phase, "outcome": outcome, "stages": stages[-32:]}, + deadline=time.time() + 2) + except CloudExError: + # Evidence upload must never prevent connecting or releasing a lease. + pass + + def _safe_selected_path(log_path): """Recover the non-sensitive selected path from forwarder lifecycle logs.""" try: @@ -646,6 +687,8 @@ def connect(profile, store, progress, attempts=3, transport="auto"): if not _valid_session_id(remote_session_id): raise CloudExError("CloudEx API returned an invalid session identifier") session["remote_session_id"] = remote_session_id + if allocated.get("diagnostics") is True: + session["diagnostics"] = True store.write_recovery(session) try: expires_at = float(allocated["expires_at"]) @@ -655,6 +698,7 @@ def connect(profile, store, progress, attempts=3, transport="auto"): runtime.mkdir(parents=True, exist_ok=True, mode=0o700) report_path.touch(mode=0o600, exist_ok=False) config = dict(allocated) + config.pop("diagnostics", None) config["transport"] = selected_transport config_path.write_text(json.dumps(config, separators=(",", ":")), encoding="utf-8") config_path.chmod(0o600) @@ -699,12 +743,14 @@ def connect(profile, store, progress, attempts=3, transport="auto"): store.write(session) store.clear_recovery(session_id) config_path.unlink(missing_ok=True) + _submit_diagnostics(api, session, store, "setup", "connected") return session except BaseException as exc: last_error = exc if _creation_definitely_rejected(exc): store.delete(session_id) raise + _submit_diagnostics(api, session, store, "setup", "failed") progress.update("Cleaning up the failed connection attempt") try: disconnect_session( @@ -802,6 +848,7 @@ def disconnect_session(session, store, progress, authorize=True, preserve_log=Fa api = CloudExAPI.from_session(session) progress.update("Stopping the local encrypted tunnel") _stop_forwarder(session) + _submit_diagnostics(api, session, store, "teardown", "closed") progress.update("Requesting DevKit tunnel cleanup") session_id = session["session_id"] remote_session_id = _remote_session_id(session, store) diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index 635f4cbf..9dc40378 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -774,3 +774,26 @@ def test_macos_forwarder_failure_includes_install_instruction(tmp_path, detail): assert "brew install wireguard-tools" in message assert message.endswith("; last stage: starting WireGuard tunnel") + + +def test_cloudex_diagnostics_posts_only_safe_stages_for_the_remote_session(tmp_path): + store = client.SessionStore(tmp_path) + local_id, sid = "a" * 32, "b" * 32 + _, _, _, log = store.runtime_paths(local_id) + log.parent.mkdir(parents=True) + log.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n" + "kerrigan-p2p-forwarder: signaling token=fixture-secret\n") + calls = [] + + class API: + def call(self, method, route, data, **kwargs): + calls.append((method, route, data)) + raise client.CloudExError("diagnostics temporarily unavailable") + + session = {"session_id": local_id, "remote_session_id": sid} + client._submit_diagnostics(API(), session, store, "setup", "failed") + assert calls == [] + session["diagnostics"] = True + client._submit_diagnostics(API(), session, store, "setup", "failed") + assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", + {"phase": "setup", "outcome": "failed", "stages": ["ICE connection state Failed"]})] From e27b7e2ca707f2fc82f48201730895fe956d7c6a Mon Sep 17 00:00:00 2001 From: Jim Fan Date: Sun, 15 Nov 2026 18:50:06 -0700 Subject: [PATCH 2/5] Report CloudEx diagnostics after a failed connection attempt --- sima_cli/cloudex/client.py | 19 +++++++++++-------- sima_cli/cloudex/commands.py | 12 +++++++----- sima_cli/cloudex/neatops.py | 18 ++++++++++++++---- tests/unit/test_cloudex_client.py | 7 ++++--- tests/unit/test_cloudex_neatops.py | 19 +++++++++++++++++-- 5 files changed, 53 insertions(+), 22 deletions(-) diff --git a/sima_cli/cloudex/client.py b/sima_cli/cloudex/client.py index be75bfb4..54e347ad 100644 --- a/sima_cli/cloudex/client.py +++ b/sima_cli/cloudex/client.py @@ -332,7 +332,7 @@ def _safe_last_stage(log_path): return stages[-1] if stages else "" -def _submit_diagnostics(api, session, store, phase, outcome): +def _submit_diagnostics(api, session, store, report_failure=None): sid = session.get("remote_session_id") if session.get("diagnostics") is not True or not _valid_session_id(sid): return @@ -355,9 +355,12 @@ def _submit_diagnostics(api, session, store, phase, outcome): except OSError: pass try: - api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", - {"phase": phase, "outcome": outcome, "stages": stages[-32:]}, - deadline=time.time() + 2) + stages = stages[-32:] + if report_failure is not None: + report_failure(api.allocation_id, sid, stages) + else: + api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", + {"stages": stages}, deadline=time.time() + 2) except CloudExError: # Evidence upload must never prevent connecting or releasing a lease. pass @@ -642,7 +645,7 @@ def _forwarder_sudo_prefix(progress=None): return prefix + ["/usr/bin/env", "PATH=" + MACOS_TUNNEL_PATH] -def connect(profile, store, progress, attempts=3, transport="auto"): +def connect(profile, store, progress, attempts=3, transport="auto", *, report_failure=None): """Create a session and return its durable local record.""" from .installer import ensure_forwarder @@ -743,14 +746,12 @@ def connect(profile, store, progress, attempts=3, transport="auto"): store.write(session) store.clear_recovery(session_id) config_path.unlink(missing_ok=True) - _submit_diagnostics(api, session, store, "setup", "connected") return session except BaseException as exc: last_error = exc if _creation_definitely_rejected(exc): store.delete(session_id) raise - _submit_diagnostics(api, session, store, "setup", "failed") progress.update("Cleaning up the failed connection attempt") try: disconnect_session( @@ -762,6 +763,9 @@ def connect(profile, store, progress, attempts=3, transport="auto"): raise CloudExError( "Connection failed and cleanup is pending: {}. Run cloudex disconnect again.".format(cleanup_error) ) from exc + finally: + if isinstance(exc, Exception): + _submit_diagnostics(api, session, store, report_failure) if not isinstance(exc, DirectHandshakeError) or number == len(choices): raise progress.update("Direct path timed out; retrying with fresh ICE candidates") @@ -848,7 +852,6 @@ def disconnect_session(session, store, progress, authorize=True, preserve_log=Fa api = CloudExAPI.from_session(session) progress.update("Stopping the local encrypted tunnel") _stop_forwarder(session) - _submit_diagnostics(api, session, store, "teardown", "closed") progress.update("Requesting DevKit tunnel cleanup") session_id = session["session_id"] remote_session_id = _remote_session_id(session, store) diff --git a/sima_cli/cloudex/commands.py b/sima_cli/cloudex/commands.py index b2d4b629..58774fa1 100644 --- a/sima_cli/cloudex/commands.py +++ b/sima_cli/cloudex/commands.py @@ -267,8 +267,9 @@ def connect_command(key_file, key_stdin, allocation, environment, transport, att raise click.UsageError("Use either --key-file or --key-stdin, not both; --allocation must be used alone.") try: if allocation: - profile = _allocation_profile(_neatops_api(environment), _allocation_id(allocation)) - _connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport) + api = _neatops_api(environment) + profile = _allocation_profile(api, _allocation_id(allocation)) + _connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport, diagnostics_api=api) return if key_file: key = _read_key_file(key_file) @@ -452,11 +453,12 @@ def register_cloudex_commands(main): main.add_command(cloudex_group) -def _connect_profile(profile, verbose=False, attempts=3, transport="auto"): +def _connect_profile(profile, verbose=False, attempts=3, transport="auto", *, diagnostics_api=None): console.print("[cyan]→[/cyan] Authorizing encrypted tunnel access") _authorize_supported_host() with StageProgress("Preparing CloudEx connection", verbose=verbose) as progress: - session = connect(profile, _store(), progress, attempts=attempts, transport=transport) + options = {"report_failure": diagnostics_api.report_cloudex_failure} if diagnostics_api else {} + session = connect(profile, _store(), progress, attempts=attempts, transport=transport, **options) progress.success("Connected to {}".format(session["device"])) path = session.get("ice_path", "unknown") path_text = "direct" if path == "direct" else "relayed" if path == "relay" else path @@ -605,7 +607,7 @@ def request_command(device, cluster, duration, environment, verbose): api.call("POST", "/devkits/allocations", {"request_id": allocation_id, "cluster_id": selected["cluster_id"], "target_id": selected["id"], "duration_seconds": DURATIONS[duration]}) profile = _allocation_profile(api, allocation_id) - _connect_profile(profile, verbose=verbose) + _connect_profile(profile, verbose=verbose, diagnostics_api=api) except CloudExError as exc: if allocation_id: console.print("Inspect: sima-cli -i cloudex devices --environment {}\nResume: sima-cli -i cloudex connect --allocation {} --environment {}\nRelease: sima-cli -i cloudex release {} --environment {}".format(environment, allocation_id, environment, allocation_id, environment), markup=False, soft_wrap=True) diff --git a/sima_cli/cloudex/neatops.py b/sima_cli/cloudex/neatops.py index 07f7eb9c..a33f1eb3 100644 --- a/sima_cli/cloudex/neatops.py +++ b/sima_cli/cloudex/neatops.py @@ -34,9 +34,9 @@ def _origin(value): return value.rstrip("/") -def _json_request(method, url, **kwargs): +def _json_request(method, url, *, timeout=20, **kwargs): try: - response = requests.request(method, url, timeout=20, allow_redirects=False, **kwargs) + response = requests.request(method, url, timeout=timeout, allow_redirects=False, **kwargs) except requests.RequestException: raise CloudExError("NeatOps is unavailable; check your connection and retry") from None try: @@ -193,7 +193,7 @@ def _read_session(self): except OSError: raise CloudExError("Sign in first with sima-cli -i cloudex login --environment " + self.environment) from None - def access_token(self): + def access_token(self, *, allow_refresh=True): auth = self.configuration()["auth"] session = self._read_session() try: @@ -204,7 +204,7 @@ def access_token(self): refresh = session.get("refresh_token") except (ValueError, KeyError, TypeError): raise LoginRequired("Sign-in policy changed; run sima-cli -i cloudex login again") from None - if not refresh: + if not refresh or not allow_refresh: raise LoginRequired("NeatOps sign-in expired; run sima-cli -i cloudex login again") tokens = self._exchange({"grant_type": "refresh_token", "refresh_token": refresh}) tokens.setdefault("refresh_token", refresh) @@ -229,6 +229,16 @@ def _authenticate(self): self.on_login_required(self) return self.access_token() + def report_cloudex_failure(self, allocation_id, session_id, stages): + token = self.access_token(allow_refresh=False) + config = self.configuration() + path = f"/devkits/allocations/{allocation_id}/diagnostics" + status, _ = _json_request("POST", config["apiBase"] + path, timeout=3, + headers={"Authorization": "Bearer " + token}, + json={"session_id": session_id, "stages": stages}) + if status != 202: + raise CloudExError("NeatOps did not accept CloudEx diagnostics") + def call(self, method, path, data=None): config = self.configuration() try: diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index 9dc40378..a50c28d7 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -176,6 +176,7 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat process = Mock(pid=123, poll=Mock(return_value=None)) allocated = { "session_id": "d" * 32, + "diagnostics": True, "expires_at": "9999999999", "target_id": "ll2", "wireguard": {"remote_address": "10.252.1.2/32"}, @@ -791,9 +792,9 @@ def call(self, method, route, data, **kwargs): raise client.CloudExError("diagnostics temporarily unavailable") session = {"session_id": local_id, "remote_session_id": sid} - client._submit_diagnostics(API(), session, store, "setup", "failed") + client._submit_diagnostics(API(), session, store) assert calls == [] session["diagnostics"] = True - client._submit_diagnostics(API(), session, store, "setup", "failed") + client._submit_diagnostics(API(), session, store) assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", - {"phase": "setup", "outcome": "failed", "stages": ["ICE connection state Failed"]})] + {"stages": ["ICE connection state Failed"]})] diff --git a/tests/unit/test_cloudex_neatops.py b/tests/unit/test_cloudex_neatops.py index 70fa9d38..9759f8e6 100644 --- a/tests/unit/test_cloudex_neatops.py +++ b/tests/unit/test_cloudex_neatops.py @@ -184,7 +184,7 @@ def test_request_selects_device_allocates_then_connects(command_api, monkeypatch command_api.call.assert_any_call("POST", "/devkits/allocations", {"request_id": AID, "cluster_id": "appliance-id", "target_id": "device-id", "duration_seconds": 14400}) profile.assert_called_once_with(command_api, AID) - connect.assert_called_once_with(PROFILE, verbose=False) + connect.assert_called_once_with(PROFILE, verbose=False, diagnostics_api=command_api) assert "allocation_secret" not in result.output @@ -231,7 +231,7 @@ def test_connect_allocation_skips_manual_key_prompt(command_api, monkeypatch): monkeypatch.setattr(commands, "_connect_profile", connect) result = CliRunner().invoke(commands.cloudex_group, ["-i", "connect", "--allocation", AID]) assert result.exit_code == 0, result.output - connect.assert_called_once_with(PROFILE, verbose=False, attempts=3, transport="auto") + connect.assert_called_once_with(PROFILE, verbose=False, attempts=3, transport="auto", diagnostics_api=command_api) def test_release_disconnects_only_matching_local_allocation(command_api, monkeypatch): @@ -582,3 +582,18 @@ def test_root_external_login_uses_portal_without_internal_initialization(monkeyp initialize.assert_not_called() portal_login.assert_called_once_with(loginDocker=False) vulcan_api.assert_not_called() + + + +def test_failure_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch): + api._save(tokens()) + request = Mock(return_value=(202, {"status": "accepted"})) + monkeypatch.setattr(neatops, "_json_request", request) + allocation, session = "a" * 32, "b" * 32 + stages = ["ICE connection state Failed"] + api.report_cloudex_failure(allocation, session, stages) + request.assert_called_once_with( + "POST", api.config["apiBase"] + f"/devkits/allocations/{allocation}/diagnostics", + timeout=3, headers={"Authorization": "Bearer private-access"}, + json={"session_id": session, "stages": stages}, + ) From 7fda0591e37b079d0282bbd3e28415b7b38009df Mon Sep 17 00:00:00 2001 From: Jim Fan Date: Sun, 15 Nov 2026 18:51:07 -0700 Subject: [PATCH 3/5] Verify failure diagnostics preserve the original connection error --- tests/unit/test_cloudex_client.py | 32 +++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index a50c28d7..9d035ed5 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -798,3 +798,35 @@ def call(self, method, route, data, **kwargs): client._submit_diagnostics(API(), session, store) assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", {"stages": ["ICE connection state Failed"]})] + + +def test_failed_connection_reports_after_attempting_cleanup(tmp_path, monkeypatch): + store = client.SessionStore(tmp_path) + profile = allocation_profile() + sid = "d" * 32 + api = Mock(url=profile["api_url"], requester=profile["requester"], + secret=profile["allocation_secret"], allocation_id=profile["allocation_id"]) + api.call.return_value = {"session_id": sid, "diagnostics": True, "expires_at": 9999999999} + monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api)) + monkeypatch.setattr(client, "authorize_admin", Mock()) + monkeypatch.setattr(client, "_forwarder_sudo_prefix", Mock(return_value=[])) + monkeypatch.setattr(installer, "ensure_forwarder", Mock(return_value=tmp_path / "forwarder")) + monkeypatch.setattr(client.subprocess, "Popen", Mock(return_value=Mock(pid=123))) + operations = [] + + def fail(_sid, _process, _report_path, log_path, *_args, **_kwargs): + log_path.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n") + raise client.CloudExError("Original tunnel failure") + + def cleanup(*_args, **_kwargs): + operations.append("cleanup") + + def report(allocation_id, session_id, stages): + operations.append((allocation_id, session_id, stages)) + raise client.CloudExError("Diagnostics unavailable") + + monkeypatch.setattr(client, "_wait_forwarder", fail) + monkeypatch.setattr(client, "disconnect_session", cleanup) + with pytest.raises(client.CloudExError, match="Original tunnel failure"): + client.connect(profile, store, Mock(), attempts=1, transport="p2p", report_failure=report) + assert operations == ["cleanup", (profile["allocation_id"], sid, ["ICE connection state Failed"])] From fbbf072900d40c041039187ed90e130f50db4d00 Mon Sep 17 00:00:00 2001 From: Jim Fan Date: Sun, 15 Nov 2026 18:53:31 -0700 Subject: [PATCH 4/5] Upload diagnostics for successful CloudEx relay connections --- sima_cli/cloudex/client.py | 17 +++++++++------- sima_cli/cloudex/commands.py | 2 +- sima_cli/cloudex/neatops.py | 4 ++-- tests/unit/test_cloudex_client.py | 31 +++++++++++++++++++----------- tests/unit/test_cloudex_neatops.py | 6 +++--- 5 files changed, 36 insertions(+), 24 deletions(-) diff --git a/sima_cli/cloudex/client.py b/sima_cli/cloudex/client.py index 54e347ad..c2806d2c 100644 --- a/sima_cli/cloudex/client.py +++ b/sima_cli/cloudex/client.py @@ -332,7 +332,7 @@ def _safe_last_stage(log_path): return stages[-1] if stages else "" -def _submit_diagnostics(api, session, store, report_failure=None): +def _submit_diagnostics(api, session, store, reason, report_diagnostics=None): sid = session.get("remote_session_id") if session.get("diagnostics") is not True or not _valid_session_id(sid): return @@ -356,11 +356,11 @@ def _submit_diagnostics(api, session, store, report_failure=None): pass try: stages = stages[-32:] - if report_failure is not None: - report_failure(api.allocation_id, sid, stages) + if report_diagnostics is not None: + report_diagnostics(api.allocation_id, sid, reason, stages) else: api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", - {"stages": stages}, deadline=time.time() + 2) + {"reason": reason, "stages": stages}, deadline=time.time() + 2) except CloudExError: # Evidence upload must never prevent connecting or releasing a lease. pass @@ -645,7 +645,7 @@ def _forwarder_sudo_prefix(progress=None): return prefix + ["/usr/bin/env", "PATH=" + MACOS_TUNNEL_PATH] -def connect(profile, store, progress, attempts=3, transport="auto", *, report_failure=None): +def connect(profile, store, progress, attempts=3, transport="auto", *, report_diagnostics=None): """Create a session and return its durable local record.""" from .installer import ensure_forwarder @@ -746,7 +746,6 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_fa store.write(session) store.clear_recovery(session_id) config_path.unlink(missing_ok=True) - return session except BaseException as exc: last_error = exc if _creation_definitely_rejected(exc): @@ -765,10 +764,14 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_fa ) from exc finally: if isinstance(exc, Exception): - _submit_diagnostics(api, session, store, report_failure) + _submit_diagnostics(api, session, store, "failed", report_diagnostics) if not isinstance(exc, DirectHandshakeError) or number == len(choices): raise progress.update("Direct path timed out; retrying with fresh ICE candidates") + else: + if session["ice_path"] == "relay": + _submit_diagnostics(api, session, store, "relay", report_diagnostics) + return session raise last_error or CloudExError("CloudEx connection failed") diff --git a/sima_cli/cloudex/commands.py b/sima_cli/cloudex/commands.py index 58774fa1..2d155f84 100644 --- a/sima_cli/cloudex/commands.py +++ b/sima_cli/cloudex/commands.py @@ -457,7 +457,7 @@ def _connect_profile(profile, verbose=False, attempts=3, transport="auto", *, di console.print("[cyan]→[/cyan] Authorizing encrypted tunnel access") _authorize_supported_host() with StageProgress("Preparing CloudEx connection", verbose=verbose) as progress: - options = {"report_failure": diagnostics_api.report_cloudex_failure} if diagnostics_api else {} + options = {"report_diagnostics": diagnostics_api.report_cloudex_diagnostics} if diagnostics_api else {} session = connect(profile, _store(), progress, attempts=attempts, transport=transport, **options) progress.success("Connected to {}".format(session["device"])) path = session.get("ice_path", "unknown") diff --git a/sima_cli/cloudex/neatops.py b/sima_cli/cloudex/neatops.py index a33f1eb3..24910888 100644 --- a/sima_cli/cloudex/neatops.py +++ b/sima_cli/cloudex/neatops.py @@ -229,13 +229,13 @@ def _authenticate(self): self.on_login_required(self) return self.access_token() - def report_cloudex_failure(self, allocation_id, session_id, stages): + def report_cloudex_diagnostics(self, allocation_id, session_id, reason, stages): token = self.access_token(allow_refresh=False) config = self.configuration() path = f"/devkits/allocations/{allocation_id}/diagnostics" status, _ = _json_request("POST", config["apiBase"] + path, timeout=3, headers={"Authorization": "Bearer " + token}, - json={"session_id": session_id, "stages": stages}) + json={"session_id": session_id, "reason": reason, "stages": stages}) if status != 202: raise CloudExError("NeatOps did not accept CloudEx diagnostics") diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index 9d035ed5..c8cfd90d 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -159,8 +159,9 @@ def test_api_signs_allocation_scoped_request(monkeypatch, environment, restored) assert "allocation_secret" not in kwargs["data"].decode() +@pytest.mark.parametrize("path", ["direct", "relay"]) @pytest.mark.parametrize("host,bootstrap", [("linux", False), ("darwin", False), ("darwin", True)]) -def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap): +def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap, path): client.sys.platform = host which = Mock(return_value="/opt/homebrew/bin/tool") if bootstrap: @@ -192,11 +193,19 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api)) popen = Mock(return_value=process) monkeypatch.setattr(client.subprocess, "Popen", popen) - wait_forwarder = Mock(return_value={"status": "connected", "path": "direct"}) + wait_forwarder = Mock(return_value={"status": "connected", "path": path}) monkeypatch.setattr(client, "_wait_forwarder", wait_forwarder) monkeypatch.setattr(client.time, "time", Mock(return_value=1000)) - result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p") + cleanup = Mock() + monkeypatch.setattr(client, "disconnect_session", cleanup) + upload = Mock(side_effect=client.CloudExError("Diagnostics unavailable")) + result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p", report_diagnostics=upload) + cleanup.assert_not_called() + if path == "relay": + upload.assert_called_once_with(api.allocation_id, allocated["session_id"], "relay", []) + else: + upload.assert_not_called() if bootstrap: assert brew.call_args.args[0] == ["/opt/homebrew/bin/brew", "install", "wireguard-tools", "wireguard-go"] @@ -204,7 +213,7 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat brew.assert_not_called() assert result["device"] == "ll2" assert result["target"] == "10.252.1.2" - assert result["ice_path"] == "direct" + assert result["ice_path"] == path assert result["expires_at"] == 9999999999.0 assert store.list()[0]["api_secret"] == allocation_profile()["allocation_secret"] assert not store.runtime_paths(result["session_id"])[1].exists() @@ -792,12 +801,12 @@ def call(self, method, route, data, **kwargs): raise client.CloudExError("diagnostics temporarily unavailable") session = {"session_id": local_id, "remote_session_id": sid} - client._submit_diagnostics(API(), session, store) + client._submit_diagnostics(API(), session, store, "failed") assert calls == [] session["diagnostics"] = True - client._submit_diagnostics(API(), session, store) + client._submit_diagnostics(API(), session, store, "failed") assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", - {"stages": ["ICE connection state Failed"]})] + {"reason": "failed", "stages": ["ICE connection state Failed"]})] def test_failed_connection_reports_after_attempting_cleanup(tmp_path, monkeypatch): @@ -821,12 +830,12 @@ def fail(_sid, _process, _report_path, log_path, *_args, **_kwargs): def cleanup(*_args, **_kwargs): operations.append("cleanup") - def report(allocation_id, session_id, stages): - operations.append((allocation_id, session_id, stages)) + def report(allocation_id, session_id, reason, stages): + operations.append((allocation_id, session_id, reason, stages)) raise client.CloudExError("Diagnostics unavailable") monkeypatch.setattr(client, "_wait_forwarder", fail) monkeypatch.setattr(client, "disconnect_session", cleanup) with pytest.raises(client.CloudExError, match="Original tunnel failure"): - client.connect(profile, store, Mock(), attempts=1, transport="p2p", report_failure=report) - assert operations == ["cleanup", (profile["allocation_id"], sid, ["ICE connection state Failed"])] + client.connect(profile, store, Mock(), attempts=1, transport="p2p", report_diagnostics=report) + assert operations == ["cleanup", (profile["allocation_id"], sid, "failed", ["ICE connection state Failed"])] diff --git a/tests/unit/test_cloudex_neatops.py b/tests/unit/test_cloudex_neatops.py index 9759f8e6..b53848a8 100644 --- a/tests/unit/test_cloudex_neatops.py +++ b/tests/unit/test_cloudex_neatops.py @@ -585,15 +585,15 @@ def test_root_external_login_uses_portal_without_internal_initialization(monkeyp -def test_failure_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch): +def test_diagnostics_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch): api._save(tokens()) request = Mock(return_value=(202, {"status": "accepted"})) monkeypatch.setattr(neatops, "_json_request", request) allocation, session = "a" * 32, "b" * 32 stages = ["ICE connection state Failed"] - api.report_cloudex_failure(allocation, session, stages) + api.report_cloudex_diagnostics(allocation, session, "failed", stages) request.assert_called_once_with( "POST", api.config["apiBase"] + f"/devkits/allocations/{allocation}/diagnostics", timeout=3, headers={"Authorization": "Bearer private-access"}, - json={"session_id": session, "stages": stages}, + json={"session_id": session, "reason": "failed", "stages": stages}, ) From fa58e3da2906215515c7032f8bd4dca8d1939b7d Mon Sep 17 00:00:00 2001 From: Jim Fan Date: Sun, 15 Nov 2026 23:04:43 -0700 Subject: [PATCH 5/5] Report direct CloudEx sessions and setup timing --- sima_cli/cloudex/client.py | 29 ++++++++++++++++++++++++----- sima_cli/cloudex/neatops.py | 7 +++++-- tests/unit/test_cloudex_client.py | 17 ++++++++++++++--- tests/unit/test_cloudex_neatops.py | 12 ++++++++---- 4 files changed, 51 insertions(+), 14 deletions(-) diff --git a/sima_cli/cloudex/client.py b/sima_cli/cloudex/client.py index c2806d2c..46c532ac 100644 --- a/sima_cli/cloudex/client.py +++ b/sima_cli/cloudex/client.py @@ -5,6 +5,7 @@ import hmac import json import os +import re import shutil import stat import subprocess @@ -23,6 +24,8 @@ MACOS_WIREGUARD_INSTALL = "Install WireGuard CLI support with: brew install wireguard-tools" CONNECTION_SETUP_TIMEOUT_SECONDS = 60 DIAGNOSTIC_LOG_LINES = 50 +ICE_SELECTION_TIMING = re.compile(r"ICE diagnostics phase=selected elapsed_ms=([0-9]{1,6}) pairs=[0-9]{1,6}") +MAX_DIAGNOSTIC_TIMING_MS = 900_000 SAFE_FORWARDER_STAGES = { "session configuration accepted", "joined authenticated signaling session", @@ -332,12 +335,18 @@ def _safe_last_stage(log_path): return stages[-1] if stages else "" -def _submit_diagnostics(api, session, store, reason, report_diagnostics=None): +def _submit_diagnostics(api, session, store, reason, report_diagnostics=None, *, setup_ms=None): sid = session.get("remote_session_id") if session.get("diagnostics") is not True or not _valid_session_id(sid): return + supports_timing = session.get("diagnostics_version") == 2 + if reason == "direct" and not supports_timing: + return log_path = store.runtime_paths(session["session_id"])[3] stages = [] + timing = {} + if supports_timing and setup_ms is not None and 0 <= setup_ms <= MAX_DIAGNOSTIC_TIMING_MS: + timing["setup_ms"] = setup_ms try: limit = 64 * 1024 fd = os.open(str(log_path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) @@ -352,15 +361,20 @@ def _submit_diagnostics(api, session, store, reason, report_diagnostics=None): event = line[len(prefix):].strip() if line.startswith(prefix) else line.strip() if event in UPLOAD_FORWARDER_STAGES and (not stages or stages[-1] != event): stages.append(event) + elif supports_timing: + selection = ICE_SELECTION_TIMING.fullmatch(event) + if selection and int(selection[1]) <= MAX_DIAGNOSTIC_TIMING_MS: + timing["ice_selection_ms"] = int(selection[1]) except OSError: pass try: stages = stages[-32:] + extra = {"timing": timing} if timing else {} if report_diagnostics is not None: - report_diagnostics(api.allocation_id, sid, reason, stages) + report_diagnostics(api.allocation_id, sid, reason, stages, **extra) else: api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", - {"reason": reason, "stages": stages}, deadline=time.time() + 2) + {"reason": reason, "stages": stages, **extra}, deadline=time.time() + 2) except CloudExError: # Evidence upload must never prevent connecting or releasing a lease. pass @@ -664,6 +678,7 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_di now = time.time() if now >= connection_deadline: break + started = time.monotonic() attempt_deadline = now + (connection_deadline - now) / (len(choices) - number + 1) session_id = uuid.uuid4().hex session = { @@ -692,6 +707,8 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_di session["remote_session_id"] = remote_session_id if allocated.get("diagnostics") is True: session["diagnostics"] = True + if allocated.get("diagnostics_version") == 2: + session["diagnostics_version"] = 2 store.write_recovery(session) try: expires_at = float(allocated["expires_at"]) @@ -702,6 +719,7 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_di report_path.touch(mode=0o600, exist_ok=False) config = dict(allocated) config.pop("diagnostics", None) + config.pop("diagnostics_version", None) config["transport"] = selected_transport config_path.write_text(json.dumps(config, separators=(",", ":")), encoding="utf-8") config_path.chmod(0o600) @@ -769,8 +787,9 @@ def connect(profile, store, progress, attempts=3, transport="auto", *, report_di raise progress.update("Direct path timed out; retrying with fresh ICE candidates") else: - if session["ice_path"] == "relay": - _submit_diagnostics(api, session, store, "relay", report_diagnostics) + if session["ice_path"] in {"direct", "relay"}: + setup_ms = round((time.monotonic() - started) * 1000) + _submit_diagnostics(api, session, store, session["ice_path"], report_diagnostics, setup_ms=setup_ms) return session raise last_error or CloudExError("CloudEx connection failed") diff --git a/sima_cli/cloudex/neatops.py b/sima_cli/cloudex/neatops.py index 24910888..c0f1b255 100644 --- a/sima_cli/cloudex/neatops.py +++ b/sima_cli/cloudex/neatops.py @@ -229,13 +229,16 @@ def _authenticate(self): self.on_login_required(self) return self.access_token() - def report_cloudex_diagnostics(self, allocation_id, session_id, reason, stages): + def report_cloudex_diagnostics(self, allocation_id, session_id, reason, stages, *, timing=None): token = self.access_token(allow_refresh=False) config = self.configuration() path = f"/devkits/allocations/{allocation_id}/diagnostics" + report = {"session_id": session_id, "reason": reason, "stages": stages} + if timing: + report["timing"] = timing status, _ = _json_request("POST", config["apiBase"] + path, timeout=3, headers={"Authorization": "Bearer " + token}, - json={"session_id": session_id, "reason": reason, "stages": stages}) + json=report) if status != 202: raise CloudExError("NeatOps did not accept CloudEx diagnostics") diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index c8cfd90d..d357039c 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -178,6 +178,7 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat allocated = { "session_id": "d" * 32, "diagnostics": True, + "diagnostics_version": 2 if path == "direct" else 1, "expires_at": "9999999999", "target_id": "ll2", "wireguard": {"remote_address": "10.252.1.2/32"}, @@ -202,10 +203,11 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat upload = Mock(side_effect=client.CloudExError("Diagnostics unavailable")) result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p", report_diagnostics=upload) cleanup.assert_not_called() - if path == "relay": - upload.assert_called_once_with(api.allocation_id, allocated["session_id"], "relay", []) + assert upload.call_args.args == (api.allocation_id, allocated["session_id"], path, []) + if path == "direct": + assert upload.call_args.kwargs["timing"]["setup_ms"] >= 0 else: - upload.assert_not_called() + assert upload.call_args.kwargs == {} if bootstrap: assert brew.call_args.args[0] == ["/opt/homebrew/bin/brew", "install", "wireguard-tools", "wireguard-go"] @@ -792,6 +794,7 @@ def test_cloudex_diagnostics_posts_only_safe_stages_for_the_remote_session(tmp_p _, _, _, log = store.runtime_paths(local_id) log.parent.mkdir(parents=True) log.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n" + "kerrigan-p2p-forwarder: ICE diagnostics phase=selected elapsed_ms=1148 pairs=6\n" "kerrigan-p2p-forwarder: signaling token=fixture-secret\n") calls = [] @@ -807,6 +810,14 @@ def call(self, method, route, data, **kwargs): client._submit_diagnostics(API(), session, store, "failed") assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", {"reason": "failed", "stages": ["ICE connection state Failed"]})] + client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520) + assert len(calls) == 1 + session["diagnostics_version"] = 2 + client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520) + assert calls[-1] == ("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", { + "reason": "direct", "stages": ["ICE connection state Failed"], + "timing": {"setup_ms": 1520, "ice_selection_ms": 1148}, + }) def test_failed_connection_reports_after_attempting_cleanup(tmp_path, monkeypatch): diff --git a/tests/unit/test_cloudex_neatops.py b/tests/unit/test_cloudex_neatops.py index b53848a8..f035a153 100644 --- a/tests/unit/test_cloudex_neatops.py +++ b/tests/unit/test_cloudex_neatops.py @@ -585,15 +585,19 @@ def test_root_external_login_uses_portal_without_internal_initialization(monkeyp -def test_diagnostics_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch): +@pytest.mark.parametrize("reason,timing", [("failed", None), ("direct", {"setup_ms": 1420})]) +def test_diagnostics_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch, reason, timing): api._save(tokens()) request = Mock(return_value=(202, {"status": "accepted"})) monkeypatch.setattr(neatops, "_json_request", request) allocation, session = "a" * 32, "b" * 32 - stages = ["ICE connection state Failed"] - api.report_cloudex_diagnostics(allocation, session, "failed", stages) + stages = ["ICE connection state Failed"] if reason == "failed" else ["WireGuard handshake confirmed"] + api.report_cloudex_diagnostics(allocation, session, reason, stages, timing=timing) + expected = {"session_id": session, "reason": reason, "stages": stages} + if timing: + expected["timing"] = timing request.assert_called_once_with( "POST", api.config["apiBase"] + f"/devkits/allocations/{allocation}/diagnostics", timeout=3, headers={"Authorization": "Bearer private-access"}, - json={"session_id": session, "reason": "failed", "stages": stages}, + json=expected, )