diff --git a/sima_cli/cloudex/client.py b/sima_cli/cloudex/client.py index 51a0220e..46c532ac 100644 --- a/sima_cli/cloudex/client.py +++ b/sima_cli/cloudex/client.py @@ -5,7 +5,9 @@ import hmac import json import os +import re import shutil +import stat import subprocess import sys import time @@ -22,6 +24,8 @@ MACOS_WIREGUARD_INSTALL = "Install WireGuard CLI support with: brew install wireguard-tools" CONNECTION_SETUP_TIMEOUT_SECONDS = 60 DIAGNOSTIC_LOG_LINES = 50 +ICE_SELECTION_TIMING = re.compile(r"ICE diagnostics phase=selected elapsed_ms=([0-9]{1,6}) pairs=[0-9]{1,6}") +MAX_DIAGNOSTIC_TIMING_MS = 900_000 SAFE_FORWARDER_STAGES = { "session configuration accepted", "joined authenticated signaling session", @@ -40,6 +44,15 @@ "signaling session conflict", "signaling rate limited", } +UPLOAD_FORWARDER_STAGES = SAFE_FORWARDER_STAGES | { + "ICE candidate gathering complete", + "ICE connection state Checking", + "ICE connection state Connected", + "ICE connection state Completed", + "ICE connection state Disconnected", + "ICE connection state Failed", + "ICE connection state Closed", +} class CloudExError(RuntimeError): @@ -322,6 +335,51 @@ def _safe_last_stage(log_path): return stages[-1] if stages else "" +def _submit_diagnostics(api, session, store, reason, report_diagnostics=None, *, setup_ms=None): + sid = session.get("remote_session_id") + if session.get("diagnostics") is not True or not _valid_session_id(sid): + return + supports_timing = session.get("diagnostics_version") == 2 + if reason == "direct" and not supports_timing: + return + log_path = store.runtime_paths(session["session_id"])[3] + stages = [] + timing = {} + if supports_timing and setup_ms is not None and 0 <= setup_ms <= MAX_DIAGNOSTIC_TIMING_MS: + timing["setup_ms"] = setup_ms + try: + limit = 64 * 1024 + fd = os.open(str(log_path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0)) + with os.fdopen(fd, "rb") as log: + info = os.fstat(log.fileno()) + if stat.S_ISREG(info.st_mode): + log.seek(max(0, info.st_size - limit)) + if log.tell(): + log.readline(limit) + for line in log.read(limit).decode("utf-8", errors="replace").splitlines(): + prefix = "kerrigan-p2p-forwarder:" + event = line[len(prefix):].strip() if line.startswith(prefix) else line.strip() + if event in UPLOAD_FORWARDER_STAGES and (not stages or stages[-1] != event): + stages.append(event) + elif supports_timing: + selection = ICE_SELECTION_TIMING.fullmatch(event) + if selection and int(selection[1]) <= MAX_DIAGNOSTIC_TIMING_MS: + timing["ice_selection_ms"] = int(selection[1]) + except OSError: + pass + try: + stages = stages[-32:] + extra = {"timing": timing} if timing else {} + if report_diagnostics is not None: + report_diagnostics(api.allocation_id, sid, reason, stages, **extra) + else: + api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics", + {"reason": reason, "stages": stages, **extra}, deadline=time.time() + 2) + except CloudExError: + # Evidence upload must never prevent connecting or releasing a lease. + pass + + def _safe_selected_path(log_path): """Recover the non-sensitive selected path from forwarder lifecycle logs.""" try: @@ -601,7 +659,7 @@ def _forwarder_sudo_prefix(progress=None): return prefix + ["/usr/bin/env", "PATH=" + MACOS_TUNNEL_PATH] -def connect(profile, store, progress, attempts=3, transport="auto"): +def connect(profile, store, progress, attempts=3, transport="auto", *, report_diagnostics=None): """Create a session and return its durable local record.""" from .installer import ensure_forwarder @@ -620,6 +678,7 @@ def connect(profile, store, progress, attempts=3, transport="auto"): now = time.time() if now >= connection_deadline: break + started = time.monotonic() attempt_deadline = now + (connection_deadline - now) / (len(choices) - number + 1) session_id = uuid.uuid4().hex session = { @@ -646,6 +705,10 @@ def connect(profile, store, progress, attempts=3, transport="auto"): if not _valid_session_id(remote_session_id): raise CloudExError("CloudEx API returned an invalid session identifier") session["remote_session_id"] = remote_session_id + if allocated.get("diagnostics") is True: + session["diagnostics"] = True + if allocated.get("diagnostics_version") == 2: + session["diagnostics_version"] = 2 store.write_recovery(session) try: expires_at = float(allocated["expires_at"]) @@ -655,6 +718,8 @@ def connect(profile, store, progress, attempts=3, transport="auto"): runtime.mkdir(parents=True, exist_ok=True, mode=0o700) report_path.touch(mode=0o600, exist_ok=False) config = dict(allocated) + config.pop("diagnostics", None) + config.pop("diagnostics_version", None) config["transport"] = selected_transport config_path.write_text(json.dumps(config, separators=(",", ":")), encoding="utf-8") config_path.chmod(0o600) @@ -699,7 +764,6 @@ def connect(profile, store, progress, attempts=3, transport="auto"): store.write(session) store.clear_recovery(session_id) config_path.unlink(missing_ok=True) - return session except BaseException as exc: last_error = exc if _creation_definitely_rejected(exc): @@ -716,9 +780,17 @@ def connect(profile, store, progress, attempts=3, transport="auto"): raise CloudExError( "Connection failed and cleanup is pending: {}. Run cloudex disconnect again.".format(cleanup_error) ) from exc + finally: + if isinstance(exc, Exception): + _submit_diagnostics(api, session, store, "failed", report_diagnostics) if not isinstance(exc, DirectHandshakeError) or number == len(choices): raise progress.update("Direct path timed out; retrying with fresh ICE candidates") + else: + if session["ice_path"] in {"direct", "relay"}: + setup_ms = round((time.monotonic() - started) * 1000) + _submit_diagnostics(api, session, store, session["ice_path"], report_diagnostics, setup_ms=setup_ms) + return session raise last_error or CloudExError("CloudEx connection failed") diff --git a/sima_cli/cloudex/commands.py b/sima_cli/cloudex/commands.py index b2d4b629..2d155f84 100644 --- a/sima_cli/cloudex/commands.py +++ b/sima_cli/cloudex/commands.py @@ -267,8 +267,9 @@ def connect_command(key_file, key_stdin, allocation, environment, transport, att raise click.UsageError("Use either --key-file or --key-stdin, not both; --allocation must be used alone.") try: if allocation: - profile = _allocation_profile(_neatops_api(environment), _allocation_id(allocation)) - _connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport) + api = _neatops_api(environment) + profile = _allocation_profile(api, _allocation_id(allocation)) + _connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport, diagnostics_api=api) return if key_file: key = _read_key_file(key_file) @@ -452,11 +453,12 @@ def register_cloudex_commands(main): main.add_command(cloudex_group) -def _connect_profile(profile, verbose=False, attempts=3, transport="auto"): +def _connect_profile(profile, verbose=False, attempts=3, transport="auto", *, diagnostics_api=None): console.print("[cyan]→[/cyan] Authorizing encrypted tunnel access") _authorize_supported_host() with StageProgress("Preparing CloudEx connection", verbose=verbose) as progress: - session = connect(profile, _store(), progress, attempts=attempts, transport=transport) + options = {"report_diagnostics": diagnostics_api.report_cloudex_diagnostics} if diagnostics_api else {} + session = connect(profile, _store(), progress, attempts=attempts, transport=transport, **options) progress.success("Connected to {}".format(session["device"])) path = session.get("ice_path", "unknown") path_text = "direct" if path == "direct" else "relayed" if path == "relay" else path @@ -605,7 +607,7 @@ def request_command(device, cluster, duration, environment, verbose): api.call("POST", "/devkits/allocations", {"request_id": allocation_id, "cluster_id": selected["cluster_id"], "target_id": selected["id"], "duration_seconds": DURATIONS[duration]}) profile = _allocation_profile(api, allocation_id) - _connect_profile(profile, verbose=verbose) + _connect_profile(profile, verbose=verbose, diagnostics_api=api) except CloudExError as exc: if allocation_id: console.print("Inspect: sima-cli -i cloudex devices --environment {}\nResume: sima-cli -i cloudex connect --allocation {} --environment {}\nRelease: sima-cli -i cloudex release {} --environment {}".format(environment, allocation_id, environment, allocation_id, environment), markup=False, soft_wrap=True) diff --git a/sima_cli/cloudex/neatops.py b/sima_cli/cloudex/neatops.py index 07f7eb9c..c0f1b255 100644 --- a/sima_cli/cloudex/neatops.py +++ b/sima_cli/cloudex/neatops.py @@ -34,9 +34,9 @@ def _origin(value): return value.rstrip("/") -def _json_request(method, url, **kwargs): +def _json_request(method, url, *, timeout=20, **kwargs): try: - response = requests.request(method, url, timeout=20, allow_redirects=False, **kwargs) + response = requests.request(method, url, timeout=timeout, allow_redirects=False, **kwargs) except requests.RequestException: raise CloudExError("NeatOps is unavailable; check your connection and retry") from None try: @@ -193,7 +193,7 @@ def _read_session(self): except OSError: raise CloudExError("Sign in first with sima-cli -i cloudex login --environment " + self.environment) from None - def access_token(self): + def access_token(self, *, allow_refresh=True): auth = self.configuration()["auth"] session = self._read_session() try: @@ -204,7 +204,7 @@ def access_token(self): refresh = session.get("refresh_token") except (ValueError, KeyError, TypeError): raise LoginRequired("Sign-in policy changed; run sima-cli -i cloudex login again") from None - if not refresh: + if not refresh or not allow_refresh: raise LoginRequired("NeatOps sign-in expired; run sima-cli -i cloudex login again") tokens = self._exchange({"grant_type": "refresh_token", "refresh_token": refresh}) tokens.setdefault("refresh_token", refresh) @@ -229,6 +229,19 @@ def _authenticate(self): self.on_login_required(self) return self.access_token() + def report_cloudex_diagnostics(self, allocation_id, session_id, reason, stages, *, timing=None): + token = self.access_token(allow_refresh=False) + config = self.configuration() + path = f"/devkits/allocations/{allocation_id}/diagnostics" + report = {"session_id": session_id, "reason": reason, "stages": stages} + if timing: + report["timing"] = timing + status, _ = _json_request("POST", config["apiBase"] + path, timeout=3, + headers={"Authorization": "Bearer " + token}, + json=report) + if status != 202: + raise CloudExError("NeatOps did not accept CloudEx diagnostics") + def call(self, method, path, data=None): config = self.configuration() try: diff --git a/tests/unit/test_cloudex_client.py b/tests/unit/test_cloudex_client.py index 635f4cbf..d357039c 100644 --- a/tests/unit/test_cloudex_client.py +++ b/tests/unit/test_cloudex_client.py @@ -159,8 +159,9 @@ def test_api_signs_allocation_scoped_request(monkeypatch, environment, restored) assert "allocation_secret" not in kwargs["data"].decode() +@pytest.mark.parametrize("path", ["direct", "relay"]) @pytest.mark.parametrize("host,bootstrap", [("linux", False), ("darwin", False), ("darwin", True)]) -def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap): +def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap, path): client.sys.platform = host which = Mock(return_value="/opt/homebrew/bin/tool") if bootstrap: @@ -176,6 +177,8 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat process = Mock(pid=123, poll=Mock(return_value=None)) allocated = { "session_id": "d" * 32, + "diagnostics": True, + "diagnostics_version": 2 if path == "direct" else 1, "expires_at": "9999999999", "target_id": "ll2", "wireguard": {"remote_address": "10.252.1.2/32"}, @@ -191,11 +194,20 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api)) popen = Mock(return_value=process) monkeypatch.setattr(client.subprocess, "Popen", popen) - wait_forwarder = Mock(return_value={"status": "connected", "path": "direct"}) + wait_forwarder = Mock(return_value={"status": "connected", "path": path}) monkeypatch.setattr(client, "_wait_forwarder", wait_forwarder) monkeypatch.setattr(client.time, "time", Mock(return_value=1000)) - result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p") + cleanup = Mock() + monkeypatch.setattr(client, "disconnect_session", cleanup) + upload = Mock(side_effect=client.CloudExError("Diagnostics unavailable")) + result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p", report_diagnostics=upload) + cleanup.assert_not_called() + assert upload.call_args.args == (api.allocation_id, allocated["session_id"], path, []) + if path == "direct": + assert upload.call_args.kwargs["timing"]["setup_ms"] >= 0 + else: + assert upload.call_args.kwargs == {} if bootstrap: assert brew.call_args.args[0] == ["/opt/homebrew/bin/brew", "install", "wireguard-tools", "wireguard-go"] @@ -203,7 +215,7 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat brew.assert_not_called() assert result["device"] == "ll2" assert result["target"] == "10.252.1.2" - assert result["ice_path"] == "direct" + assert result["ice_path"] == path assert result["expires_at"] == 9999999999.0 assert store.list()[0]["api_secret"] == allocation_profile()["allocation_secret"] assert not store.runtime_paths(result["session_id"])[1].exists() @@ -774,3 +786,67 @@ def test_macos_forwarder_failure_includes_install_instruction(tmp_path, detail): assert "brew install wireguard-tools" in message assert message.endswith("; last stage: starting WireGuard tunnel") + + +def test_cloudex_diagnostics_posts_only_safe_stages_for_the_remote_session(tmp_path): + store = client.SessionStore(tmp_path) + local_id, sid = "a" * 32, "b" * 32 + _, _, _, log = store.runtime_paths(local_id) + log.parent.mkdir(parents=True) + log.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n" + "kerrigan-p2p-forwarder: ICE diagnostics phase=selected elapsed_ms=1148 pairs=6\n" + "kerrigan-p2p-forwarder: signaling token=fixture-secret\n") + calls = [] + + class API: + def call(self, method, route, data, **kwargs): + calls.append((method, route, data)) + raise client.CloudExError("diagnostics temporarily unavailable") + + session = {"session_id": local_id, "remote_session_id": sid} + client._submit_diagnostics(API(), session, store, "failed") + assert calls == [] + session["diagnostics"] = True + client._submit_diagnostics(API(), session, store, "failed") + assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", + {"reason": "failed", "stages": ["ICE connection state Failed"]})] + client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520) + assert len(calls) == 1 + session["diagnostics_version"] = 2 + client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520) + assert calls[-1] == ("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", { + "reason": "direct", "stages": ["ICE connection state Failed"], + "timing": {"setup_ms": 1520, "ice_selection_ms": 1148}, + }) + + +def test_failed_connection_reports_after_attempting_cleanup(tmp_path, monkeypatch): + store = client.SessionStore(tmp_path) + profile = allocation_profile() + sid = "d" * 32 + api = Mock(url=profile["api_url"], requester=profile["requester"], + secret=profile["allocation_secret"], allocation_id=profile["allocation_id"]) + api.call.return_value = {"session_id": sid, "diagnostics": True, "expires_at": 9999999999} + monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api)) + monkeypatch.setattr(client, "authorize_admin", Mock()) + monkeypatch.setattr(client, "_forwarder_sudo_prefix", Mock(return_value=[])) + monkeypatch.setattr(installer, "ensure_forwarder", Mock(return_value=tmp_path / "forwarder")) + monkeypatch.setattr(client.subprocess, "Popen", Mock(return_value=Mock(pid=123))) + operations = [] + + def fail(_sid, _process, _report_path, log_path, *_args, **_kwargs): + log_path.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n") + raise client.CloudExError("Original tunnel failure") + + def cleanup(*_args, **_kwargs): + operations.append("cleanup") + + def report(allocation_id, session_id, reason, stages): + operations.append((allocation_id, session_id, reason, stages)) + raise client.CloudExError("Diagnostics unavailable") + + monkeypatch.setattr(client, "_wait_forwarder", fail) + monkeypatch.setattr(client, "disconnect_session", cleanup) + with pytest.raises(client.CloudExError, match="Original tunnel failure"): + client.connect(profile, store, Mock(), attempts=1, transport="p2p", report_diagnostics=report) + assert operations == ["cleanup", (profile["allocation_id"], sid, "failed", ["ICE connection state Failed"])] diff --git a/tests/unit/test_cloudex_neatops.py b/tests/unit/test_cloudex_neatops.py index 70fa9d38..f035a153 100644 --- a/tests/unit/test_cloudex_neatops.py +++ b/tests/unit/test_cloudex_neatops.py @@ -184,7 +184,7 @@ def test_request_selects_device_allocates_then_connects(command_api, monkeypatch command_api.call.assert_any_call("POST", "/devkits/allocations", {"request_id": AID, "cluster_id": "appliance-id", "target_id": "device-id", "duration_seconds": 14400}) profile.assert_called_once_with(command_api, AID) - connect.assert_called_once_with(PROFILE, verbose=False) + connect.assert_called_once_with(PROFILE, verbose=False, diagnostics_api=command_api) assert "allocation_secret" not in result.output @@ -231,7 +231,7 @@ def test_connect_allocation_skips_manual_key_prompt(command_api, monkeypatch): monkeypatch.setattr(commands, "_connect_profile", connect) result = CliRunner().invoke(commands.cloudex_group, ["-i", "connect", "--allocation", AID]) assert result.exit_code == 0, result.output - connect.assert_called_once_with(PROFILE, verbose=False, attempts=3, transport="auto") + connect.assert_called_once_with(PROFILE, verbose=False, attempts=3, transport="auto", diagnostics_api=command_api) def test_release_disconnects_only_matching_local_allocation(command_api, monkeypatch): @@ -582,3 +582,22 @@ def test_root_external_login_uses_portal_without_internal_initialization(monkeyp initialize.assert_not_called() portal_login.assert_called_once_with(loginDocker=False) vulcan_api.assert_not_called() + + + +@pytest.mark.parametrize("reason,timing", [("failed", None), ("direct", {"setup_ms": 1420})]) +def test_diagnostics_report_uses_cached_neatops_token_without_reauthentication(api, monkeypatch, reason, timing): + api._save(tokens()) + request = Mock(return_value=(202, {"status": "accepted"})) + monkeypatch.setattr(neatops, "_json_request", request) + allocation, session = "a" * 32, "b" * 32 + stages = ["ICE connection state Failed"] if reason == "failed" else ["WireGuard handshake confirmed"] + api.report_cloudex_diagnostics(allocation, session, reason, stages, timing=timing) + expected = {"session_id": session, "reason": reason, "stages": stages} + if timing: + expected["timing"] = timing + request.assert_called_once_with( + "POST", api.config["apiBase"] + f"/devkits/allocations/{allocation}/diagnostics", + timeout=3, headers={"Authorization": "Bearer private-access"}, + json=expected, + )