Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 74 additions & 2 deletions sima_cli/cloudex/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@
import hmac
import json
import os
import re
import shutil
import stat
import subprocess
import sys
import time
Expand All @@ -22,6 +24,8 @@
MACOS_WIREGUARD_INSTALL = "Install WireGuard CLI support with: brew install wireguard-tools"
CONNECTION_SETUP_TIMEOUT_SECONDS = 60
DIAGNOSTIC_LOG_LINES = 50
ICE_SELECTION_TIMING = re.compile(r"ICE diagnostics phase=selected elapsed_ms=([0-9]{1,6}) pairs=[0-9]{1,6}")
MAX_DIAGNOSTIC_TIMING_MS = 900_000
SAFE_FORWARDER_STAGES = {
"session configuration accepted",
"joined authenticated signaling session",
Expand All @@ -40,6 +44,15 @@
"signaling session conflict",
"signaling rate limited",
}
UPLOAD_FORWARDER_STAGES = SAFE_FORWARDER_STAGES | {
"ICE candidate gathering complete",
"ICE connection state Checking",
"ICE connection state Connected",
"ICE connection state Completed",
"ICE connection state Disconnected",
"ICE connection state Failed",
"ICE connection state Closed",
}


class CloudExError(RuntimeError):
Expand Down Expand Up @@ -322,6 +335,51 @@ def _safe_last_stage(log_path):
return stages[-1] if stages else ""


def _submit_diagnostics(api, session, store, reason, report_diagnostics=None, *, setup_ms=None):
sid = session.get("remote_session_id")
if session.get("diagnostics") is not True or not _valid_session_id(sid):
return
supports_timing = session.get("diagnostics_version") == 2
if reason == "direct" and not supports_timing:
return
log_path = store.runtime_paths(session["session_id"])[3]
stages = []
timing = {}
if supports_timing and setup_ms is not None and 0 <= setup_ms <= MAX_DIAGNOSTIC_TIMING_MS:
timing["setup_ms"] = setup_ms
try:
limit = 64 * 1024
fd = os.open(str(log_path), os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_NONBLOCK", 0))
with os.fdopen(fd, "rb") as log:
info = os.fstat(log.fileno())
if stat.S_ISREG(info.st_mode):
log.seek(max(0, info.st_size - limit))
if log.tell():
log.readline(limit)
for line in log.read(limit).decode("utf-8", errors="replace").splitlines():
prefix = "kerrigan-p2p-forwarder:"
event = line[len(prefix):].strip() if line.startswith(prefix) else line.strip()
if event in UPLOAD_FORWARDER_STAGES and (not stages or stages[-1] != event):
stages.append(event)
elif supports_timing:
selection = ICE_SELECTION_TIMING.fullmatch(event)
if selection and int(selection[1]) <= MAX_DIAGNOSTIC_TIMING_MS:
timing["ice_selection_ms"] = int(selection[1])
except OSError:
pass
try:
stages = stages[-32:]
extra = {"timing": timing} if timing else {}
if report_diagnostics is not None:
report_diagnostics(api.allocation_id, sid, reason, stages, **extra)
else:
api.call("PUT", "/v1/p2p/ice-sessions/" + sid + "/diagnostics",
{"reason": reason, "stages": stages, **extra}, deadline=time.time() + 2)
except CloudExError:
# Evidence upload must never prevent connecting or releasing a lease.
pass


def _safe_selected_path(log_path):
"""Recover the non-sensitive selected path from forwarder lifecycle logs."""
try:
Expand Down Expand Up @@ -601,7 +659,7 @@ def _forwarder_sudo_prefix(progress=None):
return prefix + ["/usr/bin/env", "PATH=" + MACOS_TUNNEL_PATH]


def connect(profile, store, progress, attempts=3, transport="auto"):
def connect(profile, store, progress, attempts=3, transport="auto", *, report_diagnostics=None):
"""Create a session and return its durable local record."""
from .installer import ensure_forwarder

Expand All @@ -620,6 +678,7 @@ def connect(profile, store, progress, attempts=3, transport="auto"):
now = time.time()
if now >= connection_deadline:
break
started = time.monotonic()
attempt_deadline = now + (connection_deadline - now) / (len(choices) - number + 1)
session_id = uuid.uuid4().hex
session = {
Expand All @@ -646,6 +705,10 @@ def connect(profile, store, progress, attempts=3, transport="auto"):
if not _valid_session_id(remote_session_id):
raise CloudExError("CloudEx API returned an invalid session identifier")
session["remote_session_id"] = remote_session_id
if allocated.get("diagnostics") is True:
session["diagnostics"] = True
if allocated.get("diagnostics_version") == 2:
session["diagnostics_version"] = 2
store.write_recovery(session)
try:
expires_at = float(allocated["expires_at"])
Expand All @@ -655,6 +718,8 @@ def connect(profile, store, progress, attempts=3, transport="auto"):
runtime.mkdir(parents=True, exist_ok=True, mode=0o700)
report_path.touch(mode=0o600, exist_ok=False)
config = dict(allocated)
config.pop("diagnostics", None)
config.pop("diagnostics_version", None)
config["transport"] = selected_transport
config_path.write_text(json.dumps(config, separators=(",", ":")), encoding="utf-8")
config_path.chmod(0o600)
Expand Down Expand Up @@ -699,7 +764,6 @@ def connect(profile, store, progress, attempts=3, transport="auto"):
store.write(session)
store.clear_recovery(session_id)
config_path.unlink(missing_ok=True)
return session
except BaseException as exc:
last_error = exc
if _creation_definitely_rejected(exc):
Expand All @@ -716,9 +780,17 @@ def connect(profile, store, progress, attempts=3, transport="auto"):
raise CloudExError(
"Connection failed and cleanup is pending: {}. Run cloudex disconnect again.".format(cleanup_error)
) from exc
finally:
if isinstance(exc, Exception):
_submit_diagnostics(api, session, store, "failed", report_diagnostics)
if not isinstance(exc, DirectHandshakeError) or number == len(choices):
raise
progress.update("Direct path timed out; retrying with fresh ICE candidates")
else:
if session["ice_path"] in {"direct", "relay"}:
setup_ms = round((time.monotonic() - started) * 1000)
_submit_diagnostics(api, session, store, session["ice_path"], report_diagnostics, setup_ms=setup_ms)
return session
raise last_error or CloudExError("CloudEx connection failed")


Expand Down
12 changes: 7 additions & 5 deletions sima_cli/cloudex/commands.py
Original file line number Diff line number Diff line change
Expand Up @@ -267,8 +267,9 @@ def connect_command(key_file, key_stdin, allocation, environment, transport, att
raise click.UsageError("Use either --key-file or --key-stdin, not both; --allocation must be used alone.")
try:
if allocation:
profile = _allocation_profile(_neatops_api(environment), _allocation_id(allocation))
_connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport)
api = _neatops_api(environment)
profile = _allocation_profile(api, _allocation_id(allocation))
_connect_profile(profile, verbose=verbose, attempts=attempts, transport=transport, diagnostics_api=api)
return
if key_file:
key = _read_key_file(key_file)
Expand Down Expand Up @@ -452,11 +453,12 @@ def register_cloudex_commands(main):
main.add_command(cloudex_group)


def _connect_profile(profile, verbose=False, attempts=3, transport="auto"):
def _connect_profile(profile, verbose=False, attempts=3, transport="auto", *, diagnostics_api=None):
console.print("[cyan]→[/cyan] Authorizing encrypted tunnel access")
_authorize_supported_host()
with StageProgress("Preparing CloudEx connection", verbose=verbose) as progress:
session = connect(profile, _store(), progress, attempts=attempts, transport=transport)
options = {"report_diagnostics": diagnostics_api.report_cloudex_diagnostics} if diagnostics_api else {}
session = connect(profile, _store(), progress, attempts=attempts, transport=transport, **options)
progress.success("Connected to {}".format(session["device"]))
path = session.get("ice_path", "unknown")
path_text = "direct" if path == "direct" else "relayed" if path == "relay" else path
Expand Down Expand Up @@ -605,7 +607,7 @@ def request_command(device, cluster, duration, environment, verbose):
api.call("POST", "/devkits/allocations", {"request_id": allocation_id,
"cluster_id": selected["cluster_id"], "target_id": selected["id"], "duration_seconds": DURATIONS[duration]})
profile = _allocation_profile(api, allocation_id)
_connect_profile(profile, verbose=verbose)
_connect_profile(profile, verbose=verbose, diagnostics_api=api)
except CloudExError as exc:
if allocation_id:
console.print("Inspect: sima-cli -i cloudex devices --environment {}\nResume: sima-cli -i cloudex connect --allocation {} --environment {}\nRelease: sima-cli -i cloudex release {} --environment {}".format(environment, allocation_id, environment, allocation_id, environment), markup=False, soft_wrap=True)
Expand Down
21 changes: 17 additions & 4 deletions sima_cli/cloudex/neatops.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,9 @@ def _origin(value):
return value.rstrip("/")


def _json_request(method, url, **kwargs):
def _json_request(method, url, *, timeout=20, **kwargs):
try:
response = requests.request(method, url, timeout=20, allow_redirects=False, **kwargs)
response = requests.request(method, url, timeout=timeout, allow_redirects=False, **kwargs)
except requests.RequestException:
raise CloudExError("NeatOps is unavailable; check your connection and retry") from None
try:
Expand Down Expand Up @@ -193,7 +193,7 @@ def _read_session(self):
except OSError:
raise CloudExError("Sign in first with sima-cli -i cloudex login --environment " + self.environment) from None

def access_token(self):
def access_token(self, *, allow_refresh=True):
auth = self.configuration()["auth"]
session = self._read_session()
try:
Expand All @@ -204,7 +204,7 @@ def access_token(self):
refresh = session.get("refresh_token")
except (ValueError, KeyError, TypeError):
raise LoginRequired("Sign-in policy changed; run sima-cli -i cloudex login again") from None
if not refresh:
if not refresh or not allow_refresh:
raise LoginRequired("NeatOps sign-in expired; run sima-cli -i cloudex login again")
tokens = self._exchange({"grant_type": "refresh_token", "refresh_token": refresh})
tokens.setdefault("refresh_token", refresh)
Expand All @@ -229,6 +229,19 @@ def _authenticate(self):
self.on_login_required(self)
return self.access_token()

def report_cloudex_diagnostics(self, allocation_id, session_id, reason, stages, *, timing=None):
token = self.access_token(allow_refresh=False)
config = self.configuration()
path = f"/devkits/allocations/{allocation_id}/diagnostics"
report = {"session_id": session_id, "reason": reason, "stages": stages}
if timing:
report["timing"] = timing
status, _ = _json_request("POST", config["apiBase"] + path, timeout=3,
headers={"Authorization": "Bearer " + token},
json=report)
if status != 202:
raise CloudExError("NeatOps did not accept CloudEx diagnostics")

def call(self, method, path, data=None):
config = self.configuration()
try:
Expand Down
84 changes: 80 additions & 4 deletions tests/unit/test_cloudex_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,8 +159,9 @@ def test_api_signs_allocation_scoped_request(monkeypatch, environment, restored)
assert "allocation_secret" not in kwargs["data"].decode()


@pytest.mark.parametrize("path", ["direct", "relay"])
@pytest.mark.parametrize("host,bootstrap", [("linux", False), ("darwin", False), ("darwin", True)])
def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap):
def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_path, monkeypatch, host, bootstrap, path):
client.sys.platform = host
which = Mock(return_value="/opt/homebrew/bin/tool")
if bootstrap:
Expand All @@ -176,6 +177,8 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat
process = Mock(pid=123, poll=Mock(return_value=None))
allocated = {
"session_id": "d" * 32,
"diagnostics": True,
"diagnostics_version": 2 if path == "direct" else 1,
"expires_at": "9999999999",
"target_id": "ll2",
"wireguard": {"remote_address": "10.252.1.2/32"},
Expand All @@ -191,19 +194,28 @@ def test_connect_persists_connected_session_and_removes_ephemeral_config(tmp_pat
monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api))
popen = Mock(return_value=process)
monkeypatch.setattr(client.subprocess, "Popen", popen)
wait_forwarder = Mock(return_value={"status": "connected", "path": "direct"})
wait_forwarder = Mock(return_value={"status": "connected", "path": path})
monkeypatch.setattr(client, "_wait_forwarder", wait_forwarder)
monkeypatch.setattr(client.time, "time", Mock(return_value=1000))

result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p")
cleanup = Mock()
monkeypatch.setattr(client, "disconnect_session", cleanup)
upload = Mock(side_effect=client.CloudExError("Diagnostics unavailable"))
result = client.connect(allocation_profile(), store, Mock(), attempts=1, transport="p2p", report_diagnostics=upload)
cleanup.assert_not_called()
assert upload.call_args.args == (api.allocation_id, allocated["session_id"], path, [])
if path == "direct":
assert upload.call_args.kwargs["timing"]["setup_ms"] >= 0
else:
assert upload.call_args.kwargs == {}

if bootstrap:
assert brew.call_args.args[0] == ["/opt/homebrew/bin/brew", "install", "wireguard-tools", "wireguard-go"]
else:
brew.assert_not_called()
assert result["device"] == "ll2"
assert result["target"] == "10.252.1.2"
assert result["ice_path"] == "direct"
assert result["ice_path"] == path
assert result["expires_at"] == 9999999999.0
assert store.list()[0]["api_secret"] == allocation_profile()["allocation_secret"]
assert not store.runtime_paths(result["session_id"])[1].exists()
Expand Down Expand Up @@ -774,3 +786,67 @@ def test_macos_forwarder_failure_includes_install_instruction(tmp_path, detail):

assert "brew install wireguard-tools" in message
assert message.endswith("; last stage: starting WireGuard tunnel")


def test_cloudex_diagnostics_posts_only_safe_stages_for_the_remote_session(tmp_path):
store = client.SessionStore(tmp_path)
local_id, sid = "a" * 32, "b" * 32
_, _, _, log = store.runtime_paths(local_id)
log.parent.mkdir(parents=True)
log.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n"
"kerrigan-p2p-forwarder: ICE diagnostics phase=selected elapsed_ms=1148 pairs=6\n"
"kerrigan-p2p-forwarder: signaling token=fixture-secret\n")
calls = []

class API:
def call(self, method, route, data, **kwargs):
calls.append((method, route, data))
raise client.CloudExError("diagnostics temporarily unavailable")

session = {"session_id": local_id, "remote_session_id": sid}
client._submit_diagnostics(API(), session, store, "failed")
assert calls == []
session["diagnostics"] = True
client._submit_diagnostics(API(), session, store, "failed")
assert calls == [("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics",
{"reason": "failed", "stages": ["ICE connection state Failed"]})]
client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520)
assert len(calls) == 1
session["diagnostics_version"] = 2
client._submit_diagnostics(API(), session, store, "direct", setup_ms=1520)
assert calls[-1] == ("PUT", f"/v1/p2p/ice-sessions/{sid}/diagnostics", {
"reason": "direct", "stages": ["ICE connection state Failed"],
"timing": {"setup_ms": 1520, "ice_selection_ms": 1148},
})


def test_failed_connection_reports_after_attempting_cleanup(tmp_path, monkeypatch):
store = client.SessionStore(tmp_path)
profile = allocation_profile()
sid = "d" * 32
api = Mock(url=profile["api_url"], requester=profile["requester"],
secret=profile["allocation_secret"], allocation_id=profile["allocation_id"])
api.call.return_value = {"session_id": sid, "diagnostics": True, "expires_at": 9999999999}
monkeypatch.setattr(client.CloudExAPI, "from_profile", Mock(return_value=api))
monkeypatch.setattr(client, "authorize_admin", Mock())
monkeypatch.setattr(client, "_forwarder_sudo_prefix", Mock(return_value=[]))
monkeypatch.setattr(installer, "ensure_forwarder", Mock(return_value=tmp_path / "forwarder"))
monkeypatch.setattr(client.subprocess, "Popen", Mock(return_value=Mock(pid=123)))
operations = []

def fail(_sid, _process, _report_path, log_path, *_args, **_kwargs):
log_path.write_text("kerrigan-p2p-forwarder: ICE connection state Failed\n")
raise client.CloudExError("Original tunnel failure")

def cleanup(*_args, **_kwargs):
operations.append("cleanup")

def report(allocation_id, session_id, reason, stages):
operations.append((allocation_id, session_id, reason, stages))
raise client.CloudExError("Diagnostics unavailable")

monkeypatch.setattr(client, "_wait_forwarder", fail)
monkeypatch.setattr(client, "disconnect_session", cleanup)
with pytest.raises(client.CloudExError, match="Original tunnel failure"):
client.connect(profile, store, Mock(), attempts=1, transport="p2p", report_diagnostics=report)
assert operations == ["cleanup", (profile["allocation_id"], sid, "failed", ["ICE connection state Failed"])]
Loading
Loading