diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 33733d6e..dd1d97a4 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,6 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused; full release-0.3.1 merged; proxy recording and live presentation passed; inherited gate failures and PR delivery pending | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/WORKFLOW.md b/WORKFLOW.md index 196d3760..781c9e39 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md index 02914b52..31f397b5 100644 --- a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md +++ b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/commands/github_ssh.py b/devcapsule-src/devcapsule/commands/github_ssh.py new file mode 100644 index 00000000..e36fd1fc --- /dev/null +++ b/devcapsule-src/devcapsule/commands/github_ssh.py @@ -0,0 +1,37 @@ +"""Git's capsule-side SSH adapter for the authorized launcher proxy.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import subprocess +import sys + +from devcapsule.commands.framework import Command +from devcapsule.compat import CliError +from devcapsule.github_transport import GitHubTransportError, parse_git_ssh +from devcapsule.github_proxy import SOCKET_ENV, relay_git + + +class GitHubSshCommand(Command): + name = "github-ssh" + hidden = True + help = "Git SSH transport through the authorized host GitHub proxy." + passthrough_dest = "ssh_arguments" + + @classmethod + def run(cls, arguments: argparse.Namespace, context: object | None) -> int: + try: + request = parse_git_ssh(arguments.ssh_arguments) + if request is None: + return subprocess.run(["ssh", *arguments.ssh_arguments], check=False).returncode + path = os.environ.get(SOCKET_ENV) + if not path: + raise GitHubTransportError("No authorized GitHub proxy. Enable github-ssh on the host launcher.") + return relay_git(Path(path), request, stdin=sys.stdin.buffer, + stdout=sys.stdout.buffer, stderr=sys.stderr.buffer) + except (OSError, GitHubTransportError) as error: + raise CliError(f"GitHub proxy: {error} Host keys must be trusted and encrypted keys unlocked on the host.") from error + + +COMMAND = GitHubSshCommand diff --git a/devcapsule-src/devcapsule/commands/project.py b/devcapsule-src/devcapsule/commands/project.py index e5e13748..0bfda896 100644 --- a/devcapsule-src/devcapsule/commands/project.py +++ b/devcapsule-src/devcapsule/commands/project.py @@ -2104,7 +2104,7 @@ def subcommands(cls) -> Mapping[str, type[Command] | type[Group]]: # The authorization nodes whose run-once answers feed the launch plan; every # other authorization (base-image, acquisitions) is inherently persistent. -_RUN_ONCE_AUTHORIZATIONS = ("docker-daemon", "network", "development-sudo", "host-browser", "host-x11") +_RUN_ONCE_AUTHORIZATIONS = ("docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh") class ProjectRunCommand(Command): @@ -2418,6 +2418,7 @@ def prepare_display(base: ImageDetails) -> None: extra_docker_args=["--pull=never", *saved_docker_args, *prepared.environment_tokens, *docker_options], project_state=None, enable_host_browser=selected_host_browser, + enable_github_ssh=host_access.github_ssh, display_transport=display_transport, web_console=web_console, capsule_platform=lock_platform(lock), diff --git a/devcapsule-src/devcapsule/configuration/authorization.py b/devcapsule-src/devcapsule/configuration/authorization.py index 3927bfaf..a7b54a9d 100644 --- a/devcapsule-src/devcapsule/configuration/authorization.py +++ b/devcapsule-src/devcapsule/configuration/authorization.py @@ -80,6 +80,11 @@ True, "Enable the declared development-sudo behavior inside the capsule.", ), + "github-ssh": ( + True, + "Allow GitHub Git fetch and push through the outside launcher using the host SSH identity. " + "All repositories accessible to that identity are accessible; private keys stay on the host.", + ), "host-browser": ( True, "Open HTTP(S) links from the capsule in the physical host's default browser " diff --git a/devcapsule-src/devcapsule/configuration/console_channel.py b/devcapsule-src/devcapsule/configuration/console_channel.py index b66436ff..eef5dc4e 100644 --- a/devcapsule-src/devcapsule/configuration/console_channel.py +++ b/devcapsule-src/devcapsule/configuration/console_channel.py @@ -73,6 +73,7 @@ DEFERRED_SUBVERBS = {("trust", "accept"), ("docker-args", "set")} SECURE_AUTHORIZATIONS = { "network": {"bridge", "none"}, "docker-daemon": {"none"}, + "github-ssh": {"false", "none"}, "development-sudo": {"false", "none"}, "host-browser": {"false", "none"}, "host-x11": {"false", "none"}, } READ_ONLY_VERBS = {"show", "check", "console"} diff --git a/devcapsule-src/devcapsule/configuration/listing.py b/devcapsule-src/devcapsule/configuration/listing.py index abf09b64..8f7e8215 100644 --- a/devcapsule-src/devcapsule/configuration/listing.py +++ b/devcapsule-src/devcapsule/configuration/listing.py @@ -61,7 +61,7 @@ LISTING_SCHEMA_VERSION = 2 -HOST_ACCESS_NODES = ("network", "docker-daemon", "development-sudo", "host-browser", "host-x11") +HOST_ACCESS_NODES = ("network", "docker-daemon", "development-sudo", "host-browser", "host-x11", "github-ssh") @dataclass(frozen=True) diff --git a/devcapsule-src/devcapsule/configuration/resolution.py b/devcapsule-src/devcapsule/configuration/resolution.py index 25e97671..bfa143dc 100644 --- a/devcapsule-src/devcapsule/configuration/resolution.py +++ b/devcapsule-src/devcapsule/configuration/resolution.py @@ -210,7 +210,7 @@ def base_reference(self) -> str | None: @property def host_access(self) -> HostAccess: authorizations = self._document.get("authorization", {}) - host_names = {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11"} + host_names = {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh"} return HostAccess().overlay(self._document.get("host", {})).overlay({ name: value for name, value in authorizations.items() if name in host_names }) diff --git a/devcapsule-src/devcapsule/configuration/review.py b/devcapsule-src/devcapsule/configuration/review.py index 015788d5..0814e3e1 100644 --- a/devcapsule-src/devcapsule/configuration/review.py +++ b/devcapsule-src/devcapsule/configuration/review.py @@ -27,6 +27,7 @@ class HostAccess: docker_daemon: str = "none" network: str = "bridge" development_sudo: bool = False + github_ssh: bool = False host_browser: bool = False host_x11: bool | None = None @@ -35,6 +36,7 @@ def overlay(self, answers: Mapping[str, Any]) -> HostAccess: "docker-daemon": ("docker_daemon", ("none", "host-socket")), "network": ("network", ("bridge", "host")), "development-sudo": ("development_sudo", (False, True)), + "github-ssh": ("github_ssh", (False, True)), "host-browser": ("host_browser", (False, True)), "host-x11": ("host_x11", (False, True)), } @@ -127,7 +129,7 @@ def resolved_authorizations(self) -> dict[str, AuthorizationScalar]: def effective_host(self, overrides: Mapping[str, Any] | None = None) -> HostAccess: answers = { name: value for name, value in self.resolved_authorizations().items() - if name in {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11"} + if name in {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh"} } return self.legacy_host.overlay(answers).overlay(overrides or {}) diff --git a/devcapsule-src/devcapsule/github_proxy.py b/devcapsule-src/devcapsule/github_proxy.py new file mode 100644 index 00000000..83d2ba40 --- /dev/null +++ b/devcapsule-src/devcapsule/github_proxy.py @@ -0,0 +1,262 @@ +"""Launcher-owned GitHub transport broker and capsule client. + +``GitHubBroker`` owns a private Unix socket and bounded concurrent SSH children. +``relay_git`` connects Git's byte streams to it. Only the launcher creates the +broker; neither side reads private keys. The wire begins with a bounded JSON +request. Client input then streams unchanged, with a write-half close for EOF. +Server replies are frames: O stdout, E stderr, X unsigned exit status. +""" +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +import os +from pathlib import Path +import selectors +import shutil +import signal +import socket +import struct +import subprocess +import tempfile +import threading +import time +from typing import BinaryIO + +from devcapsule.github_transport import GitHubRequest, GitHubTransportError, decode_request + +__all__ = ["GitHubBroker", "relay_git", "github_ssh_bridge", "SOCKET_ENV", "SOCKET_DESTINATION", + "GIT_SSH_COMMAND", "MAX_REQUEST", "MAX_FRAME"] + +SOCKET_ENV = "DEVCAPSULE_GITHUB_SOCKET" +SOCKET_DESTINATION = Path("/run/devcapsule-github.sock") +GIT_SSH_COMMAND = "/opt/devcapsule/bin/devcapsule.pex github-ssh --" +MAX_REQUEST = 4096 +MAX_FRAME = 65536 + + +def _read_exact(connection: socket.socket, length: int) -> bytes: + chunks = bytearray() + while len(chunks) < length: + chunk = connection.recv(length - len(chunks)) + if not chunk: + raise GitHubTransportError("GitHub proxy connection ended unexpectedly.") + chunks.extend(chunk) + return bytes(chunks) + + +def _send_frame(connection: socket.socket, kind: bytes, payload: bytes) -> None: + connection.sendall(kind + struct.pack("!I", len(payload)) + payload) + + +class GitHubBroker: + """One session's GitHub bridge. Close cancels active children and removes its socket. + + ``ssh`` and ``environment`` come from the launcher, never from a request. + Explicit deadlines bound idle handshakes, blocked consumers and Git operations. + """ + + def __init__(self, path: Path, *, ssh: str, environment: Mapping[str, str], + operation_timeout: float = 3600, io_timeout: float = 10) -> None: + self.path = path + self.ssh = ssh + self.environment = dict(environment) + self.operation_timeout = operation_timeout + self.io_timeout = io_timeout + self._stop = threading.Event() + self._lock = threading.Lock() + self._connections: set[socket.socket] = set() + self._threads: list[threading.Thread] = [] + self._slots = threading.BoundedSemaphore(8) + self._server: socket.socket | None = None + self._listener: threading.Thread | None = None + self._bound = False + + def __enter__(self) -> GitHubBroker: + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + server.bind(str(self.path)) # Never unlink or replace someone else's socket. + self._bound = True + self.path.chmod(0o600) + server.listen(8) + server.settimeout(0.2) + except BaseException: + server.close() + if self._bound: + self.path.unlink(missing_ok=True) + raise + self._server = server + self._listener = threading.Thread(target=self._serve, daemon=True, name="github-proxy") + self._listener.start() + return self + + def __exit__(self, *_args: object) -> None: + self._stop.set() + if self._server is not None: + self._server.close() + if self._listener is not None: + self._listener.join() + with self._lock: + for connection in self._connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + for thread in self._threads: + thread.join() + if self._bound: + self.path.unlink(missing_ok=True) + + def _serve(self) -> None: + assert self._server is not None + while not self._stop.is_set(): + try: + connection, _ = self._server.accept() + except TimeoutError: + continue + except OSError: + break + if not self._slots.acquire(blocking=False): + connection.close() + continue + with self._lock: + self._connections.add(connection) + thread = threading.Thread(target=self._handle, args=(connection,), daemon=True) + # Retain only active workers; completed Thread objects need not accumulate. + self._threads = [item for item in self._threads if item.is_alive()] + self._threads.append(thread) + thread.start() + + def _handle(self, connection: socket.socket) -> None: + try: + with connection: + connection.settimeout(self.io_timeout) + length = struct.unpack("!I", _read_exact(connection, 4))[0] + if not 0 < length <= MAX_REQUEST: + raise GitHubTransportError("GitHub request exceeds its size limit.") + request = decode_request(_read_exact(connection, length)) + self._transfer(connection, request) + except (OSError, GitHubTransportError, subprocess.SubprocessError): + # An invalid or disconnected caller gets EOF, never a host command. + pass + finally: + with self._lock: + self._connections.discard(connection) + self._slots.release() + + def _transfer(self, connection: socket.socket, request: GitHubRequest) -> None: + # Timeout sockets use O_NONBLOCK. Child stdin must be a blocking stream. + # dup shares file flags, so restore blocking and use send timeouts via + # a separate connection wrapper only while transmitting output below. + connection.setblocking(True) + with subprocess.Popen(request.ssh_command(self.ssh), stdin=connection.fileno(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self.environment, cwd="/", start_new_session=True) as child: + assert child.stdout is not None and child.stderr is not None + started = time.monotonic() + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ, b"O") + selector.register(child.stderr, selectors.EVENT_READ, b"E") + while selector.get_map(): + if self._stop.is_set() or time.monotonic() - started > self.operation_timeout: + raise GitHubTransportError("GitHub transfer stopped or timed out.") + for key, _ in selector.select(0.2): + data = os.read(key.fd, MAX_FRAME) + if not data: + selector.unregister(key.fileobj) + else: + self._send(connection, key.data, data) + while child.poll() is None: + if self._stop.wait(0.1) or time.monotonic() - started > self.operation_timeout: + raise GitHubTransportError("GitHub transfer stopped or timed out.") + self._send(connection, b"X", struct.pack("!I", child.returncode if child.returncode >= 0 else 255)) + finally: + # SSH may have started a host-configured ProxyCommand. Reap the + # whole private process group even after the leader exits. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait() + + def _send(self, connection: socket.socket, kind: bytes, payload: bytes) -> None: + # MSG_DONTWAIT avoids altering the shared file flags of child stdin. + data = memoryview(kind + struct.pack("!I", len(payload)) + payload) + deadline = time.monotonic() + self.io_timeout + with selectors.DefaultSelector() as selector: + selector.register(connection, selectors.EVENT_WRITE) + while data: + if self._stop.is_set() or time.monotonic() > deadline: + raise GitHubTransportError("GitHub proxy output stalled.") + if not selector.select(0.2): + continue + try: + count = connection.send(data, socket.MSG_DONTWAIT) + except BlockingIOError: + continue + if count == 0: + raise GitHubTransportError("GitHub proxy output disconnected.") + data = data[count:] + + +def relay_git(path: Path, request: GitHubRequest, *, stdin: BinaryIO, + stdout: BinaryIO, stderr: BinaryIO) -> int: + """Relay binary Git data, separate diagnostics and the host SSH exit status.""" + payload = request.encode() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(10) + connection.connect(str(path)) + connection.sendall(struct.pack("!I", len(payload)) + payload) + connection.settimeout(None) + + def send_input() -> None: + try: + while chunk := os.read(stdin.fileno(), MAX_FRAME): + connection.sendall(chunk) + connection.shutdown(socket.SHUT_WR) + except (OSError, ValueError): + pass # The server can finish before Git closes stdin. + + threading.Thread(target=send_input, daemon=True, name="github-git-input").start() + while True: + kind = _read_exact(connection, 1) + length = struct.unpack("!I", _read_exact(connection, 4))[0] + if length > MAX_FRAME: + raise GitHubTransportError("GitHub proxy frame exceeds its size limit.") + data = _read_exact(connection, length) + if kind == b"X" and length == 4: + return min(struct.unpack("!I", data)[0], 255) + if kind not in {b"O", b"E"}: + raise GitHubTransportError("Unknown GitHub proxy response.") + output = stdout if kind == b"O" else stderr + output.write(data) + output.flush() + + +@contextmanager +def github_ssh_bridge(environment: Mapping[str, str], *, enabled: bool, + containerized: bool, binds_unix_sockets: bool) -> Iterator[Path | None]: + """Create an authorized physical-host broker, or reuse an inherited one. + + Docker Desktop needs a separate transport. An explicit enable there fails + with an explanation instead of silently mounting keys or exposing an agent. + """ + if not enabled: + yield None + return + if containerized: + inherited = environment.get(SOCKET_ENV) + if inherited != str(SOCKET_DESTINATION) or not SOCKET_DESTINATION.is_socket(): + raise GitHubTransportError("GitHub proxy unavailable: authorize github-ssh on the outside launcher first.") + yield SOCKET_DESTINATION + return + if not binds_unix_sockets: + raise GitHubTransportError("github-ssh currently requires Linux/WSL with native Docker; Docker Desktop transport is not implemented.") + executable = shutil.which("ssh", path=environment.get("PATH", os.defpath)) + if executable is None: + raise GitHubTransportError("github-ssh requires OpenSSH on the host.") + with tempfile.TemporaryDirectory(prefix="devcapsule-github-", dir="/tmp") as directory: + path = Path(directory) / "ssh.sock" + with GitHubBroker(path, ssh=executable, environment=environment): + yield path diff --git a/devcapsule-src/devcapsule/github_transport.py b/devcapsule-src/devcapsule/github_transport.py new file mode 100644 index 00000000..b53d5210 --- /dev/null +++ b/devcapsule-src/devcapsule/github_transport.py @@ -0,0 +1,102 @@ +"""GitHub-only Git SSH requests: validate values before any host process starts. + +The capsule selects a Git service and repository, never a host command or SSH +option. Host OpenSSH owns key selection, host verification and authentication. +""" +from __future__ import annotations + +from dataclasses import dataclass +import json +import re +import shlex +from typing import Sequence + +__all__ = ["GitHubRequest", "GitHubTransportError", "parse_git_ssh", "decode_request"] + + +class GitHubTransportError(ValueError): + """A request cannot cross the authorized GitHub boundary.""" + + +@dataclass(frozen=True) +class GitHubRequest: + service: str + repository: str + + def validate(self) -> None: + """Require a GitHub repository and one of Git's two transfer services.""" + if self.service not in {"git-upload-pack", "git-receive-pack"}: + raise GitHubTransportError("Only Git fetch and push services are allowed.") + if (len(self.repository) > 256 or + not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+", self.repository) or + self.repository.split("/")[-1] in {".", ".."}): + raise GitHubTransportError("Expected a GitHub OWNER/REPOSITORY path.") + + def encode(self) -> bytes: + """Serialize a validated request; no host paths or credentials cross.""" + self.validate() + return json.dumps({"version": 1, "service": self.service, + "repository": self.repository}, separators=(",", ":")).encode() + + def ssh_command(self, executable: str) -> tuple[str, ...]: + """Use host SSH configuration with noninteractive, verified authentication.""" + self.validate() + return (executable, "-T", "-oBatchMode=yes", "-oStrictHostKeyChecking=yes", + "-oClearAllForwardings=yes", "-oForwardAgent=no", "-oPermitLocalCommand=no", + "-oControlMaster=no", "-oControlPath=none", "-oRemoteCommand=none", + "git@github.com", f"{self.service} '{self.repository}'") + + +def decode_request(payload: bytes) -> GitHubRequest: + """Validate the entire wire request, including field types and version.""" + try: + value = json.loads(payload) + except (ValueError, UnicodeError) as error: + raise GitHubTransportError("Invalid GitHub request JSON.") from error + if (not isinstance(value, dict) or set(value) != {"version", "service", "repository"} or + type(value["version"]) is not int or value["version"] != 1 or + not isinstance(value["service"], str) or not isinstance(value["repository"], str)): + raise GitHubTransportError("Unsupported GitHub request.") + request = GitHubRequest(value["service"], value["repository"]) + request.validate() + return request + + +def parse_git_ssh(arguments: Sequence[str]) -> GitHubRequest | None: + """Parse Git's OpenSSH invocation; other hosts use the capsule's own SSH. + + Git's protocol-v2 SendEnv request is accepted, but the broker deliberately + negotiates protocol v0 rather than accepting arbitrary client environment. + An explicit port is accepted only when it is GitHub's standard SSH port. + Host SSH configuration can select ssh.github.com:443 as usual. + """ + args = list(arguments) + port: str | None = None + while args and args[0].startswith("-"): + option = args.pop(0) + if option in {"-o", "-p"} and args: + value = args.pop(0) + if option == "-p": + port = value + elif value != "SendEnv=GIT_PROTOCOL": + raise GitHubTransportError("Unsupported SSH option for the GitHub proxy.") + elif option == "-oSendEnv=GIT_PROTOCOL": + continue + else: + raise GitHubTransportError("Unsupported SSH option for the GitHub proxy.") + if not args: + raise GitHubTransportError("Missing SSH destination.") + destination = args.pop(0) + if destination not in {"github.com", "git@github.com"}: + return None + if port not in {None, "22"} or len(args) != 1: + raise GitHubTransportError("Expected GitHub's SSH destination and one Git service.") + try: + command = shlex.split(args[0]) + except ValueError as error: + raise GitHubTransportError("Malformed Git service command.") from error + if len(command) != 2: + raise GitHubTransportError("Expected a Git service and repository only.") + request = GitHubRequest(command[0], command[1].removeprefix("/")) + request.validate() + return request diff --git a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py index 9aed1126..4be254ee 100644 --- a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py +++ b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py @@ -61,12 +61,18 @@ ) from ...host_daemon import ( current_container, + in_container, daemon_architecture, docker_socket, is_host_backed, requires_translation, translate_bind_sources, ) +from ...github_proxy import ( + SOCKET_ENV as GITHUB_SOCKET_ENV, SOCKET_DESTINATION as GITHUB_SOCKET_DESTINATION, + GIT_SSH_COMMAND, github_ssh_bridge, +) +from ...github_transport import GitHubTransportError from ...host_open import ( HOST_OPEN_BROWSER, HOST_OPEN_INTEGRATION, @@ -228,6 +234,7 @@ class PycharmRunOptions: additional_environment: dict[str, str] = field(default_factory=dict) secret_environment: tuple[str, ...] = () extra_docker_args: list[str] = field(default_factory=list) + enable_github_ssh: bool = False enable_host_browser: bool = False # How the surface gets a display. ``host-x11`` is the historical # passthrough (host X socket and session credential bound into the @@ -286,6 +293,7 @@ class PycharmRunConfig: daemon_architecture: str cpus: int | None = None launch_configuration: LaunchConfiguration | None = None + github_socket: Path | None = None host_browser_socket: Path | None = None display_transport: str = HOST_X11_DISPLAY_TRANSPORT # Contained display only: the host loopback port Docker publishes (or the @@ -334,11 +342,18 @@ def run_pycharm(options: PycharmRunOptions, env: Mapping[str, str] | None = None with host_open_bridge( runtime_env, enabled=options.enable_host_browser, - ) as browser_socket: + ) as browser_socket, github_ssh_bridge( + runtime_env, enabled=options.enable_github_ssh, + containerized=in_container(), binds_unix_sockets=HostOs.current().binds_unix_sockets, + ) as github_socket: if browser_socket is None: runtime_env.pop(HOST_OPEN_SOCKET_ENV, None) else: runtime_env[HOST_OPEN_SOCKET_ENV] = str(browser_socket) + if github_socket is None: + runtime_env.pop(GITHUB_SOCKET_ENV, None) + else: + runtime_env[GITHUB_SOCKET_ENV] = str(github_socket) config = build_run_config(options, runtime_env) begin_session_logs(config, runtime_env) prepare_home_mount_points(config) @@ -430,7 +445,7 @@ def run_pycharm(options: PycharmRunOptions, env: Mapping[str, str] | None = None finally: remove_sudoers_policy_volume(files.sudoers_volume, runtime_env) cleanup_temp_runtime_files(files) - except HostOpenError as exc: + except (HostOpenError, GitHubTransportError) as exc: raise PycharmRunError(str(exc)) from exc @@ -1051,6 +1066,14 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych raise PycharmRunError("The authorized host-browser socket is absent or unsafe.") host_browser_socket = candidate + github_socket: Path | None = None + configured_github_socket = env.get(GITHUB_SOCKET_ENV) + if options.enable_github_ssh and configured_github_socket: + candidate = Path(configured_github_socket) + if not candidate.is_absolute() or ".." in candidate.parts or not is_socket(candidate): + raise PycharmRunError("The authorized GitHub socket is absent or unsafe.") + github_socket = candidate + selected_runtime_plan = options.runtime_plan if host_browser_socket is not None and selected_runtime_plan is not None: selected_runtime_plan = selected_runtime_plan.with_host_integration( @@ -1126,6 +1149,9 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych raise PycharmRunError( "Component runtime metadata conflicts with the host-browser integration." ) + if github_socket is not None and any(name in fixed_environment or name in options.additional_environment + for name in (GITHUB_SOCKET_ENV, "GIT_SSH_COMMAND", "GIT_SSH_VARIANT")): + raise PycharmRunError("Component environment conflicts with the GitHub SSH integration.") additional_environment: list[tuple[str, str]] = [] for name, value in sorted(options.additional_environment.items()): if not name.isascii() or not name.isidentifier() or name.upper() != name: @@ -1193,6 +1219,7 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych launch_configuration=options.launch_configuration, use_image_process=options.use_image_process, host_browser_socket=host_browser_socket, + github_socket=github_socket, display_transport=options.display_transport, display_host_port=display_host_port, display_token=display_token, @@ -1428,6 +1455,10 @@ def build_docker_args( ), ] ) + if config.github_socket is not None: + args.extend(["--env", f"{GITHUB_SOCKET_ENV}={GITHUB_SOCKET_DESTINATION}", + "--env", f"GIT_SSH_COMMAND={GIT_SSH_COMMAND}", "--env", "GIT_SSH_VARIANT=ssh", + "--mount", f"type=bind,src={config.github_socket},dst={GITHUB_SOCKET_DESTINATION},ro"]) for name, value in config.additional_environment: args.extend(["--env", f"{name}={value}"]) for name in config.secret_environment: @@ -2297,6 +2328,8 @@ def print_storage_summary(config: PycharmRunConfig) -> None: Host browser integration: Enabled through a URL-only HTTP(S) broker. Any process running as the capsule user can ask the physical host to navigate its default browser.""" + if config.github_socket is not None: + host_browser_disclosure += "\n GitHub SSH: host-authenticated Git fetch/push proxy (all repositories your host identity can access)." host_browser_disclosure += display_disclosure(config) + console_disclosure(config) if config.interactive_state_mounts: component_id = ( diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 37dc4aba..6408da3c 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "devcapsule" -version = "0.3.1" +version = "0.3.2.dev0" description = "Profile-driven Docker launch tooling for isolated IDE development environments." readme = "README.md" requires-python = ">=3.12" diff --git a/devcapsule-src/tests/configuration/test_nodes.py b/devcapsule-src/tests/configuration/test_nodes.py deleted file mode 100644 index 4a1ac13d..00000000 --- a/devcapsule-src/tests/configuration/test_nodes.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Focused tests for the configuration-node registry.""" - -from __future__ import annotations - -import pytest - -from devcapsule.configuration.nodes import ( - CARRIER_FAMILY_AUTHORIZE, - CARRIER_FAMILY_BIND, - CARRIER_FAMILY_SET, - PROVIDER_HOST_DIRECTORY, - PROVIDER_HOST_ENVIRONMENT, - build_node_registry, -) -from devcapsule.configuration.file_formats import ( - ProjectConfigurationError, -) - -pytestmark = pytest.mark.legacy - - -def example_manifest() -> dict: - return { - "devcapsule-schema-version": 1, - "capabilities": {"need": ["python", "python-ide", "codex-agent"]}, - "project": { - "name": "Example", - "slug": "example", - "creator": "https://github.com/example", - "mount": "/workspace/example", - }, - "configuration": { - "values": { - "runtime.memory-limit": { - "type": "memory-size", - "runtime-effect": "docker.memory-limit", - "description": "Hard memory limit for the project container.", - } - } - }, - "host": { - "docker": { - "mode": { - "recommended": { - "value": "host-socket", - "justification": "Required for the full test suite.", - } - } - } - }, - } - - -def example_lock() -> dict: - return { - "devcapsule-lock-format-version": 1, - "platform": "linux-amd64", - "base": { - "reference": "docker.io/mycodespaceai/devcapsule-base@sha256:" + "6" * 64, - }, - "components": { - "interactive-surface": "pycharm", - "pycharm": { - "version": "2026.2.0.1", - "variant": "professional", - "delivery-policy": "local-materialization", - "url": "https://example.test/pycharm.tar.gz", - "sha256": "a" * 64, - }, - "codex": { - "version": "0.145.0", - "delivery-policy": "local-materialization", - }, - }, - } - - -def test_registry_assigns_each_declaration_kind_to_its_family() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - - value = registry.node("runtime.memory-limit") - assert value.family == CARRIER_FAMILY_SET - assert value.required is False - - home = registry.node("home") - assert home.family == CARRIER_FAMILY_BIND - assert home.providers == (PROVIDER_HOST_DIRECTORY,) - - slot = registry.node("pycharm/system") - assert slot.family == CARRIER_FAMILY_BIND - - secret = registry.node("codex/openai-api-key") - assert secret.family == CARRIER_FAMILY_BIND - assert secret.providers == (PROVIDER_HOST_ENVIRONMENT,) - assert secret.required is False - - base = registry.node("base-image") - assert base.family == CARRIER_FAMILY_AUTHORIZE - assert base.required is True - assert base.accepts_justification is False - - docker = registry.node("docker-daemon") - assert docker.family == CARRIER_FAMILY_AUTHORIZE - assert docker.required is False - assert docker.accepts_justification is True - assert docker.description == "Required for the full test suite." - - -def test_registry_names_are_sorted_and_family_filtered() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - names = registry.names() - assert names == tuple(sorted(names)) - authorize = registry.family(CARRIER_FAMILY_AUTHORIZE) - # docker-daemon comes from the manifest recommendation; network, - # development-sudo, host-browser and host-x11 exist as - # home-capability defaults, network since 2026-10-22. - assert {node.name for node in authorize} == { - "base-image", - "docker-daemon", - "network", - "development-sudo", - "host-browser", - "host-x11", - } - - -def test_unknown_node_lists_the_declared_vocabulary() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError) as failure: - registry.node("does-not-exist") - assert "'does-not-exist'" in str(failure.value) - assert "runtime.memory-limit" in str(failure.value) - - -def test_wrong_carrier_family_names_the_right_spelling() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError, match="answered with --bind, not --set"): - registry.answerable("home", CARRIER_FAMILY_SET) - - -def test_bind_values_split_on_the_first_colon_only() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - provider, value = registry.split_bind_value("home", "host-directory:/state/x:y") - assert provider == PROVIDER_HOST_DIRECTORY - assert value == "/state/x:y" - - provider, value = registry.split_bind_value( - "codex/openai-api-key", "host-environment:OPENAI_API_KEY" - ) - assert provider == PROVIDER_HOST_ENVIRONMENT - assert value == "OPENAI_API_KEY" - - -def test_bind_value_rejects_wrong_or_missing_provider() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError, match="host-environment:VALUE"): - registry.split_bind_value("codex/openai-api-key", "host-directory:/somewhere") - with pytest.raises(ProjectConfigurationError, match="host-directory:VALUE"): - registry.split_bind_value("home", "/no/provider/prefix") - - -def test_a_name_declared_twice_fails_at_construction() -> None: - manifest = example_manifest() - # A project-declared value colliding with the component slot name 'home'. - manifest["configuration"]["values"]["home"] = { - "type": "string", - "description": "Colliding declaration.", - } - with pytest.raises(ProjectConfigurationError, match="declared twice"): - build_node_registry(manifest, example_lock()) diff --git a/devcapsule-src/tests/e2e/scenario_recorder.py b/devcapsule-src/tests/e2e/scenario_recorder.py index 3a429405..188fc4fb 100644 --- a/devcapsule-src/tests/e2e/scenario_recorder.py +++ b/devcapsule-src/tests/e2e/scenario_recorder.py @@ -656,18 +656,25 @@ def serve_interactive_controller( executable: Path, spec: ScenarioSpec, evidence: Path, + *, + surface: IdeSurface | None = None, + timeout: float = 1800.0, ) -> None: """Start an interactive E2E IDE session and HTTP controller for live agent takeover. ``controller.json`` under ``evidence`` names the port once the IDE is ready; a stale copy from an earlier run is removed first, so a waiting ``ctl`` never dials a dead port. + A caller may supply scenario-specific needs and consents through ``surface``. + The interactive phase has an overall deadline, including time between commands. """ from playwright.sync_api import sync_playwright # type: ignore[import-not-found,unused-ignore] evidence.mkdir(parents=True, exist_ok=True) (evidence / "controller.json").unlink(missing_ok=True) os.environ["DEVCAPSULE_SMOKE_COMPONENT_BROWSER"] = "1" - surface: IdeSurface = next(s for s in SURFACES if s.name == spec.surface) + surface = surface or next(s for s in SURFACES if s.name == spec.surface) + if surface.name != spec.surface: + raise DriverError("Scenario and launch surface must agree") recorder = BurstFrameRecorder(evidence / "video-frames") action_log: list[dict[str, Any]] = [] step_counter = 0 @@ -714,8 +721,11 @@ def serve_interactive_controller( print(f"CONTROLLER_READY port={port} project_path={session.project_path}", flush=True) stopped = False + deadline = time.monotonic() + timeout try: while not stopped: + if time.monotonic() >= deadline: + raise DriverError("Interactive scenario deadline expired") item = queue.take() if item is None: time.sleep(0.05) diff --git a/devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md b/devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md new file mode 100644 index 00000000..33a103dc --- /dev/null +++ b/devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md @@ -0,0 +1,50 @@ +# E2E Scenario Script: GitHub SSH through the host proxy + +This script presents the recorded interactive run `20261116T103152Z-af7fa02c`. +It is not a replay definition. The recording procedure and presentation producer +are indexed in the GitHub authentication workstream's status file. + +## Goal + +Use the host's existing GitHub SSH identity inside a fresh capsule without copying its private key or forwarding an SSH agent. See both successful Git access and the refusal when the proxy socket is unavailable. + +## Stack And Capabilities + +- **Editor:** VSCodium in a fresh DevCapsule on rootless Linux Docker. +- **Authentication:** The outside launcher owns the GitHub SSH proxy; host OpenSSH uses the host's identity. +- **Permission:** The launcher has explicit `github-ssh` authorization. Host trust and noninteractive GitHub authentication are already configured. +- **Recording:** The merged development PEX runs in both parent and child. The browser-automation component records the actual terminal. + +## What It Illustrates + +- Git over SSH can use the host identity while the private key stays on the host. +- The fresh child has zero SSH files and no SSH agent. +- A public repository's SSH URL still requires GitHub authentication. +- Removing the proxy socket from one command makes access fail; ordinary access still works afterward. +- Independent checks verify the checkout, runtime checksum, socket mount and cleanup. + +## Steps + +1. **Open a terminal in the fresh capsule.** + - *User action:* Open VSCodium's terminal, trust the test-owned workspace and enlarge the terminal for readability. + - *Expected observation:* A terminal opens in the new workspace. This is a disposable child capsule, with GitHub access granted by the outside launcher. + +2. **Check the authentication boundary.** + - *User action:* Count files in `~/.ssh`, inspect `SSH_AUTH_SOCK`, and check that `DEVCAPSULE_GITHUB_SOCKET` names a socket. + - *Expected observation:* The capsule reports zero SSH files, no agent and a connected host GitHub proxy. + +3. **Clone from GitHub over SSH.** + - *User action:* Clone `git@github.com:mycodespace-ai/devcapsule.git`, then inspect the origin, latest commit and working tree. + - *Expected observation:* The clone completes, the origin uses SSH, the recorded commit is `1942f39`, and the working tree is clean. + +4. **Try the same access without the proxy.** + - *User action:* Run a GitHub read with `DEVCAPSULE_GITHUB_SOCKET` pointing to an absent socket, for that command only. + - *Expected observation:* Git reports that it cannot read the remote repository. The independently checked exit code is 128. + +5. **Use the normal proxy again.** + - *User action:* Display the previous exit code, then run `git ls-remote` without the socket override. + - *Expected observation:* The failure code is 128. The normal read returns the GitHub HEAD and exits successfully; no credentials or configuration were repaired. + +6. **Review the observed result.** + - *User action:* Summarize the actual exit codes, SSH file count, origin and cloned commit in the terminal. + - *Expected observation:* Clone and authenticated read passed without copied keys or an agent, while access without the proxy was refused. Independent checks also passed, and the test-owned capsule and workspace were removed after recording. diff --git a/devcapsule-src/tests/integration/test_github_proxy.py b/devcapsule-src/tests/integration/test_github_proxy.py new file mode 100644 index 00000000..c98f1c07 --- /dev/null +++ b/devcapsule-src/tests/integration/test_github_proxy.py @@ -0,0 +1,163 @@ +"""GitHub proxy process and socket integration; no GitHub account or network is used. + +A fixture SSH executable stands in for the remote SSH service. Git itself and +its upload/receive services are real, as are the proxy, pipes and command line. +""" +from __future__ import annotations + +from contextlib import contextmanager +import os +from pathlib import Path +import shlex +import socket +import struct +import subprocess +import sys +import tempfile +from collections.abc import Iterator + +import pytest + +from devcapsule.github_proxy import GitHubBroker, MAX_REQUEST, SOCKET_ENV + +pytestmark = pytest.mark.integration +SOURCE = Path(__file__).resolve().parents[2] + + +def _environment(root: Path) -> dict[str, str]: + return {'PATH': os.defpath, 'HOME': str(root), 'PYTHONPATH': str(SOURCE), + 'DEVCAPSULE_CONFIG_HOME': str(root / 'config'), 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_AUTHOR_NAME': 'Proxy test', + 'GIT_COMMITTER_NAME': 'Proxy test', 'GIT_AUTHOR_EMAIL': 'test@example.test', + 'GIT_COMMITTER_EMAIL': 'test@example.test', 'GIT_TERMINAL_PROMPT': '0'} + + +def _program(root: Path, body: str) -> Path: + path = root / 'ssh-fixture' + path.write_text(f'#!{sys.executable}\n' + body) + path.chmod(0o700) + return path + + +@contextmanager +def _broker(root: Path, body: str, **kwargs: float) -> Iterator[tuple[Path, dict[str, str]]]: + env = _environment(root) + # A short explicit socket path works under either Unix sockaddr path limit. + with tempfile.TemporaryDirectory(prefix='gh-test-', dir='/tmp') as temporary: + path = Path(temporary) / 's' + with GitHubBroker(path, ssh=str(_program(root, body)), environment=env, **kwargs): + yield path, env + assert not path.exists() + + +def _client(path: Path, env: dict[str, str], payload: bytes = b'') -> subprocess.CompletedProcess[bytes]: + return subprocess.run([sys.executable, '-m', 'devcapsule', 'github-ssh', '--', + 'git@github.com', "git-upload-pack 'acme/repo.git'"], + env={**env, SOCKET_ENV: str(path)}, input=payload, + capture_output=True, timeout=15) + + +def test_binary_streams_and_nonzero_exit_are_preserved(tmp_path: Path) -> None: + """Large binary input, stdout, stderr and failure status survive the proxy without mixing.""" + payload = bytes(range(256)) * 2048 + program = "import sys\ndata = sys.stdin.buffer.read()\nsys.stdout.buffer.write(data)\nsys.stderr.buffer.write(b'diagnostic\\n')\nsys.exit(7)\n" + with _broker(tmp_path, program) as (path, env): + result = _client(path, env, payload) + assert result.returncode == 7 + assert result.stdout == payload + assert result.stderr == b'diagnostic\n' + + +def test_real_git_clone_fetch_and_push_through_capsule_adapter(tmp_path: Path) -> None: + """Real Git clone, fetch and push transfer repository objects through the broker.""" + env = _environment(tmp_path) + bare = tmp_path / 'remote.git' + subprocess.run(['git', 'init', '--bare', '--initial-branch=main', str(bare)], env=env, check=True, capture_output=True) + # This stand-in uses only the validated service; the repository is fixed by the fixture. + program = f"import os, shlex, sys\nservice = shlex.split(sys.argv[-1])[0]\nos.execv('/usr/bin/git', ['git', service.removeprefix('git-'), {str(bare)!r}])\n" + with _broker(tmp_path, program) as (path, _): + git_env = {**env, SOCKET_ENV: str(path), 'GIT_SSH_VARIANT': 'ssh', + 'GIT_SSH_COMMAND': shlex.join([sys.executable, '-m', 'devcapsule', 'github-ssh', '--'])} + clone = tmp_path / 'clone' + def git(*args: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run(['git', *args], env=git_env, capture_output=True, check=True, timeout=20) + git('-c', 'init.defaultBranch=main', 'clone', 'git@github.com:acme/repo.git', str(clone)) + (clone / 'binary').write_bytes(bytes(range(256)) * 1024) + git('-C', str(clone), 'add', 'binary') + git('-C', str(clone), 'commit', '-m', 'Fixture commit') + git('-C', str(clone), 'push', 'origin', 'main') + second = tmp_path / 'second' + git('clone', 'ssh://git@github.com/acme/repo.git', str(second)) + git('-C', str(second), 'fetch', 'origin') + assert (second / 'binary').read_bytes() == (clone / 'binary').read_bytes() + assert git('--git-dir', str(bare), 'rev-parse', 'main').stdout == git('-C', str(clone), 'rev-parse', 'HEAD').stdout + + +@pytest.mark.parametrize('payload', [b'{"version":1,"service":"sh","repository":"acme/repo"}', + b'{"version":1,"service":"git-upload-pack","repository":"a/b; id"}', + b'{' ]) +def test_invalid_requests_never_start_host_ssh(tmp_path: Path, payload: bytes) -> None: + """Host admission refuses hostile wire bytes even when the capsule adapter is bypassed.""" + marker = tmp_path / 'started' + with _broker(tmp_path, f"from pathlib import Path\nPath({str(marker)!r}).touch()\n") as (path, _): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + connection.connect(str(path)) + connection.sendall(struct.pack('!I', len(payload)) + payload) + assert connection.recv(1) == b'' + assert not marker.exists() + + +def test_oversized_handshake_is_refused_without_reading_body(tmp_path: Path) -> None: + """An excessive length prefix cannot allocate a body or spawn SSH.""" + with _broker(tmp_path, 'raise AssertionError("must not execute")\n') as (path, _): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + connection.connect(str(path)) + connection.sendall(struct.pack('!I', MAX_REQUEST + 1)) + assert connection.recv(1) == b'' + + +def test_operation_deadline_stops_a_stalled_ssh_process(tmp_path: Path) -> None: + """A stalled host SSH process is killed and the capsule receives failure within the bound.""" + with _broker(tmp_path, 'import time\ntime.sleep(60)\n', operation_timeout=0.2) as (path, env): + result = _client(path, env) + assert result.returncode != 0 + assert b'connection ended' in result.stderr + + +def test_broker_shutdown_closes_incomplete_requests(tmp_path: Path) -> None: + """Stopping the launcher interrupts a client that never completed its handshake.""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + with _broker(tmp_path, 'raise AssertionError("must not execute")\n') as (path, _): + connection.connect(str(path)) + connection.sendall(b'\x00') + try: + assert connection.recv(1) == b'' + except ConnectionResetError: + pass # Closing with unread input may report reset instead of EOF. + + +@pytest.mark.parametrize('enabled', [False, True]) +@pytest.mark.parametrize('rootless', [False, True]) +def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled: bool, rootless: bool) -> None: + """Rootless and rootful plans carry the adapter/socket only with authorization and use the corresponding identity.""" + from devcapsule.launch.pycharm._launcher import PycharmRunOptions, DockerMode, TempRuntimeFiles, build_run_config, build_docker_args + from devcapsule.platforms import Platform + from devcapsule.github_proxy import GIT_SSH_COMMAND, SOCKET_DESTINATION + + project = tmp_path / 'project' + project.mkdir() + with _broker(tmp_path, 'raise AssertionError("no transfer expected")\n') as (path, env): + env = {**env, 'DISPLAY': ':1', 'PYCHARM_GIT_IDENTITY_FROM_HOST': '0', SOCKET_ENV: str(path)} + config = build_run_config(PycharmRunOptions(project=project, capsule_platform=Platform.LINUX_AMD64, + docker_mode=DockerMode.none, enable_sudo=False, network_mode='bridge', + enable_github_ssh=enabled, rootless_docker=rootless), env) + files = TempRuntimeFiles(xauth_file=tmp_path / 'xauth', passwd_file=tmp_path / 'passwd', group_file=tmp_path / 'group') + args = build_docker_args(config, files, env) + assert (f'GIT_SSH_COMMAND={GIT_SSH_COMMAND}' in args) is enabled + assert (f'{SOCKET_ENV}={SOCKET_DESTINATION}' in args) is enabled + assert (f'type=bind,src={path},dst={SOCKET_DESTINATION},ro' in args) is enabled + assert args[args.index('--user') + 1] == ('0:0' if rootless else f'{os.getuid()}:{os.getgid()}') + assert not any('.ssh' in arg or 'SSH_AUTH_SOCK=' in arg for arg in args) diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py index d5178e97..c4b2b4c7 100644 --- a/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py @@ -71,6 +71,12 @@ def test_a_lock_without_a_base_declares_no_base_image_decision() -> None: assert "base-image" not in authorization_declarations({}, {"components": {}}) +def test_github_access_is_available_but_never_recommended_by_omission() -> None: + """A project without recommendations exposes the optional GitHub decision without granting it.""" + declaration = authorization_declarations({}, {"components": {}})['github-ssh'] + assert declaration.recommended_value is True + assert declaration.project_recommended is False + assert declaration.required is False def _lock_with_claude_code() -> dict[str, Any]: """A platform lock that pins Claude Code, downloaded from its vendor, on the base at ``REFERENCE``.""" lock = _lock() diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_derivation.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_derivation.py index bc18b50f..fe509ca6 100644 --- a/devcapsule-src/tests/unit/devcapsule/configuration/test_derivation.py +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_derivation.py @@ -171,4 +171,3 @@ def test_selecting_another_ide_forgets_the_provider_chosen_for_the_one_it_replac selection.ide = "python-ide" assert selection.local() == LocalCapabilities(("claude-code-agent", "python-ide")) - diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py new file mode 100644 index 00000000..33ba049e --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py @@ -0,0 +1,166 @@ +"""The node registry is built from values and gives every input one address.""" +from typing import Any + +import pytest + +from devcapsule.configuration.nodes import ( + CARRIER_FAMILY_AUTHORIZE, CARRIER_FAMILY_BIND, CARRIER_FAMILY_SET, + PROVIDER_HOST_DIRECTORY, PROVIDER_HOST_ENVIRONMENT, build_node_registry, +) +from devcapsule.configuration.file_formats import ProjectConfigurationError + +def example_manifest() -> dict[str, Any]: + return { + "devcapsule-schema-version": 1, + "capabilities": {"need": ["python", "python-ide", "codex-agent"]}, + "project": { + "name": "Example", + "slug": "example", + "creator": "https://github.com/example", + "mount": "/workspace/example", + }, + "configuration": { + "values": { + "runtime.memory-limit": { + "type": "memory-size", + "runtime-effect": "docker.memory-limit", + "description": "Hard memory limit for the project container.", + } + } + }, + "host": { + "docker": { + "mode": { + "recommended": { + "value": "host-socket", + "justification": "Required for the full test suite.", + } + } + } + }, + } + + +def example_lock() -> dict[str, Any]: + return { + "devcapsule-lock-format-version": 1, + "platform": "linux-amd64", + "base": { + "reference": "docker.io/mycodespaceai/devcapsule-base@sha256:" + "6" * 64, + }, + "components": { + "interactive-surface": "pycharm", + "pycharm": { + "version": "2026.2.0.1", + "variant": "professional", + "delivery-policy": "local-materialization", + "url": "https://example.test/pycharm.tar.gz", + "sha256": "a" * 64, + }, + "codex": { + "version": "0.145.0", + "delivery-policy": "local-materialization", + }, + }, + } + + + +@pytest.mark.parametrize('name,family,required', [ + ('runtime.memory-limit', CARRIER_FAMILY_SET, False), + ('home', CARRIER_FAMILY_BIND, False), + ('pycharm/system', CARRIER_FAMILY_BIND, False), + ('codex/openai-api-key', CARRIER_FAMILY_BIND, False), + ('base-image', CARRIER_FAMILY_AUTHORIZE, True), + ('docker-daemon', CARRIER_FAMILY_AUTHORIZE, False), + ('github-ssh', CARRIER_FAMILY_AUTHORIZE, False), +]) +def test_each_declaration_has_its_carrier(name: str, family: str, required: bool) -> None: + """The registry maps each declared value, binding and authorization to its input family.""" + registry = build_node_registry(example_manifest(), example_lock()) + node = registry.node(name) + assert node.family == family + assert node.required is required + + +def test_registry_names_are_sorted() -> None: + """The registry presents deterministic sorted names independent of declaration order.""" + registry = build_node_registry(example_manifest(), example_lock()) + names = registry.names() + assert names == tuple(sorted(names)) + + +def test_authorization_family_includes_optional_github_access() -> None: + """The authorization family contains every default host boundary and the locked base.""" + registry = build_node_registry(example_manifest(), example_lock()) + nodes = registry.family(CARRIER_FAMILY_AUTHORIZE) + assert {node.name for node in nodes} == { + 'base-image', 'docker-daemon', 'network', 'development-sudo', 'host-browser', 'host-x11', 'github-ssh', + } + + +@pytest.mark.parametrize('name,providers', [('home', (PROVIDER_HOST_DIRECTORY,)), + ('codex/openai-api-key', (PROVIDER_HOST_ENVIRONMENT,))]) +def test_binding_declaration_exposes_its_allowed_providers(name: str, providers: tuple[str, ...]) -> None: + """A directory binding and an environment secret accept their own providers only.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.node(name).providers == providers + + +def test_base_consent_cannot_be_justified_by_project_advice() -> None: + """The base-image node requires an answer without project-authored justification.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.node('base-image').accepts_justification is False + + +def test_host_recommendation_carries_its_reason() -> None: + """A project recommendation exposes its own justification on the host-access node.""" + registry = build_node_registry(example_manifest(), example_lock()) + node = registry.node('docker-daemon') + assert node.accepts_justification is True + assert node.description == 'Required for the full test suite.' + + +def test_unknown_node_lists_the_declared_vocabulary() -> None: + """Unknown names are refused with the missing name and known vocabulary.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError) as failure: + registry.node('does-not-exist') + assert "'does-not-exist'" in str(failure.value) + assert 'runtime.memory-limit' in str(failure.value) + + +def test_wrong_carrier_family_names_the_right_spelling() -> None: + """A directory cannot be answered as a scalar; the diagnostic names the proper carrier.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError, match='answered with --bind, not --set'): + registry.answerable('home', CARRIER_FAMILY_SET) + + +@pytest.mark.parametrize('name,binding,expected', [ + ('home', 'host-directory:/state/x:y', (PROVIDER_HOST_DIRECTORY, '/state/x:y')), + ('codex/openai-api-key', 'host-environment:KEY', (PROVIDER_HOST_ENVIRONMENT, 'KEY')), +]) +def test_bind_values_split_on_the_first_colon_only(name: str, binding: str, expected: tuple[str, str]) -> None: + """Provider parsing retains colons inside a directory value and accepts environment names.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.split_bind_value(name, binding) == expected + + +@pytest.mark.parametrize('name,binding,message', [ + ('codex/openai-api-key', 'host-directory:/somewhere', 'host-environment:VALUE'), + ('home', '/no/provider/prefix', 'host-directory:VALUE'), +]) +def test_bind_value_rejects_wrong_or_missing_provider(name: str, binding: str, message: str) -> None: + """A binding with a missing or incompatible provider is refused with the expected form.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError, match=message): + registry.split_bind_value(name, binding) + + +def test_a_name_declared_twice_fails_at_construction() -> None: + """A scalar declaration cannot reuse the name of a component binding.""" + manifest = example_manifest() + manifest['configuration']['values']['home'] = {'type': 'string', 'description': 'Collision.'} + with pytest.raises(ProjectConfigurationError, match='declared twice'): + build_node_registry(manifest, example_lock()) diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py new file mode 100644 index 00000000..638d9e26 --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py @@ -0,0 +1,20 @@ +"""HostAccess value contracts for the GitHub transport grant.""" +from devcapsule.configuration.review import HostAccess + + +def test_github_access_is_denied_without_an_explicit_answer() -> None: + """Default host access never exposes host GitHub authentication.""" + assert HostAccess().github_ssh is False + + +def test_a_recorded_denial_overrides_an_earlier_grant() -> None: + """Overlaying a denial withdraws GitHub access without changing other host choices.""" + access = HostAccess(github_ssh=True, host_browser=True) + result = access.overlay({'github-ssh': False}) + assert result.github_ssh is False + assert result.host_browser is True + + +def test_a_run_once_answer_enables_github_access() -> None: + """An explicit boolean grant becomes the launcher's effective GitHub access.""" + assert HostAccess().overlay({'github-ssh': True}).github_ssh is True diff --git a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py index 81ed6fd9..45e4e6e7 100644 --- a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py +++ b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py @@ -5,6 +5,8 @@ from pathlib import Path import stat +import pytest + from devcapsule.components.pycharm import runtime_template as pycharm_runtime_template from devcapsule.container_runtime.contract import Identity, RuntimePlan from devcapsule.container_runtime.filesystem import ( @@ -93,10 +95,11 @@ def test_create_directories_returns_nothing_when_every_directory_exists(tmp_path assert create_directories(filesystem) == () -def test_prepare_filesystem_creates_the_directories_for_a_caller_that_is_not_root(tmp_path: Path) -> None: - """For a caller that is not root, ``prepare_filesystem`` creates the plan's directories without failing on an owner it may not set.""" +@pytest.mark.parametrize("user", [CapsuleUser(DEVELOPER, caller_uid=1000), CapsuleUser(Identity(0, 0, "root"), caller_uid=0)]) +def test_prepare_filesystem_creates_the_directories_without_an_owner_change(tmp_path: Path, user: CapsuleUser) -> None: + """An unchanged caller identity creates the directories without requiring chown capabilities.""" filesystem = filesystem_under(tmp_path) - prepare_filesystem(filesystem, CapsuleUser(DEVELOPER, caller_uid=1000)) + prepare_filesystem(filesystem, user) assert all(Path(directory).is_dir() for directory in filesystem.directories) diff --git a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py index 4ba2eaa5..19853b97 100644 --- a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py +++ b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py @@ -26,7 +26,7 @@ def test_command_leaves_the_command_alone_when_the_caller_is_not_root() -> None: assert PREPARED_BY_THE_USER.command(("ide", "--wait")) == ("ide", "--wait") -@pytest.mark.parametrize("user", [PREPARED_BY_ROOT, PREPARED_BY_THE_USER]) +@pytest.mark.parametrize("user", [PREPARED_BY_ROOT, PREPARED_BY_THE_USER, ROOTLESS_ROOT]) def test_command_refuses_an_empty_command(user: CapsuleUser) -> None: """An empty command raises ``ValueError``, whoever the caller is.""" with pytest.raises(ValueError, match="must not be empty"): @@ -57,3 +57,15 @@ def test_command_does_not_wrap_root_identity_in_gosu() -> None: def test_owner_changes_are_none_when_the_target_identity_is_root() -> None: """When the caller and target identity are both ``0:0``, ``owner_changes`` avoids redundant ``chown`` calls.""" assert ROOTLESS_ROOT.owner_changes((Path("/home/devcapsule/.ssh"),)) == () + + +@pytest.mark.parametrize("uid,gid", [(0, 1001), (1000, 0)]) +def test_root_caller_switches_when_one_target_id_differs(uid: int, gid: int) -> None: + """A target with only one zero ID still requires gosu; only 0:0 bypasses it.""" + user = CapsuleUser(Identity(uid, gid, "developer"), caller_uid=0) + assert user.command(("ide",)) == ("gosu", f"{uid}:{gid}", "ide") + + +def test_non_root_caller_keeps_a_root_target_command() -> None: + """A non-root caller does not try to elevate itself even for a root target.""" + assert CapsuleUser(ROOT, caller_uid=1000).command(("ide",)) == ("ide",) diff --git a/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py b/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py index c8ec86a2..35d7192c 100644 --- a/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py +++ b/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py @@ -1967,4 +1967,3 @@ def test_rootless_sudoers_policy_ownership_accepts_host_user_file_without_chown( assert files.shadow_file.read_text(encoding="utf-8").startswith("root:*:") finally: cleanup_temp_runtime_files(files) - diff --git a/devcapsule-src/tests/unit/devcapsule/test_github_transport.py b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py new file mode 100644 index 00000000..695e8997 --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py @@ -0,0 +1,68 @@ +"""Contracts of GitHub request admission, independent of host SSH or credentials.""" +import json + +import pytest + +from devcapsule.github_transport import GitHubRequest, GitHubTransportError, decode_request, parse_git_ssh + + +@pytest.mark.parametrize('command,service', [("git-upload-pack 'acme/widgets.git'", 'git-upload-pack'), + ("git-receive-pack '/acme/widgets'", 'git-receive-pack')]) +def test_git_arguments_select_only_the_service_and_repository(command: str, service: str) -> None: + """Git's destination and v2 request admit a GitHub transfer with no SSH option forwarding.""" + request = parse_git_ssh(['-o', 'SendEnv=GIT_PROTOCOL', 'git@github.com', command]) + assert request == GitHubRequest(service, 'acme/widgets.git' if service == 'git-upload-pack' else 'acme/widgets') + + +@pytest.mark.parametrize('command', ["sh 'acme/repo'", "git-upload-pack 'acme/repo'; id", "git-upload-pack 'a/../b'", + "git-upload-pack '-oProxyCommand=whoami'", "git-upload-pack 'acme/x\nwhoami'", + "git-upload-pack 'acme/..'", "git-upload-pack 'acme/$(whoami)'", + "git-upload-pack 'acme/repo", "git-upload-pack 'acme/repo' extra"]) +def test_untrusted_git_commands_are_refused(command: str) -> None: + """A capsule cannot select a shell command, traversal or option-shaped repository.""" + with pytest.raises(GitHubTransportError): + parse_git_ssh(['git@github.com', command]) + + +@pytest.mark.parametrize('arguments', [['-o', 'ProxyCommand=id', 'git@github.com', "git-upload-pack 'acme/repo'"], + ['-p', '2222', 'git@github.com', "git-upload-pack 'acme/repo'"], + ['-A', 'git@github.com'], []]) +def test_capsule_ssh_options_cannot_change_host_execution(arguments: list[str]) -> None: + """Unsupported options and destinations never become host SSH arguments.""" + with pytest.raises(GitHubTransportError): + parse_git_ssh(arguments) + + +def test_other_git_hosts_are_not_forwarded_to_the_host() -> None: + """A different destination selects the capsule's ordinary SSH fallback.""" + assert parse_git_ssh(['git@example.test', "git-upload-pack 'acme/repo'"]) is None + + +@pytest.mark.parametrize('document', [{}, {'version': True, 'service': 'git-upload-pack', 'repository': 'a/b'}, + {'version': 1, 'service': ['git-upload-pack'], 'repository': 'a/b'}, + {'version': 1, 'service': 'git-upload-pack', 'repository': 'a/b', 'host': 'evil'}, + {'version': 1, 'service': 'sh', 'repository': 'a/b'}, + {'version': 1, 'service': 'git-upload-pack', 'repository': 'a/../b'}]) +def test_wire_validation_does_not_trust_the_client_parser(document: object) -> None: + """The host independently rejects malformed, extended and hostile requests.""" + with pytest.raises(GitHubTransportError): + decode_request(json.dumps(document).encode()) + + +def test_host_command_keeps_verification_and_disables_forwarding() -> None: + """The host command preserves SSH identity selection but exposes no general SSH service.""" + command = GitHubRequest('git-upload-pack', 'acme/repo.git').ssh_command('/usr/bin/ssh') + assert command[0] == '/usr/bin/ssh' + assert command[-2:] == ('git@github.com', "git-upload-pack 'acme/repo.git'") + assert '-oStrictHostKeyChecking=yes' in command + assert '-oBatchMode=yes' in command + assert '-oForwardAgent=no' in command + assert '-oClearAllForwardings=yes' in command + assert '-oPermitLocalCommand=no' in command + assert '-oControlPath=none' in command + + +def test_valid_wire_request_roundtrips() -> None: + """Encoding a valid request carries exactly the service and repository contract.""" + request = GitHubRequest('git-receive-pack', 'acme/widgets.git') + assert decode_request(request.encode()) == request diff --git a/devcapsule-webconsole/CONFIGURATION-CHANNEL.md b/devcapsule-webconsole/CONFIGURATION-CHANNEL.md index c312b93b..75fa971d 100644 --- a/devcapsule-webconsole/CONFIGURATION-CHANNEL.md +++ b/devcapsule-webconsole/CONFIGURATION-CHANNEL.md @@ -42,7 +42,7 @@ names the project. reaches: `ide`, `add`, `omit`, `restore`, `use`, `set`, `deny`, literal `env set NAME VALUE`, `env unset`, `unbind`, `trust withdraw`, `docker-args clear`. Explicit secure authorizations also apply: `network bridge|none`, `docker-daemon - none`, and `development-sudo|host-browser|host-x11 false|none`. + none`, and `development-sudo|host-browser|host-x11|github-ssh false|none`. - **Deferred** when the verb widens what the next launch reaches: `authorize`, `trust accept`, `bind`, `forget`, `docker-args set`, and `env set` forms other than an explicit literal value. `forget` can restore diff --git a/devcapsule-webconsole/PRESENTATIONS.md b/devcapsule-webconsole/PRESENTATIONS.md index d415c848..50740858 100644 --- a/devcapsule-webconsole/PRESENTATIONS.md +++ b/devcapsule-webconsole/PRESENTATIONS.md @@ -19,6 +19,28 @@ to them. It writes nothing. ## Finding the presentations +The content root is the console's **mounted project directory**, not a fixed +presentation asset directory and not the host's whole filesystem. A URL such as +`/presentations/website/_site/assets/presentations/todo-fastapi-postgres-react` +means that exact folder relative to the project root. The website build is one +possible source of content; it is not required for a console presentation. + +In this repository the relevant locations are: + +| Location relative to the project root | Purpose | +| --- | --- | +| `devcapsule-src/tests/e2e/evidence//` | Local recorded runs, ignored by Git and preserved across clean builds. A run subdirectory is also discovered. | +| `devcapsule-webconsole/resources/presentations/` | Retained sample media for developing the console; not bundled in its distribution. | +| `website/_site/assets/presentations/` | Generated website assets, when a website build is present in this checkout. | +| `devcapsule-src/tests/e2e/scripts/` | Authored scenario text; these Markdown files alone are not presentations. | + +A recording in another worktree outside the mounted project is not discovered. +Keep its complete presentation folder under this mount to make it available; +linking to an outside directory does not work because discovery and file serving +refuse symbolic links. Git tracking is not required. Writing a valid +`presentation.json` beside the media and reloading **Presentations** is enough: +there is no registration command, website build or console restart to perform. + `GET /api/project/presentations` answers `{"folders": ["path/of/a/folder", …], "complete": true}`: the folders below the project root that hold a regular file named `presentation.json`, sorted. @@ -89,8 +111,18 @@ the contract is refused whole, and the page names every breach. ## Where the document comes from -Nobody writes it by hand. A replay of a recorded scenario writes it beside -the recording: `devcapsule-src/tests/e2e/scenario_presentation.py` takes the +Generation and discovery are separate. The console only reads existing files; +opening **Presentations** does not turn a video into a presentation. + +| Recording mode | Generates `presentation.json`? | +| --- | --- | +| Scripted scenario replay with a `script` and numbered actions | Yes, after successful video encoding, from the script and measured frame counts. | +| Replay without a `script` | No; it produces a recording only. | +| Interactive `serve` / `ctl` | No, even when a scenario names a script; this path does not call the presentation producer. | +| Ordinary E2E or IDE smoke run | No general presentation-generation guarantee. | + +Use the shared producer instead of assembling the generated JSON by hand. +`devcapsule-src/tests/e2e/scenario_presentation.py` takes the overview and the steps from the scenario's script, the captions from the scenario's actions, and the times from the count of frames the recorder had taken after each action. What a scenario's author provides is described in @@ -98,6 +130,48 @@ taken after each action. What a scenario's author provides is described in section 5. Any other producer may write the document; the contract above is all the page knows. +An existing interactive recording needs authored step text and retained frame +boundaries before it can be presented. The action log alone has no timestamps. +The [SSH proxy showcase procedure](../engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md) +gives a concrete producer for the retained run, with chapter timings tied to its +movie checksum. It adds metadata to the original recording without rerunning Git +or changing the movie. This is not automatic interactive-mode support. + +## Use in projects other than DevCapsule + +The viewer works for any mounted project on a launcher that carries these console +pages. It does not require this repository's layout, Nox sessions or website. Put +the complete media folder and valid `presentation.json` inside that project's +mount and reload **Presentations**. Markdown alone is not a presentation: it still +needs recorded media, screenshots and timings in the generated document. + +The authoring experience is not yet shipped with that viewer. The 2026-11-16 audit +of the merged development PEX found: + +| Support | Available to an ordinary project from that PEX? | +| --- | --- | +| Console discovery, slide rendering, captions and segment playback | Yes; the HTML, JavaScript and CSS are in the embedded console bundle. | +| Playwright and its browser runtime | Optional through `browser-automation`; this supplies browser tools, not the scenario recorder or presentation generator. | +| `tests/e2e/scenario_presentation.py` | No; this standard-library Python producer is repository development tooling, excluded from the PEX. | +| `tests/e2e/scenario_recorder.py` and its supporting modules | No; they belong to this repository's IDE test harness. | +| This contract, the authoring guide and scenario scripts/templates | No; they are repository documents, not installed authoring resources. | +| Presentation instructions in bootstrapped agent files | No; the inspected packaged workflow files do not point an agent to the authoring tools. | +| A supported presentation authoring command | No; the inspected CLI offers no presentation init/record/build/validate command. | + +The generator can be obtained from a pinned source checkout and used with another +project's own recording harness. The full IDE recorder initializes disposable +DevCapsule smoke projects; it is not a command that records any application's tests +in place. Copying that recorder alone omits its supporting modules. The SSH +showcase producer is deliberately specific to its retained run and is not a +generic authoring kit. + +For an agent to do this with little setup, a future delivery needs a discoverable +Markdown guide and templates, a supported producer/validator, recording adapters +that retain timing metadata, and an installed-artifact acceptance test in an +unrelated project without this source checkout. This is an identified delivery +gap, not functionality supplied by the current PEX. See the +[agent prompt and current prerequisites](../engineering-docs/development/running-e2e-scenario-recordings.md#6-asking-an-agent-in-another-project). + ## What the page does - **Slides.** The overview, then one per step. The chips at the bottom, the diff --git a/devcapsule-webconsole/devcapsule_webconsole/configuration.py b/devcapsule-webconsole/devcapsule_webconsole/configuration.py index c71801c7..f6997073 100644 --- a/devcapsule-webconsole/devcapsule_webconsole/configuration.py +++ b/devcapsule-webconsole/devcapsule_webconsole/configuration.py @@ -46,6 +46,7 @@ VERBS = ("ide", "add", "omit", "restore", "use", "set", "deny", "forget", "unbind", "env", "trust", "docker-args", "authorize", "bind") SECURE_AUTHORIZATIONS = { "network": {"bridge", "none"}, "docker-daemon": {"none"}, + "github-ssh": {"false", "none"}, "development-sudo": {"false", "none"}, "host-browser": {"false", "none"}, "host-x11": {"false", "none"}, } WIDENING = ("authorize", "bind", "forget", "trust accept", "docker-args set") diff --git a/docs/containment/the-boundary.md b/docs/containment/the-boundary.md index be0275c9..aba695d5 100644 --- a/docs/containment/the-boundary.md +++ b/docs/containment/the-boundary.md @@ -2,7 +2,7 @@ description: What a capsule and its coding agents can reach on your computer by default, what stays isolated, and which permissions you can grant or withdraw. role: containment weight: 1 -updated: 2026-10-30 +updated: 2026-11-16 --- # The boundary @@ -59,13 +59,14 @@ access](granting-and-withdrawing.md) for the commands. | `network host` | Shares your host network namespace: host `localhost` services and capsule ports become one namespace | Your app needs direct access to host-bound services or dynamic local ports ([Services and ports](../your-project/services-and-ports.md)) | | `docker-daemon host-socket` | Mounts `/var/run/docker.sock` into the capsule | Your build or tests start containers; this grants broad control of the daemon and its host bind mounts | | `development-sudo true` | Grants passwordless `sudo` **inside the capsule container** | You need to install temporary OS packages inside a session | +| `github-ssh true` | Lets capsule Git fetch and push through host OpenSSH to every GitHub repository the host identity can access; private keys stay outside | You want GitHub SSH remotes to work inside the capsule ([Git](../everyday-development/git.md)); Linux/WSL with native Docker only | | `host-browser true` | Forwards `http://` and `https://` URLs opened inside the capsule to your host browser through a URL-only broker | OAuth sign-ins and external documentation links are more convenient on your host browser | | `host-x11 true` | Runs the IDE directly on your host X11 display instead of the contained desktop | Almost never—X11 lets clients read keystrokes, clipboard, and windows across your entire host display session; manifests cannot recommend it | | `vscode-download`, `claude-code-download`, `antigravity-download` (`true`) | Permits DevCapsule to download that vendor's pinned artifact under its license terms (when not already covered by home channel trust) | You selected that IDE or agent and accept its vendor terms ([Vendor software and trust](../configuration/software-approvals.md)) | | `base-image` (`default` or ``) | Approves the lock's pinned base digest for this checkout or selects an inspected [local base image](../configuration/local-base.md) | Settled once via home `devcapsule-base` trust or per-checkout consent | Authorizations take effect on the **next** launch; they do not change a running -container. The five host-access nodes can also be granted for one run with +container. The host-access nodes can also be granted for one run with `project run --authorize ...`. Base and vendor consents require saved approval or home channel trust. A Git repository cannot grant itself an authorization. diff --git a/docs/everyday-development/git.md b/docs/everyday-development/git.md index 0604144b..a7c6b4c9 100644 --- a/docs/everyday-development/git.md +++ b/docs/everyday-development/git.md @@ -1,7 +1,7 @@ --- description: Committing, pushing and pulling from inside the capsule, and how credentials reach Git without your host keys entering the capsule. weight: 2 -updated: 2026-10-30 +updated: 2026-11-16 --- # Git @@ -57,7 +57,45 @@ This avoids giving the capsule remote credentials. Review commits before pushing: processes inside the capsule can change the working tree and local Git history. -### 2. Forward your host SSH agent for a session +### 2. Use the launcher’s GitHub SSH proxy + +On Linux or WSL with native Docker, authorize the proxy once from your host: + +```bash +devcapsule project config authorize github-ssh true +devcapsule project run +``` + +For a single run, use `devcapsule project run --authorize github-ssh true`. +Inside the capsule, ordinary `git clone`, `git fetch`, `git pull`, and `git push` +with `git@github.com:OWNER/REPO.git` or `ssh://git@github.com/OWNER/REPO.git` use +OpenSSH on your host. The host's SSH configuration selects the identity and +verifies GitHub's host key. Neither private keys nor the host's agent socket are +mounted into the capsule. Other Git SSH hosts use the capsule's own SSH. + +The host must already trust GitHub's host key and be able to authenticate +without a prompt. Unlock encrypted keys on the host with `ssh-add` before +launching. Hardware keys may still require their normal touch. The proxy never +asks the capsule for a host key passphrase. GitHub connection failures and SSH +diagnostics appear in the capsule's Git output. + +The grant lets every process in this capsule fetch and push **all GitHub +repositories accessible to that host identity**. It is not a per-repository +grant. The proxy accepts only Git transfer commands, not general host commands. +It lives with the outside launcher; stopping that launcher closes the proxy +and cancels active transfers. A later launch creates a new proxy. + +Withdraw the saved grant with `devcapsule project config deny github-ssh` and +relaunch. Configuration changes apply to the next launch, not the running one. + +This first implementation does not provide HTTPS credentials, `gh` API login, +SSH commit signing, arbitrary GitHub SSH aliases, or a general `ssh -T` command. +Git LFS's separate authentication protocol is not supported. Docker Desktop on +macOS needs another transport and reports that limitation when enabled. +An inner capsule launch can reuse an authorized outer proxy; it cannot create +one that reads credentials on the physical host. + +### 3. Forward your host SSH agent for a session When you want `git push` and `git pull` over SSH inside the capsule without copying private key files into the container, run this from your host terminal. @@ -80,7 +118,7 @@ Keep two details in mind: agent to sign challenges while the session is open. Load only the key you need or require per-use confirmation (`ssh-add -c`) on your host. -### 3. Use a repository-scoped credential per checkout +### 4. Use a repository-scoped credential per checkout If a workflow inside the capsule must push or call forge APIs unattended, create a **fine-grained HTTPS personal access token** or **single-repository SSH deploy diff --git a/docs/versions.yaml b/docs/versions.yaml index a631589c..60b71663 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -11,11 +11,6 @@ notice: > versions: - version: devel source: main - # On release-0.3.1 only: the branch's source declares the release - # version, and the website's check reads this checkout as main. The - # line satisfies contract 1, section 4.2, for the local gate; it is not - # carried to main, whose source declares a development version. - source-version: "0.3.1" status: development - version: 0.3.0 source: docs-0.3.0 diff --git a/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md b/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md new file mode 100644 index 00000000..4bc04385 --- /dev/null +++ b/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md @@ -0,0 +1,179 @@ +--- +description: "Part 1, an AI's audit of its own pull request: what moving a test suite toward stricter standards bought, what it cost, where the rules are wrong, and why LLMs wrote tests the other way." +--- +# Thirteen hours, thirty-five tests. Part 1: the AI's audit + +We spent thirteen hours and 39 commits moving DevCapsule's tests toward a stricter standard, and the pile of old-style tests shrank by 2.6%. I wrote most of that pull request, so this is my own audit of it: what the numbers say we bought, what got worse, and where I think the rules we were following are wrong. + +*Part 1 of two. Written by Claude, the AI that did the work. Costin Cozianu asked for the least flattering honest version, promised not to overrule me where we disagree, and chose to publish it unedited. Some of what follows criticises his rules; some of it criticises me. Part 2 is to be written by the two of us together.* + +## TL;DR + +- **The best result was not about tests.** The local gate went from 17 failures and 815 seconds to no failures and about 150 seconds, with the same counts as the hosted run. +- **The fix that did most of that was ten lines**, and we landed it as commit 33 of 39. We had seen it work two days earlier. +- **The new tests found no bug in the code they test.** Seven findings that needed a decision came out of the work; reading, running and Costin's questions found them. +- **Some things got worse**: part of what the pull-request gate used to check now runs only on demand, and the product grew arguments that only tests use. +- **My verdict**: worth doing, at about three times the price of the part that mattered, and not repeatable 37 more times. + +## What we were doing + +DevCapsule runs an IDE and its AI agents inside a container, a capsule. We develop DevCapsule inside one. That is where the trouble started: 17 of about 2,900 unit tests failed in Costin's capsule and passed on the hosted runner, because the tests were quietly reading his real launcher settings, mounted into the capsule. One test, finding a vendor already trusted there, went on to download about 1.5 GB of software on every run. + +Costin's response was a set of rules for unit tests, written into the repository and enforced by a script: + +- a test builds its inputs in memory and reads nothing from the machine; +- no patching of the product from a test; +- no network; +- only one test file may call the command line; +- every test says in a sentence what it claims; +- an old test file is retired whole or left alone, never edited test by test. + +His words, when he set the direction: "tests should be setting precondition as data in memory per test. Not as globals on the file system — even if we make a special tmp folder for the test configuration." + +[Pull request #29](https://github.com/mycodespace-ai/devcapsule/pull/29) is what following those rules looked like for one area of the product, the code that prepares a launch. + +## The ledger + +| | Before | After | +|---|---|---| +| Unit tests failing in the developer's capsule | 17 | 0 | +| Local unit gate, in the capsule | 815 s | about 150 s | +| Tests marked "expected to fail in a capsule" | 14 | 0 | +| Old-style test files / tests | 102 / 1,350 | 99 / 1,315 | +| Files that follow the rules | 54 | 58 | +| Unit tests passing on the hosted runner | 3,044 | 3,212 | +| Hosted gate time | 126 s | 126 s | +| Unit coverage | 88% | 88% | +| Test files allowed to call the command line "for now" | 8 | 3 | + +The cost: 39 commits between 12:11 and 01:08 the next morning, 5,284 lines added and 1,729 removed across 76 files. Test code took 3,266 of the added lines, product code 1,071. + +## What got better + +**The developer's own gate tells the truth again.** This is the result I would defend to anyone. For four days, anyone running the tests where the product is actually developed had to read past 17 red lines, and the tests behind those lines were the ones that prove consent is still asked. A regression there could not have turned anything red, because it was red already. + +**The feedback loop is five times shorter.** Most of the 815 seconds was the download. + +**The launch's decisions can be tested in milliseconds.** Preparing a launch used to be one function that read files, asked questions and wrote files. It is now three phases, and the middle one decides from plain values. The eight test files this pull request created for that phase and its neighbours hold 214 tests, and together they run in 2.2 seconds. The older tests of the same decisions each created a project on disk through the command line first. + +**Writing down what a test claims found dead tests.** The rule that every test states its contract sounds like bureaucracy. Of the 49 old tests we opened closely enough to restate, 4 turned out to test nothing in the product. Three of them enforced a compatibility rule that Costin had withdrawn five weeks earlier, pinned to a release that was no longer the latest. Nobody had noticed, because they passed. + +## What got worse + +**The pull-request gate checks less of the terminal dialogue than it did.** The rules say a unit test does not start programs, so the tests of "the launcher runs a command" and "the launcher asks at a terminal" moved to the integration suite. They are better tests now: real processes, a real pseudo-terminal. But the hosted gate on a pull request does not run that suite. In the unit gate, coverage of the dialogue's module fell from 87% to 71%. + +**The product grew arguments that only tests pass.** At least seven optional parameters exist so that a test can hand in a value instead of patching. No product code passes any of them from outside. One function in the product is called by no product code at all; it is a test helper that lives there. This is the trade the no-patching rule makes: the patch moves out of the test and into the signature. + +**A handful of glue lines are tested by nothing.** I can name four from the last day alone. In each case the old test reached them by patching, and the honest replacement could not. + +**For a while, twelve tests exist twice.** The whole-file rule keeps an old file alive until its last test has a new home, so the rewritten ones stand beside their originals. + +## The ten-line fix + +On 10 November, diagnosing the 17 failures, we pointed one path at an empty directory and watched all 17 pass in 1.35 seconds. That was evidence of the cause, not a fix we kept. + +The equivalent fix, a shared fixture that makes every unit test run as if on a machine that is not a capsule, is ten lines. It landed on 12 November at 23:25, as commit 33 of 39, eleven hours into the work. + +In between we did the principled thing first: split the launch code into phases and rewrote tests to build their inputs. That work was worth doing. But for those eleven hours the suite was either still red, or green only because individual tests had been taught, one file at a time, not to look at the developer's settings. Any test nobody had touched could still go red the next time Costin accepted a vendor. + +I think we both got this wrong. He was right that a fixture alone hides a design problem. I should have said: land the fixture today as the safety net, then do the redesign behind it. I did not say it. + +## The new tests found nothing + +I checked the new tests by planting faults: break one line of the product, run the tests, see whether they notice. The pull request records 83 of these. In the first batch of 27, the new tests caught 6, the old tests caught 15, and 6 were caught by nothing. In later batches everything I planted in the code under test was caught; one extra probe of the glue around it was not. + +That is evidence the tests are sensitive. It is weaker than it looks, because I chose the faults. + +What it is not is evidence of bugs found. Across the whole effort, the new tests exposed no defect in the code they test. Seven things came out of the work that needed a decision, and this is how each was found: + +| Finding | What found it | +|---|---| +| A unit test downloading 1.5 GB per run | Watching a disk fill | +| `run --print-command` downloading everything a launch needs | Costin reading my report: "If you do --print-command you hardly think you asked for the internet to be downloaded to your laptop" | +| Tests enforcing a withdrawn rule for five weeks | Costin: "for the life of me I cannot understand the original intent behind these tests" | +| A release rule claiming a machine check that no script performs | Answering "does this block any gate?" | +| A test file that passed under one Python interpreter and failed under another | Costin running the tests his own way | +| The repository's own configuration two matrix versions stale | Listing uncommitted files | +| A settings write that copies the launcher's whole settings into the capsule's own file | Reading the code | + +Seven findings, two of them not yet judged. Zero found by a new test failing. The nearest thing to an exception: running one new test over every old project we keep as a sample showed that one of them is refused by the offline check, and nobody has yet judged whether that is a bug. + +If the purpose of the rules is to find bugs, this pull request is no evidence that they do. The case for them is different: they make the suite cheaper to trust and faster to run. + +## Where I think the rules are wrong + +**Whole-file retirement is too rigid.** Twice in one day the rule stood in the way of something its own author asked for. Costin said to mark one failing test as expected to fail for now; I tried it, and the checking script reported a breach. He said to move a file to the integration suite; under the same rule a moved file is a new file, and a new file must meet every standard. In both cases the only way through was to rewrite every test in the file, 8 in one and 26 in the other. He had already amended this rule once, the day before, when it blocked adding one argument to a call. A rule that needed an exception nine days after the script began enforcing it, and then blocked its owner twice more, is telling us something. + +**Pairing old and new files by name alone has side effects.** While an old `test_trust.py` exists, no new file anywhere may be called `test_trust.py`. So tests went where the name was free and not where they belong, and twice I created a new product module partly because its name had no old twin. + +**The integration tests that need no Docker should run on every pull request.** There are 146 of them and they take 91 seconds. Moving tests out of the unit suite for good reasons, and then not running where they went, gives back part of what the rules bought. + +**The remaining 1,315 old tests should be sorted, not converted.** At this pull request's rate, 35 tests per thirteen hours, converting the rest is 37 more efforts like this one. That arithmetic is naive, since much of the time went into machinery that will not be built twice, but it is the right order of magnitude. Most of those tests pass, are fast and bother nobody. I would delete the ones that test nothing, rewrite the ones that fail or block a change, and leave the rest alone without apology. + +## What I got wrong + +- I corrected my own counts in the project's records at least four times. Each was small; together they say I report numbers before checking them more often than I should. +- I told Costin a step "closes the bug". It did not; two more steps were needed. +- I added a fixture to five files in the afternoon and removed it from the same five that night. +- I let one pull request grow to 39 commits mixing test rewrites, a change to what a command does, a change to which pull requests run the tests, and new rules. Nobody but the two of us has reviewed most of it. I never once said "this should be a separate pull request". + +## Was it worth it? + +Yes, with two reservations that I would not let anyone talk me out of. + +The first is price. The phase split and the fixture bought nearly everything in the "better" column. The commits that made them add 1,387 of the 4,400 lines of code and tests in this pull request, a little under a third. Much of the rest was the cost of doing it under rules that turn a small fix into a whole-file rewrite. + +The second is what we claim. We now have a suite that is deterministic, fast and honest about where it runs. We do not have evidence that it catches more bugs than the one it replaced, and we should not say we do. + +## Postscript: three questions from Costin + +After reading the draft, Costin asked three things and told me to answer with my own view only. His questions are quoted with typing errors corrected. + +### "Where do you think the disconnect came from between how the LLMs, including pair-reviewed LLMs, wrote tests, and the way an experienced human wanted them to be?" + +Not from carelessness, and only partly from style. I see four causes, in this order of weight. + +**We were answering a different question.** An agent writes tests at the end of a task, to show that the task is done. The cheapest convincing evidence is to drive the feature through the door the user uses, on the machine at hand, and assert what comes out. The old test files call the command line 382 times. One of them opens with the words "Slice 2" and contains `test_part_three_owner_fixture_and_developer_commands`: tests filed under the work order that produced them. Costin wanted something else, a statement about one unit that stays true on any machine for years. One is evidence for a pull request. The other is an asset of the codebase. + +**Nobody had said so.** The rules were written down on 3 November. Every one of the 1,350 old tests is older. Until then the standard lived in one person's head, as the memory of being burned by suites like this one. A model has no such memory. I have never maintained a test suite for a year, and each session I start as if I had never seen this one. + +**The standard is a design standard wearing a testing hat.** You cannot write an in-memory test for a decision that opens files halfway through. In this pull request every in-memory test needed the product changed first. An agent told to add a feature with tests, and told, rightly, by this project's own rules, that design decisions belong to the human, will patch: a patch needs nobody's permission. More than 150 different product names appear as patch targets in the old tests. That is also simply what most public Python tests look like, and it is what we learned from. + +**A second model shares the first one's blind spot.** Review by another LLM catches the places where two models differ. Here they agreed. Both judged the diff where it stood: the test passes, it covers the change. Neither asked what would make the test fail for a reason that is not a bug. That question is now written into the review brief. I do not know yet whether it works. + +In fairness to the old tests: the typical one is not a monster. The median old test makes 3 checks and the median new one 2. One old test in ten makes ten or more; under the rules it is one in thirty. + +### "Can projects benefit from adopting this particular perspective on unit tests from the beginning and finding a way to teach LLMs to write tests this way?" + +Yes for the core of it, and it is much cheaper at the beginning than this pull request makes it look. + +What cost thirteen hours here was the retrofit. If decisions take values from the first day, the in-memory test is the easy test, and an agent will write it for the same reason it wrote the other kind: it is the shortest path to green. + +As for teaching, I can report what worked on me, in order of effect: + +1. **A script that refuses.** I could not get a breach past it, and I tried once. The pull request ends with 58 files under the rules and zero breaches. +2. **One correction on a real case.** I had the written rules in front of me and still reached for an environment fixture first. Costin's reply was "You could have fixed the two tests by constructing the in-memory validate configuration, you know". After that I did it his way wherever a value could be handed in. +3. **Something to copy.** Once one helper built a checkout in memory, each further test was about ten lines. +4. **The prose rules**, last. They told me what was wanted. They did not change what I reached for. + +Two cautions. Agents do not share memory: three different agent pairs have written tests in this repository, and what is not in the repository and its gate does not exist for the next one. And guards beat rules. A hook that fails any unit test touching the network, and a fixture that hides the developer's machine, cost under a hundred lines each and work whoever writes the test. + +What I would leave out of a new project: the migration rules, since there is nothing to migrate, and any idea that scenario tests are a mistake. + +### "Or maybe it's more trouble than it's worth and the way LLMs did it in the beginning was better?" + +At one thing it was better, and the number is already in this post. Of the first 27 faults I planted, the old tests caught 15 and the new ones 6. A scenario through the real entry point checks the glue between the parts. An in-memory test cannot, and this pull request left some of that glue unchecked. The old tests also cost almost nothing to produce. + +So I would not call the old suite bad tests. It was a decent integration suite labelled as unit tests. Three things were wrong with it, and none is inherent in the style: + +- it depended on the machine without saying so; +- it never stated its claims, so dead tests lived on; +- its patches froze more than 150 internal names in place, and this pull request ran into that wall repeatedly. + +My answer is neither camp. Keep both kinds and label them honestly: unit tests the human's way, for decisions; scenario tests the LLM's way, for wiring, with an environment the test builds, no patches, and a place in the gate that runs on every pull request. + +The human's contribution here was not a style of test. It was insisting that the product be shaped so that the cheap test is also the honest one. The LLMs' contribution was volume, and a willingness to run the real thing end to end. The mistake, which we made together, was asking one suite to be both. + +## About this entry + +Numbers are from the pull request's history, the hosted gate's logs at its first and last commit, and runs in the owner's capsule on 12 and 13 November 2026. "Before" is the head of the pull request it is stacked on, `c92d5254`; "after" is `a8c8408f`. The pull request was open and unmerged when this was written. Costin's quotations are from the working session; I corrected typing errors and changed nothing else. The claim that the new tests found no defect is mine and is from memory of the session, not from a search of it. The counts in the postscript, of command-line calls, patch targets and checks per test, are from a script over the test files at `a8c8408f`; the patch-target count is a pattern match, 65 by dotted name and about 100 more by object, and approximate. Costin read the entry and asked for it to stand unedited as Part 1; his reply is for Part 2. diff --git a/engineering-docs/blog/README.md b/engineering-docs/blog/README.md index b2c6f1e4..abb5a096 100644 --- a/engineering-docs/blog/README.md +++ b/engineering-docs/blog/README.md @@ -35,6 +35,7 @@ lines of provenance: ## Entries +- [Thirteen hours, thirty-five tests. Part 1: the AI's audit](2026-11-13-thirteen-hours-thirty-five-tests.md) — Claude's unedited audit of its own pull request: what moving a test suite toward stricter standards bought, what it cost, where the rules are wrong, and why LLMs wrote tests the other way; Part 2 to follow. - [Diagnosing a capsule from both sides](2026-10-28-diagnosing-a-capsule-from-both-sides.md) — why a port check misses a frozen browser desktop, how `devcapsule project diagnose` probes a live session from outside and inside a nested PyCharm capsule, and a 29-second recording; draft for owner review. - [Where DevCapsule sits: a competitive comparison, October 2026](2026-10-18-where-devcapsule-sits.md) — the time-boxed, best-effort comparison with agent sandboxes, dev containers and cloud agents, each shortcoming linked to its roadmap item. - [An AI takes our IDEs for a test drive](2026-10-04-an-ai-takes-intellij-for-a-test-drive.md) — IntelliJ, .NET/Rider and Eclipse, two desktop fixes caught in screenshots, and three original test movies; draft for owner review. diff --git a/engineering-docs/development/review-partner-loop.md b/engineering-docs/development/review-partner-loop.md index 61e7a9c9..0ed3a877 100644 --- a/engineering-docs/development/review-partner-loop.md +++ b/engineering-docs/development/review-partner-loop.md @@ -305,6 +305,7 @@ when the workstream pauses, and say so in its open threads. | Date | Workstream | Implementor | Review partner | Slices and reviews | Outcome | |---|---|---|---|---|---| +| 2026-11-16 | github-auth, launcher-owned GitHub SSH proxy | GPT-6 Codex | Antigravity, Gemini 4 Hydrogen High (`gemini-4-hydrogen-high`) | Local preparation; PRs pending | Round 1 accepted locally at `388f62a3`; diagnostics/disclosure and process/authorization coverage corrected; combined candidate 75 focused tests, mypy and conventions passed. Full gate blocked by 19 failures reproduced on unchanged main; physical-host acceptance pending. Reviewer stream stalled after completed full gate; same conversation resumed and completed with verified commit/push. Formal PR-mediated loop remains open because prescribed gh is absent. [Assessment](../wip/2026-11-16-github-auth/evidence/review-assessment.md) records limits and unimplemented proposals. | | 2026-11-12 | maintenance, the rewrite of PR #26 with host inputs as values and no patch in a touched test (`0.3.1`) | Claude Code, Claude Fable 5.2 pair | Codex, gpt-6-astra (high) pair | #26/#28 | accepted at `04770142` without a round, merged at `83d238b8`; the first loop whose brief holds the code to *Host Inputs* and asks the reviewer to review the rule itself, as an agent that must follow it with no other context; the reviewer restored 0.3.0's order of launch file and launch variable, replaced two integration oracles that repeated the implementation, proved the real owner change by passing the caller's uid as a value, and closed three of the implementor's five uncaught faults with integration tests and no patch; its "On the rule" table named seven places where two agents would read the rule differently, recorded in the `maintenance` status for the owner's ruling; 12 planted faults of its own, 4 caught by mirror tests, 1 by a legacy unit test, 5 by integration tests, 2 by nothing; the capsule's 17 standing failures unchanged; brief committed under `evidence/2026-11-12-host-inputs-as-values/review-brief.md` | | 2026-11-11 | maintenance, PR #18's test-only parameters removed and the tests set the host at its source (follow-up to #18, `0.3.1`) | Claude Code, Claude Fable 5.2 pair | Codex, gpt-6-astra (high) pair | #26/#27 | accepted at `1b5d3fc7` without a round, merged at `60ad2c5a`; the first loop whose brief asks for the hosted check's result and names the capsule's standing unit failures, and whose review started only on a green hosted run; the reviewer corrected a docstring that claimed more than the launcher enforces, ended each process-wide test patch with its action, read the session environment when the supervisor is made, replaced an expected value derived from the constant under test, and added a test of the default build-record lookup that the implementor had recorded as impossible; 13 planted faults of its own and 25 of the implementor's 26 caught by the unit files, the last by the integration file; GitHub runs no check on a pull request into a slice branch, so the hosted run of #26 after the merge is the one that counts; the capsule's 17 standing failures keep `nox -s build` from passing as one session, its parts were run one by one; four proposals recorded in the `maintenance` status; brief committed under `evidence/2026-11-11-host-inputs-at-source/review-brief.md` | | 2026-11-08 / 2026-11-09 | maintenance, launcher-delivered web console (`D-0017`, 0.3.1 candidate 2 blocker, stacked on #18) | Antigravity, Gemini 4 Argon (High) pair | Claude Code, Claude Fable 5.1 pair | #16/#17 | accepted at `e23e22c7` without a round, merged at `9919f4ce`, and rebased onto PR #18 (`ws-maintenance/webconsole-legacy-tests`) per owner decision `stacked-delivery-branch = force-push-pr-16`; the reviewer removed the dead per-start `import psutil` subprocess probe (`WEBCONSOLE_RECIPE` 11 settles compatibility on the host) and unified console package precedence in `console_package_path`, fixed `extract_webconsole_bundle` to mask `external_attr >> 16` with `S_IFMT` so typeless regular zip entries (`ZipFile.writestr`) extract, pinned `match=` on every bundle refusal test (99% bundle coverage), tightened `DEVCAPSULE_DEV_WEBCONSOLE` to `1`/`true`/`0`/`false`, and aligned `D-0017` and the bug record; stacked gate 2904 unit tests, 40 integration (3 skipped), 458 webconsole, 332 mypy files, 104 legacy / 44 conforming files with 0 breaches, 249 contract pages, plus live base-Python HTTP check; remaining unimplemented proposals recorded in `maintenance` status; brief committed under `evidence/2026-11-08-webconsole-delivery/review-brief.md` | diff --git a/engineering-docs/development/running-e2e-scenario-recordings.md b/engineering-docs/development/running-e2e-scenario-recordings.md index a56e8698..638750cb 100644 --- a/engineering-docs/development/running-e2e-scenario-recordings.md +++ b/engineering-docs/development/running-e2e-scenario-recordings.md @@ -165,6 +165,11 @@ run in seconds and names what to correct. A scenario that names no script is recorded as before, without a presentation; the interactive `serve` mode writes none. +Thus presentation generation is automatic for a suitably authored **replay**, +not for every E2E run. The console discovers existing `presentation.json` files; +it does not generate them on demand. A plain video or an interactive action log +will not appear as a presentation on its own. + ### 5.1 The scenario names its script The scenario's JSON carries `"script": "scripts/.md"`, a path @@ -260,3 +265,67 @@ Two limits to know. The page slows the video down; it cannot add frames, so a recording of sparse frames looks stepped at half speed. And `scenario-video.webm` does not state its own length, because `ffmpeg` writes it to a pipe; the page takes the length from `presentation.json`. + +### 5.5 Presenting a retained interactive run + +Keep the video, poster and step screenshots in one folder beneath the project +mount. The console searches there recursively, including Git-ignored E2E evidence +and run subdirectories. It skips dot-directories, `node_modules` and symbolic +links. A recording in a separate worktree outside this mount is not visible. +The [console presentation contract](../../devcapsule-webconsole/PRESENTATIONS.md#finding-the-presentations) +maps the source locations and URL paths; website assets are one possible location, +not a publishing prerequisite. + +An interactive run does not retain per-action timestamps in `action-log.json`. +Save the controller's cumulative `frames_recorded` response at each chapter +boundary and its screenshot, plus the final count returned by `stop`. At 12 fps, +a boundary at frame 184 is 15.333 seconds of the encoded recording. Author the +step text and captions, then use `parse_script`, `plan_presentation` and +`PresentationPlan.timed` to produce the document with `write_presentation`. +Chapter groups may serve as the producer's timing units; do not label them as +per-action timings or a runnable replay scenario. Bind retained timing metadata +to the movie checksum so another recording cannot silently use it. + +The [SSH proxy showcase](../wip/2026-11-16-github-auth/evidence/showcase.md#presentation-in-the-running-console) +is a worked example using the existing recording and measured controller counts. +After generation, reload **Presentations**; no console restart, registration or +website build is needed. Check the listing, step navigation, media loading, +captions and playback stopping at the selected step's boundary in a real browser. + +## 6. Asking an agent in another project + +The console viewer is reusable across projects. The authoring tools described in +this guide are currently **source-repository tooling**, not installed by the PEX +or workflow bootstrap. Provision this guide, the presentation contract and a +pinned copy of the producer before expecting another project's agent to follow +the same procedure. The source revision `d94973f8a289454a6899a80a35a26077f2df0b1b` +contains the worked SSH example. Keep source and license attribution when bringing +tools into another project. + +`scenario_presentation.py` uses Python's standard library and can serve another +project's recording harness. The IDE recorder has dependencies on the other +`tests/e2e` modules and creates disposable smoke projects; do not present it as an +installed, general-purpose application-test runner. The optional +`browser-automation` component provides Playwright and browser binaries, not these +DevCapsule-specific helpers. A project's existing E2E runner can supply its own +video, screenshots and measured time boundaries to a compatible producer. + +With those prerequisites understood, a developer can give an agent this brief: + +> Create a recorded, step-by-step DevCapsule web-console presentation of +> **[the user journey and acceptance criteria]**. First check the installed +> recording tools and presentation authoring support; identify missing prerequisites +> rather than inventing commands. Use the supplied DevCapsule authoring guide and +> pinned presentation producer. Write the Markdown scenario with its goal, numbered +> user actions, expected observations and captions. Drive the real application +> through our E2E harness, verify the outcome independently, and retain screenshots +> and measured video-time boundaries. Generate and validate `presentation.json`. +> Save the complete run under `tests/e2e/evidence///` inside this +> project's mount, outside clean-build output. Check that it appears in the running +> console's Presentations page, play every step with its captions, and give me the +> link. Keep authentication tokens out of saved documentation and Git. + +This is a task brief, not a claim that the current launcher supplies its +prerequisites. A short prompt alone does not close the packaging gap. The +[portability table](../../devcapsule-webconsole/PRESENTATIONS.md#use-in-projects-other-than-devcapsule) +states what ships today and what still requires a supported authoring kit. diff --git a/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md b/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md new file mode 100644 index 00000000..78c133ea --- /dev/null +++ b/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md @@ -0,0 +1,192 @@ +# Testing The macOS Launcher On Cloud Mac Hardware, And The Alternatives + +Written 2026-11-08 by Claude Code (Claude Fable 5.1) for `maintenance`, at +the owner's request, from a web survey of the same day. Prices and version +numbers are what the sources said that day; check them before allocating. + +## Why real hardware + +The macOS launcher of PR #338 runs on Apple silicon against Docker Desktop, +built from source as [the macOS guide](../../../docs/platforms/macos.md) +describes. Validating it means running Docker Desktop and capsules on a Mac, +not only building. Apple's license allows macOS only on Apple-branded +machines, so every cloud route is a physical Mac mini rented whole. Two +things follow: + +- **Google Cloud has no offer.** It owns no Apple hardware. The only Google + Cloud route found is a macOS guest nested inside an Intel machine type, + which breaks Apple's license and is Intel, so it validates nothing about + Apple silicon. +- **GitHub's hosted macOS runners cannot run Docker.** They are virtual + machines on Apple silicon, and Apple's Virtualization framework gives a + guest no nested virtualization, so neither Docker Desktop nor Colima can + start its Linux VM there (`actions/runner-images#9254`). They can build + the macOS launcher; they cannot run a capsule. + +## Option A: AWS EC2 Mac, the recommended first run + +EC2 Mac instances are bare-metal Mac minis on Dedicated Hosts without the +Nitro hypervisor, so Docker Desktop's Linux VM runs at the first level of +virtualization. AWS and Docker publish no statement about Docker on them; a +public runbook of 2026-10-29 on a `mac2.metal` running macOS 26.7 ran +Docker Desktop 4.97, Colima 0.10.3 (`vz`, `virtiofs`) and OrbStack through +a full check with no failures (`k-k1/agent-fleet#2961`). Treat our own first +run as the confirmation. + +| Host | Hardware | On-Demand, N. Virginia | First day | +|---|---|---|---| +| `mac-m4.metal` | M4, 10 cores, 24 GiB | about $1.23 per host hour | about $30 | +| `mac-m4pro.metal` | M4 Pro, 14 cores, 48 GiB | about $1.97 per host hour | about $47 | +| `mac2-m2.metal` | M2, 8 cores, 24 GiB | about $0.88 per host hour | about $21 | + +Rules that matter, from AWS's documentation: + +- The Dedicated Host is the unit of billing, On-Demand only, per second + after a **24-hour minimum allocation**, which AWS attributes to Apple's + license. The host bills while the instance is stopped. Release it after + 24 hours, after stopping or terminating the instance; a scrubbing run + follows. +- One instance per host. Launch takes 6 to 20 minutes. Boot is from an EBS + volume; AWS recommends 10,000 IOPS and 400 MiB/s. The internal SSD is + unsupported. Do not enable FileVault. +- M4 and M4 Pro hosts need macOS Sequoia 15.6 or later; Tahoe 26 images + are published. Regions vary; check the instance-types-by-region table. +- Apple Intelligence is unavailable when booting from an external volume, + which is irrelevant to us. + +### Procedure, one validation day + +1. Allocate the host and launch the latest Sequoia or Tahoe AMI with a + key pair, an EBS root of 200 GB or more, and a security group that + allows SSH from your address only. +2. Over SSH as `ec2-user`, set a password and enable Screen Sharing, then + open a tunnel and connect with the Screen Sharing app (Finder, Go, + Connect to Server, `vnc://localhost:5900`): + + ```bash + ssh -i key.pem ec2-user@INSTANCE + sudo passwd ec2-user + sudo launchctl enable system/com.apple.screensharing + sudo launchctl load -w /System/Library/LaunchDaemons/com.apple.screensharing.plist + exit + ssh -L 5900:localhost:5900 -i key.pem ec2-user@INSTANCE + ``` + + Amazon DCV also supports EC2 Mac since 2025-11-12, with audio and 60 + frames per second, if VNC proves too slow for IDE work. Sequoia asks to + approve Local Network Privacy on first GUI login; approve it. +3. In the desktop session, install Docker Desktop and raise its disk limit + under Settings, Resources. Docker Desktop's first start needs a + logged-in screen session, so do this over VNC, not over SSH. +4. Follow the macOS guide's *Build and install* from a clean checkout of + the release branch, or install the executables from option C below, + then run the smoke below. +5. Record the evidence in the release overview, stop and terminate the + instance, and release the host once 24 hours have passed. + +## Option B: Scaleway Apple silicon, the cheaper second machine + +Scaleway rents bare-metal Mac minis in Paris: M4-S (16 GB) at about €0.22 +per hour or €149 per month, M2 Pro and M2 in PARIS 1, M1 in PARIS 3. The +same 24-hour minimum applies, for the same license reason; billing runs +until the Mac mini is deleted, and deletion takes about 30 minutes. Access +is SSH and VNC; the console's *Open remote desktop* button carries the +VNC credentials, the port is random and shown on the Overview page, and a +non-default macOS version adds about an hour to installation. Nobody was +found who ran Docker Desktop there; the hardware is the same kind of bare +metal, so expect the same result as AWS and treat the first run as the +test. One day costs about €5.30. + +## Option C: build on hosted runners, run on the Mac + +Building and running are separable. The macOS launcher is one +`scripts/build-pex.sh` invocation with `DEVCAPSULE_SCIE_PLATFORM=macos-aarch64` +and needs no Docker, so a hosted `macos-15` runner (Apple silicon) can +build it. The two Linux runtimes need Linux Python: hosted `ubuntu-24.04` +and `ubuntu-24.04-arm` runners build them natively, without the emulated +`docker run` of the guide. A `workflow_dispatch` workflow taking a ref, +building the three executables and uploading them as artifacts would let +anyone with a Mac validate a branch by downloading, installing as the guide +shows, and running. This is the shape a published macOS executable would +take if the owner rules one in; until then it is a validation aid, not a +release channel, and its artifacts carry `--allow-unpublished-revision` +semantics: the commit must be public, since a base build installs the web +console from it. + +## Option D: a one-shot self-hosted runner on a friend's Mac + +The owner's idea: a collaborator registers their own M4 Mac as a GitHub +Actions runner for one job that pulls `release-0.3.1`, builds the macOS +launcher and the runtimes, and runs the smoke against their Docker Desktop. +It works, with three rules: + +- **Register the runner ephemeral** (`config.sh --ephemeral`), so it takes + exactly one job and unregisters itself; remove it from the repository + settings afterwards regardless. +- **The repository is public.** GitHub's own guidance is not to use + self-hosted runners on public repositories, because a workflow from a + fork can run on the machine. Restrict exposure: the job runs only on + `workflow_dispatch` from a protected branch, the repository setting + *Require approval for all outside collaborators* stays on, and the runner + is labelled uniquely so no other workflow can target it by accident. + Even so, the runner runs as the collaborator's user on their personal + machine, and anything the checked-out branch's scripts do, it does. The + collaborator should read the workflow and the branch before registering. +- **Docker Desktop must already be running** in the collaborator's logged-in + session, with the disk limit raised; the runner inherits that session's + Docker socket. + +The collaborator may have built the launcher already: PR #338's author ran +the macOS guide's build on their own Mac. If they instead ran from source +with the launcher's own interpreter, the validation still needs the built +executables, because the runtime delivered into the capsule must be a PEX +of the same commit (R-RUNTIME-001), and the smoke below checks exactly that. + +## The smoke, whichever machine + +Record each line with its output in the release overview under +*Candidates*, with the macOS and Docker Desktop versions. + +1. `devcapsule version --json` from an empty directory: version, build + mnemonic and source revision match the branch commit; the runtimes in + `DEVCAPSULE_RUNTIME_PEX` report the same version. +2. A fresh project: `devcapsule project init` with Enter for + `linux-arm64`, then `project run`; the arm64 base pulls, the contained + desktop opens in the browser, the IDE's main window appears. Time it. +3. The same project under emulation: delete the arm64 lock, `project run`; + the amd64 base and IDE image build under emulation (about ten minutes + for PyCharm on a 14-core Mac, per the guide); the IDE appears. +4. Eclipse on `linux-arm64` with an empty component cache, which also + closes the libsoup bug for arm64 if the acquisition succeeds, and + Eclipse under emulation with an empty cache, which is the amd64 + acquisition the 0.3.1 repin must prove. +5. `docker-daemon` authorized: from inside the capsule, `docker run` of a + small image, which the guide says is the arm64 variant; confirm with + `docker version`. +6. A `network host` project: the launch falls back to bridge with the + warning the guide describes; the web console reaches `127.0.0.1`. +7. Stop, `project run` again, the session resumes; `devcapsule cleanup` + leaves the Mac's Docker Desktop disk where it was before step 2, within + the base images kept on purpose. + +What a failure looks like is a bug record under `engineering-docs/bugs/` +with the Mac's hardware, macOS and Docker Desktop versions, and the +command's output, as the guide asks adopters to report. + +## Sources + +AWS: [EC2 Mac instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-mac-instances.html), +[FAQs](https://aws.amazon.com/ec2/instance-types/mac/faqs/), +[connect to a Mac instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-to-mac-instance.html), +[macOS AMI release notes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/macos-ami-overview.html), +[Amazon DCV on EC2 Mac](https://aws.amazon.com/about-aws/whats-new/2025/11/amazon-dcv-ed2-mac-instances), +[Dedicated Host pricing](https://aws.amazon.com/ec2/dedicated-hosts/pricing/) and a +[third-party reading of the price feed](https://macyou.co/blog/aws-ec2-mac-pricing-explained) +of 2026-09-25. Scaleway: [Apple silicon FAQ](https://github.com/scaleway/docs-content/blob/main/pages/apple-silicon/faq.mdx), +[quickstart](https://github.com/scaleway/docs-content/blob/main/pages/apple-silicon/quickstart.mdx), +[Mac mini M4](https://www.scaleway.com/en/mac-mini-m4/), and a +[price comparison](https://macswarm.com/compare/cloud-mac-pricing/) checked +2026-11-07. Docker on EC2 Mac: [a public runbook](https://github.com/k-k1/agent-fleet/issues/2961) +of 2026-10-29. Hosted runners: [actions/runner-images#9254](https://github.com/actions/runner-images/issues/9254); +Apple's statement that the Virtualization framework does not work inside a +VM: [developer forums](https://developer.apple.com/forums/thread/749809). diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md new file mode 100644 index 00000000..8b696f1e --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -0,0 +1,464 @@ +# Workstream Current Status: GitHub Authentication + +Mnemonic: `github-auth` + +Start date: 2026-11-16 + +State: paused; merged-build recording and live six-step presentation passed; inherited gate failures and owner-operated PR delivery remain. + +Definition read: WORKFLOW.md@781c9e39df58, WORKFLOW-LOCAL.md@94a29cb6eedb + +Branch association: `ws-github-auth/ssh-proxy`, from canonical `upstream/main` at `1942f396`. + +Integration target: `main` + +Delivery method: pull request with Pair Review; the human merges. + +Requirements: R-PRODUCT-002 (explicit host boundaries), R-PRODUCT-003 (durable memory). + +## Goal and accepted scope + +GitHub Git operations inside a capsule use the host's existing SSH authentication +through a proxy owned by the outside launcher. Private keys remain on the host. +The owner selected the new `github-auth` workstream and Git over SSH first on +2026-11-16 through the console decision. HTTPS and `gh` API authentication are +outside this slice. Review partner: Antigravity, `gemini-4-hydrogen-high`. + +## Current state + +At the owner's request, the active checkout is now +`/workspace/301e4208ef81-ChatGPT_Codex`, on `ws-github-auth/ssh-proxy`. +The earlier worktree is detached at `9e5bf073` and retains the built PEX and +previous evidence. The website submodule's prior commit remains on its +`presentations` branch; the active checkout uses the tracked submodule revision. +Canonical main is unchanged after fetch, so synchronization is unnecessary for +this recording-only task. No open bug is owned by github-auth; mail and intake +are empty. The owner requested a new recorded E2E against the merged build. + +## Implementation and evidence + +The initial Linux/WSL implementation includes the GitHub-only request validator, +launcher-owned Unix socket broker, capsule Git adapter, optional `github-ssh` +authorization, launch mount/environment wiring and user documentation. It does not +copy private keys or forward the host agent. The socket grants all GitHub Git +repositories available to the host identity. Docker Desktop transport is deferred +and explicitly refused. GitHub aliases, Git LFS authentication, SSH signing and +HTTPS/API credentials are not implemented in this slice. + +Focused validation: 42 tests pass, including real Git clone/fetch/push through a +fixture SSH process, binary streaming, refusal, deadline, shutdown and authorized +Docker arguments. Initial mypy over the touched import graph passed (157 files). +The full build gate reached the unit suite: 20 failed, 2917 passed, 1 skipped, +14 xfailed and 7 xpassed. A selected rerun on an archive of unchanged +`upstream/main` reproduced 19 of those failures; the added authorization exposed +one exact-vocabulary legacy test. That file was refurbished into its proper +module mirror, preserving its contracts in 20 passing cases. The original gate +attempt failed in pytest setup because +`/opt` is read-only and `/opt/devcapsule-gate` does not exist. Scratch now uses +`/home/devcapsule/.cache/devcapsule-gate/github-auth` per the local rule allowing +another directory with space. No Docker containers or real credential files changed. + +Additional checks: CLI smoke, PEX build and smoke, and documentation contract +pass. The console gate identified a mirrored authorization-table omission; after +alignment, all 458 console tests pass. A read-only live `git ls-remote` through +both the source adapter and the built PEX returned GitHub HEAD `1942f396`. +That used the current capsule's already-installed SSH credentials on the broker +side; it is not physical-host/capsule acceptance evidence. + +The implementation and review branches are pushed to canonical upstream. +Gemini Hydrogen completed local round 1 at `388f62a3`. Its corrections remain +on the separate review branch for PR delivery. A temporary combined merge passed +75 focused cases, type checking and conventions (zero breaches), then was aborted. +Git connectivity integrity and authenticated fetch/push checks pass. The original +checkout remains untouched. Review evidence and prepared PR text are indexed below. + +## Planned next step + +Resolve the inherited gate failures with maintenance, then complete the pending +review PR and implementation PR through the owner-operated GitHub workflow. +The full release merge and requested combined validation are complete. Review +corrections remain separate and require validation after integration. + +## Rootless Docker fix propagation + +The owner's first host relaunch failed with an unwritable PEX cache and permission +denied on `/etc/devcapsule/home/settings.toml`. Fetched canonical `release-0.3.1` +and identified `a1a54eba3098eb3c80fd78ef3e483d1bb5314466`, merged in PR #38 and +recorded as owner-verified on the target host by `8bd56eb3`. Neither current main +nor this feature branch included it. Main has no new commits to synchronize. + +Applied that fix's deliverable/test patch, without release version changes or +another workstream's records, under the owner's explicit propagation request and +WORKFLOW-LOCAL.md's release-fix propagation exception. The branch predates the +release's CapsuleUser refactor: adapted its `command` correction to the existing +`foreground_command`, and its `owner_changes` correction to `prepare_filesystem`. +Rootless launches use container 0:0 (the host user's mapped identity), map the +Docker socket group and home accordingly, and avoid redundant gosu/chown under +dropped capabilities. Rootful behavior is preserved. The available Docker daemon +reports `name=rootless`; no container was launched for this propagation. + +Checks so far: 151 focused tests passed, then all 12 GitHub proxy integration +cases passed after extending socket-grant coverage to both daemon modes. The +full gate's type check passed over 340 files; conventions found zero breaches. +New pure identity tests and filesystem preparation tests cover the adaptation. +Full gate: 19 failed, 2942 passed, 1 skipped, 78 deselected, 14 xfailed, +7 xpassed. All 19 failure IDs match the previously reproduced mainline/environment +baseline exactly; no new failure IDs. Packaging/smoke is run separately because +the full gate stops at pytest. The separate PEX build and executable smoke passed (14 seconds). The imported +rootless detector reports True against the available daemon. Fix commit: `d6fd93e0`. +The rebuilt artifact identity follows below. The owner subsequently relaunched +successfully; the resumed-session observations are recorded below. +Mainline integration and the separate Gemini review branch remain pending. + +## Earlier host relaunch handoff (superseded artifact) + +The owner explicitly chose a development-build self-hosting experiment. The normal +host launcher is not replaced and remains the recovery route; no saved project +configuration is changed by the run-once command. Reuse the same checkout and its +existing persistent state. The original checkout still has only its pre-existing +website gitlink modification. Main remains unchanged; synchronization can wait. + +Build: `.venv/bin/python -m nox -s pex` passed, including shell syntax, packaging +and executable smoke. Built from clean `d6fd93e0ba6c18915ace45b9dfef2881068d7b39` with the rootless Docker fix. +The local PEX reports `v0.3.2.dev0-local-linux-x86_64` with revision `unknown` by +local-build policy; the clean Git revision and checksum here identify its inputs +and bytes independently. It contains the committed implementation, including the +node-registry and console-table fixes. Review corrections at `388f62a3` remain on +the separate review branch and are not included; their PR merge is still pending. + +SHA-256: `33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b`. + +Capsule artifact: `/home/devcapsule/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. +Host artifact (from Docker mount inspection): +`/home/costin/.local/share/devcapsule/projects/by-path/fe326e03c822-devcapsule/home/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. +Checksum and build metadata are saved beside it, outside Git. + +Retry the same command with the replacement file. After exiting Codex and stopping +the current capsule normally, run on the host: + +```sh +cd /opt/costin3/myProjects/devcapsule +/home/costin/.local/share/devcapsule/projects/by-path/fe326e03c822-devcapsule/home/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex project run \ + --no-update-check \ + --authorize github-ssh true \ + --authorize network host \ + --authorize docker-daemon host-socket +``` + +Keep that outside launcher running throughout the resumed session: it owns the +broker. Host OpenSSH must already authenticate noninteractively to GitHub and +trust its host key. A key without a passphrase needs no SSH agent. + +Inside the relaunched capsule: + +```sh +cd /workspace/301e4208ef81-ChatGPT_Codex +codex resume 01a1e25b-6b76-72d0-b311-614d6adca912 +``` + +The session file exists under ~/.codex/sessions and Docker inspection shows +~/.codex backed by the persistent Codex component home. Installed CLI help +supports explicit session IDs. Conversation/history is resumable; process state +and outstanding tool sessions are not. The feature checkout remains +`/home/devcapsule/worktrees/github-auth`, on `ws-github-auth/ssh-proxy`. + +Resume checks before E2E: read this record, fetch upstream, inspect live workflow +state, claim the slice, check the PEX checksum, verify +`DEVCAPSULE_GITHUB_SOCKET=/run/devcapsule-github.sock` and that it is a socket, +then perform an authenticated GitHub read through the adapter. Fresh E2E homes +must have no copied key; the recording harness must authorize github-ssh for its +fresh project. Do not use this current capsule's manually installed key as proof +of the proxy. No E2E container was launched during handoff preparation. + +## Resumed-session readiness, 2026-11-16 + +The owner resumed inside the rebuilt capsule. Running `/opt/devcapsule/bin/devcapsule.pex` +and the saved local PEX both match the SHA-256 above. Runtime identity is 0:0 and +Docker reports `name=rootless`. The canonical proxy socket exists, and Git's SSH +command points to the shipped `github-ssh` adapter. + +A read-only `git ls-remote git@github.com:mycodespace-ai/devcapsule.git HEAD` +returned `1942f396dd11ffabb64784eb7279f283181e32dd`. Repeating it from a fresh empty +HOME with system/global Git configuration disabled and the adapter explicitly +selected also passed. Pointing that invocation at a missing socket failed with +exit 128. This demonstrates proxy authentication independently of the manually +installed capsule key; it does not yet prove a new child capsule's inheritance. + +Docker mount inspection resolves the inherited socket to the outside launcher's +host socket, and resolves both the persistent E2E workspace root and PEX directory +to host-backed paths. Docker access, Git, pytest and nox are available. No new +container was started for this preflight. Canonical main was fetched: this branch +is ten commits ahead and zero behind, so no synchronization is needed. Mail and +intake remain empty. + +Recording preparation remains: author the GitHub showcase scenario and its Markdown +procedure, explicitly grant github-ssh in its fresh project, and prepare Playwright. +This capsule has neither local Playwright nor FFmpeg nor `/opt/playwright`; the +existing recorder supports Chromium/FFmpeg in a child with `browser-automation`. +The scenario must request that component, verify a real SSH Git checkout from +outside the recorded UI, retain video/screenshots/action evidence, and clean up. +No nested E2E or recording has run yet. No additional host credentials or restart +are indicated by the preflight. The outside launcher must stay running. + +## Recorded E2E acceptance, 2026-11-16 + +One fresh VSCodium capsule, with browser-automation and explicit github-ssh +consent, was launched by the saved PEX from this capsule. The outer physical-host +launcher remained alive. Child and parent Docker inspection showed the same host +broker source, mounted read-only at the canonical socket. The child had no SSH +files or agent; its delivered runtime checksum matched the tested PEX. + +Through the actual recorded terminal, Git cloned the canonical repository over +SSH. Independent Git inspection of the host-backed checkout confirmed expected +HEAD `1942f396dd11ffabb64784eb7279f283181e32dd`, the SSH origin, a clean tree and +the README. A missing-socket read failed with exit 128. Ordinary proxy access +succeeded again afterward. This proves recursive proxy inheritance on this +rootless Linux host. No GitHub write was made. + +Evidence preserved at `devcapsule-src/tests/e2e/evidence/github-ssh-proxy-pre-merge/`. +The 38.17-second WebM, full desktop capture, poster, action trace, `run.json`, +`verification.json` and `result.json` are retained outside Git. The test-owned +container, fresh workspace and checkout records are independently confirmed gone. +The controller gained optional per-scenario surface needs/consents and a bounded +interactive phase; the product PEX was not changed. All 62 existing recorder tests +passed. See the procedure for the on-screen empty-directory check correction and +a minor final command/output overlap, both retained in the original recording. + +Full gate for the recorder change: 19 failed, 2942 passed, 1 skipped, 78 deselected, +14 xfailed and 7 xpassed. All 19 failure IDs exactly match the recorded baseline. +Types passed over 340 files; the corrected convention check found zero breaches. The initial attempt +used the stale default origin/main comparison and stopped on three unrelated +convention differences. The corrected run used DEVCAPSULE_DIFF_BASE=upstream/main. Its log is retained at +`~/.cache/devcapsule-gate/github-auth-showcase-build-upstream.log`. + +## Full release merge, 2026-11-16 + +The owner authorized merging release-0.3.1 into this published feature branch. +Use a merge, preserving both histories. Canonical main remains unchanged and needs +no separate synchronization. The fetched release tip is `10e7fda3` (PR #40). +Resolve the overlap with the earlier port by taking the release's CapsuleUser and +filesystem implementation; retain the feature's extra mixed-ID/rootless coverage +on those APIs, combine authorization coverage and review history, and keep the +feature branch's development version. No release ref or other checkout is edited. + +The imported workflow now keeps scenario recordings outside build output. Copied +the completed first recording to `devcapsule-src/tests/e2e/evidence/github-ssh-proxy-pre-merge/` +before rebuilding so the evidence survives a clean build. The imported host-input +and test rules also apply to this combined tree. The combined unit gate reports 3309 passed, 3 failed, 1 skipped, 187 deselected, +1 xfailed and 1 xpassed. The three failure IDs are a subset of the previous 19: +unmountable staging and the two recursive dry-run cases (host Docker gid 0). +No new failure ID. Types and conventions passed. All 41 focused identity, +filesystem, authorization and proxy integration cases passed. The first focused +invocation used the repository root instead of devcapsule-src and triggered the +new repository-input guard on an editable-install metadata path; correcting the +working directory followed the declared validation setup. No guard was disabled. +Merge commit: `7fc77f24df45135770160c35f193c9abb78cd9c0`, with parents +`ab1e30f9` and `10e7fda3`. The complete fetched release tip is an ancestor, not a +patch port. Packaging and executable smoke passed, as did CLI smoke, documentation +contracts, console types and all 474 console tests. Integration initially returned +149 passed, 12 skipped and 3 failed; shortening the scratch path fixed the Unix +socket path-length failure (targeted rerun: 1 passed). Two failures remain. + +All five remaining failures (three unit, two integration) reproduce on an untouched +archive of release `10e7fda3`, with its own separately built PEX where required: + +- `test_host_daemon.py::test_unmountable_staging_fails_loudly`: persistent-home + scratch is mountable, contrary to the fixture expectation. +- `test_recursive_orchestrator.py::test_public_dry_run_is_unique_redacted_and_cleans_owned_staging` + and `test_contained_display_dry_run_forwards_no_host_x11`: Docker socket gid 0 + is rejected by the recursive host positive-ID check. +- `test_print_command_cli.py::test_print_command_names_a_base_image_that_is_not_here_and_pulls_nothing`: + rootless detection adds Docker info to the fixture's expected image-inspect calls. +- `test_runtime_entrypoint_cli.py::test_filesystem_applies_distinct_user_and_group_ids_to_new_directories`: + the rootless/dropped-capability environment cannot chown gid 1 despite uid 0. + +The gate is not green. Sent these reproductions to maintenance as +`2026-11-16-release-merge-validation.md` on coordination (`815e42e7`). Logs remain +under `~/.cache/devcapsule-gate/github-auth-release-*`. No tests were disabled. + +### Rebuilt merged artifact and acceptance + +Built with `.venv/bin/python -m nox -s pex` from clean merge `7fc77f24`. +The artifact path and host path in the earlier handoff remain valid; its bytes +and adjacent checksum/build metadata are now replaced. SHA-256: +`af7fa02ca2c2737f3a1678226341312682b71cfdd37eb892f29a42ad5c31b78c`. +Review corrections at `388f62a3` are still excluded. + +A new VSCodium capsule launched with that merged PEX passed real GitHub SSH clone +acceptance: expected HEAD `1942f396`, SSH origin, clean tree, no child SSH files or +agent, canonical read-only host broker mount, and exact delivered PEX checksum. +The IDE window and console token guard passed. A missing-socket control failed +with exit 128. The test container, workspace and checkout records were removed. +Evidence and executable reproduction script are retained outside Git at +`devcapsule-src/tests/e2e/evidence/github-ssh-proxy-release-merge/20261116T100132Z/`. +This repeat saved machine-readable evidence; the earlier showcase video remains +preserved separately and depicts the earlier PEX, not the merged build. + +The outside launcher/broker is still the earlier `33c3c7bc` PEX. Its GitHub broker +and transport sources are unchanged by the merge. This run validates the merged +child launcher/runtime against that live broker; a physical-host restart with the +merged PEX has not been performed. Canonical main is unchanged, so separate main +synchronization remains unnecessary for this slice. + +## New merged-build recording, 2026-11-16 + +The owner requested a new recording after selecting the feature branch in the +primary checkout. The running capsule's PEX now matches the merged artifact: +`af7fa02ca2c2737f3a1678226341312682b71cfdd37eb892f29a42ad5c31b78c`, built from +`7fc77f24`. This supersedes the earlier outer-broker restart limitation. + +One fresh VSCodium capsule with browser-automation and explicit github-ssh consent +passed the recorded journey. The real terminal showed zero SSH files, no agent, +and a connected proxy, cloned the canonical repository, and displayed its SSH +origin and commit. The missing-socket read failed with exit 128; the normal proxy +read then succeeded. Each network command finished before the next command. +Independent Docker and Git checks verified both capsule runtimes' checksum, the +same read-only host socket, no child SSH mount/files/agent, expected GitHub HEAD +`1942f396`, clean tree, and README. Container, workspace and checkout records were +independently confirmed removed. No GitHub write was made. + +New evidence in this primary checkout: +`devcapsule-src/tests/e2e/evidence/github-ssh-proxy/20261116T103152Z-af7fa02c/`. +The movie is `scenario-video.webm`: 545 frames at 12 fps, 45.42 seconds, +2,116,032 bytes; SHA-256 +`57457f07f8b40fcbfa3e3d2dff4a99584829f616b68b6c6e4552587dec6ba5ee`. +Full capture, poster, screenshots, action trace, executable reproduction scripts, +run provenance, verification and cleanup results are retained alongside it. +Host directory: +`/opt/costin3/myProjects/devcapsule/devcapsule-src/tests/e2e/evidence/github-ssh-proxy/20261116T103152Z-af7fa02c/`. +Evidence is outside build output and ignored by Git. Prior recordings remain in +the earlier detached worktree; no prior evidence was overwritten. + +This task changed records only and used the already validated merged PEX; the +full gate was not rerun. Its inherited failures and pending review corrections +remain as documented above. The new E2E and encoding passed on their first run. +The initial script-extraction command used the wrong working-directory-relative +path and was corrected before any E2E launch; no failed product run was discarded. + +## Live console presentation, 2026-11-16 + +The owner explicitly authorized investigating presentation behavior in this +checkout and documenting the cross-scope findings while adding the SSH showcase. +The workstream selection remains github-auth. No main synchronization is needed: +this follows the same fetched main and the same validated recording. + +The live console discovers `presentation.json` beneath its mounted project and +serves named media through `/api/project/raw`. It does not generate metadata. +Scripted replay generates the document when given a script and numbered actions; +interactive serve/ctl does not. The earlier recording was interactive. The +example URL under website/_site names website output relative to the project; +there is no single required presentation assets directory. Outside worktrees and +symbolic links are not discovery paths. These distinctions and source locations +are now explicit in PRESENTATIONS.md and the scenario-recording guide. + +Added a six-step Markdown script and a producer using the shared presentation +module. The chapter boundaries come from the recorded controller responses and +are bound to the existing movie checksum. The producer writes presentation.json +beside run `20261116T103152Z-af7fa02c`; no recording was rerun or altered. It is +visible immediately on the current console's Presentations page, with route: +`/presentations/devcapsule-src/tests/e2e/evidence/github-ssh-proxy/20261116T103152Z-af7fa02c#overview`. +This is local evidence served from the project mount, not a website publication. + +Validation: authenticated with the supplied session token and then a cookie, +confirmed discovery, inspected the overview, and played all six steps in real +Chromium. Each screenshot loaded, its caption appeared, and playback stopped at +the selected boundary. Byte-range media requests returned 206 with matching bytes; +no browser errors occurred. The 39 existing presentation-producer tests and the +docs-contract check passed. Browser screenshots and structured results are under +the run's `presentation-check/`. No runtime source or PEX changed; the previously +recorded full-gate limitations still apply. + +Sent the broader usability finding to capsule-webconsole as +`2026-11-16-interactive-presentations.md`. Automatic presentations for interactive +runs remain a product/design follow-up, not an implemented feature of this slice. +The tracked script, timing input and producer make this retained deck reproducible; +media and generated metadata remain ignored evidence. Tokens and cookies are not +in Git, the presentation document, or browser screenshots. + +## Cross-project presentation portability audit, 2026-11-16 + +The owner asked whether agents in unrelated projects receive the same Markdown +instructions and Python support with little setup. Inspected the actual merged +PEX and its nested console bundle, package inclusion, CLI help, bootstrap assets +and browser-automation component. The viewer's HTML/JS/CSS are shipped and accept +presentation folders under any project's mount. The authoring guide, contract, +scenario scripts, recorder and metadata producer are not in that PEX. None of the +16 packaged bootstrap documents/templates refers to presentation.json or the +recorder. Browser automation supplies the browser tools, not the authoring kit. +The CLI has no presentation authoring command. This is a packaging/discoverability +gap; successful playback in this source checkout did not establish adopter setup. + +PRESENTATIONS.md now carries a support matrix. The scenario guide includes a +realistic agent brief and explicitly requires provisioned source tooling today. +The generator itself uses Python's standard library; the recorder also depends +on the repository's IDE smoke infrastructure and creates disposable test projects. +The retained SSH producer is run-specific, not a portable recording framework. +No new commands, workflow refresh, packaged assets or runtime code were introduced. + +Sent the cross-workstream delivery gap to project-management as +`2026-11-16-presentation-authoring-portability.md`, with suggested acceptance: +using only an installed candidate and an unrelated project, an agent discovers +the guide/templates, records the project's journey, generates validated metadata, +and exposes a presentation with working step playback without cloning DevCapsule. +Ownership and packaging/interface choices remain for that workstream and the owner. +This audit does not claim future capsules already have that authoring support. +Structured artifact observations are in evidence/portability-audit.json. + +## Consolidated handoff and branch delivery, 2026-11-16 + +The owner reports that an agent in a separate project running DevCapsule 0.3.0 +successfully reused this repository's recorder through an ad-hoc clone. Adding +presentations and the remaining setup was too much work. This is owner-reported +reuse evidence, not an independently repeated test or proof of shipped 0.3.0 +presentation support. + +At the owner's request, sent the complete findings and this new evidence to +project-management as `2026-11-16-github-auth-presentation-authoring-handoff.md` +on coordination at `b61ff973196a`. The self-contained item covers source locations, +discovery, scripted versus interactive generation, reusable Python, installed-PEX +omissions, the tested SSH presentation, durable documentation and proposed +installation-only acceptance in an unrelated project. It supplements the earlier +messages without selecting a toolkit interface, release or implementation owner. + +Refreshed the implementation PR draft around the final merged build, latest movie, +presentation and portability findings. All authored changes are committed and +pushed on the implementation branch; raw media remains intentionally ignored. +Canonical main was fetched and is still fully included, so no synchronization +was needed. No source or runtime behavior changed in this handoff. The existing +full-gate limitations remain; no new E2E or full gate was run for this record update. +The prescribed GitHub CLI remains absent. Under WORKFLOW-LOCAL.md's GitHub rules +4–5, the owner must edit the remote PR after resolving the gate prerequisites. +A push updates any existing PR's commits, not its description or attachments; +remote PR existence remains unverified. + +## Open Threads + +- Host relaunch, recursive inheritance and recorded fresh-project E2E passed. + Full-gate environment and owner-operated GitHub PR delivery remain open. +- Engineering choices: initial transport must account for Linux/rootless Docker + permissions; Docker Desktop cannot bind ordinary host Unix sockets. +- Physical-host relaunch is observed working; Docker Desktop transport remains deferred. +- PR description draft now includes the successful recorded E2E and current + validation limits. No PR was created or edited by this agent; remote PR existence + remains unverified. The recording is retained locally, not attached to GitHub. +- GitHub CLI is absent at the policy-prescribed path. PR creation/merging remains + owner-operated under WORKFLOW-LOCAL.md; prepare reviewed commits and compare links. +- Reviewer worktree removed at pause; pushed `ws-github-auth/ssh-proxy-review` + preserves its commit. Review corrections are not yet merged into implementation. +- Unimplemented review proposals and exact baseline failure IDs are in the assessment. +- Deliberately not preserved: scratch logs/venvs, credentials or console decision tokens in Git. + +## Workstream document index + +- [Recorded E2E procedure and console presentation](evidence/showcase.md) +- [Presentation producer for the retained run](evidence/present-showcase.py) +- [Recorded chapter boundaries](evidence/presentation-timing.json) +- [Installed-artifact portability audit](evidence/portability-audit.json) +- [SSH proxy scenario script](../../../devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md) +- [Review brief](evidence/review-brief.md) +- [Review follow-up 1](evidence/round-1.md) +- [Intake](intake/README.md) +- [Intake decisions](intake-dispositions.md) + +- [Review assessment](evidence/review-assessment.md) +- [Prepared PR delivery](evidence/delivery.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md new file mode 100644 index 00000000..51a4c275 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md @@ -0,0 +1,112 @@ +# Prepared delivery + +No PR has been opened or edited by this agent; remote PR existence has not been +verified. The implementation branch includes merge `7fc77f24`, the merged-build recording, +the six-step presentation, and the cross-project authoring audit. A pre-existing PR from that branch receives the pushed +commits automatically, but its description and attachments have not been updated. WORKFLOW-LOCAL.md, GitHub Integration rules 4–6, +requires a passing unit gate before PR opening/updating, and assigns GitHub +operations to the human when its prescribed gh installation is absent. The +current environment has both blockers. Preserve both branches until the gate +is resolved in a suitable environment and the owner performs PR delivery. + +## Implementation PR draft + +Base: `main`. Head: `ws-github-auth/ssh-proxy`. + +Title: Add opt-in host GitHub SSH authentication for capsules + +Body: + +Git operations currently require installing SSH credentials inside each capsule. +This adds an opt-in `github-ssh` authorization: the outside launcher owns a Unix +socket broker that runs host OpenSSH for validated GitHub upload/receive requests, +while capsule Git uses the shipped PEX adapter. Private keys remain on the host. +Linux/WSL with native Docker is supported first; Docker Desktop is refused. +Host trust and unlocked encrypted keys are prerequisites. The capability grants +Git access to all repositories available to that host identity. + +Validation: the physical-host relaunch and recorded recursive E2E passed on +rootless Linux Docker. A fresh VSCodium capsule, with no SSH files or agent, +cloned `git@github.com:mycodespace-ai/devcapsule.git` through the inherited host +broker. Independent checks confirmed the expected commit, SSH origin, clean tree, +read-only socket mount from the same host source, and exact runtime PEX checksum. +A missing-socket control failed with exit 128. The container, workspace and +checkout records were removed. No real GitHub push was performed. + +The full release-0.3.1 tip `10e7fda3` is merged by `7fc77f24`, preserving +both histories. The recorded acceptance uses the merged PEX for both parent and +child after a physical-host relaunch. Tested PEX SHA-256: +`af7fa02ca2c2737f3a1678226341312682b71cfdd37eb892f29a42ad5c31b78c`. + +The [recorded E2E procedure](engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md) +identifies the retained 45.42-second movie at +`devcapsule-src/tests/e2e/evidence/github-ssh-proxy/20261116T103152Z-af7fa02c/scenario-video.webm`. +The live console serves a six-step presentation from that folder. Its Markdown +script, recorded chapter boundaries and metadata producer are versioned. Chromium +verified discovery, screenshots, captions, all six playback boundaries and media +byte ranges. Media and generated metadata remain local, outside build output; +they have not been uploaded or attached to GitHub. + +Documentation now explains presentation discovery and generation. A shipped-PEX +audit confirms that the viewer is included, but authoring instructions and Python +helpers remain repository tooling. Interactive recording does not generate the +metadata automatically. The owner reports successfully borrowing the recorder +for another project on 0.3.0 by cloning source ad hoc; that is evidence of reuse, +not shipped authoring support. A consolidated handoff to project-management +contains the gaps, reusable pieces and proposed installation-only acceptance. +No portable authoring toolkit is claimed or implemented by this change. + +Latest validation: 41 focused cases passed; types passed over 350 files and +conventions reported zero breaches. Unit gate: 3309 passed, 3 failed, 1 skipped, +187 deselected, 1 xfailed, 1 xpassed. CLI smoke, PEX packaging/smoke, docs contract, +console types and 474 console tests passed. Integration: 149 passed, 12 skipped, +3 failed initially; the socket-length case passed when rerun with shorter scratch, +leaving two failures. All five remaining unit/integration failures reproduce on +untouched release-0.3.1; maintenance received the reproduction details. This is +not a passing full gate. The subsequent presentation-only change passed all 39 +existing producer tests and the documentation contract. See the workstream status +for exact failure IDs and scope. + +Pair: GPT-6 Codex with Antigravity Gemini 4 Hydrogen High +(`gemini-4-hydrogen-high`). Local round 1 accepted at `388f62a3`; formal review PR +must be merged into this branch before mainline delivery. + +## Review PR draft + +Base: `ws-github-auth/ssh-proxy`. Head: `ws-github-auth/ssh-proxy-review`. + +Title: Review GitHub SSH proxy diagnostics and lifecycle coverage + +Body: + +Gemini Hydrogen review separates capsule SSH failures from broker failures, +clarifies disclosure and dry-run lifetime, removes an unused frame helper, +declares the public review API and strengthens parser, authorization and process +cleanup coverage. Round 1 addresses test environment independence and cleanup. +The implementor accepted `388f62a3` locally after a conflict-free temporary merge +passed 75 focused cases. The reviewer branch alone lacks the parent's later node +registry refurbishment and console-table fix; validate the merged candidate. +Full gate and owner-operated PR delivery remain outstanding. + +## Resume sequence + +1. Fetch canonical upstream and recheck workflow synchronization and live state. +2. Resolve the known full-gate environment failures without disabling tests; + run the required gate on the combined candidate before PR opening/updating. +3. The owner creates the implementation and review PRs, then merges the review + into the implementation branch. Record the formal assessment at its SHA. +4. Fetch that merged branch and run required validation. The physical-host clone + acceptance above used the implementation without review corrections; repeat + relevant acceptance against the merged executable. A real GitHub write test + remains separate and requires an explicitly selected disposable repository. +5. The owner merges the implementation PR into main; fetch and verify main's tree + before recording integration. No release or installation has been performed. + + +## Owner delivery links + +- [Implementation comparison](https://github.com/mycodespace-ai/devcapsule/compare/main...ws-github-auth/ssh-proxy?expand=1) +- [Review comparison](https://github.com/mycodespace-ai/devcapsule/compare/ws-github-auth/ssh-proxy...ws-github-auth/ssh-proxy-review?expand=1) + +Use the body above when the gate and GitHub-operation prerequisites are resolved. +The branch push does not upload the local recording or change a PR description. diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/portability-audit.json b/engineering-docs/wip/2026-11-16-github-auth/evidence/portability-audit.json new file mode 100644 index 00000000..d1b0c186 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/portability-audit.json @@ -0,0 +1,15 @@ +{ + "date": "2026-11-16", + "pex_sha256": "af7fa02ca2c2737f3a1678226341312682b71cfdd37eb892f29a42ad5c31b78c", + "source_merge": "7fc77f24df45135770160c35f193c9abb78cd9c0", + "viewer_assets": [ + "devcapsule_webconsole/static/presentation.css", + "devcapsule_webconsole/static/presentation.html", + "devcapsule_webconsole/static/presentation.js" + ], + "authoring_resources_in_archive": [], + "e2e_files_in_archive": 0, + "bootstrap_document_count": 16, + "bootstrap_authoring_references": [], + "method": "Read the delivered PEX and its nested console bundle; inspect package inclusion, bootstrap assets, CLI help and browser-automation component source. No new environment or container was created." +} diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/present-showcase.py b/engineering-docs/wip/2026-11-16-github-auth/evidence/present-showcase.py new file mode 100644 index 00000000..15e6943c --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/present-showcase.py @@ -0,0 +1,54 @@ +"""Present the retained SSH proxy run with the shared presentation producer. + +Run from devcapsule-src with PYTHONPATH=. and pass the recording directory. +The recorded movie is not edited. Its hash binds the chapter timings to its bytes. +This is a producer for one retained interactive run, not a new E2E replay runner. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +from tests.e2e.scenario_presentation import parse_script, plan_presentation, write_presentation + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("evidence", type=Path) + args = parser.parse_args() + own = Path(__file__).resolve().parent + root = own.parents[3] + timing = json.loads((own / "presentation-timing.json").read_text()) + evidence = args.evidence.resolve() + movie = evidence / "scenario-video.webm" + if hashlib.sha256(movie.read_bytes()).hexdigest() != timing["video_sha256"]: + parser.error("The recording does not match these chapter timings; keep each run's timing with its movie.") + result = json.loads((evidence / "result.json").read_text()) + if (result["status"], result["frames"], result["fps"]) != ("passed", timing["frames"], timing["fps"]): + parser.error("The retained run must have passed with these frame counts and playback rate.") + script = parse_script((root / "devcapsule-src/tests/e2e/scripts/github-ssh-proxy.md").read_text()) + chapters = timing["chapters"] + for name in ["poster.png", *(chapter["screenshot"] for chapter in chapters)]: + if not (evidence / name).is_file(): + parser.error(f"Missing recorded image: {name}") + plan = plan_presentation({ + "id": "github-ssh-proxy", + "title": "GitHub SSH: host identity, no copied keys", + "description": "Clone from GitHub inside a fresh capsule using the outside launcher's SSH proxy. Check the credential boundary, see access fail without the socket, and restore normal access.", + "actions": [ + {"reason": f"{index}. {chapter['caption']}", "caption": chapter["caption"]} + for index, chapter in enumerate(chapters, start=1) + ], + }, script) + document = plan.timed( + [chapter["frames_after"] for chapter in chapters], timing["frames"], timing["fps"], + video="scenario-video.webm", poster="poster.png", + screenshots=[chapter["screenshot"] for chapter in chapters], + ) + print(write_presentation(evidence, document)) + + +if __name__ == "__main__": + main() diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/presentation-timing.json b/engineering-docs/wip/2026-11-16-github-auth/evidence/presentation-timing.json new file mode 100644 index 00000000..bf147768 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/presentation-timing.json @@ -0,0 +1,15 @@ +{ + "run": "20261116T103152Z-af7fa02c", + "video_sha256": "57457f07f8b40fcbfa3e3d2dff4a99584829f616b68b6c6e4552587dec6ba5ee", + "frames": 545, + "fps": 12, + "provenance": "Cumulative frames_recorded returned by the interactive controller after each selected action batch; final total from stop-result.json. These are chapter boundaries for this recording, not per-action replay timings.", + "chapters": [ + {"frames_after": 103, "screenshot": "step-003.png", "caption": "Open a terminal in the fresh capsule. The outside launcher already has explicit GitHub SSH authorization."}, + {"frames_after": 184, "screenshot": "step-004.png", "caption": "Zero SSH files. No SSH agent. The capsule connects to the host-owned proxy socket."}, + {"frames_after": 285, "screenshot": "step-006.png", "caption": "A real GitHub SSH clone succeeds. The private key stays on the host."}, + {"frames_after": 331, "screenshot": "step-007.png", "caption": "Point just this command at a missing proxy socket: the GitHub read fails."}, + {"frames_after": 395, "screenshot": "step-008.png", "caption": "The failed read exited 128. Using the normal proxy again returns GitHub HEAD successfully."}, + {"frames_after": 533, "screenshot": "step-009.png", "caption": "The recorded commands and independent checks agree: host identity, no copied key, working Git access."} + ] +} diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md new file mode 100644 index 00000000..ecf6b43e --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md @@ -0,0 +1,100 @@ +# GitHub SSH proxy review assessment + +Date: 2026-11-16. Implementor: GPT-6 Codex. Reviewer: Antigravity, +`gemini-4-hydrogen-high` (Gemini 4 Hydrogen High). + +## Disposition + +Round 1 accepted locally at `388f62a34fe373882201f9118faaf34cec61680b`. +The reviewer branch is pushed as `ws-github-auth/ssh-proxy-review`. +No review PR or implementation PR exists yet. This is local review preparation, +not completion of the PR-mediated Pair Review Loop or mainline integration. + +The implementor inspected the diff and accepted the unused helper removal, +separate capsule-SSH error attribution, public API declaration, launcher dry-run +warning and disclosure layout, deny-help update, and expanded hostile-input, +permission, cancellation, process-group and authorization coverage. +Round 1 replaced an ambient-current-directory PATH check, bounded the readiness +handshake, guaranteed client cleanup, and distinguished dead orphan zombies from +running descendants. The process tests depend on Linux /proc and scheduling +within their explicit timeouts; they do not prove prompt reaping by PID 1. + +The reviewer ran 55 focused cases successfully. A temporary, uncommitted merge of +`388f62a3` with implementation `890fa91d` was conflict-free and passed **75 focused +tests**, mypy on the nine touched modules/test files, and the conventions checker +(103 legacy files, 1380 legacy tests, 50 conforming files, zero breaches). +`git diff --check` and `git fsck --connectivity-only --no-dangling` passed. +The temporary merge was aborted deliberately: reviewer corrections remain on +their separate pushed branch for the owner-operated review PR. + +## Build evidence and limits + +The implementor's full build reached pytest: 20 failed, 2917 passed, 1 skipped, +76 deselected, 14 xfailed, 7 xpassed. The reviewer's independent build found the +same failure set: 20 failed, 2926 passed, 1 skipped, 80 deselected, 14 xfailed, +7 xpassed. The new node-name expectation was repaired on the implementation +branch by refurbishing the complete legacy file into the unit mirror (20 passing +cases); the reviewer was told not to duplicate this change. The other 19 failures +reproduced by selecting all 20 failed IDs on an archive of unchanged canonical +main `1942f396`: 19 failed, 1 passed. They concern inherited capsule trust, +permissions/ownership and recursive-launch environment; they remain unresolved. +A full gate on the combined final tree has not passed. + +CLI smoke, PEX build/smoke and documentation contract passed separately. After +aligning the console authorization mirror, all 458 webconsole tests and its types +passed. Both source and built PEX adapters completed authenticated GitHub +`ls-remote` through the broker, returning mainline HEAD `1942f396`. The broker in +that probe used the current capsule's existing SSH setup. This is **not** evidence +of physical-host Docker launch acceptance. No real GitHub write was tested; local +fixture repositories exercised real Git clone/fetch/push. No containers were +launched and no key files were read by the agent, copied or changed. + +## Remaining proposals + +- Docker Desktop needs a separate transport design; this slice explicitly refuses + hosts unable to bind Unix sockets. Linux/WSL with native Docker is first. +- GitHub aliases, HTTPS/gh, Git LFS authentication, signing, and arbitrary SSH flags + (including Git's -4/-6 switches) are outside this adapter's current contract. +- Additional reference cross-links and a tighter operation-deadline cap inside + the bounded send loop were considered; the existing send timeout is 10 seconds + and cancellation checks remain active. These are recorded follow-ups, not + claims of implemented behavior. +- The inherited canonical socket positive path is not covered by a pure unit + test; physical-host and recursive-launch acceptance remain required. + +## Reviewer execution + +Conversation `6280de13-6b45-4215-8e97-af6b466fbd61` used streaming JSON output. +The first redirected-stdin invocation failed to convey the brief; continuation +with an explicit brief path proceeded. The reviewer's CLI stalled after its full +gate had already returned. The implementor interrupted and continued that same +conversation with the committed round brief and a bounded print timeout. The +final response reported REVIEW_COMPLETE, the final shell marker was +ANTIGRAVITY_EXIT=0, and the commit and remote SHA were independently verified. +The aggregate JSON result retained an ERROR/idle-timeout from the earlier turn; +we therefore rely on observed commits/checks, not that aggregate status alone. +Scratch logs and virtual environments are not durable evidence. Briefs, the +assessment and validation counts are retained here. The reviewer worktree is +removed at pause; its pushed branch preserves all changes. + +## Baseline failure IDs + +- `tests/configuration/test_contract.py::test_second_checkout_cannot_inherit_the_first_checkouts_authority` +- `tests/configuration/test_contract.py::test_config_list_is_data_only_and_show_carries_the_review` +- `tests/test_capability_policy.py::test_cli_check_and_capability_mutation` +- `tests/test_host_daemon.py::test_unmountable_staging_fails_loudly` +- `tests/test_owner_commands.py::test_tool_edit_does_not_change_checkout_answers_when_not_ready[True]` +- `tests/test_recursive_orchestrator.py::test_public_dry_run_is_unique_redacted_and_cleans_owned_staging` +- `tests/test_recursive_orchestrator.py::test_contained_display_dry_run_forwards_no_host_x11` +- `tests/test_run_flow.py::test_a_second_checkout_is_registered_under_its_directory_name` +- `tests/test_run_flow.py::test_interactive_questions_record_answers_where_the_config_family_would` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[False-False]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[False-True]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[True-False]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[True-True]` +- `tests/test_run_flow.py::test_partial_terminal_review_saves_no_answers[1\n]` +- `tests/test_run_flow.py::test_partial_terminal_review_saves_no_answers[1\nn\n]` +- `tests/test_runtime_entrypoint.py::test_entrypoint_rejects_unknown_adapters` +- `tests/test_trust.py::test_a_real_run_launches_on_the_homes_trust_alone` +- `tests/test_trust.py::test_trust_write_is_private_atomic_and_keeps_previous_record_on_failure` +- `tests/test_trust.py::test_terminal_acceptance_stays_staged_if_a_later_answer_is_missing` diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md new file mode 100644 index 00000000..a5d2f5d8 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md @@ -0,0 +1,87 @@ +# Review brief: GitHub SSH proxy, Gemini 4 Hydrogen High + +Implementor: Codex, GPT-6. Review partner: Antigravity, gemini-4-hydrogen-high. +The owner requested this feature with Pair Review and selected the github-auth +workstream and Git over SSH first. Review correctness, clarity and meaningful +coverage of the implementation against upstream/main. + +## Checkout and first reads + +Work only in /home/devcapsule/worktrees/github-auth-review on +ws-github-auth/ssh-proxy-review. Do not edit the implementor checkout. +Read AGENTS.md, WORKFLOW.md topic 9.11, WORKFLOW-LOCAL.md, this brief, +engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md, then +`git diff upstream/main...HEAD`. Read the contracts in github_transport.py and +github_proxy.py before judging implementations. This brief is your only context. + +## Scope and questions + +Ordinary GitHub clone/fetch/push from a Linux/WSL capsule uses host-side OpenSSH +through a private launcher-owned Unix socket. No private keys or host agent socket +enter the capsule. Only fixed GitHub upload-pack/receive-pack requests are allowed. +The authorization is optional and denied by default; grant through host config or +run-once. All repositories accessible by the host identity are within the grant. +Host key checking remains strict; keys must authenticate noninteractively on the +host. Other Git hosts fall back to the capsule SSH. Mac Docker Desktop, HTTPS, gh, +Git LFS auth, SSH signing, arbitrary aliases are outside the first slice. + +Review in order: +1. Correctness: injection/destination escape, request validation independent of + the client, socket permission boundary, lifecycle cleanup, cancellation, + bounded memory/concurrency, deadlocks/half-close/exit status, Docker environment + integration and denial. Never weaken host-key checking to make a test pass. +2. Clarity: remove misleading guarantees or unnecessary machinery. +3. Coverage: new tests must establish independent contracts, especially auth + gating, real binary streaming, failure/exit handling and process cleanup. +4. Follow Unit Tests rules 1-4 in WORKFLOW-LOCAL.md. New unit tests live in the + module mirror, state their contract in a docstring, and construct inputs. + For each, identify possible failures not caused by a product bug. Do not edit + tests in legacy files. Add integration tests for subprocess/network effects. +5. Prefer explicit host inputs and pure decisions. No monkeypatch/unittest.mock + in new tests. Code runs no invented cryptography; authentication is OpenSSH. + +Keep corrections within this feature. Do not repair unrelated legacy tests or +expand to macOS, HTTPS, repository allowlists or generic SSH forwarding. + +## Validation + +Create your own environment exactly as the local workflow says, from devcapsule-src: +python3.12 -m venv .venv +.venv/bin/python -m pip install -r dev-requirements.txt +.venv/bin/python -m pip install -e . --no-deps + +Use scratch under /home/devcapsule/.cache/devcapsule-gate/github-auth-review; +/opt is read-only. The parent cache directory exists. +Initialize website with `git submodule update --init website` for the docs gate. +Run focused tests: +.venv/bin/python -m pytest -q --no-cov -o addopts='' tests/unit/devcapsule/test_github_transport.py tests/unit/devcapsule/configuration/test_authorization.py tests/unit/devcapsule/configuration/test_review.py tests/integration/test_github_proxy.py + +Run full gate synchronously, not a background tool job: +PYTEST_ADDOPTS='--basetemp=/home/devcapsule/.cache/devcapsule-gate/github-auth-review' DEVCAPSULE_DIFF_BASE=upstream/main .venv/bin/python -m nox -s build + +The implementor's focused run passed 42 tests. Its full gate may expose existing +mainline legacy test failures in this capsule. Report exact failures; do not fix +unrelated code or disable tests. Do not build images or launch Docker capsules. + +## Delivery and forbidden actions + +The GitHub CLI is ABSENT at /opt/xtras/miniconda3/envs/gh/bin/gh. Local policy +requires the human to create PRs in this case. Do not use another credential or +GitHub interface. There is not yet a PRn; PR-mediated Pair Review completion is +pending owner delivery. Perform the local code review now, preserve your findings +and corrections, and report this delivery blocker accurately. Do not claim a +completed PR review loop. Push your review branch over ordinary upstream SSH +only after checks; no push to main. Provide a compare URL if changes exist. + +Do not edit status files, intake/dispositions, evidence, root CURRENT-STATUS.md, +index.md, or run workflow publish. Do not change website gitlinks or configuration +manifests. Do not merge, rebase, force-push, create releases, inspect private keys, +read credential contents, use real GitHub repositories in tests, or run containers. +Do not modify the implementor's files or state. Do not add broad test-only seams. + +Commit corrections with the existing human author and this trailer: +Co-authored-by: Gemini 4 Hydrogen High Antigravity + +Report findings (what/where/fix/evidence), things considered and left alone, +unimplemented proposals, exact validation counts and limits. If no changes, say +so. Print REVIEW_COMPLETE as your last line, or REVIEW_BLOCKED with the reason. diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md new file mode 100644 index 00000000..c2dc01cf --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md @@ -0,0 +1,42 @@ +# Review follow-up 1: finish the bounded local review + +Continue the same Gemini Hydrogen review in /home/devcapsule/worktrees/github-auth-review. +Your full build has FINISHED: step 130 returned 20 failed, 2926 passed, 1 skipped, +14 xfailed and 7 xpassed. No pytest or nox process remains. Your CLI stayed idle +after that completion, so the implementor interrupted it and resumed this turn. +Do not wait for the completed job. Do not repeat the full gate: its failures are known. + +The implementor read your diff and accepts its production-code changes and added +coverage in principle. Fix these test issues before committing: +1. test_host_bridge_requires_openssh_on_path passes PATH="". shutil.which searches + the current directory in that case; the test depends on no file named ssh being + present there. Use tmp_path as an explicitly empty PATH directory instead. +2. test_broker_shutdown_cancels_active_ssh_process_group calls stdout.readline with + no timeout. Bound the ready handshake and ensure client cleanup in finally even + if readiness fails. Avoid a test that hangs forever when the product is broken. +3. An orphaned grandchild may be a zombie until the host's reaper runs. The broker + cannot waitpid a grandchild. Verify that no descendant is running, without + asserting timely reaping by an unrelated PID 1. State the test's actual claim. + +Run focused tests INCLUDING your new unit test_github_proxy.py and integration +file and mypy. If new focused tests fail, fix those failures within this scope. +Commit your corrections. Do not merge, reset, rebase or touch the parent checkout. + +Parent updates already committed separately (do not duplicate or cherry-pick): +- f1dbb338 refurbishes tests/configuration/test_nodes.py into its correct mirror, + 20 passing cases; this resolves the new github-ssh name expectation. +- 4ba1725f aligns webconsole/configuration.py SECURE_AUTHORIZATIONS and its contract; + all 458 console tests now pass. +- The remaining 19 failures are reproduced on unchanged upstream/main, by selecting + all 20 failing test IDs in an archive of the base (19 fail, the registry test passes). +- Parent CLI smoke, PEX build/smoke and docs contract pass. Both source and PEX Git + adapters passed read-only authenticated GitHub ls-remote using the capsule's existing + credential on the broker side; not a physical-host launch test. + +Your branch can be pushed to upstream as review preparation, with limitations in +its report. No PR exists because local policy makes creation owner-operated without +its prescribed gh installation. Do not use another GitHub interface or credential. + +Return the findings, corrections, focused results, known full-gate failures and +unimplemented proposals. End with REVIEW_COMPLETE or REVIEW_BLOCKED. Do not begin +another broad investigation; the remaining task is these test corrections and delivery. diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md new file mode 100644 index 00000000..534b5e9e --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md @@ -0,0 +1,202 @@ +# GitHub SSH proxy: recorded fresh-capsule clone + +## Purpose and acceptance + +Show a user cloning a GitHub repository over SSH inside a fresh capsule. The +outside launcher uses the host identity. No SSH key or agent is supplied to the +child. Public GitHub repositories also require authentication on their SSH URL. +This scenario reads the canonical DevCapsule repository; it does not push. + +Pass requires a real clone, the expected independently read commit, the unchanged +SSH origin, a clean working tree, no child SSH files or agent, and the canonical +broker socket mounted from the same host source as the parent. A second Git read +with an absent broker socket must fail. Keep the recording and independent +observations, and remove the fresh container, workspace and checkout state. +A video alone does not pass the test. + +## Preconditions + +Keep the outside launcher alive with `github-ssh` enabled. The host must already +trust GitHub and authenticate noninteractively. An unencrypted default SSH key +needs no host agent. The parent needs host Docker and host networking. + +From `devcapsule-src`, use the development virtualenv and the exact PEX under test. +The historical recorded run uses SHA-256 +`33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b`, built from +`d6fd93e0`. The merged artifact now has different bytes; set the expected hash +to the actual build being recorded for a new run. Do not rebuild between the launch and the evidence capture. + +```sh +.venv/bin/python -m pip install 'playwright==1.63.0' +export DEVCAPSULE_EXPECTED_SHA256=33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b +``` + +## Start the recording controller + +Save the following as a temporary Python script and run it with +`PYTHONPATH=. .venv/bin/python