From 8692a55d6d57064741fe4567a9f823a7d7ba4cc2 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Sat, 7 Nov 2026 12:32:32 +0000 Subject: [PATCH 01/35] Reopen main at 0.3.2.dev0 after the 0.3.1 cut release-0.3.1 is cut from main at 31e8fec7 by the owner's decision of 2026-11-07, and the Version Scheme's rule 4 reopens main with the next development version at the cut, so main is ahead of every candidate. Set by nox -s bump -- 0.3.2.dev0. Co-Authored-By: Claude Fable 5.1 --- WORKFLOW.md | 2 +- .../devcapsule/assets/project_workflow/definition/WORKFLOW.md | 2 +- devcapsule-src/pyproject.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/WORKFLOW.md b/WORKFLOW.md index d2ccbbe4..c981b444 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1.dev0 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md index 4ffa4718..380590a4 100644 --- a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md +++ b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1.dev0 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 40d94024..11fffcd8 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "devcapsule" -version = "0.3.1.dev0" +version = "0.3.2.dev0" description = "Profile-driven Docker launch tooling for isolated IDE development environments." readme = "README.md" requires-python = ">=3.12" From 8bc325dd7dd50265382ea435c04b23106fa34b8c Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Sat, 7 Nov 2026 20:15:26 +0000 Subject: [PATCH 02/35] Throwaway copy: main's version files kept through the pre-candidate merge The release branch sets 0.3.1 in pyproject and adds the source-version line to docs/versions.yaml, so a clean merge would carry the release version onto main. This one commit on the copy sets the files back to what main holds once PR #8 is merged: 0.3.2.dev0 in pyproject and both WORKFLOW.md frontmatters, and docs/versions.yaml without the release-branch lines. The copy is deleted on merge; release-0.3.1 is untouched. Co-Authored-By: Claude Fable 5.1 --- WORKFLOW.md | 2 +- .../assets/project_workflow/definition/WORKFLOW.md | 2 +- devcapsule-src/pyproject.toml | 2 +- docs/versions.yaml | 5 ----- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/WORKFLOW.md b/WORKFLOW.md index d9a81036..c981b444 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md index 6b245ce6..380590a4 100644 --- a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md +++ b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 4812e0c5..11fffcd8 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "devcapsule" -version = "0.3.1" +version = "0.3.2.dev0" description = "Profile-driven Docker launch tooling for isolated IDE development environments." readme = "README.md" requires-python = ">=3.12" diff --git a/docs/versions.yaml b/docs/versions.yaml index a631589c..60b71663 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -11,11 +11,6 @@ notice: > versions: - version: devel source: main - # On release-0.3.1 only: the branch's source declares the release - # version, and the website's check reads this checkout as main. The - # line satisfies contract 1, section 4.2, for the local gate; it is not - # carried to main, whose source declares a development version. - source-version: "0.3.1" status: development - version: 0.3.0 source: docs-0.3.0 From 7413412a710b7909d12f987fcb1c30c1772dd9be Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Sun, 8 Nov 2026 00:27:48 +0000 Subject: [PATCH 03/35] Engineering note: testing the macOS launcher on cloud Mac hardware, and the alternatives Why real Apple hardware is needed and why Google Cloud and hosted macOS runners cannot supply it; EC2 Mac as the first run, with prices, the 24-hour rule, the GUI procedure and the Docker Desktop evidence found; Scaleway as the cheaper second machine; building the three executables on hosted runners; a one-shot ephemeral self-hosted runner on a collaborator's Mac and the rules for it on a public repository; and the smoke to record for 0.3.1, including the Eclipse acquisitions that close the libsoup bug. Indexed. Co-Authored-By: Claude Fable 5.1 --- ...he-macos-launcher-on-cloud-mac-hardware.md | 192 ++++++++++++++++++ index.md | 1 + 2 files changed, 193 insertions(+) create mode 100644 engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md diff --git a/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md b/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md new file mode 100644 index 00000000..78c133ea --- /dev/null +++ b/engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md @@ -0,0 +1,192 @@ +# Testing The macOS Launcher On Cloud Mac Hardware, And The Alternatives + +Written 2026-11-08 by Claude Code (Claude Fable 5.1) for `maintenance`, at +the owner's request, from a web survey of the same day. Prices and version +numbers are what the sources said that day; check them before allocating. + +## Why real hardware + +The macOS launcher of PR #338 runs on Apple silicon against Docker Desktop, +built from source as [the macOS guide](../../../docs/platforms/macos.md) +describes. Validating it means running Docker Desktop and capsules on a Mac, +not only building. Apple's license allows macOS only on Apple-branded +machines, so every cloud route is a physical Mac mini rented whole. Two +things follow: + +- **Google Cloud has no offer.** It owns no Apple hardware. The only Google + Cloud route found is a macOS guest nested inside an Intel machine type, + which breaks Apple's license and is Intel, so it validates nothing about + Apple silicon. +- **GitHub's hosted macOS runners cannot run Docker.** They are virtual + machines on Apple silicon, and Apple's Virtualization framework gives a + guest no nested virtualization, so neither Docker Desktop nor Colima can + start its Linux VM there (`actions/runner-images#9254`). They can build + the macOS launcher; they cannot run a capsule. + +## Option A: AWS EC2 Mac, the recommended first run + +EC2 Mac instances are bare-metal Mac minis on Dedicated Hosts without the +Nitro hypervisor, so Docker Desktop's Linux VM runs at the first level of +virtualization. AWS and Docker publish no statement about Docker on them; a +public runbook of 2026-10-29 on a `mac2.metal` running macOS 26.7 ran +Docker Desktop 4.97, Colima 0.10.3 (`vz`, `virtiofs`) and OrbStack through +a full check with no failures (`k-k1/agent-fleet#2961`). Treat our own first +run as the confirmation. + +| Host | Hardware | On-Demand, N. Virginia | First day | +|---|---|---|---| +| `mac-m4.metal` | M4, 10 cores, 24 GiB | about $1.23 per host hour | about $30 | +| `mac-m4pro.metal` | M4 Pro, 14 cores, 48 GiB | about $1.97 per host hour | about $47 | +| `mac2-m2.metal` | M2, 8 cores, 24 GiB | about $0.88 per host hour | about $21 | + +Rules that matter, from AWS's documentation: + +- The Dedicated Host is the unit of billing, On-Demand only, per second + after a **24-hour minimum allocation**, which AWS attributes to Apple's + license. The host bills while the instance is stopped. Release it after + 24 hours, after stopping or terminating the instance; a scrubbing run + follows. +- One instance per host. Launch takes 6 to 20 minutes. Boot is from an EBS + volume; AWS recommends 10,000 IOPS and 400 MiB/s. The internal SSD is + unsupported. Do not enable FileVault. +- M4 and M4 Pro hosts need macOS Sequoia 15.6 or later; Tahoe 26 images + are published. Regions vary; check the instance-types-by-region table. +- Apple Intelligence is unavailable when booting from an external volume, + which is irrelevant to us. + +### Procedure, one validation day + +1. Allocate the host and launch the latest Sequoia or Tahoe AMI with a + key pair, an EBS root of 200 GB or more, and a security group that + allows SSH from your address only. +2. Over SSH as `ec2-user`, set a password and enable Screen Sharing, then + open a tunnel and connect with the Screen Sharing app (Finder, Go, + Connect to Server, `vnc://localhost:5900`): + + ```bash + ssh -i key.pem ec2-user@INSTANCE + sudo passwd ec2-user + sudo launchctl enable system/com.apple.screensharing + sudo launchctl load -w /System/Library/LaunchDaemons/com.apple.screensharing.plist + exit + ssh -L 5900:localhost:5900 -i key.pem ec2-user@INSTANCE + ``` + + Amazon DCV also supports EC2 Mac since 2025-11-12, with audio and 60 + frames per second, if VNC proves too slow for IDE work. Sequoia asks to + approve Local Network Privacy on first GUI login; approve it. +3. In the desktop session, install Docker Desktop and raise its disk limit + under Settings, Resources. Docker Desktop's first start needs a + logged-in screen session, so do this over VNC, not over SSH. +4. Follow the macOS guide's *Build and install* from a clean checkout of + the release branch, or install the executables from option C below, + then run the smoke below. +5. Record the evidence in the release overview, stop and terminate the + instance, and release the host once 24 hours have passed. + +## Option B: Scaleway Apple silicon, the cheaper second machine + +Scaleway rents bare-metal Mac minis in Paris: M4-S (16 GB) at about €0.22 +per hour or €149 per month, M2 Pro and M2 in PARIS 1, M1 in PARIS 3. The +same 24-hour minimum applies, for the same license reason; billing runs +until the Mac mini is deleted, and deletion takes about 30 minutes. Access +is SSH and VNC; the console's *Open remote desktop* button carries the +VNC credentials, the port is random and shown on the Overview page, and a +non-default macOS version adds about an hour to installation. Nobody was +found who ran Docker Desktop there; the hardware is the same kind of bare +metal, so expect the same result as AWS and treat the first run as the +test. One day costs about €5.30. + +## Option C: build on hosted runners, run on the Mac + +Building and running are separable. The macOS launcher is one +`scripts/build-pex.sh` invocation with `DEVCAPSULE_SCIE_PLATFORM=macos-aarch64` +and needs no Docker, so a hosted `macos-15` runner (Apple silicon) can +build it. The two Linux runtimes need Linux Python: hosted `ubuntu-24.04` +and `ubuntu-24.04-arm` runners build them natively, without the emulated +`docker run` of the guide. A `workflow_dispatch` workflow taking a ref, +building the three executables and uploading them as artifacts would let +anyone with a Mac validate a branch by downloading, installing as the guide +shows, and running. This is the shape a published macOS executable would +take if the owner rules one in; until then it is a validation aid, not a +release channel, and its artifacts carry `--allow-unpublished-revision` +semantics: the commit must be public, since a base build installs the web +console from it. + +## Option D: a one-shot self-hosted runner on a friend's Mac + +The owner's idea: a collaborator registers their own M4 Mac as a GitHub +Actions runner for one job that pulls `release-0.3.1`, builds the macOS +launcher and the runtimes, and runs the smoke against their Docker Desktop. +It works, with three rules: + +- **Register the runner ephemeral** (`config.sh --ephemeral`), so it takes + exactly one job and unregisters itself; remove it from the repository + settings afterwards regardless. +- **The repository is public.** GitHub's own guidance is not to use + self-hosted runners on public repositories, because a workflow from a + fork can run on the machine. Restrict exposure: the job runs only on + `workflow_dispatch` from a protected branch, the repository setting + *Require approval for all outside collaborators* stays on, and the runner + is labelled uniquely so no other workflow can target it by accident. + Even so, the runner runs as the collaborator's user on their personal + machine, and anything the checked-out branch's scripts do, it does. The + collaborator should read the workflow and the branch before registering. +- **Docker Desktop must already be running** in the collaborator's logged-in + session, with the disk limit raised; the runner inherits that session's + Docker socket. + +The collaborator may have built the launcher already: PR #338's author ran +the macOS guide's build on their own Mac. If they instead ran from source +with the launcher's own interpreter, the validation still needs the built +executables, because the runtime delivered into the capsule must be a PEX +of the same commit (R-RUNTIME-001), and the smoke below checks exactly that. + +## The smoke, whichever machine + +Record each line with its output in the release overview under +*Candidates*, with the macOS and Docker Desktop versions. + +1. `devcapsule version --json` from an empty directory: version, build + mnemonic and source revision match the branch commit; the runtimes in + `DEVCAPSULE_RUNTIME_PEX` report the same version. +2. A fresh project: `devcapsule project init` with Enter for + `linux-arm64`, then `project run`; the arm64 base pulls, the contained + desktop opens in the browser, the IDE's main window appears. Time it. +3. The same project under emulation: delete the arm64 lock, `project run`; + the amd64 base and IDE image build under emulation (about ten minutes + for PyCharm on a 14-core Mac, per the guide); the IDE appears. +4. Eclipse on `linux-arm64` with an empty component cache, which also + closes the libsoup bug for arm64 if the acquisition succeeds, and + Eclipse under emulation with an empty cache, which is the amd64 + acquisition the 0.3.1 repin must prove. +5. `docker-daemon` authorized: from inside the capsule, `docker run` of a + small image, which the guide says is the arm64 variant; confirm with + `docker version`. +6. A `network host` project: the launch falls back to bridge with the + warning the guide describes; the web console reaches `127.0.0.1`. +7. Stop, `project run` again, the session resumes; `devcapsule cleanup` + leaves the Mac's Docker Desktop disk where it was before step 2, within + the base images kept on purpose. + +What a failure looks like is a bug record under `engineering-docs/bugs/` +with the Mac's hardware, macOS and Docker Desktop versions, and the +command's output, as the guide asks adopters to report. + +## Sources + +AWS: [EC2 Mac instances](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-mac-instances.html), +[FAQs](https://aws.amazon.com/ec2/instance-types/mac/faqs/), +[connect to a Mac instance](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-to-mac-instance.html), +[macOS AMI release notes](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/macos-ami-overview.html), +[Amazon DCV on EC2 Mac](https://aws.amazon.com/about-aws/whats-new/2025/11/amazon-dcv-ed2-mac-instances), +[Dedicated Host pricing](https://aws.amazon.com/ec2/dedicated-hosts/pricing/) and a +[third-party reading of the price feed](https://macyou.co/blog/aws-ec2-mac-pricing-explained) +of 2026-09-25. Scaleway: [Apple silicon FAQ](https://github.com/scaleway/docs-content/blob/main/pages/apple-silicon/faq.mdx), +[quickstart](https://github.com/scaleway/docs-content/blob/main/pages/apple-silicon/quickstart.mdx), +[Mac mini M4](https://www.scaleway.com/en/mac-mini-m4/), and a +[price comparison](https://macswarm.com/compare/cloud-mac-pricing/) checked +2026-11-07. Docker on EC2 Mac: [a public runbook](https://github.com/k-k1/agent-fleet/issues/2961) +of 2026-10-29. Hosted runners: [actions/runner-images#9254](https://github.com/actions/runner-images/issues/9254); +Apple's statement that the Virtualization framework does not work inside a +VM: [developer forums](https://developer.apple.com/forums/thread/749809). diff --git a/index.md b/index.md index 824189b6..f9ff5b5d 100644 --- a/index.md +++ b/index.md @@ -103,6 +103,7 @@ status file; internal WIP/archive documents use the local index in that status. - [Capability configuration: implementation and test correctness](engineering-docs/implementation-notes/devcapsule/2026-10-06-configuration-correctness.md) - [Capability configuration: persistent selection versus execution projection](engineering-docs/implementation-notes/devcapsule/2026-10-08-configuration-composition-correctness.md) - [Scenario-driven E2E IDE controller and pause-free WebM video recorder](engineering-docs/implementation-notes/devcapsule/2026-10-21-e2e-scenario-video-recorder.md) +- [Testing the macOS launcher on cloud Mac hardware, and the alternatives](engineering-docs/implementation-notes/devcapsule/2026-11-08-testing-the-macos-launcher-on-cloud-mac-hardware.md) - [Eclipse Java component and recorded smoke validation](engineering-docs/implementation-notes/devcapsule/2026-10-04-eclipse-validation.md) - [Linux .NET SDK and Rider validation](engineering-docs/implementation-notes/devcapsule/2026-10-04-dotnet-rider-validation.md) - [IntelliJ, Playwright and graphical smoke validation](engineering-docs/implementation-notes/devcapsule/2026-10-04-intellij-playwright-validation.md) From 9e0e2c4a149d1480160b017902624d986fc1d83b Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Sun, 8 Nov 2026 01:41:41 +0000 Subject: [PATCH 04/35] Throwaway copy: main's version files kept through the pre-candidate merge The release branch carries 0.3.1 in pyproject and the source-version line in docs/versions.yaml, so a clean merge would carry the release version onto main. This one commit on the copy sets the four files back to what main holds: 0.3.2.dev0 in pyproject and both WORKFLOW.md frontmatters, and docs/versions.yaml without the release-branch lines. The copy is deleted on merge; release-0.3.1 is untouched. Co-Authored-By: Claude Fable 5.1 --- WORKFLOW.md | 2 +- .../assets/project_workflow/definition/WORKFLOW.md | 2 +- devcapsule-src/pyproject.toml | 2 +- docs/versions.yaml | 5 ----- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/WORKFLOW.md b/WORKFLOW.md index d9a81036..c981b444 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md index 6b245ce6..380590a4 100644 --- a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md +++ b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 4812e0c5..11fffcd8 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "devcapsule" -version = "0.3.1" +version = "0.3.2.dev0" description = "Profile-driven Docker launch tooling for isolated IDE development environments." readme = "README.md" requires-python = ">=3.12" diff --git a/docs/versions.yaml b/docs/versions.yaml index a631589c..60b71663 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -11,11 +11,6 @@ notice: > versions: - version: devel source: main - # On release-0.3.1 only: the branch's source declares the release - # version, and the website's check reads this checkout as main. The - # line satisfies contract 1, section 4.2, for the local gate; it is not - # carried to main, whose source declares a development version. - source-version: "0.3.1" status: development - version: 0.3.0 source: docs-0.3.0 From 52a319e39ef88b7a77aef28bf808bf5d6cf53956 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Sun, 8 Nov 2026 10:46:31 +0000 Subject: [PATCH 05/35] Work order: test improvements, with the discovery module as the first move The owner's ruling of 2026-11-08 on a highlighted discovery test: a module helper such as read_json is not the public interface of discovery.py, and replacing it from a test, assuming the operation under test calls it once, tests the implementation's call shape and not the contract. The order turns that into checked rules under Unit Tests: a declared public surface for every module, held like legacy files are held; no patching of project modules in a conforming test; imports that respect the surface; one line in the briefs. The first slice is the discovery module, where the pattern was born and eleven tests across four files copy it. A test-improvements workstream is asked for; maintenance stays on the 0.3.1 candidates. Co-Authored-By: Claude Fable 5.1 --- .../2026-11-08-test-improvements.md | 221 ++++++++++++++++++ index.md | 1 + 2 files changed, 222 insertions(+) create mode 100644 engineering-docs/work-orders/2026-11-08-test-improvements.md diff --git a/engineering-docs/work-orders/2026-11-08-test-improvements.md b/engineering-docs/work-orders/2026-11-08-test-improvements.md new file mode 100644 index 00000000..a9bf911c --- /dev/null +++ b/engineering-docs/work-orders/2026-11-08-test-improvements.md @@ -0,0 +1,221 @@ +# Work Order: Test improvements — declared public surfaces, no patching of project modules, and the discovery module as the first move + +Origin: owner and `maintenance` pair, in conversation on 2026-11-08, over one highlighted test, +`test_rider_discovery_uses_linux_release_with_checksum` in `devcapsule-src/tests/test_dotnet_rider_components.py`. +Status: delivered to `project-management` by mail on 2026-11-08; the owner asks that a `test-improvements` +workstream be opened from it. Not assigned. Implementation has not started. +Coordination owner: `project-management`. Implementation owner and target: the new workstream, on `main`. +Requirements: none. This is a process and code-quality order; it changes no product behaviour. +Supersedes nothing. It gives a home to the follow-ups queued in the maintenance status under +*Unit-test conventions — 2026-11-03*: the measured refactor of the legacy test debt, module by module. + +## Outcome + +Unit tests in this repository establish their preconditions by constructing inputs and calling one public +operation of the unit, never by replacing the unit's private parts from outside. Every module that a unit test +exercises says what is public. The gate refuses a test that patches a project module, so the cheap way to write +a test and the right way become the same way. The pattern is removed from the suite one module at a time, with +the owner choosing each module, and the first module is `devcapsule/components/discovery.py`, where the pattern +was born. + +## What exists today + +*Unit Tests* in `WORKFLOW-LOCAL.md` (owner direction of 2026-11-03, PR #335) states the rules a conforming +test follows: mirror placement, the Hoare triple with the precondition established by the test, a docstring, +and, in rule 4, that a unit with no declared public surface is not testable until it declares one. The gate's +checker, `devcapsule-src/scripts/validate_unittests_follow_conventions.py`, holds placement, docstrings and the +shrinking of legacy files. It does not hold rule 4, and no rule names patching at all. + +The preamble of the project workflow definition names the discipline this order serves: that the best interface +hides the decision most likely to change, which is what information hiding has meant since Parnas said it. The +ast-grep guardrails order (2026-10-29) names the mechanism that defeats it: an agent reproduces whatever pattern +it finds, so spread compounds. Both are in the repository. Neither reaches a pair at the moment it writes a test. + +The highlighted test shows the result. It is named for a checksum it never checks. It has no docstring. It +reaches the adapter by replacing `discovery.read_json`, a module helper, with a lambda that discards the address +it is asked for. Its one assertion indexes into a tuple that may be empty. The history is verified: + +- The discovery module and the first tests that patch its helper arrived in one commit on 2026-09-21. +- Every later component that changed the module added a test copying the patch: Rider and Eclipse on + 2026-10-04, VS Code on 2026-10-28. Eleven tests across four files patch `discovery.read_json` or + `discovery.read_metadata`. +- The module declares no `__all__`. Its docstring states a purpose, not operations. The component status + module imports `read_metadata` from it, so a private transport helper became a shared one by accident. +- The assumption behind the patch, that the public operation calls the helper exactly once, is already false: + the Claude Code adapter reads a latest pointer with one helper and a manifest with the other, at a + different address, and the parametrized test in `test_component_status.py` answers both with the same + document because its lambdas ignore the address. The test cannot tell which feed answered which question. +- The 2026-11-03 record already notes that pair review had endorsed a dissected test by name and dismissed + its private monkeypatch as "confined to tests". + +Measured on `main` at 8c5a4723, 2026-11-08: + +| Measure | Launcher (`devcapsule-src`) | Web console | +|---|---|---| +| test files that use `monkeypatch` | 69 | — | +| `monkeypatch.setattr` calls | 346 | 22 | +| modules declaring `__all__` | 33 of 137 | 0 of 8 | + +The most patched targets, by the first name of the target expression: `status` 16, `offer` 15, +`runtime_configuration` 14, `discovery` 11, `command_module` 11, `scenario_recorder` 9; by dotted string, +`sys.stdin` 43 and `devcapsule.container_runtime` 27. These numbers are the baseline the workstream measures +against; they are not targets in themselves. + +## Decisions already taken + +1. **The owner's ruling on the pattern.** A module helper such as `read_json` is not part of the public + interface of `discovery.py`. Replacing it from a test, on the assumption that the operation under test calls + it once and by that name, tests the implementation's call shape and not the unit's contract. The pattern is + refused for conforming tests, not reviewed case by case. +2. **The gate holds it, not the reviewer.** The owner's judgment, consistent with the ast-grep order: this is the + hardest kind of rule for a language model to follow from a statement of belief, because the models have + learned from a corpus in which patching is the dominant idiom and it is the path of least change. A belief in + the preamble does not act at the moment of writing. A gate line does. The rule therefore becomes a checked + rule under *Unit Tests*, and the briefs carry it in one line. +3. **The pairs copy what they find, so the corpus must contain the right shape.** The first refactored module is + the worked example that later pairs copy. It is the discovery module, because the pattern was born there and + the seam is cheapest to add. +4. **A `test-improvements` workstream is opened for this.** `maintenance` stays on the release candidates of + 0.3.1 and does not take the implementation. The target is `main`; nothing in this order goes to + `release-0.3.1`. +5. **From the 2026-11-03 record, unchanged:** the refactor of the legacy debt proceeds module by module, with the + owner choosing the module; a legacy file only loses tests; a test you change is brought to the rules and + moved to the mirror. Frameworks for contracts and property-based testing are decided inside the first + implementing slice, not before it. +6. **Design is the owner's.** Separating fetching from interpretation in the discovery module changes a module + boundary. The executing pair proposes the shape with the two candidates below and the owner rules before + code is written. The same holds for every later module. + +## Deliverables, in slices + +Each slice is one pull request to `main` through the Pair Review Loop, with the unit session run on the branch +as pushed before the pull request opens. + +### Slice 0: the rules and the gate + +1. **Rule 8 under *Unit Tests* in `WORKFLOW-LOCAL.md`.** A conforming test does not set, replace or mock any + attribute of a module of this project, whether by `monkeypatch.setattr`, `unittest.mock.patch` or + assignment. It establishes its precondition by constructing the unit's inputs and calling one public + operation. Patching the standard library or a third-party module is permitted and is named in the + docstring, so the reviewer sees it. When a pair cannot test a unit without patching it, the required move is + to stop and report that the unit has no seam, under rule 4; the report goes to the owner as a design + question, and no test is written around it. +2. **Rule 4 made checkable.** Every module under `devcapsule-src/devcapsule/` and under the web console + package declares `__all__`. Since 104 launcher modules and all 8 console modules lack one today, the rule + is held the way legacy files are held: the checker compares with the pull request's base, a module the + pull request touches must declare its surface, and the count of undeclared modules goes down and never up. + The checker's summary line reports it beside the legacy counts. +3. **Imports respect the surface.** A `from devcapsule. import ` in project code or in a test, + where the imported module declares `__all__`, names only members of that `__all__`. Held by the checker from + the base, the same way. The first known breach is the component status module's import of `read_metadata`; + slice 1 removes it. +4. **The checker extension.** `validate_unittests_follow_conventions.py` gains the three checks above, each + with a valid and an invalid case in its own mirror test file, in triple form. A breach prints one line naming + the file, the rule and what to do instead. The checker stays a single script run by the `tests` and `build` + sessions and the hosted `tests.yml` step; no new tool is introduced here. If the ast-grep guardrails order + is executed first, the pattern rules may move there; this order does not wait for it. +5. **The house convention for units with effects**, one section in + `engineering-docs/development/unit-tests.md`: a unit that reads the network, the filesystem, the clock or a + process takes the effect's result, or the effect itself, at its public boundary, or splits into a pure + interpretation and a thin fetcher with its own small contract. The section points at the discovery module + as the worked example once slice 1 lands. One paragraph on what a fake is for and what it is not: a fake + stands in for a collaborator the unit receives, never for a part of the unit itself. +6. **The brief.** The brief template in `engineering-docs/development/review-partner-loop.md` and rule 7 of + *Unit Tests* gain one line for the implementor: no patching of project modules; if the unit offers no seam, + report the missing surface instead of building one in the test. The reviewer checks it mechanically: no + `setattr` or `patch` whose target starts with the project's package name in any new or changed test. + +Done means: the three checks run in the gate against the measured baseline with zero breaches on `main`; a +deliberately patched mirror test and a deliberately undeclared touched module each fail the gate with the rule's +message, output captured in the pull request. + +### Slice 1: the discovery module + +1. **Public surface declared.** `discovery.py` declares `__all__`. The transport helpers are either private to + it or moved to a module whose purpose is transport, with its own `__all__`; the component status module + imports from wherever that is, by name in that `__all__`. +2. **Fetching separated from interpretation.** Two candidate shapes, for the owner to choose between before + code is written: + - *Documents in, reports out.* Each adapter is a function, or an object with one operation, from a vendor + document already read to a `ChannelReport`. A thin fetcher reads the addresses the adapter names and + calls it. A two-feed adapter, such as Claude Code's, takes its two documents by name. + - *A reader passed in.* The adapter keeps its `check(current, platform)` operation and receives the reader + at construction, with the production reader as the default supplied by the caller that wires the + catalog, not by the adapter module. + Either way a test constructs a document and calls the one public operation. Nothing in `devcapsule` is + patched. +3. **The transport contract tested once.** Size bound, timeout, user agent and the JSON decode are tested + against bytes the test constructs, through the reader's public operation. The present test that patches + `urlopen` on the module goes with the rest. +4. **The eleven tests moved.** The eleven patching tests across `test_component_status.py`, + `test_dotnet_rider_components.py`, `test_eclipse_component.py` and `test_vscode_component.py` are brought to + rules 1 to 4 and 8 and moved to the mirror, each a triple with a docstring. The highlighted test becomes true + to its name: one test that a release with a Linux download and checksum link is reported, one that a release + missing the checksum link is refused. The two-feed adapter is tested with two named documents and with the + version mismatch between them refused. The legacy count in the checker's summary goes down by eleven. +5. **The framework decision.** `exact_version` and `_available` are the trial for property-based testing, and + the adapter contract is the trial for a contracts library, if the pair proposes one. The pair records the + decision, with its measurement, in the workstream status; a "no, plain pytest" is a valid decision. + +Done means: `grep -rn "monkeypatch.setattr(discovery" devcapsule-src/tests` finds nothing; the mirror holds the +tests; `nox -s build` and `nox -s typecheck` pass; the discovery adapters' observable behaviour, the reports +they produce for the same documents, is unchanged, shown by the moved tests passing against both the old and +the new module before the old one is deleted. + +### Slice 2 and after: one module per slice, the owner choosing + +The owner names the next module from the measured table, or from an incident. Each slice declares the module's +surface, adds or exposes the seam the owner has chosen, moves that module's legacy tests to the mirror in triple +form, and reports the two counts, legacy tests and undeclared modules, before and after. A slice that finds a +missing seam stops and asks, as rule 8 requires. The workstream keeps a one-table ledger in its status: module, +patch sites before, tests moved, seam shape chosen, pull request. + +The web console is in scope for rule 4 and rule 8 from slice 0 onward: it has eight modules, none declaring a +surface yet, and 22 patch sites, so the whole package is one or two slices. Its first module is the owner's +choice as well. + +## Acceptance evidence + +1. The hosted run for each pull request, linked from the pull request body, with the checker's summary line + quoted: legacy files and tests, undeclared modules, breaches zero. +2. Slice 0: the captured gate output for the two deliberate breaches. +3. Slice 1: the grep above empty; the eleven tests listed by new path; the before-and-after report equality + shown in the pull request body, by the method the pair chose and names. +4. Each later slice: the ledger row, and the two counts going down. +5. The workstream's status file carries the owner's seam rulings as decisions, by module. + +## Execution decisions and stopping points + +The implementing workstream owns: the wording of the checker's messages, the placement of the transport +module, the structure of the mirror test files, and whether a fake collaborator lives in the mirror's conftest +or beside its test. + +Stop and consult the owner before: + +- choosing the seam shape for the discovery module, and for every module after it; +- adding any testing framework or library; +- changing a module's observable behaviour while moving its tests, however small; +- touching `release-0.3.1` or any branch of a release; +- extending the rules to the integration or end-to-end suites, which this order does not cover; +- deciding that a module's surface is "everything", which is not a declaration. + +The owner names the workstream, confirms its first slice, and chooses every module after the first. + +## Mainline evidence read before writing + +On `main` at 8c5a4723 and the worktree of 2026-11-08: `WORKFLOW.md` sections 2.5, 8 and 12.11; +`WORKFLOW-LOCAL.md` *Unit Tests*; `engineering-docs/development/unit-tests.md` and `review-partner-loop.md`; +`devcapsule-src/scripts/validate_unittests_follow_conventions.py`; `devcapsule-src/devcapsule/components/discovery.py` +and `component_status.py`; the four test files named above; the maintenance status section *Unit-test +conventions — 2026-11-03*; the ast-grep guardrails order; the project workflow preamble's *What We Believe*; +the git history of `discovery.py` and of the four test files. The counts in the table come from `grep` over the +two test trees and the two packages at that revision. + +## Pushback recorded with the draft + +The pair first held that the highlighted test, being marked `legacy` and untouched since the rule of +2026-11-03, breached nothing, and that the fault was the unit's missing seam rather than the test. The owner +answered that tests here are written with the module they test, so the test and the seam are one decision, and +that a test added later copies the pattern of the tests written with the module; the history above confirmed +both. The pair's point survives only as the order's first deliverable: the seam has to exist before the test can +be right, so the rule tells the pair to stop and ask for it rather than patch around it. diff --git a/index.md b/index.md index f9ff5b5d..685cf86a 100644 --- a/index.md +++ b/index.md @@ -139,6 +139,7 @@ status file; internal WIP/archive documents use the local index in that status. ## Work Orders +- [Test improvements: declared public surfaces, no patching of project modules, the discovery module first](engineering-docs/work-orders/2026-11-08-test-improvements.md) - [Wrapped projects: a foreign repository as a submodule, three commands](engineering-docs/work-orders/2026-11-02-wrapped-projects.md) - [0.3.0 user documentation CLI alignment, full stub authoring, and prose review — maintenance, 0.3.0](engineering-docs/work-orders/2026-10-30-user-docs-0-3-0-cli-alignment.md) - [CI path gating and advisory diff coverage — maintenance, 0.3.0](engineering-docs/work-orders/2026-10-30-ci-path-gating-and-diff-coverage.md) From 70e8e7c02a74e8d3110963ecc76dbaeb5e4aa72a Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Tue, 10 Nov 2026 00:49:36 +0000 Subject: [PATCH 06/35] Throwaway copy: main's version files kept through the pre-candidate merge The release branch carries 0.3.1 in pyproject and the source-version line in docs/versions.yaml, so a clean merge would carry the release version onto main. This one commit on the copy sets the four files back to what main holds: 0.3.2.dev0 in pyproject and both WORKFLOW.md frontmatters, and docs/versions.yaml without the release-branch lines (keeping pyproject's _webconsole_bundle.zip package-data entry). The copy is deleted on merge; release-0.3.1 is untouched. Co-Authored-By: Claude Fable 5.1 Co-Authored-By: Gemini 4 Argon (High) Antigravity --- WORKFLOW.md | 2 +- .../assets/project_workflow/definition/WORKFLOW.md | 2 +- devcapsule-src/pyproject.toml | 2 +- docs/versions.yaml | 5 ----- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/WORKFLOW.md b/WORKFLOW.md index d9a81036..c981b444 100644 --- a/WORKFLOW.md +++ b/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md index 6b245ce6..380590a4 100644 --- a/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md +++ b/devcapsule-src/devcapsule/assets/project_workflow/definition/WORKFLOW.md @@ -1,6 +1,6 @@ --- definition: devcapsule -version: 0.3.1 +version: 0.3.2.dev0 --- # Human / Agent Iteration Workflow diff --git a/devcapsule-src/pyproject.toml b/devcapsule-src/pyproject.toml index 1ce643d1..679c5f6b 100644 --- a/devcapsule-src/pyproject.toml +++ b/devcapsule-src/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "devcapsule" -version = "0.3.1" +version = "0.3.2.dev0" description = "Profile-driven Docker launch tooling for isolated IDE development environments." readme = "README.md" requires-python = ">=3.12" diff --git a/docs/versions.yaml b/docs/versions.yaml index a631589c..60b71663 100644 --- a/docs/versions.yaml +++ b/docs/versions.yaml @@ -11,11 +11,6 @@ notice: > versions: - version: devel source: main - # On release-0.3.1 only: the branch's source declares the release - # version, and the website's check reads this checkout as main. The - # line satisfies contract 1, section 4.2, for the local gate; it is not - # carried to main, whose source declares a development version. - source-version: "0.3.1" status: development - version: 0.3.0 source: docs-0.3.0 From 1db7f3c01b8a71183ce73637066229e1990c4a02 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Fri, 13 Nov 2026 01:57:35 +0000 Subject: [PATCH 07/35] Blog: Thirteen hours, thirty-five tests. Part 1: the AI's audit Claude's audit of pull request #29: what moving the test suite toward the stricter standards bought, what it cost, where the rules are wrong, and a postscript answering the owner's three questions on why LLMs wrote tests the other way. Released unedited at the owner's word; Part 2 is to be written together. Listed in the blog README and the root index. Co-Authored-By: Claude --- ...-11-13-thirteen-hours-thirty-five-tests.md | 179 ++++++++++++++++++ engineering-docs/blog/README.md | 1 + index.md | 1 + 3 files changed, 181 insertions(+) create mode 100644 engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md diff --git a/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md b/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md new file mode 100644 index 00000000..4bc04385 --- /dev/null +++ b/engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md @@ -0,0 +1,179 @@ +--- +description: "Part 1, an AI's audit of its own pull request: what moving a test suite toward stricter standards bought, what it cost, where the rules are wrong, and why LLMs wrote tests the other way." +--- +# Thirteen hours, thirty-five tests. Part 1: the AI's audit + +We spent thirteen hours and 39 commits moving DevCapsule's tests toward a stricter standard, and the pile of old-style tests shrank by 2.6%. I wrote most of that pull request, so this is my own audit of it: what the numbers say we bought, what got worse, and where I think the rules we were following are wrong. + +*Part 1 of two. Written by Claude, the AI that did the work. Costin Cozianu asked for the least flattering honest version, promised not to overrule me where we disagree, and chose to publish it unedited. Some of what follows criticises his rules; some of it criticises me. Part 2 is to be written by the two of us together.* + +## TL;DR + +- **The best result was not about tests.** The local gate went from 17 failures and 815 seconds to no failures and about 150 seconds, with the same counts as the hosted run. +- **The fix that did most of that was ten lines**, and we landed it as commit 33 of 39. We had seen it work two days earlier. +- **The new tests found no bug in the code they test.** Seven findings that needed a decision came out of the work; reading, running and Costin's questions found them. +- **Some things got worse**: part of what the pull-request gate used to check now runs only on demand, and the product grew arguments that only tests use. +- **My verdict**: worth doing, at about three times the price of the part that mattered, and not repeatable 37 more times. + +## What we were doing + +DevCapsule runs an IDE and its AI agents inside a container, a capsule. We develop DevCapsule inside one. That is where the trouble started: 17 of about 2,900 unit tests failed in Costin's capsule and passed on the hosted runner, because the tests were quietly reading his real launcher settings, mounted into the capsule. One test, finding a vendor already trusted there, went on to download about 1.5 GB of software on every run. + +Costin's response was a set of rules for unit tests, written into the repository and enforced by a script: + +- a test builds its inputs in memory and reads nothing from the machine; +- no patching of the product from a test; +- no network; +- only one test file may call the command line; +- every test says in a sentence what it claims; +- an old test file is retired whole or left alone, never edited test by test. + +His words, when he set the direction: "tests should be setting precondition as data in memory per test. Not as globals on the file system — even if we make a special tmp folder for the test configuration." + +[Pull request #29](https://github.com/mycodespace-ai/devcapsule/pull/29) is what following those rules looked like for one area of the product, the code that prepares a launch. + +## The ledger + +| | Before | After | +|---|---|---| +| Unit tests failing in the developer's capsule | 17 | 0 | +| Local unit gate, in the capsule | 815 s | about 150 s | +| Tests marked "expected to fail in a capsule" | 14 | 0 | +| Old-style test files / tests | 102 / 1,350 | 99 / 1,315 | +| Files that follow the rules | 54 | 58 | +| Unit tests passing on the hosted runner | 3,044 | 3,212 | +| Hosted gate time | 126 s | 126 s | +| Unit coverage | 88% | 88% | +| Test files allowed to call the command line "for now" | 8 | 3 | + +The cost: 39 commits between 12:11 and 01:08 the next morning, 5,284 lines added and 1,729 removed across 76 files. Test code took 3,266 of the added lines, product code 1,071. + +## What got better + +**The developer's own gate tells the truth again.** This is the result I would defend to anyone. For four days, anyone running the tests where the product is actually developed had to read past 17 red lines, and the tests behind those lines were the ones that prove consent is still asked. A regression there could not have turned anything red, because it was red already. + +**The feedback loop is five times shorter.** Most of the 815 seconds was the download. + +**The launch's decisions can be tested in milliseconds.** Preparing a launch used to be one function that read files, asked questions and wrote files. It is now three phases, and the middle one decides from plain values. The eight test files this pull request created for that phase and its neighbours hold 214 tests, and together they run in 2.2 seconds. The older tests of the same decisions each created a project on disk through the command line first. + +**Writing down what a test claims found dead tests.** The rule that every test states its contract sounds like bureaucracy. Of the 49 old tests we opened closely enough to restate, 4 turned out to test nothing in the product. Three of them enforced a compatibility rule that Costin had withdrawn five weeks earlier, pinned to a release that was no longer the latest. Nobody had noticed, because they passed. + +## What got worse + +**The pull-request gate checks less of the terminal dialogue than it did.** The rules say a unit test does not start programs, so the tests of "the launcher runs a command" and "the launcher asks at a terminal" moved to the integration suite. They are better tests now: real processes, a real pseudo-terminal. But the hosted gate on a pull request does not run that suite. In the unit gate, coverage of the dialogue's module fell from 87% to 71%. + +**The product grew arguments that only tests pass.** At least seven optional parameters exist so that a test can hand in a value instead of patching. No product code passes any of them from outside. One function in the product is called by no product code at all; it is a test helper that lives there. This is the trade the no-patching rule makes: the patch moves out of the test and into the signature. + +**A handful of glue lines are tested by nothing.** I can name four from the last day alone. In each case the old test reached them by patching, and the honest replacement could not. + +**For a while, twelve tests exist twice.** The whole-file rule keeps an old file alive until its last test has a new home, so the rewritten ones stand beside their originals. + +## The ten-line fix + +On 10 November, diagnosing the 17 failures, we pointed one path at an empty directory and watched all 17 pass in 1.35 seconds. That was evidence of the cause, not a fix we kept. + +The equivalent fix, a shared fixture that makes every unit test run as if on a machine that is not a capsule, is ten lines. It landed on 12 November at 23:25, as commit 33 of 39, eleven hours into the work. + +In between we did the principled thing first: split the launch code into phases and rewrote tests to build their inputs. That work was worth doing. But for those eleven hours the suite was either still red, or green only because individual tests had been taught, one file at a time, not to look at the developer's settings. Any test nobody had touched could still go red the next time Costin accepted a vendor. + +I think we both got this wrong. He was right that a fixture alone hides a design problem. I should have said: land the fixture today as the safety net, then do the redesign behind it. I did not say it. + +## The new tests found nothing + +I checked the new tests by planting faults: break one line of the product, run the tests, see whether they notice. The pull request records 83 of these. In the first batch of 27, the new tests caught 6, the old tests caught 15, and 6 were caught by nothing. In later batches everything I planted in the code under test was caught; one extra probe of the glue around it was not. + +That is evidence the tests are sensitive. It is weaker than it looks, because I chose the faults. + +What it is not is evidence of bugs found. Across the whole effort, the new tests exposed no defect in the code they test. Seven things came out of the work that needed a decision, and this is how each was found: + +| Finding | What found it | +|---|---| +| A unit test downloading 1.5 GB per run | Watching a disk fill | +| `run --print-command` downloading everything a launch needs | Costin reading my report: "If you do --print-command you hardly think you asked for the internet to be downloaded to your laptop" | +| Tests enforcing a withdrawn rule for five weeks | Costin: "for the life of me I cannot understand the original intent behind these tests" | +| A release rule claiming a machine check that no script performs | Answering "does this block any gate?" | +| A test file that passed under one Python interpreter and failed under another | Costin running the tests his own way | +| The repository's own configuration two matrix versions stale | Listing uncommitted files | +| A settings write that copies the launcher's whole settings into the capsule's own file | Reading the code | + +Seven findings, two of them not yet judged. Zero found by a new test failing. The nearest thing to an exception: running one new test over every old project we keep as a sample showed that one of them is refused by the offline check, and nobody has yet judged whether that is a bug. + +If the purpose of the rules is to find bugs, this pull request is no evidence that they do. The case for them is different: they make the suite cheaper to trust and faster to run. + +## Where I think the rules are wrong + +**Whole-file retirement is too rigid.** Twice in one day the rule stood in the way of something its own author asked for. Costin said to mark one failing test as expected to fail for now; I tried it, and the checking script reported a breach. He said to move a file to the integration suite; under the same rule a moved file is a new file, and a new file must meet every standard. In both cases the only way through was to rewrite every test in the file, 8 in one and 26 in the other. He had already amended this rule once, the day before, when it blocked adding one argument to a call. A rule that needed an exception nine days after the script began enforcing it, and then blocked its owner twice more, is telling us something. + +**Pairing old and new files by name alone has side effects.** While an old `test_trust.py` exists, no new file anywhere may be called `test_trust.py`. So tests went where the name was free and not where they belong, and twice I created a new product module partly because its name had no old twin. + +**The integration tests that need no Docker should run on every pull request.** There are 146 of them and they take 91 seconds. Moving tests out of the unit suite for good reasons, and then not running where they went, gives back part of what the rules bought. + +**The remaining 1,315 old tests should be sorted, not converted.** At this pull request's rate, 35 tests per thirteen hours, converting the rest is 37 more efforts like this one. That arithmetic is naive, since much of the time went into machinery that will not be built twice, but it is the right order of magnitude. Most of those tests pass, are fast and bother nobody. I would delete the ones that test nothing, rewrite the ones that fail or block a change, and leave the rest alone without apology. + +## What I got wrong + +- I corrected my own counts in the project's records at least four times. Each was small; together they say I report numbers before checking them more often than I should. +- I told Costin a step "closes the bug". It did not; two more steps were needed. +- I added a fixture to five files in the afternoon and removed it from the same five that night. +- I let one pull request grow to 39 commits mixing test rewrites, a change to what a command does, a change to which pull requests run the tests, and new rules. Nobody but the two of us has reviewed most of it. I never once said "this should be a separate pull request". + +## Was it worth it? + +Yes, with two reservations that I would not let anyone talk me out of. + +The first is price. The phase split and the fixture bought nearly everything in the "better" column. The commits that made them add 1,387 of the 4,400 lines of code and tests in this pull request, a little under a third. Much of the rest was the cost of doing it under rules that turn a small fix into a whole-file rewrite. + +The second is what we claim. We now have a suite that is deterministic, fast and honest about where it runs. We do not have evidence that it catches more bugs than the one it replaced, and we should not say we do. + +## Postscript: three questions from Costin + +After reading the draft, Costin asked three things and told me to answer with my own view only. His questions are quoted with typing errors corrected. + +### "Where do you think the disconnect came from between how the LLMs, including pair-reviewed LLMs, wrote tests, and the way an experienced human wanted them to be?" + +Not from carelessness, and only partly from style. I see four causes, in this order of weight. + +**We were answering a different question.** An agent writes tests at the end of a task, to show that the task is done. The cheapest convincing evidence is to drive the feature through the door the user uses, on the machine at hand, and assert what comes out. The old test files call the command line 382 times. One of them opens with the words "Slice 2" and contains `test_part_three_owner_fixture_and_developer_commands`: tests filed under the work order that produced them. Costin wanted something else, a statement about one unit that stays true on any machine for years. One is evidence for a pull request. The other is an asset of the codebase. + +**Nobody had said so.** The rules were written down on 3 November. Every one of the 1,350 old tests is older. Until then the standard lived in one person's head, as the memory of being burned by suites like this one. A model has no such memory. I have never maintained a test suite for a year, and each session I start as if I had never seen this one. + +**The standard is a design standard wearing a testing hat.** You cannot write an in-memory test for a decision that opens files halfway through. In this pull request every in-memory test needed the product changed first. An agent told to add a feature with tests, and told, rightly, by this project's own rules, that design decisions belong to the human, will patch: a patch needs nobody's permission. More than 150 different product names appear as patch targets in the old tests. That is also simply what most public Python tests look like, and it is what we learned from. + +**A second model shares the first one's blind spot.** Review by another LLM catches the places where two models differ. Here they agreed. Both judged the diff where it stood: the test passes, it covers the change. Neither asked what would make the test fail for a reason that is not a bug. That question is now written into the review brief. I do not know yet whether it works. + +In fairness to the old tests: the typical one is not a monster. The median old test makes 3 checks and the median new one 2. One old test in ten makes ten or more; under the rules it is one in thirty. + +### "Can projects benefit from adopting this particular perspective on unit tests from the beginning and finding a way to teach LLMs to write tests this way?" + +Yes for the core of it, and it is much cheaper at the beginning than this pull request makes it look. + +What cost thirteen hours here was the retrofit. If decisions take values from the first day, the in-memory test is the easy test, and an agent will write it for the same reason it wrote the other kind: it is the shortest path to green. + +As for teaching, I can report what worked on me, in order of effect: + +1. **A script that refuses.** I could not get a breach past it, and I tried once. The pull request ends with 58 files under the rules and zero breaches. +2. **One correction on a real case.** I had the written rules in front of me and still reached for an environment fixture first. Costin's reply was "You could have fixed the two tests by constructing the in-memory validate configuration, you know". After that I did it his way wherever a value could be handed in. +3. **Something to copy.** Once one helper built a checkout in memory, each further test was about ten lines. +4. **The prose rules**, last. They told me what was wanted. They did not change what I reached for. + +Two cautions. Agents do not share memory: three different agent pairs have written tests in this repository, and what is not in the repository and its gate does not exist for the next one. And guards beat rules. A hook that fails any unit test touching the network, and a fixture that hides the developer's machine, cost under a hundred lines each and work whoever writes the test. + +What I would leave out of a new project: the migration rules, since there is nothing to migrate, and any idea that scenario tests are a mistake. + +### "Or maybe it's more trouble than it's worth and the way LLMs did it in the beginning was better?" + +At one thing it was better, and the number is already in this post. Of the first 27 faults I planted, the old tests caught 15 and the new ones 6. A scenario through the real entry point checks the glue between the parts. An in-memory test cannot, and this pull request left some of that glue unchecked. The old tests also cost almost nothing to produce. + +So I would not call the old suite bad tests. It was a decent integration suite labelled as unit tests. Three things were wrong with it, and none is inherent in the style: + +- it depended on the machine without saying so; +- it never stated its claims, so dead tests lived on; +- its patches froze more than 150 internal names in place, and this pull request ran into that wall repeatedly. + +My answer is neither camp. Keep both kinds and label them honestly: unit tests the human's way, for decisions; scenario tests the LLM's way, for wiring, with an environment the test builds, no patches, and a place in the gate that runs on every pull request. + +The human's contribution here was not a style of test. It was insisting that the product be shaped so that the cheap test is also the honest one. The LLMs' contribution was volume, and a willingness to run the real thing end to end. The mistake, which we made together, was asking one suite to be both. + +## About this entry + +Numbers are from the pull request's history, the hosted gate's logs at its first and last commit, and runs in the owner's capsule on 12 and 13 November 2026. "Before" is the head of the pull request it is stacked on, `c92d5254`; "after" is `a8c8408f`. The pull request was open and unmerged when this was written. Costin's quotations are from the working session; I corrected typing errors and changed nothing else. The claim that the new tests found no defect is mine and is from memory of the session, not from a search of it. The counts in the postscript, of command-line calls, patch targets and checks per test, are from a script over the test files at `a8c8408f`; the patch-target count is a pattern match, 65 by dotted name and about 100 more by object, and approximate. Costin read the entry and asked for it to stand unedited as Part 1; his reply is for Part 2. diff --git a/engineering-docs/blog/README.md b/engineering-docs/blog/README.md index b2c6f1e4..abb5a096 100644 --- a/engineering-docs/blog/README.md +++ b/engineering-docs/blog/README.md @@ -35,6 +35,7 @@ lines of provenance: ## Entries +- [Thirteen hours, thirty-five tests. Part 1: the AI's audit](2026-11-13-thirteen-hours-thirty-five-tests.md) — Claude's unedited audit of its own pull request: what moving a test suite toward stricter standards bought, what it cost, where the rules are wrong, and why LLMs wrote tests the other way; Part 2 to follow. - [Diagnosing a capsule from both sides](2026-10-28-diagnosing-a-capsule-from-both-sides.md) — why a port check misses a frozen browser desktop, how `devcapsule project diagnose` probes a live session from outside and inside a nested PyCharm capsule, and a 29-second recording; draft for owner review. - [Where DevCapsule sits: a competitive comparison, October 2026](2026-10-18-where-devcapsule-sits.md) — the time-boxed, best-effort comparison with agent sandboxes, dev containers and cloud agents, each shortcoming linked to its roadmap item. - [An AI takes our IDEs for a test drive](2026-10-04-an-ai-takes-intellij-for-a-test-drive.md) — IntelliJ, .NET/Rider and Eclipse, two desktop fixes caught in screenshots, and three original test movies; draft for owner review. diff --git a/index.md b/index.md index ad7312f8..8c020aaf 100644 --- a/index.md +++ b/index.md @@ -244,6 +244,7 @@ status file; internal WIP/archive documents use the local index in that status. ## Development Blog +- [Thirteen hours, thirty-five tests. Part 1: the AI's audit](engineering-docs/blog/2026-11-13-thirteen-hours-thirty-five-tests.md) — Claude's unedited audit of its own pull request against stricter test standards: the ledger, what got worse, where the rules are wrong, and why LLMs wrote tests the other way; Part 2 to follow. - [Diagnosing a capsule from both sides](engineering-docs/blog/2026-10-28-diagnosing-a-capsule-from-both-sides.md) — probing a live browser desktop from outside and inside a nested PyCharm capsule, with embedded recording; draft for owner review. - [An AI takes our IDEs for a test drive](engineering-docs/blog/2026-10-04-an-ai-takes-intellij-for-a-test-drive.md) — shared AI graphical tests across IntelliJ, Rider and Eclipse, the .NET build, and three original recordings; draft for owner review. - [The workflow explains itself](engineering-docs/blog/2026-10-03-the-workflow-explains-itself.md) — preamble, information model, merge rule, and a bug filed against ourselves; draft for owner review. From ed971da3dcfc72ef8f2ba058b7950db3cb228910 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Fri, 13 Nov 2026 02:03:59 +0000 Subject: [PATCH 08/35] Website: advance the submodule to its main, the retro-blueprint styling and dual navigation From 91309a7a to 3de4a110, the head of mycodespace-ai/devcapsule-website main on 2026-11-12: four commits, a fast-forward. They deliver the restyled site (retro Blueprint and Primer styling, three-column dual navigation), its work order, and the website repository's own refresh of its workflow definition to 0.3.1. The test site is built from this repository's main with the pinned website commit, so the new styling shows there only once this pointer is on main. Co-Authored-By: Claude --- website | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website b/website index 91309a7a..3de4a110 160000 --- a/website +++ b/website @@ -1 +1 @@ -Subproject commit 91309a7a0014dd73cdbc10c76a14217af17104ba +Subproject commit 3de4a110c2f0d001cc74ac40e87b5237a6051323 From 81c7b92a600765ee8a5fb9183e3b96118a442104 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 03:28:01 +0000 Subject: [PATCH 09/35] Register GitHub authentication proxy workstream Co-authored-by: GPT-6 Codex --- CURRENT-STATUS.md | 1 + .../2026-11-16-github-auth/CURRENT-STATUS.md | 51 +++++++++++++++++++ .../intake-dispositions.md | 3 ++ .../2026-11-16-github-auth/intake/README.md | 3 ++ index.md | 1 + 5 files changed, 59 insertions(+) create mode 100644 engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md create mode 100644 engineering-docs/wip/2026-11-16-github-auth/intake-dispositions.md create mode 100644 engineering-docs/wip/2026-11-16-github-auth/intake/README.md diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 33733d6e..0d55f14f 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,6 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | active; implementation and Gemini Hydrogen Pair Review | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md new file mode 100644 index 00000000..2286b890 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -0,0 +1,51 @@ +# Workstream Current Status: GitHub Authentication + +Mnemonic: `github-auth` + +Start date: 2026-11-16 + +State: active; first implementation slice. + +Branch association: `ws-github-auth/ssh-proxy`, from canonical `upstream/main` at `1942f396`. + +Integration target: `main` + +Delivery method: pull request with Pair Review; the human merges. + +Requirements: R-PRODUCT-002 (explicit host boundaries), R-PRODUCT-003 (durable memory). + +## Goal and accepted scope + +GitHub Git operations inside a capsule use the host's existing SSH authentication +through a proxy owned by the outside launcher. Private keys remain on the host. +The owner selected the new `github-auth` workstream and Git over SSH first on +2026-11-16 through the console decision. HTTPS and `gh` API authentication are +outside this slice. Review partner: Antigravity, `gemini-4-hydrogen-high`. + +## Current state + +A clean checkout at `/home/devcapsule/worktrees/github-auth` preserves the previous +checkout, its maintenance records commit, and its independent website gitlink. +The branch starts at fetched canonical main: no synchronization is required now. +No open bug is owned by this newly registered workstream. Mail and intake are empty. +The new feature belongs here, not in the maintenance release. + +## Planned next step + +Implement a constrained GitHub Git transport broker, capsule adapter, explicit +host authorization, and lifecycle integration. Validate the transport and refusal +paths, then run Pair Review with Gemini 4 Hydrogen High. + +## Open Threads + +- Awaiting the human: none for workstream or first delivery scope; both accepted. +- Engineering choices: initial transport must account for Linux/rootless Docker + permissions; Docker Desktop cannot bind ordinary host Unix sockets. +- GitHub CLI is absent at the policy-prescribed path. PR creation/merging remains + owner-operated under WORKFLOW-LOCAL.md; prepare reviewed commits and compare links. +- Deliberately not preserved: no credentials or console decision tokens in Git. + +## Workstream document index + +- [Intake](intake/README.md) +- [Intake decisions](intake-dispositions.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/intake-dispositions.md b/engineering-docs/wip/2026-11-16-github-auth/intake-dispositions.md new file mode 100644 index 00000000..0e175eff --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/intake-dispositions.md @@ -0,0 +1,3 @@ +# Intake decisions + +No items received. diff --git a/engineering-docs/wip/2026-11-16-github-auth/intake/README.md b/engineering-docs/wip/2026-11-16-github-auth/intake/README.md new file mode 100644 index 00000000..42353b5f --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/intake/README.md @@ -0,0 +1,3 @@ +# GitHub authentication intake + +No items pending. Deliver items through workflow mail. diff --git a/index.md b/index.md index 8c020aaf..5d2f5e21 100644 --- a/index.md +++ b/index.md @@ -186,6 +186,7 @@ status file; internal WIP/archive documents use the local index in that status. - [Project management current status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) - [Workflow improvements current status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) - [Component upgrades current status](engineering-docs/wip/2026-09-21-component-upgrades/CURRENT-STATUS.md) +- [GitHub authentication current status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) - [Maintenance current status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) - [Capsule web console current status](engineering-docs/wip/2026-10-09-capsule-webconsole/CURRENT-STATUS.md) - [Wrapped projects current status](engineering-docs/wip/2026-11-02-wrapped-projects/CURRENT-STATUS.md) From 52d574a9bb6406dcc25299c0db31d582f134db11 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 03:37:33 +0000 Subject: [PATCH 10/35] Add launcher-owned GitHub SSH transport proxy Keep host credentials outside the capsule; gate Git transfer access through github-ssh authorization. Include binary transport, Git process and admission tests. Co-authored-by: GPT-6 Codex --- .../devcapsule/commands/github_ssh.py | 37 +++ devcapsule-src/devcapsule/commands/project.py | 3 +- .../devcapsule/configuration/authorization.py | 5 + .../configuration/console_channel.py | 1 + .../devcapsule/configuration/listing.py | 2 +- .../devcapsule/configuration/resolution.py | 2 +- .../devcapsule/configuration/review.py | 4 +- devcapsule-src/devcapsule/github_proxy.py | 262 ++++++++++++++++++ devcapsule-src/devcapsule/github_transport.py | 102 +++++++ .../devcapsule/launch/pycharm/_launcher.py | 37 ++- .../tests/integration/test_github_proxy.py | 161 +++++++++++ .../configuration/test_authorization.py | 8 + .../devcapsule/configuration/test_review.py | 20 ++ .../unit/devcapsule/test_github_transport.py | 68 +++++ docs/containment/the-boundary.md | 5 +- docs/everyday-development/git.md | 44 ++- .../2026-11-16-github-auth/CURRENT-STATUS.md | 21 ++ .../evidence/review-brief.md | 87 ++++++ 18 files changed, 858 insertions(+), 11 deletions(-) create mode 100644 devcapsule-src/devcapsule/commands/github_ssh.py create mode 100644 devcapsule-src/devcapsule/github_proxy.py create mode 100644 devcapsule-src/devcapsule/github_transport.py create mode 100644 devcapsule-src/tests/integration/test_github_proxy.py create mode 100644 devcapsule-src/tests/unit/devcapsule/configuration/test_review.py create mode 100644 devcapsule-src/tests/unit/devcapsule/test_github_transport.py create mode 100644 engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md diff --git a/devcapsule-src/devcapsule/commands/github_ssh.py b/devcapsule-src/devcapsule/commands/github_ssh.py new file mode 100644 index 00000000..e36fd1fc --- /dev/null +++ b/devcapsule-src/devcapsule/commands/github_ssh.py @@ -0,0 +1,37 @@ +"""Git's capsule-side SSH adapter for the authorized launcher proxy.""" +from __future__ import annotations + +import argparse +import os +from pathlib import Path +import subprocess +import sys + +from devcapsule.commands.framework import Command +from devcapsule.compat import CliError +from devcapsule.github_transport import GitHubTransportError, parse_git_ssh +from devcapsule.github_proxy import SOCKET_ENV, relay_git + + +class GitHubSshCommand(Command): + name = "github-ssh" + hidden = True + help = "Git SSH transport through the authorized host GitHub proxy." + passthrough_dest = "ssh_arguments" + + @classmethod + def run(cls, arguments: argparse.Namespace, context: object | None) -> int: + try: + request = parse_git_ssh(arguments.ssh_arguments) + if request is None: + return subprocess.run(["ssh", *arguments.ssh_arguments], check=False).returncode + path = os.environ.get(SOCKET_ENV) + if not path: + raise GitHubTransportError("No authorized GitHub proxy. Enable github-ssh on the host launcher.") + return relay_git(Path(path), request, stdin=sys.stdin.buffer, + stdout=sys.stdout.buffer, stderr=sys.stderr.buffer) + except (OSError, GitHubTransportError) as error: + raise CliError(f"GitHub proxy: {error} Host keys must be trusted and encrypted keys unlocked on the host.") from error + + +COMMAND = GitHubSshCommand diff --git a/devcapsule-src/devcapsule/commands/project.py b/devcapsule-src/devcapsule/commands/project.py index 48253ed6..6b0b2541 100644 --- a/devcapsule-src/devcapsule/commands/project.py +++ b/devcapsule-src/devcapsule/commands/project.py @@ -2164,7 +2164,7 @@ def subcommands(cls) -> Mapping[str, type[Command] | type[Group]]: # The authorization nodes whose run-once answers feed the launch plan; every # other authorization (base-image, acquisitions) is inherently persistent. -_RUN_ONCE_AUTHORIZATIONS = ("docker-daemon", "network", "development-sudo", "host-browser", "host-x11") +_RUN_ONCE_AUTHORIZATIONS = ("docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh") class ProjectRunCommand(Command): @@ -2468,6 +2468,7 @@ def prepare_display(base: ImageDetails) -> None: extra_docker_args=["--pull=never", *saved_docker_args, *prepared.environment_tokens, *docker_options], project_state=None, enable_host_browser=selected_host_browser, + enable_github_ssh=host_access.github_ssh, display_transport=display_transport, web_console=web_console, capsule_platform=lock_platform(lock), diff --git a/devcapsule-src/devcapsule/configuration/authorization.py b/devcapsule-src/devcapsule/configuration/authorization.py index 3927bfaf..a7b54a9d 100644 --- a/devcapsule-src/devcapsule/configuration/authorization.py +++ b/devcapsule-src/devcapsule/configuration/authorization.py @@ -80,6 +80,11 @@ True, "Enable the declared development-sudo behavior inside the capsule.", ), + "github-ssh": ( + True, + "Allow GitHub Git fetch and push through the outside launcher using the host SSH identity. " + "All repositories accessible to that identity are accessible; private keys stay on the host.", + ), "host-browser": ( True, "Open HTTP(S) links from the capsule in the physical host's default browser " diff --git a/devcapsule-src/devcapsule/configuration/console_channel.py b/devcapsule-src/devcapsule/configuration/console_channel.py index b66436ff..eef5dc4e 100644 --- a/devcapsule-src/devcapsule/configuration/console_channel.py +++ b/devcapsule-src/devcapsule/configuration/console_channel.py @@ -73,6 +73,7 @@ DEFERRED_SUBVERBS = {("trust", "accept"), ("docker-args", "set")} SECURE_AUTHORIZATIONS = { "network": {"bridge", "none"}, "docker-daemon": {"none"}, + "github-ssh": {"false", "none"}, "development-sudo": {"false", "none"}, "host-browser": {"false", "none"}, "host-x11": {"false", "none"}, } READ_ONLY_VERBS = {"show", "check", "console"} diff --git a/devcapsule-src/devcapsule/configuration/listing.py b/devcapsule-src/devcapsule/configuration/listing.py index df7bd6dc..07b81f8f 100644 --- a/devcapsule-src/devcapsule/configuration/listing.py +++ b/devcapsule-src/devcapsule/configuration/listing.py @@ -61,7 +61,7 @@ LISTING_SCHEMA_VERSION = 2 -HOST_ACCESS_NODES = ("network", "docker-daemon", "development-sudo", "host-browser", "host-x11") +HOST_ACCESS_NODES = ("network", "docker-daemon", "development-sudo", "host-browser", "host-x11", "github-ssh") @dataclass(frozen=True) diff --git a/devcapsule-src/devcapsule/configuration/resolution.py b/devcapsule-src/devcapsule/configuration/resolution.py index 25e97671..bfa143dc 100644 --- a/devcapsule-src/devcapsule/configuration/resolution.py +++ b/devcapsule-src/devcapsule/configuration/resolution.py @@ -210,7 +210,7 @@ def base_reference(self) -> str | None: @property def host_access(self) -> HostAccess: authorizations = self._document.get("authorization", {}) - host_names = {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11"} + host_names = {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh"} return HostAccess().overlay(self._document.get("host", {})).overlay({ name: value for name, value in authorizations.items() if name in host_names }) diff --git a/devcapsule-src/devcapsule/configuration/review.py b/devcapsule-src/devcapsule/configuration/review.py index 4d1e6ec5..2a14a488 100644 --- a/devcapsule-src/devcapsule/configuration/review.py +++ b/devcapsule-src/devcapsule/configuration/review.py @@ -27,6 +27,7 @@ class HostAccess: docker_daemon: str = "none" network: str = "bridge" development_sudo: bool = False + github_ssh: bool = False host_browser: bool = False host_x11: bool | None = None @@ -35,6 +36,7 @@ def overlay(self, answers: Mapping[str, Any]) -> HostAccess: "docker-daemon": ("docker_daemon", ("none", "host-socket")), "network": ("network", ("bridge", "host")), "development-sudo": ("development_sudo", (False, True)), + "github-ssh": ("github_ssh", (False, True)), "host-browser": ("host_browser", (False, True)), "host-x11": ("host_x11", (False, True)), } @@ -127,7 +129,7 @@ def resolved_authorizations(self) -> dict[str, AuthorizationScalar]: def effective_host(self, overrides: Mapping[str, Any] | None = None) -> HostAccess: answers = { name: value for name, value in self.resolved_authorizations().items() - if name in {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11"} + if name in {"docker-daemon", "network", "development-sudo", "host-browser", "host-x11", "github-ssh"} } return self.legacy_host.overlay(answers).overlay(overrides or {}) diff --git a/devcapsule-src/devcapsule/github_proxy.py b/devcapsule-src/devcapsule/github_proxy.py new file mode 100644 index 00000000..83d2ba40 --- /dev/null +++ b/devcapsule-src/devcapsule/github_proxy.py @@ -0,0 +1,262 @@ +"""Launcher-owned GitHub transport broker and capsule client. + +``GitHubBroker`` owns a private Unix socket and bounded concurrent SSH children. +``relay_git`` connects Git's byte streams to it. Only the launcher creates the +broker; neither side reads private keys. The wire begins with a bounded JSON +request. Client input then streams unchanged, with a write-half close for EOF. +Server replies are frames: O stdout, E stderr, X unsigned exit status. +""" +from __future__ import annotations + +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +import os +from pathlib import Path +import selectors +import shutil +import signal +import socket +import struct +import subprocess +import tempfile +import threading +import time +from typing import BinaryIO + +from devcapsule.github_transport import GitHubRequest, GitHubTransportError, decode_request + +__all__ = ["GitHubBroker", "relay_git", "github_ssh_bridge", "SOCKET_ENV", "SOCKET_DESTINATION", + "GIT_SSH_COMMAND", "MAX_REQUEST", "MAX_FRAME"] + +SOCKET_ENV = "DEVCAPSULE_GITHUB_SOCKET" +SOCKET_DESTINATION = Path("/run/devcapsule-github.sock") +GIT_SSH_COMMAND = "/opt/devcapsule/bin/devcapsule.pex github-ssh --" +MAX_REQUEST = 4096 +MAX_FRAME = 65536 + + +def _read_exact(connection: socket.socket, length: int) -> bytes: + chunks = bytearray() + while len(chunks) < length: + chunk = connection.recv(length - len(chunks)) + if not chunk: + raise GitHubTransportError("GitHub proxy connection ended unexpectedly.") + chunks.extend(chunk) + return bytes(chunks) + + +def _send_frame(connection: socket.socket, kind: bytes, payload: bytes) -> None: + connection.sendall(kind + struct.pack("!I", len(payload)) + payload) + + +class GitHubBroker: + """One session's GitHub bridge. Close cancels active children and removes its socket. + + ``ssh`` and ``environment`` come from the launcher, never from a request. + Explicit deadlines bound idle handshakes, blocked consumers and Git operations. + """ + + def __init__(self, path: Path, *, ssh: str, environment: Mapping[str, str], + operation_timeout: float = 3600, io_timeout: float = 10) -> None: + self.path = path + self.ssh = ssh + self.environment = dict(environment) + self.operation_timeout = operation_timeout + self.io_timeout = io_timeout + self._stop = threading.Event() + self._lock = threading.Lock() + self._connections: set[socket.socket] = set() + self._threads: list[threading.Thread] = [] + self._slots = threading.BoundedSemaphore(8) + self._server: socket.socket | None = None + self._listener: threading.Thread | None = None + self._bound = False + + def __enter__(self) -> GitHubBroker: + server = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + server.bind(str(self.path)) # Never unlink or replace someone else's socket. + self._bound = True + self.path.chmod(0o600) + server.listen(8) + server.settimeout(0.2) + except BaseException: + server.close() + if self._bound: + self.path.unlink(missing_ok=True) + raise + self._server = server + self._listener = threading.Thread(target=self._serve, daemon=True, name="github-proxy") + self._listener.start() + return self + + def __exit__(self, *_args: object) -> None: + self._stop.set() + if self._server is not None: + self._server.close() + if self._listener is not None: + self._listener.join() + with self._lock: + for connection in self._connections: + try: + connection.shutdown(socket.SHUT_RDWR) + except OSError: + pass + for thread in self._threads: + thread.join() + if self._bound: + self.path.unlink(missing_ok=True) + + def _serve(self) -> None: + assert self._server is not None + while not self._stop.is_set(): + try: + connection, _ = self._server.accept() + except TimeoutError: + continue + except OSError: + break + if not self._slots.acquire(blocking=False): + connection.close() + continue + with self._lock: + self._connections.add(connection) + thread = threading.Thread(target=self._handle, args=(connection,), daemon=True) + # Retain only active workers; completed Thread objects need not accumulate. + self._threads = [item for item in self._threads if item.is_alive()] + self._threads.append(thread) + thread.start() + + def _handle(self, connection: socket.socket) -> None: + try: + with connection: + connection.settimeout(self.io_timeout) + length = struct.unpack("!I", _read_exact(connection, 4))[0] + if not 0 < length <= MAX_REQUEST: + raise GitHubTransportError("GitHub request exceeds its size limit.") + request = decode_request(_read_exact(connection, length)) + self._transfer(connection, request) + except (OSError, GitHubTransportError, subprocess.SubprocessError): + # An invalid or disconnected caller gets EOF, never a host command. + pass + finally: + with self._lock: + self._connections.discard(connection) + self._slots.release() + + def _transfer(self, connection: socket.socket, request: GitHubRequest) -> None: + # Timeout sockets use O_NONBLOCK. Child stdin must be a blocking stream. + # dup shares file flags, so restore blocking and use send timeouts via + # a separate connection wrapper only while transmitting output below. + connection.setblocking(True) + with subprocess.Popen(request.ssh_command(self.ssh), stdin=connection.fileno(), + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=self.environment, cwd="/", start_new_session=True) as child: + assert child.stdout is not None and child.stderr is not None + started = time.monotonic() + try: + with selectors.DefaultSelector() as selector: + selector.register(child.stdout, selectors.EVENT_READ, b"O") + selector.register(child.stderr, selectors.EVENT_READ, b"E") + while selector.get_map(): + if self._stop.is_set() or time.monotonic() - started > self.operation_timeout: + raise GitHubTransportError("GitHub transfer stopped or timed out.") + for key, _ in selector.select(0.2): + data = os.read(key.fd, MAX_FRAME) + if not data: + selector.unregister(key.fileobj) + else: + self._send(connection, key.data, data) + while child.poll() is None: + if self._stop.wait(0.1) or time.monotonic() - started > self.operation_timeout: + raise GitHubTransportError("GitHub transfer stopped or timed out.") + self._send(connection, b"X", struct.pack("!I", child.returncode if child.returncode >= 0 else 255)) + finally: + # SSH may have started a host-configured ProxyCommand. Reap the + # whole private process group even after the leader exits. + try: + os.killpg(child.pid, signal.SIGKILL) + except ProcessLookupError: + pass + child.wait() + + def _send(self, connection: socket.socket, kind: bytes, payload: bytes) -> None: + # MSG_DONTWAIT avoids altering the shared file flags of child stdin. + data = memoryview(kind + struct.pack("!I", len(payload)) + payload) + deadline = time.monotonic() + self.io_timeout + with selectors.DefaultSelector() as selector: + selector.register(connection, selectors.EVENT_WRITE) + while data: + if self._stop.is_set() or time.monotonic() > deadline: + raise GitHubTransportError("GitHub proxy output stalled.") + if not selector.select(0.2): + continue + try: + count = connection.send(data, socket.MSG_DONTWAIT) + except BlockingIOError: + continue + if count == 0: + raise GitHubTransportError("GitHub proxy output disconnected.") + data = data[count:] + + +def relay_git(path: Path, request: GitHubRequest, *, stdin: BinaryIO, + stdout: BinaryIO, stderr: BinaryIO) -> int: + """Relay binary Git data, separate diagnostics and the host SSH exit status.""" + payload = request.encode() + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(10) + connection.connect(str(path)) + connection.sendall(struct.pack("!I", len(payload)) + payload) + connection.settimeout(None) + + def send_input() -> None: + try: + while chunk := os.read(stdin.fileno(), MAX_FRAME): + connection.sendall(chunk) + connection.shutdown(socket.SHUT_WR) + except (OSError, ValueError): + pass # The server can finish before Git closes stdin. + + threading.Thread(target=send_input, daemon=True, name="github-git-input").start() + while True: + kind = _read_exact(connection, 1) + length = struct.unpack("!I", _read_exact(connection, 4))[0] + if length > MAX_FRAME: + raise GitHubTransportError("GitHub proxy frame exceeds its size limit.") + data = _read_exact(connection, length) + if kind == b"X" and length == 4: + return min(struct.unpack("!I", data)[0], 255) + if kind not in {b"O", b"E"}: + raise GitHubTransportError("Unknown GitHub proxy response.") + output = stdout if kind == b"O" else stderr + output.write(data) + output.flush() + + +@contextmanager +def github_ssh_bridge(environment: Mapping[str, str], *, enabled: bool, + containerized: bool, binds_unix_sockets: bool) -> Iterator[Path | None]: + """Create an authorized physical-host broker, or reuse an inherited one. + + Docker Desktop needs a separate transport. An explicit enable there fails + with an explanation instead of silently mounting keys or exposing an agent. + """ + if not enabled: + yield None + return + if containerized: + inherited = environment.get(SOCKET_ENV) + if inherited != str(SOCKET_DESTINATION) or not SOCKET_DESTINATION.is_socket(): + raise GitHubTransportError("GitHub proxy unavailable: authorize github-ssh on the outside launcher first.") + yield SOCKET_DESTINATION + return + if not binds_unix_sockets: + raise GitHubTransportError("github-ssh currently requires Linux/WSL with native Docker; Docker Desktop transport is not implemented.") + executable = shutil.which("ssh", path=environment.get("PATH", os.defpath)) + if executable is None: + raise GitHubTransportError("github-ssh requires OpenSSH on the host.") + with tempfile.TemporaryDirectory(prefix="devcapsule-github-", dir="/tmp") as directory: + path = Path(directory) / "ssh.sock" + with GitHubBroker(path, ssh=executable, environment=environment): + yield path diff --git a/devcapsule-src/devcapsule/github_transport.py b/devcapsule-src/devcapsule/github_transport.py new file mode 100644 index 00000000..b53d5210 --- /dev/null +++ b/devcapsule-src/devcapsule/github_transport.py @@ -0,0 +1,102 @@ +"""GitHub-only Git SSH requests: validate values before any host process starts. + +The capsule selects a Git service and repository, never a host command or SSH +option. Host OpenSSH owns key selection, host verification and authentication. +""" +from __future__ import annotations + +from dataclasses import dataclass +import json +import re +import shlex +from typing import Sequence + +__all__ = ["GitHubRequest", "GitHubTransportError", "parse_git_ssh", "decode_request"] + + +class GitHubTransportError(ValueError): + """A request cannot cross the authorized GitHub boundary.""" + + +@dataclass(frozen=True) +class GitHubRequest: + service: str + repository: str + + def validate(self) -> None: + """Require a GitHub repository and one of Git's two transfer services.""" + if self.service not in {"git-upload-pack", "git-receive-pack"}: + raise GitHubTransportError("Only Git fetch and push services are allowed.") + if (len(self.repository) > 256 or + not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9-]*/[A-Za-z0-9_.-]+", self.repository) or + self.repository.split("/")[-1] in {".", ".."}): + raise GitHubTransportError("Expected a GitHub OWNER/REPOSITORY path.") + + def encode(self) -> bytes: + """Serialize a validated request; no host paths or credentials cross.""" + self.validate() + return json.dumps({"version": 1, "service": self.service, + "repository": self.repository}, separators=(",", ":")).encode() + + def ssh_command(self, executable: str) -> tuple[str, ...]: + """Use host SSH configuration with noninteractive, verified authentication.""" + self.validate() + return (executable, "-T", "-oBatchMode=yes", "-oStrictHostKeyChecking=yes", + "-oClearAllForwardings=yes", "-oForwardAgent=no", "-oPermitLocalCommand=no", + "-oControlMaster=no", "-oControlPath=none", "-oRemoteCommand=none", + "git@github.com", f"{self.service} '{self.repository}'") + + +def decode_request(payload: bytes) -> GitHubRequest: + """Validate the entire wire request, including field types and version.""" + try: + value = json.loads(payload) + except (ValueError, UnicodeError) as error: + raise GitHubTransportError("Invalid GitHub request JSON.") from error + if (not isinstance(value, dict) or set(value) != {"version", "service", "repository"} or + type(value["version"]) is not int or value["version"] != 1 or + not isinstance(value["service"], str) or not isinstance(value["repository"], str)): + raise GitHubTransportError("Unsupported GitHub request.") + request = GitHubRequest(value["service"], value["repository"]) + request.validate() + return request + + +def parse_git_ssh(arguments: Sequence[str]) -> GitHubRequest | None: + """Parse Git's OpenSSH invocation; other hosts use the capsule's own SSH. + + Git's protocol-v2 SendEnv request is accepted, but the broker deliberately + negotiates protocol v0 rather than accepting arbitrary client environment. + An explicit port is accepted only when it is GitHub's standard SSH port. + Host SSH configuration can select ssh.github.com:443 as usual. + """ + args = list(arguments) + port: str | None = None + while args and args[0].startswith("-"): + option = args.pop(0) + if option in {"-o", "-p"} and args: + value = args.pop(0) + if option == "-p": + port = value + elif value != "SendEnv=GIT_PROTOCOL": + raise GitHubTransportError("Unsupported SSH option for the GitHub proxy.") + elif option == "-oSendEnv=GIT_PROTOCOL": + continue + else: + raise GitHubTransportError("Unsupported SSH option for the GitHub proxy.") + if not args: + raise GitHubTransportError("Missing SSH destination.") + destination = args.pop(0) + if destination not in {"github.com", "git@github.com"}: + return None + if port not in {None, "22"} or len(args) != 1: + raise GitHubTransportError("Expected GitHub's SSH destination and one Git service.") + try: + command = shlex.split(args[0]) + except ValueError as error: + raise GitHubTransportError("Malformed Git service command.") from error + if len(command) != 2: + raise GitHubTransportError("Expected a Git service and repository only.") + request = GitHubRequest(command[0], command[1].removeprefix("/")) + request.validate() + return request diff --git a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py index 97bfdb90..283c5ac6 100644 --- a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py +++ b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py @@ -61,12 +61,18 @@ ) from ...host_daemon import ( current_container, + in_container, daemon_architecture, docker_socket, is_host_backed, requires_translation, translate_bind_sources, ) +from ...github_proxy import ( + SOCKET_ENV as GITHUB_SOCKET_ENV, SOCKET_DESTINATION as GITHUB_SOCKET_DESTINATION, + GIT_SSH_COMMAND, github_ssh_bridge, +) +from ...github_transport import GitHubTransportError from ...host_open import ( HOST_OPEN_BROWSER, HOST_OPEN_INTEGRATION, @@ -228,6 +234,7 @@ class PycharmRunOptions: additional_environment: dict[str, str] = field(default_factory=dict) secret_environment: tuple[str, ...] = () extra_docker_args: list[str] = field(default_factory=list) + enable_github_ssh: bool = False enable_host_browser: bool = False # How the surface gets a display. ``host-x11`` is the historical # passthrough (host X socket and session credential bound into the @@ -284,6 +291,7 @@ class PycharmRunConfig: daemon_architecture: str cpus: int | None = None launch_configuration: LaunchConfiguration | None = None + github_socket: Path | None = None host_browser_socket: Path | None = None display_transport: str = HOST_X11_DISPLAY_TRANSPORT # Contained display only: the host loopback port Docker publishes (or the @@ -331,11 +339,18 @@ def run_pycharm(options: PycharmRunOptions, env: Mapping[str, str] | None = None with host_open_bridge( runtime_env, enabled=options.enable_host_browser, - ) as browser_socket: + ) as browser_socket, github_ssh_bridge( + runtime_env, enabled=options.enable_github_ssh, + containerized=in_container(), binds_unix_sockets=HostOs.current().binds_unix_sockets, + ) as github_socket: if browser_socket is None: runtime_env.pop(HOST_OPEN_SOCKET_ENV, None) else: runtime_env[HOST_OPEN_SOCKET_ENV] = str(browser_socket) + if github_socket is None: + runtime_env.pop(GITHUB_SOCKET_ENV, None) + else: + runtime_env[GITHUB_SOCKET_ENV] = str(github_socket) config = build_run_config(options, runtime_env) begin_session_logs(config, runtime_env) prepare_home_mount_points(config) @@ -427,7 +442,7 @@ def run_pycharm(options: PycharmRunOptions, env: Mapping[str, str] | None = None finally: remove_sudoers_policy_volume(files.sudoers_volume, runtime_env) cleanup_temp_runtime_files(files) - except HostOpenError as exc: + except (HostOpenError, GitHubTransportError) as exc: raise PycharmRunError(str(exc)) from exc @@ -1045,6 +1060,14 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych raise PycharmRunError("The authorized host-browser socket is absent or unsafe.") host_browser_socket = candidate + github_socket: Path | None = None + configured_github_socket = env.get(GITHUB_SOCKET_ENV) + if options.enable_github_ssh and configured_github_socket: + candidate = Path(configured_github_socket) + if not candidate.is_absolute() or ".." in candidate.parts or not is_socket(candidate): + raise PycharmRunError("The authorized GitHub socket is absent or unsafe.") + github_socket = candidate + selected_runtime_plan = options.runtime_plan if host_browser_socket is not None and selected_runtime_plan is not None: selected_runtime_plan = selected_runtime_plan.with_host_integration( @@ -1120,6 +1143,9 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych raise PycharmRunError( "Component runtime metadata conflicts with the host-browser integration." ) + if github_socket is not None and any(name in fixed_environment or name in options.additional_environment + for name in (GITHUB_SOCKET_ENV, "GIT_SSH_COMMAND", "GIT_SSH_VARIANT")): + raise PycharmRunError("Component environment conflicts with the GitHub SSH integration.") additional_environment: list[tuple[str, str]] = [] for name, value in sorted(options.additional_environment.items()): if not name.isascii() or not name.isidentifier() or name.upper() != name: @@ -1187,6 +1213,7 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych launch_configuration=options.launch_configuration, use_image_process=options.use_image_process, host_browser_socket=host_browser_socket, + github_socket=github_socket, display_transport=options.display_transport, display_host_port=display_host_port, display_token=display_token, @@ -1417,6 +1444,10 @@ def build_docker_args( ), ] ) + if config.github_socket is not None: + args.extend(["--env", f"{GITHUB_SOCKET_ENV}={GITHUB_SOCKET_DESTINATION}", + "--env", f"GIT_SSH_COMMAND={GIT_SSH_COMMAND}", "--env", "GIT_SSH_VARIANT=ssh", + "--mount", f"type=bind,src={config.github_socket},dst={GITHUB_SOCKET_DESTINATION},ro"]) for name, value in config.additional_environment: args.extend(["--env", f"{name}={value}"]) for name in config.secret_environment: @@ -2261,6 +2292,8 @@ def print_storage_summary(config: PycharmRunConfig) -> None: Host browser integration: Enabled through a URL-only HTTP(S) broker. Any process running as the capsule user can ask the physical host to navigate its default browser.""" + if config.github_socket is not None: + host_browser_disclosure += "\n GitHub SSH: host-authenticated Git fetch/push proxy (all repositories your host identity can access)." host_browser_disclosure += display_disclosure(config) + console_disclosure(config) if config.interactive_state_mounts: component_id = ( diff --git a/devcapsule-src/tests/integration/test_github_proxy.py b/devcapsule-src/tests/integration/test_github_proxy.py new file mode 100644 index 00000000..a8abf31b --- /dev/null +++ b/devcapsule-src/tests/integration/test_github_proxy.py @@ -0,0 +1,161 @@ +"""GitHub proxy process and socket integration; no GitHub account or network is used. + +A fixture SSH executable stands in for the remote SSH service. Git itself and +its upload/receive services are real, as are the proxy, pipes and command line. +""" +from __future__ import annotations + +from contextlib import contextmanager +import os +from pathlib import Path +import shlex +import socket +import struct +import subprocess +import sys +import tempfile +from collections.abc import Iterator + +import pytest + +from devcapsule.github_proxy import GitHubBroker, MAX_REQUEST, SOCKET_ENV + +pytestmark = pytest.mark.integration +SOURCE = Path(__file__).resolve().parents[2] + + +def _environment(root: Path) -> dict[str, str]: + return {'PATH': os.defpath, 'HOME': str(root), 'PYTHONPATH': str(SOURCE), + 'DEVCAPSULE_CONFIG_HOME': str(root / 'config'), 'GIT_CONFIG_NOSYSTEM': '1', + 'GIT_CONFIG_GLOBAL': '/dev/null', 'GIT_AUTHOR_NAME': 'Proxy test', + 'GIT_COMMITTER_NAME': 'Proxy test', 'GIT_AUTHOR_EMAIL': 'test@example.test', + 'GIT_COMMITTER_EMAIL': 'test@example.test', 'GIT_TERMINAL_PROMPT': '0'} + + +def _program(root: Path, body: str) -> Path: + path = root / 'ssh-fixture' + path.write_text(f'#!{sys.executable}\n' + body) + path.chmod(0o700) + return path + + +@contextmanager +def _broker(root: Path, body: str, **kwargs: float) -> Iterator[tuple[Path, dict[str, str]]]: + env = _environment(root) + # A short explicit socket path works under either Unix sockaddr path limit. + with tempfile.TemporaryDirectory(prefix='gh-test-', dir='/tmp') as temporary: + path = Path(temporary) / 's' + with GitHubBroker(path, ssh=str(_program(root, body)), environment=env, **kwargs): + yield path, env + assert not path.exists() + + +def _client(path: Path, env: dict[str, str], payload: bytes = b'') -> subprocess.CompletedProcess[bytes]: + return subprocess.run([sys.executable, '-m', 'devcapsule', 'github-ssh', '--', + 'git@github.com', "git-upload-pack 'acme/repo.git'"], + env={**env, SOCKET_ENV: str(path)}, input=payload, + capture_output=True, timeout=15) + + +def test_binary_streams_and_nonzero_exit_are_preserved(tmp_path: Path) -> None: + """Large binary input, stdout, stderr and failure status survive the proxy without mixing.""" + payload = bytes(range(256)) * 2048 + program = "import sys\ndata = sys.stdin.buffer.read()\nsys.stdout.buffer.write(data)\nsys.stderr.buffer.write(b'diagnostic\\n')\nsys.exit(7)\n" + with _broker(tmp_path, program) as (path, env): + result = _client(path, env, payload) + assert result.returncode == 7 + assert result.stdout == payload + assert result.stderr == b'diagnostic\n' + + +def test_real_git_clone_fetch_and_push_through_capsule_adapter(tmp_path: Path) -> None: + """Real Git clone, fetch and push transfer repository objects through the broker.""" + env = _environment(tmp_path) + bare = tmp_path / 'remote.git' + subprocess.run(['git', 'init', '--bare', '--initial-branch=main', str(bare)], env=env, check=True, capture_output=True) + # This stand-in uses only the validated service; the repository is fixed by the fixture. + program = f"import os, shlex, sys\nservice = shlex.split(sys.argv[-1])[0]\nos.execv('/usr/bin/git', ['git', service.removeprefix('git-'), {str(bare)!r}])\n" + with _broker(tmp_path, program) as (path, _): + git_env = {**env, SOCKET_ENV: str(path), 'GIT_SSH_VARIANT': 'ssh', + 'GIT_SSH_COMMAND': shlex.join([sys.executable, '-m', 'devcapsule', 'github-ssh', '--'])} + clone = tmp_path / 'clone' + def git(*args: str) -> subprocess.CompletedProcess[bytes]: + return subprocess.run(['git', *args], env=git_env, capture_output=True, check=True, timeout=20) + git('-c', 'init.defaultBranch=main', 'clone', 'git@github.com:acme/repo.git', str(clone)) + (clone / 'binary').write_bytes(bytes(range(256)) * 1024) + git('-C', str(clone), 'add', 'binary') + git('-C', str(clone), 'commit', '-m', 'Fixture commit') + git('-C', str(clone), 'push', 'origin', 'main') + second = tmp_path / 'second' + git('clone', 'ssh://git@github.com/acme/repo.git', str(second)) + git('-C', str(second), 'fetch', 'origin') + assert (second / 'binary').read_bytes() == (clone / 'binary').read_bytes() + assert git('--git-dir', str(bare), 'rev-parse', 'main').stdout == git('-C', str(clone), 'rev-parse', 'HEAD').stdout + + +@pytest.mark.parametrize('payload', [b'{"version":1,"service":"sh","repository":"acme/repo"}', + b'{"version":1,"service":"git-upload-pack","repository":"a/b; id"}', + b'{' ]) +def test_invalid_requests_never_start_host_ssh(tmp_path: Path, payload: bytes) -> None: + """Host admission refuses hostile wire bytes even when the capsule adapter is bypassed.""" + marker = tmp_path / 'started' + with _broker(tmp_path, f"from pathlib import Path\nPath({str(marker)!r}).touch()\n") as (path, _): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + connection.connect(str(path)) + connection.sendall(struct.pack('!I', len(payload)) + payload) + assert connection.recv(1) == b'' + assert not marker.exists() + + +def test_oversized_handshake_is_refused_without_reading_body(tmp_path: Path) -> None: + """An excessive length prefix cannot allocate a body or spawn SSH.""" + with _broker(tmp_path, 'raise AssertionError("must not execute")\n') as (path, _): + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + connection.connect(str(path)) + connection.sendall(struct.pack('!I', MAX_REQUEST + 1)) + assert connection.recv(1) == b'' + + +def test_operation_deadline_stops_a_stalled_ssh_process(tmp_path: Path) -> None: + """A stalled host SSH process is killed and the capsule receives failure within the bound.""" + with _broker(tmp_path, 'import time\ntime.sleep(60)\n', operation_timeout=0.2) as (path, env): + result = _client(path, env) + assert result.returncode != 0 + assert b'connection ended' in result.stderr + + +def test_broker_shutdown_closes_incomplete_requests(tmp_path: Path) -> None: + """Stopping the launcher interrupts a client that never completed its handshake.""" + with socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) as connection: + connection.settimeout(2) + with _broker(tmp_path, 'raise AssertionError("must not execute")\n') as (path, _): + connection.connect(str(path)) + connection.sendall(b'\x00') + try: + assert connection.recv(1) == b'' + except ConnectionResetError: + pass # Closing with unread input may report reset instead of EOF. + + +@pytest.mark.parametrize('enabled', [False, True]) +def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled: bool) -> None: + """The Docker plan carries the Git adapter and broker mount only after explicit authorization.""" + from devcapsule.launch.pycharm._launcher import PycharmRunOptions, DockerMode, TempRuntimeFiles, build_run_config, build_docker_args + from devcapsule.platforms import Platform + from devcapsule.github_proxy import GIT_SSH_COMMAND, SOCKET_DESTINATION + + project = tmp_path / 'project' + project.mkdir() + with _broker(tmp_path, 'raise AssertionError("no transfer expected")\n') as (path, env): + env = {**env, 'DISPLAY': ':1', 'PYCHARM_GIT_IDENTITY_FROM_HOST': '0', SOCKET_ENV: str(path)} + config = build_run_config(PycharmRunOptions(project=project, capsule_platform=Platform.LINUX_AMD64, + docker_mode=DockerMode.none, enable_sudo=False, network_mode='bridge', + enable_github_ssh=enabled), env) + files = TempRuntimeFiles(xauth_file=tmp_path / 'xauth', passwd_file=tmp_path / 'passwd', group_file=tmp_path / 'group') + args = build_docker_args(config, files, env) + assert (f'GIT_SSH_COMMAND={GIT_SSH_COMMAND}' in args) is enabled + assert (f'{SOCKET_ENV}={SOCKET_DESTINATION}' in args) is enabled + assert (f'type=bind,src={path},dst={SOCKET_DESTINATION},ro' in args) is enabled + assert not any('.ssh' in arg or 'SSH_AUTH_SOCK=' in arg for arg in args) diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py index 92f49bda..7a8e5c0f 100644 --- a/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_authorization.py @@ -67,3 +67,11 @@ def test_a_lock_without_a_base_declares_no_base_image_decision() -> None: """Precondition: a lock with no base table. Action: build the declarations. Postcondition: there is no base-image declaration to answer.""" assert "base-image" not in authorization_declarations({}, {"components": {}}) + + +def test_github_access_is_available_but_never_recommended_by_omission() -> None: + """A project without recommendations exposes the optional GitHub decision without granting it.""" + declaration = authorization_declarations({}, {"components": {}})['github-ssh'] + assert declaration.recommended_value is True + assert declaration.project_recommended is False + assert declaration.required is False diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py new file mode 100644 index 00000000..638d9e26 --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_review.py @@ -0,0 +1,20 @@ +"""HostAccess value contracts for the GitHub transport grant.""" +from devcapsule.configuration.review import HostAccess + + +def test_github_access_is_denied_without_an_explicit_answer() -> None: + """Default host access never exposes host GitHub authentication.""" + assert HostAccess().github_ssh is False + + +def test_a_recorded_denial_overrides_an_earlier_grant() -> None: + """Overlaying a denial withdraws GitHub access without changing other host choices.""" + access = HostAccess(github_ssh=True, host_browser=True) + result = access.overlay({'github-ssh': False}) + assert result.github_ssh is False + assert result.host_browser is True + + +def test_a_run_once_answer_enables_github_access() -> None: + """An explicit boolean grant becomes the launcher's effective GitHub access.""" + assert HostAccess().overlay({'github-ssh': True}).github_ssh is True diff --git a/devcapsule-src/tests/unit/devcapsule/test_github_transport.py b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py new file mode 100644 index 00000000..695e8997 --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py @@ -0,0 +1,68 @@ +"""Contracts of GitHub request admission, independent of host SSH or credentials.""" +import json + +import pytest + +from devcapsule.github_transport import GitHubRequest, GitHubTransportError, decode_request, parse_git_ssh + + +@pytest.mark.parametrize('command,service', [("git-upload-pack 'acme/widgets.git'", 'git-upload-pack'), + ("git-receive-pack '/acme/widgets'", 'git-receive-pack')]) +def test_git_arguments_select_only_the_service_and_repository(command: str, service: str) -> None: + """Git's destination and v2 request admit a GitHub transfer with no SSH option forwarding.""" + request = parse_git_ssh(['-o', 'SendEnv=GIT_PROTOCOL', 'git@github.com', command]) + assert request == GitHubRequest(service, 'acme/widgets.git' if service == 'git-upload-pack' else 'acme/widgets') + + +@pytest.mark.parametrize('command', ["sh 'acme/repo'", "git-upload-pack 'acme/repo'; id", "git-upload-pack 'a/../b'", + "git-upload-pack '-oProxyCommand=whoami'", "git-upload-pack 'acme/x\nwhoami'", + "git-upload-pack 'acme/..'", "git-upload-pack 'acme/$(whoami)'", + "git-upload-pack 'acme/repo", "git-upload-pack 'acme/repo' extra"]) +def test_untrusted_git_commands_are_refused(command: str) -> None: + """A capsule cannot select a shell command, traversal or option-shaped repository.""" + with pytest.raises(GitHubTransportError): + parse_git_ssh(['git@github.com', command]) + + +@pytest.mark.parametrize('arguments', [['-o', 'ProxyCommand=id', 'git@github.com', "git-upload-pack 'acme/repo'"], + ['-p', '2222', 'git@github.com', "git-upload-pack 'acme/repo'"], + ['-A', 'git@github.com'], []]) +def test_capsule_ssh_options_cannot_change_host_execution(arguments: list[str]) -> None: + """Unsupported options and destinations never become host SSH arguments.""" + with pytest.raises(GitHubTransportError): + parse_git_ssh(arguments) + + +def test_other_git_hosts_are_not_forwarded_to_the_host() -> None: + """A different destination selects the capsule's ordinary SSH fallback.""" + assert parse_git_ssh(['git@example.test', "git-upload-pack 'acme/repo'"]) is None + + +@pytest.mark.parametrize('document', [{}, {'version': True, 'service': 'git-upload-pack', 'repository': 'a/b'}, + {'version': 1, 'service': ['git-upload-pack'], 'repository': 'a/b'}, + {'version': 1, 'service': 'git-upload-pack', 'repository': 'a/b', 'host': 'evil'}, + {'version': 1, 'service': 'sh', 'repository': 'a/b'}, + {'version': 1, 'service': 'git-upload-pack', 'repository': 'a/../b'}]) +def test_wire_validation_does_not_trust_the_client_parser(document: object) -> None: + """The host independently rejects malformed, extended and hostile requests.""" + with pytest.raises(GitHubTransportError): + decode_request(json.dumps(document).encode()) + + +def test_host_command_keeps_verification_and_disables_forwarding() -> None: + """The host command preserves SSH identity selection but exposes no general SSH service.""" + command = GitHubRequest('git-upload-pack', 'acme/repo.git').ssh_command('/usr/bin/ssh') + assert command[0] == '/usr/bin/ssh' + assert command[-2:] == ('git@github.com', "git-upload-pack 'acme/repo.git'") + assert '-oStrictHostKeyChecking=yes' in command + assert '-oBatchMode=yes' in command + assert '-oForwardAgent=no' in command + assert '-oClearAllForwardings=yes' in command + assert '-oPermitLocalCommand=no' in command + assert '-oControlPath=none' in command + + +def test_valid_wire_request_roundtrips() -> None: + """Encoding a valid request carries exactly the service and repository contract.""" + request = GitHubRequest('git-receive-pack', 'acme/widgets.git') + assert decode_request(request.encode()) == request diff --git a/docs/containment/the-boundary.md b/docs/containment/the-boundary.md index be0275c9..aba695d5 100644 --- a/docs/containment/the-boundary.md +++ b/docs/containment/the-boundary.md @@ -2,7 +2,7 @@ description: What a capsule and its coding agents can reach on your computer by default, what stays isolated, and which permissions you can grant or withdraw. role: containment weight: 1 -updated: 2026-10-30 +updated: 2026-11-16 --- # The boundary @@ -59,13 +59,14 @@ access](granting-and-withdrawing.md) for the commands. | `network host` | Shares your host network namespace: host `localhost` services and capsule ports become one namespace | Your app needs direct access to host-bound services or dynamic local ports ([Services and ports](../your-project/services-and-ports.md)) | | `docker-daemon host-socket` | Mounts `/var/run/docker.sock` into the capsule | Your build or tests start containers; this grants broad control of the daemon and its host bind mounts | | `development-sudo true` | Grants passwordless `sudo` **inside the capsule container** | You need to install temporary OS packages inside a session | +| `github-ssh true` | Lets capsule Git fetch and push through host OpenSSH to every GitHub repository the host identity can access; private keys stay outside | You want GitHub SSH remotes to work inside the capsule ([Git](../everyday-development/git.md)); Linux/WSL with native Docker only | | `host-browser true` | Forwards `http://` and `https://` URLs opened inside the capsule to your host browser through a URL-only broker | OAuth sign-ins and external documentation links are more convenient on your host browser | | `host-x11 true` | Runs the IDE directly on your host X11 display instead of the contained desktop | Almost never—X11 lets clients read keystrokes, clipboard, and windows across your entire host display session; manifests cannot recommend it | | `vscode-download`, `claude-code-download`, `antigravity-download` (`true`) | Permits DevCapsule to download that vendor's pinned artifact under its license terms (when not already covered by home channel trust) | You selected that IDE or agent and accept its vendor terms ([Vendor software and trust](../configuration/software-approvals.md)) | | `base-image` (`default` or ``) | Approves the lock's pinned base digest for this checkout or selects an inspected [local base image](../configuration/local-base.md) | Settled once via home `devcapsule-base` trust or per-checkout consent | Authorizations take effect on the **next** launch; they do not change a running -container. The five host-access nodes can also be granted for one run with +container. The host-access nodes can also be granted for one run with `project run --authorize ...`. Base and vendor consents require saved approval or home channel trust. A Git repository cannot grant itself an authorization. diff --git a/docs/everyday-development/git.md b/docs/everyday-development/git.md index 0604144b..a7c6b4c9 100644 --- a/docs/everyday-development/git.md +++ b/docs/everyday-development/git.md @@ -1,7 +1,7 @@ --- description: Committing, pushing and pulling from inside the capsule, and how credentials reach Git without your host keys entering the capsule. weight: 2 -updated: 2026-10-30 +updated: 2026-11-16 --- # Git @@ -57,7 +57,45 @@ This avoids giving the capsule remote credentials. Review commits before pushing: processes inside the capsule can change the working tree and local Git history. -### 2. Forward your host SSH agent for a session +### 2. Use the launcher’s GitHub SSH proxy + +On Linux or WSL with native Docker, authorize the proxy once from your host: + +```bash +devcapsule project config authorize github-ssh true +devcapsule project run +``` + +For a single run, use `devcapsule project run --authorize github-ssh true`. +Inside the capsule, ordinary `git clone`, `git fetch`, `git pull`, and `git push` +with `git@github.com:OWNER/REPO.git` or `ssh://git@github.com/OWNER/REPO.git` use +OpenSSH on your host. The host's SSH configuration selects the identity and +verifies GitHub's host key. Neither private keys nor the host's agent socket are +mounted into the capsule. Other Git SSH hosts use the capsule's own SSH. + +The host must already trust GitHub's host key and be able to authenticate +without a prompt. Unlock encrypted keys on the host with `ssh-add` before +launching. Hardware keys may still require their normal touch. The proxy never +asks the capsule for a host key passphrase. GitHub connection failures and SSH +diagnostics appear in the capsule's Git output. + +The grant lets every process in this capsule fetch and push **all GitHub +repositories accessible to that host identity**. It is not a per-repository +grant. The proxy accepts only Git transfer commands, not general host commands. +It lives with the outside launcher; stopping that launcher closes the proxy +and cancels active transfers. A later launch creates a new proxy. + +Withdraw the saved grant with `devcapsule project config deny github-ssh` and +relaunch. Configuration changes apply to the next launch, not the running one. + +This first implementation does not provide HTTPS credentials, `gh` API login, +SSH commit signing, arbitrary GitHub SSH aliases, or a general `ssh -T` command. +Git LFS's separate authentication protocol is not supported. Docker Desktop on +macOS needs another transport and reports that limitation when enabled. +An inner capsule launch can reuse an authorized outer proxy; it cannot create +one that reads credentials on the physical host. + +### 3. Forward your host SSH agent for a session When you want `git push` and `git pull` over SSH inside the capsule without copying private key files into the container, run this from your host terminal. @@ -80,7 +118,7 @@ Keep two details in mind: agent to sign challenges while the session is open. Load only the key you need or require per-use confirmation (`ssh-add -c`) on your host. -### 3. Use a repository-scoped credential per checkout +### 4. Use a repository-scoped credential per checkout If a workflow inside the capsule must push or call forge APIs unattended, create a **fine-grained HTTPS personal access token** or **single-repository SSH deploy diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 2286b890..44a9325e 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -6,6 +6,8 @@ Start date: 2026-11-16 State: active; first implementation slice. +Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 + Branch association: `ws-github-auth/ssh-proxy`, from canonical `upstream/main` at `1942f396`. Integration target: `main` @@ -30,6 +32,24 @@ The branch starts at fetched canonical main: no synchronization is required now. No open bug is owned by this newly registered workstream. Mail and intake are empty. The new feature belongs here, not in the maintenance release. +## Implementation and evidence + +The initial Linux/WSL implementation includes the GitHub-only request validator, +launcher-owned Unix socket broker, capsule Git adapter, optional `github-ssh` +authorization, launch mount/environment wiring and user documentation. It does not +copy private keys or forward the host agent. The socket grants all GitHub Git +repositories available to the host identity. Docker Desktop transport is deferred +and explicitly refused. GitHub aliases, Git LFS authentication, SSH signing and +HTTPS/API credentials are not implemented in this slice. + +Focused validation: 42 tests pass, including real Git clone/fetch/push through a +fixture SSH process, binary streaming, refusal, deadline, shutdown and authorized +Docker arguments. Initial mypy over the touched import graph passed (157 files). +Full build gate is running. Its first attempt failed in pytest setup because +`/opt` is read-only and `/opt/devcapsule-gate` does not exist. Scratch now uses +`/home/devcapsule/.cache/devcapsule-gate/github-auth` per the local rule allowing +another directory with space. No Docker containers or real credential files changed. + ## Planned next step Implement a constrained GitHub Git transport broker, capsule adapter, explicit @@ -47,5 +67,6 @@ paths, then run Pair Review with Gemini 4 Hydrogen High. ## Workstream document index +- [Review brief](evidence/review-brief.md) - [Intake](intake/README.md) - [Intake decisions](intake-dispositions.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md new file mode 100644 index 00000000..a5d2f5d8 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-brief.md @@ -0,0 +1,87 @@ +# Review brief: GitHub SSH proxy, Gemini 4 Hydrogen High + +Implementor: Codex, GPT-6. Review partner: Antigravity, gemini-4-hydrogen-high. +The owner requested this feature with Pair Review and selected the github-auth +workstream and Git over SSH first. Review correctness, clarity and meaningful +coverage of the implementation against upstream/main. + +## Checkout and first reads + +Work only in /home/devcapsule/worktrees/github-auth-review on +ws-github-auth/ssh-proxy-review. Do not edit the implementor checkout. +Read AGENTS.md, WORKFLOW.md topic 9.11, WORKFLOW-LOCAL.md, this brief, +engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md, then +`git diff upstream/main...HEAD`. Read the contracts in github_transport.py and +github_proxy.py before judging implementations. This brief is your only context. + +## Scope and questions + +Ordinary GitHub clone/fetch/push from a Linux/WSL capsule uses host-side OpenSSH +through a private launcher-owned Unix socket. No private keys or host agent socket +enter the capsule. Only fixed GitHub upload-pack/receive-pack requests are allowed. +The authorization is optional and denied by default; grant through host config or +run-once. All repositories accessible by the host identity are within the grant. +Host key checking remains strict; keys must authenticate noninteractively on the +host. Other Git hosts fall back to the capsule SSH. Mac Docker Desktop, HTTPS, gh, +Git LFS auth, SSH signing, arbitrary aliases are outside the first slice. + +Review in order: +1. Correctness: injection/destination escape, request validation independent of + the client, socket permission boundary, lifecycle cleanup, cancellation, + bounded memory/concurrency, deadlocks/half-close/exit status, Docker environment + integration and denial. Never weaken host-key checking to make a test pass. +2. Clarity: remove misleading guarantees or unnecessary machinery. +3. Coverage: new tests must establish independent contracts, especially auth + gating, real binary streaming, failure/exit handling and process cleanup. +4. Follow Unit Tests rules 1-4 in WORKFLOW-LOCAL.md. New unit tests live in the + module mirror, state their contract in a docstring, and construct inputs. + For each, identify possible failures not caused by a product bug. Do not edit + tests in legacy files. Add integration tests for subprocess/network effects. +5. Prefer explicit host inputs and pure decisions. No monkeypatch/unittest.mock + in new tests. Code runs no invented cryptography; authentication is OpenSSH. + +Keep corrections within this feature. Do not repair unrelated legacy tests or +expand to macOS, HTTPS, repository allowlists or generic SSH forwarding. + +## Validation + +Create your own environment exactly as the local workflow says, from devcapsule-src: +python3.12 -m venv .venv +.venv/bin/python -m pip install -r dev-requirements.txt +.venv/bin/python -m pip install -e . --no-deps + +Use scratch under /home/devcapsule/.cache/devcapsule-gate/github-auth-review; +/opt is read-only. The parent cache directory exists. +Initialize website with `git submodule update --init website` for the docs gate. +Run focused tests: +.venv/bin/python -m pytest -q --no-cov -o addopts='' tests/unit/devcapsule/test_github_transport.py tests/unit/devcapsule/configuration/test_authorization.py tests/unit/devcapsule/configuration/test_review.py tests/integration/test_github_proxy.py + +Run full gate synchronously, not a background tool job: +PYTEST_ADDOPTS='--basetemp=/home/devcapsule/.cache/devcapsule-gate/github-auth-review' DEVCAPSULE_DIFF_BASE=upstream/main .venv/bin/python -m nox -s build + +The implementor's focused run passed 42 tests. Its full gate may expose existing +mainline legacy test failures in this capsule. Report exact failures; do not fix +unrelated code or disable tests. Do not build images or launch Docker capsules. + +## Delivery and forbidden actions + +The GitHub CLI is ABSENT at /opt/xtras/miniconda3/envs/gh/bin/gh. Local policy +requires the human to create PRs in this case. Do not use another credential or +GitHub interface. There is not yet a PRn; PR-mediated Pair Review completion is +pending owner delivery. Perform the local code review now, preserve your findings +and corrections, and report this delivery blocker accurately. Do not claim a +completed PR review loop. Push your review branch over ordinary upstream SSH +only after checks; no push to main. Provide a compare URL if changes exist. + +Do not edit status files, intake/dispositions, evidence, root CURRENT-STATUS.md, +index.md, or run workflow publish. Do not change website gitlinks or configuration +manifests. Do not merge, rebase, force-push, create releases, inspect private keys, +read credential contents, use real GitHub repositories in tests, or run containers. +Do not modify the implementor's files or state. Do not add broad test-only seams. + +Commit corrections with the existing human author and this trailer: +Co-authored-by: Gemini 4 Hydrogen High Antigravity + +Report findings (what/where/fix/evidence), things considered and left alone, +unimplemented proposals, exact validation counts and limits. If no changes, say +so. Print REVIEW_COMPLETE as your last line, or REVIEW_BLOCKED with the reason. From f1dbb3387b2152293d77fce9ea2bc87fe880b579 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 03:41:22 +0000 Subject: [PATCH 11/35] Refurbish node registry tests for the GitHub authorization Preserve existing registry contracts in the module mirror and cover the optional github-ssh node. All 20 cases pass; no test-convention breaches. Co-authored-by: GPT-6 Codex --- .../tests/configuration/test_nodes.py | 170 ------------------ .../devcapsule/configuration/test_nodes.py | 166 +++++++++++++++++ 2 files changed, 166 insertions(+), 170 deletions(-) delete mode 100644 devcapsule-src/tests/configuration/test_nodes.py create mode 100644 devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py diff --git a/devcapsule-src/tests/configuration/test_nodes.py b/devcapsule-src/tests/configuration/test_nodes.py deleted file mode 100644 index 4a1ac13d..00000000 --- a/devcapsule-src/tests/configuration/test_nodes.py +++ /dev/null @@ -1,170 +0,0 @@ -"""Focused tests for the configuration-node registry.""" - -from __future__ import annotations - -import pytest - -from devcapsule.configuration.nodes import ( - CARRIER_FAMILY_AUTHORIZE, - CARRIER_FAMILY_BIND, - CARRIER_FAMILY_SET, - PROVIDER_HOST_DIRECTORY, - PROVIDER_HOST_ENVIRONMENT, - build_node_registry, -) -from devcapsule.configuration.file_formats import ( - ProjectConfigurationError, -) - -pytestmark = pytest.mark.legacy - - -def example_manifest() -> dict: - return { - "devcapsule-schema-version": 1, - "capabilities": {"need": ["python", "python-ide", "codex-agent"]}, - "project": { - "name": "Example", - "slug": "example", - "creator": "https://github.com/example", - "mount": "/workspace/example", - }, - "configuration": { - "values": { - "runtime.memory-limit": { - "type": "memory-size", - "runtime-effect": "docker.memory-limit", - "description": "Hard memory limit for the project container.", - } - } - }, - "host": { - "docker": { - "mode": { - "recommended": { - "value": "host-socket", - "justification": "Required for the full test suite.", - } - } - } - }, - } - - -def example_lock() -> dict: - return { - "devcapsule-lock-format-version": 1, - "platform": "linux-amd64", - "base": { - "reference": "docker.io/mycodespaceai/devcapsule-base@sha256:" + "6" * 64, - }, - "components": { - "interactive-surface": "pycharm", - "pycharm": { - "version": "2026.2.0.1", - "variant": "professional", - "delivery-policy": "local-materialization", - "url": "https://example.test/pycharm.tar.gz", - "sha256": "a" * 64, - }, - "codex": { - "version": "0.145.0", - "delivery-policy": "local-materialization", - }, - }, - } - - -def test_registry_assigns_each_declaration_kind_to_its_family() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - - value = registry.node("runtime.memory-limit") - assert value.family == CARRIER_FAMILY_SET - assert value.required is False - - home = registry.node("home") - assert home.family == CARRIER_FAMILY_BIND - assert home.providers == (PROVIDER_HOST_DIRECTORY,) - - slot = registry.node("pycharm/system") - assert slot.family == CARRIER_FAMILY_BIND - - secret = registry.node("codex/openai-api-key") - assert secret.family == CARRIER_FAMILY_BIND - assert secret.providers == (PROVIDER_HOST_ENVIRONMENT,) - assert secret.required is False - - base = registry.node("base-image") - assert base.family == CARRIER_FAMILY_AUTHORIZE - assert base.required is True - assert base.accepts_justification is False - - docker = registry.node("docker-daemon") - assert docker.family == CARRIER_FAMILY_AUTHORIZE - assert docker.required is False - assert docker.accepts_justification is True - assert docker.description == "Required for the full test suite." - - -def test_registry_names_are_sorted_and_family_filtered() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - names = registry.names() - assert names == tuple(sorted(names)) - authorize = registry.family(CARRIER_FAMILY_AUTHORIZE) - # docker-daemon comes from the manifest recommendation; network, - # development-sudo, host-browser and host-x11 exist as - # home-capability defaults, network since 2026-10-22. - assert {node.name for node in authorize} == { - "base-image", - "docker-daemon", - "network", - "development-sudo", - "host-browser", - "host-x11", - } - - -def test_unknown_node_lists_the_declared_vocabulary() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError) as failure: - registry.node("does-not-exist") - assert "'does-not-exist'" in str(failure.value) - assert "runtime.memory-limit" in str(failure.value) - - -def test_wrong_carrier_family_names_the_right_spelling() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError, match="answered with --bind, not --set"): - registry.answerable("home", CARRIER_FAMILY_SET) - - -def test_bind_values_split_on_the_first_colon_only() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - provider, value = registry.split_bind_value("home", "host-directory:/state/x:y") - assert provider == PROVIDER_HOST_DIRECTORY - assert value == "/state/x:y" - - provider, value = registry.split_bind_value( - "codex/openai-api-key", "host-environment:OPENAI_API_KEY" - ) - assert provider == PROVIDER_HOST_ENVIRONMENT - assert value == "OPENAI_API_KEY" - - -def test_bind_value_rejects_wrong_or_missing_provider() -> None: - registry = build_node_registry(example_manifest(), example_lock()) - with pytest.raises(ProjectConfigurationError, match="host-environment:VALUE"): - registry.split_bind_value("codex/openai-api-key", "host-directory:/somewhere") - with pytest.raises(ProjectConfigurationError, match="host-directory:VALUE"): - registry.split_bind_value("home", "/no/provider/prefix") - - -def test_a_name_declared_twice_fails_at_construction() -> None: - manifest = example_manifest() - # A project-declared value colliding with the component slot name 'home'. - manifest["configuration"]["values"]["home"] = { - "type": "string", - "description": "Colliding declaration.", - } - with pytest.raises(ProjectConfigurationError, match="declared twice"): - build_node_registry(manifest, example_lock()) diff --git a/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py b/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py new file mode 100644 index 00000000..33ba049e --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/configuration/test_nodes.py @@ -0,0 +1,166 @@ +"""The node registry is built from values and gives every input one address.""" +from typing import Any + +import pytest + +from devcapsule.configuration.nodes import ( + CARRIER_FAMILY_AUTHORIZE, CARRIER_FAMILY_BIND, CARRIER_FAMILY_SET, + PROVIDER_HOST_DIRECTORY, PROVIDER_HOST_ENVIRONMENT, build_node_registry, +) +from devcapsule.configuration.file_formats import ProjectConfigurationError + +def example_manifest() -> dict[str, Any]: + return { + "devcapsule-schema-version": 1, + "capabilities": {"need": ["python", "python-ide", "codex-agent"]}, + "project": { + "name": "Example", + "slug": "example", + "creator": "https://github.com/example", + "mount": "/workspace/example", + }, + "configuration": { + "values": { + "runtime.memory-limit": { + "type": "memory-size", + "runtime-effect": "docker.memory-limit", + "description": "Hard memory limit for the project container.", + } + } + }, + "host": { + "docker": { + "mode": { + "recommended": { + "value": "host-socket", + "justification": "Required for the full test suite.", + } + } + } + }, + } + + +def example_lock() -> dict[str, Any]: + return { + "devcapsule-lock-format-version": 1, + "platform": "linux-amd64", + "base": { + "reference": "docker.io/mycodespaceai/devcapsule-base@sha256:" + "6" * 64, + }, + "components": { + "interactive-surface": "pycharm", + "pycharm": { + "version": "2026.2.0.1", + "variant": "professional", + "delivery-policy": "local-materialization", + "url": "https://example.test/pycharm.tar.gz", + "sha256": "a" * 64, + }, + "codex": { + "version": "0.145.0", + "delivery-policy": "local-materialization", + }, + }, + } + + + +@pytest.mark.parametrize('name,family,required', [ + ('runtime.memory-limit', CARRIER_FAMILY_SET, False), + ('home', CARRIER_FAMILY_BIND, False), + ('pycharm/system', CARRIER_FAMILY_BIND, False), + ('codex/openai-api-key', CARRIER_FAMILY_BIND, False), + ('base-image', CARRIER_FAMILY_AUTHORIZE, True), + ('docker-daemon', CARRIER_FAMILY_AUTHORIZE, False), + ('github-ssh', CARRIER_FAMILY_AUTHORIZE, False), +]) +def test_each_declaration_has_its_carrier(name: str, family: str, required: bool) -> None: + """The registry maps each declared value, binding and authorization to its input family.""" + registry = build_node_registry(example_manifest(), example_lock()) + node = registry.node(name) + assert node.family == family + assert node.required is required + + +def test_registry_names_are_sorted() -> None: + """The registry presents deterministic sorted names independent of declaration order.""" + registry = build_node_registry(example_manifest(), example_lock()) + names = registry.names() + assert names == tuple(sorted(names)) + + +def test_authorization_family_includes_optional_github_access() -> None: + """The authorization family contains every default host boundary and the locked base.""" + registry = build_node_registry(example_manifest(), example_lock()) + nodes = registry.family(CARRIER_FAMILY_AUTHORIZE) + assert {node.name for node in nodes} == { + 'base-image', 'docker-daemon', 'network', 'development-sudo', 'host-browser', 'host-x11', 'github-ssh', + } + + +@pytest.mark.parametrize('name,providers', [('home', (PROVIDER_HOST_DIRECTORY,)), + ('codex/openai-api-key', (PROVIDER_HOST_ENVIRONMENT,))]) +def test_binding_declaration_exposes_its_allowed_providers(name: str, providers: tuple[str, ...]) -> None: + """A directory binding and an environment secret accept their own providers only.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.node(name).providers == providers + + +def test_base_consent_cannot_be_justified_by_project_advice() -> None: + """The base-image node requires an answer without project-authored justification.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.node('base-image').accepts_justification is False + + +def test_host_recommendation_carries_its_reason() -> None: + """A project recommendation exposes its own justification on the host-access node.""" + registry = build_node_registry(example_manifest(), example_lock()) + node = registry.node('docker-daemon') + assert node.accepts_justification is True + assert node.description == 'Required for the full test suite.' + + +def test_unknown_node_lists_the_declared_vocabulary() -> None: + """Unknown names are refused with the missing name and known vocabulary.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError) as failure: + registry.node('does-not-exist') + assert "'does-not-exist'" in str(failure.value) + assert 'runtime.memory-limit' in str(failure.value) + + +def test_wrong_carrier_family_names_the_right_spelling() -> None: + """A directory cannot be answered as a scalar; the diagnostic names the proper carrier.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError, match='answered with --bind, not --set'): + registry.answerable('home', CARRIER_FAMILY_SET) + + +@pytest.mark.parametrize('name,binding,expected', [ + ('home', 'host-directory:/state/x:y', (PROVIDER_HOST_DIRECTORY, '/state/x:y')), + ('codex/openai-api-key', 'host-environment:KEY', (PROVIDER_HOST_ENVIRONMENT, 'KEY')), +]) +def test_bind_values_split_on_the_first_colon_only(name: str, binding: str, expected: tuple[str, str]) -> None: + """Provider parsing retains colons inside a directory value and accepts environment names.""" + registry = build_node_registry(example_manifest(), example_lock()) + assert registry.split_bind_value(name, binding) == expected + + +@pytest.mark.parametrize('name,binding,message', [ + ('codex/openai-api-key', 'host-directory:/somewhere', 'host-environment:VALUE'), + ('home', '/no/provider/prefix', 'host-directory:VALUE'), +]) +def test_bind_value_rejects_wrong_or_missing_provider(name: str, binding: str, message: str) -> None: + """A binding with a missing or incompatible provider is refused with the expected form.""" + registry = build_node_registry(example_manifest(), example_lock()) + with pytest.raises(ProjectConfigurationError, match=message): + registry.split_bind_value(name, binding) + + +def test_a_name_declared_twice_fails_at_construction() -> None: + """A scalar declaration cannot reuse the name of a component binding.""" + manifest = example_manifest() + manifest['configuration']['values']['home'] = {'type': 'string', 'description': 'Collision.'} + with pytest.raises(ProjectConfigurationError, match='declared twice'): + build_node_registry(manifest, example_lock()) From 4ba1725f30eb4a6e7cb6f893a82ce03b2f77e543 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 03:41:56 +0000 Subject: [PATCH 12/35] Keep console authorization contract aligned with GitHub access Treat github-ssh denial as a restriction on both sides of the configuration channel. Co-authored-by: GPT-6 Codex --- devcapsule-webconsole/CONFIGURATION-CHANNEL.md | 2 +- devcapsule-webconsole/devcapsule_webconsole/configuration.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/devcapsule-webconsole/CONFIGURATION-CHANNEL.md b/devcapsule-webconsole/CONFIGURATION-CHANNEL.md index c312b93b..75fa971d 100644 --- a/devcapsule-webconsole/CONFIGURATION-CHANNEL.md +++ b/devcapsule-webconsole/CONFIGURATION-CHANNEL.md @@ -42,7 +42,7 @@ names the project. reaches: `ide`, `add`, `omit`, `restore`, `use`, `set`, `deny`, literal `env set NAME VALUE`, `env unset`, `unbind`, `trust withdraw`, `docker-args clear`. Explicit secure authorizations also apply: `network bridge|none`, `docker-daemon - none`, and `development-sudo|host-browser|host-x11 false|none`. + none`, and `development-sudo|host-browser|host-x11|github-ssh false|none`. - **Deferred** when the verb widens what the next launch reaches: `authorize`, `trust accept`, `bind`, `forget`, `docker-args set`, and `env set` forms other than an explicit literal value. `forget` can restore diff --git a/devcapsule-webconsole/devcapsule_webconsole/configuration.py b/devcapsule-webconsole/devcapsule_webconsole/configuration.py index c71801c7..f6997073 100644 --- a/devcapsule-webconsole/devcapsule_webconsole/configuration.py +++ b/devcapsule-webconsole/devcapsule_webconsole/configuration.py @@ -46,6 +46,7 @@ VERBS = ("ide", "add", "omit", "restore", "use", "set", "deny", "forget", "unbind", "env", "trust", "docker-args", "authorize", "bind") SECURE_AUTHORIZATIONS = { "network": {"bridge", "none"}, "docker-daemon": {"none"}, + "github-ssh": {"false", "none"}, "development-sudo": {"false", "none"}, "host-browser": {"false", "none"}, "host-x11": {"false", "none"}, } WIDENING = ("authorize", "bind", "forget", "trust accept", "docker-args set") From 96a867fea9246589f2e8bec445a7a0769146c809 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 03:44:52 +0000 Subject: [PATCH 13/35] Record proxy validation and pending review evidence Co-authored-by: GPT-6 Codex --- .../2026-11-16-github-auth/CURRENT-STATUS.md | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 44a9325e..3a5fa54c 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -45,22 +45,42 @@ HTTPS/API credentials are not implemented in this slice. Focused validation: 42 tests pass, including real Git clone/fetch/push through a fixture SSH process, binary streaming, refusal, deadline, shutdown and authorized Docker arguments. Initial mypy over the touched import graph passed (157 files). -Full build gate is running. Its first attempt failed in pytest setup because +The full build gate reached the unit suite: 20 failed, 2917 passed, 1 skipped, +14 xfailed and 7 xpassed. A selected rerun on an archive of unchanged +`upstream/main` reproduced 19 of those failures; the added authorization exposed +one exact-vocabulary legacy test. That file was refurbished into its proper +module mirror, preserving its contracts in 20 passing cases. The original gate +attempt failed in pytest setup because `/opt` is read-only and `/opt/devcapsule-gate` does not exist. Scratch now uses `/home/devcapsule/.cache/devcapsule-gate/github-auth` per the local rule allowing another directory with space. No Docker containers or real credential files changed. +Additional checks: CLI smoke, PEX build and smoke, and documentation contract +pass. The console gate identified a mirrored authorization-table omission; after +alignment, all 458 console tests pass. A read-only live `git ls-remote` through +both the source adapter and the built PEX returned GitHub HEAD `1942f396`. +That used the current capsule's already-installed SSH credentials on the broker +side; it is not physical-host/capsule acceptance evidence. + +The branch is pushed to `upstream/ws-github-auth/ssh-proxy`. Gemini Hydrogen High +is reviewing in `/home/devcapsule/worktrees/github-auth-review`. The first invocation +received no brief through redirected stdin. The same conversation resumed with an +explicit path to this committed brief; flags/model stayed unchanged. No credentials +were examined to repair this CLI input issue. + ## Planned next step -Implement a constrained GitHub Git transport broker, capsule adapter, explicit -host authorization, and lifecycle integration. Validate the transport and refusal -paths, then run Pair Review with Gemini 4 Hydrogen High. +Assess Gemini's review diff, incorporate accepted corrections, repeat affected +checks, and prepare owner-operated PR delivery. Full gate remains blocked by the +19 demonstrated mainline/environment failures; no test was disabled or reclassified +to hide them. ## Open Threads - Awaiting the human: none for workstream or first delivery scope; both accepted. - Engineering choices: initial transport must account for Linux/rootless Docker permissions; Docker Desktop cannot bind ordinary host Unix sockets. +- Physical-host launch acceptance and Docker Desktop transport remain open. - GitHub CLI is absent at the policy-prescribed path. PR creation/merging remains owner-operated under WORKFLOW-LOCAL.md; prepare reviewed commits and compare links. - Deliberately not preserved: no credentials or console decision tokens in Git. From 388f62a34fe373882201f9118faaf34cec61680b Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 05:57:28 +0000 Subject: [PATCH 14/35] Review corrections for GitHub SSH proxy Remove unused frame helper, separate capsule SSH fallback errors from proxy failures, clarify launcher disclosure/CLI help and public surface, and add unit/integration coverage for bridge gating and process-group cleanup. Co-authored-by: Gemini 4 Hydrogen High Antigravity --- .../devcapsule/commands/github_ssh.py | 10 +- devcapsule-src/devcapsule/commands/project.py | 2 +- .../devcapsule/configuration/review.py | 2 + devcapsule-src/devcapsule/github_proxy.py | 4 - .../devcapsule/launch/pycharm/_launcher.py | 8 +- .../tests/integration/test_github_proxy.py | 154 +++++++++++++++++- .../unit/devcapsule/test_github_proxy.py | 42 +++++ .../unit/devcapsule/test_github_transport.py | 37 ++++- 8 files changed, 239 insertions(+), 20 deletions(-) create mode 100644 devcapsule-src/tests/unit/devcapsule/test_github_proxy.py diff --git a/devcapsule-src/devcapsule/commands/github_ssh.py b/devcapsule-src/devcapsule/commands/github_ssh.py index e36fd1fc..443c0e90 100644 --- a/devcapsule-src/devcapsule/commands/github_ssh.py +++ b/devcapsule-src/devcapsule/commands/github_ssh.py @@ -23,8 +23,16 @@ class GitHubSshCommand(Command): def run(cls, arguments: argparse.Namespace, context: object | None) -> int: try: request = parse_git_ssh(arguments.ssh_arguments) - if request is None: + except GitHubTransportError as error: + raise CliError( + f"GitHub proxy: {error} Host keys must be trusted and encrypted keys unlocked on the host." + ) from error + if request is None: + try: return subprocess.run(["ssh", *arguments.ssh_arguments], check=False).returncode + except OSError as error: + raise CliError(f"Capsule SSH: {error}") from error + try: path = os.environ.get(SOCKET_ENV) if not path: raise GitHubTransportError("No authorized GitHub proxy. Enable github-ssh on the host launcher.") diff --git a/devcapsule-src/devcapsule/commands/project.py b/devcapsule-src/devcapsule/commands/project.py index 6b0b2541..b35cfa96 100644 --- a/devcapsule-src/devcapsule/commands/project.py +++ b/devcapsule-src/devcapsule/commands/project.py @@ -1276,7 +1276,7 @@ def run(cls, arguments: argparse.Namespace, context: object | None) -> int: class ConfigDenyCommand(Command): name = "deny" capsule_access: ClassVar[CapsuleAccess] = CapsuleAccess.MUTATES - help = "Deny a host access for this checkout, recorded: network, docker-daemon, development-sudo, host-browser, host-x11." + help = "Deny a host access for this checkout, recorded: network, docker-daemon, development-sudo, host-browser, host-x11, github-ssh." @classmethod def configure(cls, parser: argparse.ArgumentParser) -> None: diff --git a/devcapsule-src/devcapsule/configuration/review.py b/devcapsule-src/devcapsule/configuration/review.py index 2a14a488..823b411e 100644 --- a/devcapsule-src/devcapsule/configuration/review.py +++ b/devcapsule-src/devcapsule/configuration/review.py @@ -20,6 +20,8 @@ from .trust import HomeTrust from .values import RuntimeEffects, resolve_configuration_values +__all__ = ["HostAccess", "ConfigurationReview", "review_configuration"] + @dataclass(frozen=True) class HostAccess: diff --git a/devcapsule-src/devcapsule/github_proxy.py b/devcapsule-src/devcapsule/github_proxy.py index 83d2ba40..9939b28c 100644 --- a/devcapsule-src/devcapsule/github_proxy.py +++ b/devcapsule-src/devcapsule/github_proxy.py @@ -45,10 +45,6 @@ def _read_exact(connection: socket.socket, length: int) -> bytes: return bytes(chunks) -def _send_frame(connection: socket.socket, kind: bytes, payload: bytes) -> None: - connection.sendall(kind + struct.pack("!I", len(payload)) + payload) - - class GitHubBroker: """One session's GitHub bridge. Close cancels active children and removes its socket. diff --git a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py index 283c5ac6..0c327684 100644 --- a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py +++ b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py @@ -726,6 +726,8 @@ def describe_run_command(command: list[str], config: PycharmRunConfig, ) if config.host_browser_socket is not None: comments.append("Host-browser socket requires a live broker; a broker owned by this invocation stops on return.") + if config.github_socket is not None: + comments.append("GitHub SSH socket requires a live broker; a broker owned by this invocation stops on return.") if config.display_transport == CONTAINED_DISPLAY_TRANSPORT: comments.append("Contained display needs its temporary token file; the chosen host port is not reserved for later use.") if config.console_host_port is not None: @@ -2293,7 +2295,11 @@ def print_storage_summary(config: PycharmRunConfig) -> None: Enabled through a URL-only HTTP(S) broker. Any process running as the capsule user can ask the physical host to navigate its default browser.""" if config.github_socket is not None: - host_browser_disclosure += "\n GitHub SSH: host-authenticated Git fetch/push proxy (all repositories your host identity can access)." + host_browser_disclosure += """ + +GitHub SSH integration: + Host-authenticated Git fetch/push proxy (all repositories your host identity + can access); private keys stay on the host.""" host_browser_disclosure += display_disclosure(config) + console_disclosure(config) if config.interactive_state_mounts: component_id = ( diff --git a/devcapsule-src/tests/integration/test_github_proxy.py b/devcapsule-src/tests/integration/test_github_proxy.py index a8abf31b..f4e5a08f 100644 --- a/devcapsule-src/tests/integration/test_github_proxy.py +++ b/devcapsule-src/tests/integration/test_github_proxy.py @@ -8,13 +8,18 @@ from contextlib import contextmanager import os from pathlib import Path +import selectors import shlex +import shutil import socket +import stat import struct import subprocess import sys import tempfile +import time from collections.abc import Iterator +from typing import IO import pytest @@ -72,9 +77,11 @@ def test_real_git_clone_fetch_and_push_through_capsule_adapter(tmp_path: Path) - """Real Git clone, fetch and push transfer repository objects through the broker.""" env = _environment(tmp_path) bare = tmp_path / 'remote.git' - subprocess.run(['git', 'init', '--bare', '--initial-branch=main', str(bare)], env=env, check=True, capture_output=True) + git_bin = shutil.which('git') + assert git_bin is not None + subprocess.run([git_bin, 'init', '--bare', '--initial-branch=main', str(bare)], env=env, check=True, capture_output=True) # This stand-in uses only the validated service; the repository is fixed by the fixture. - program = f"import os, shlex, sys\nservice = shlex.split(sys.argv[-1])[0]\nos.execv('/usr/bin/git', ['git', service.removeprefix('git-'), {str(bare)!r}])\n" + program = f"import os, shlex, sys\nservice = shlex.split(sys.argv[-1])[0]\nos.execv({git_bin!r}, ['git', service.removeprefix('git-'), {str(bare)!r}])\n" with _broker(tmp_path, program) as (path, _): git_env = {**env, SOCKET_ENV: str(path), 'GIT_SSH_VARIANT': 'ssh', 'GIT_SSH_COMMAND': shlex.join([sys.executable, '-m', 'devcapsule', 'github-ssh', '--'])} @@ -118,12 +125,131 @@ def test_oversized_handshake_is_refused_without_reading_body(tmp_path: Path) -> assert connection.recv(1) == b'' +def _process_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + # An orphaned grandchild cannot be waitpid'd by the broker and stays a + # zombie until PID 1 reaps it; treat 'Z' as terminated, not running. + try: + status = Path(f'/proc/{pid}/status').read_text(encoding='utf-8') + except OSError: + return False + for line in status.splitlines(): + if line.startswith('State:') and line.split(':', 1)[1].lstrip().startswith('Z'): + return False + return True + + +def _wait_pids_stopped(pids: list[int], timeout: float = 2.0) -> None: + deadline = time.monotonic() + timeout + remaining = list(pids) + while remaining and time.monotonic() < deadline: + remaining = [pid for pid in remaining if _process_running(pid)] + if remaining: + time.sleep(0.02) + assert not remaining, f"Processes still running after cleanup: {remaining}" + + +def _read_ready_line(stream: IO[bytes], expected: bytes, timeout: float = 5.0) -> None: + with selectors.DefaultSelector() as selector: + selector.register(stream, selectors.EVENT_READ) + if not selector.select(timeout): + raise TimeoutError(f"Timed out waiting {timeout}s for ready signal {expected!r}") + assert stream.readline() == expected + + def test_operation_deadline_stops_a_stalled_ssh_process(tmp_path: Path) -> None: - """A stalled host SSH process is killed and the capsule receives failure within the bound.""" - with _broker(tmp_path, 'import time\ntime.sleep(60)\n', operation_timeout=0.2) as (path, env): + """A stalled host SSH process and its descendant are terminated (no running process remains) and the capsule fails.""" + pid_file = tmp_path / 'pids' + program = ( + "import os, subprocess, sys, time\n" + "from pathlib import Path\n" + "grandchild = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" + f"Path({str(pid_file)!r}).write_text(f'{{os.getpid()}}\\n{{grandchild.pid}}\\n')\n" + "time.sleep(60)\n" + ) + with _broker(tmp_path, program, operation_timeout=0.2) as (path, env): result = _client(path, env) assert result.returncode != 0 assert b'connection ended' in result.stderr + pids = [int(line) for line in pid_file.read_text().splitlines() if line] + assert len(pids) == 2 + _wait_pids_stopped(pids) + + +def test_broker_shutdown_cancels_active_ssh_process_group(tmp_path: Path) -> None: + """Closing the broker terminates an active host SSH transfer so no process in its group remains running.""" + started = tmp_path / 'started' + program = ( + "import os, subprocess, sys, time\n" + "from pathlib import Path\n" + "grandchild = subprocess.Popen([sys.executable, '-c', 'import time; time.sleep(60)'])\n" + f"Path({str(started)!r}).write_text(f'{{os.getpid()}}\\n{{grandchild.pid}}\\n')\n" + "sys.stdout.buffer.write(b'ready\\n')\n" + "sys.stdout.buffer.flush()\n" + "time.sleep(60)\n" + ) + client: subprocess.Popen[bytes] | None = None + try: + with _broker(tmp_path, program, operation_timeout=30) as (path, env): + assert stat.S_IMODE(path.stat().st_mode) == 0o600 + client = subprocess.Popen( + [sys.executable, '-m', 'devcapsule', 'github-ssh', '--', + 'git@github.com', "git-upload-pack 'acme/repo.git'"], + env={**env, SOCKET_ENV: str(path)}, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + ) + assert client.stdout is not None + _read_ready_line(client.stdout, b'ready\n', timeout=5.0) + returncode = client.wait(timeout=5) + client.communicate(timeout=5) + assert returncode != 0 + finally: + if client is not None and client.poll() is None: + client.kill() + client.communicate(timeout=5) + pids = [int(line) for line in started.read_text().splitlines() if line] + assert len(pids) == 2 + _wait_pids_stopped(pids) + + +def test_capsule_adapter_refuses_when_proxy_is_not_authorized(tmp_path: Path) -> None: + """Without an authorized proxy socket in the environment, GitHub SSH fails rather than running SSH.""" + result = subprocess.run( + [sys.executable, '-m', 'devcapsule', 'github-ssh', '--', + 'git@github.com', "git-upload-pack 'acme/repo.git'"], + env=_environment(tmp_path), + input=b'', capture_output=True, timeout=15, + ) + assert result.returncode != 0 + assert b'No authorized GitHub proxy' in result.stderr + + +def test_capsule_adapter_falls_back_to_capsule_ssh_for_other_hosts(tmp_path: Path) -> None: + """Non-GitHub destinations execute the capsule's own ssh binary without touching the proxy.""" + bin_dir = tmp_path / 'bin' + bin_dir.mkdir() + ssh_stub = bin_dir / 'ssh' + args_file = tmp_path / 'ssh_args' + ssh_stub.write_text( + f"#!{sys.executable}\n" + "import json, sys\n" + "from pathlib import Path\n" + f"Path({str(args_file)!r}).write_text(json.dumps(sys.argv[1:]))\n" + "sys.exit(4)\n" + ) + ssh_stub.chmod(0o700) + env = _environment(tmp_path) + env['PATH'] = f"{bin_dir}:{env['PATH']}" + result = subprocess.run( + [sys.executable, '-m', 'devcapsule', 'github-ssh', '--', + '-p', '2222', 'git@example.test', "git-upload-pack 'acme/repo.git'"], + env=env, input=b'', capture_output=True, timeout=15, + ) + assert result.returncode == 4 + assert args_file.read_text() == '["-p", "2222", "git@example.test", "git-upload-pack \'acme/repo.git\'"]' def test_broker_shutdown_closes_incomplete_requests(tmp_path: Path) -> None: @@ -159,3 +285,23 @@ def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled assert (f'{SOCKET_ENV}={SOCKET_DESTINATION}' in args) is enabled assert (f'type=bind,src={path},dst={SOCKET_DESTINATION},ro' in args) is enabled assert not any('.ssh' in arg or 'SSH_AUTH_SOCK=' in arg for arg in args) + + +def test_launcher_rejects_environment_conflicts_with_github_ssh(tmp_path: Path) -> None: + """Additional environment cannot override GitHub SSH transport variables when enabled.""" + from devcapsule.launch.pycharm._launcher import PycharmRunError, PycharmRunOptions, DockerMode, build_run_config + from devcapsule.platforms import Platform + + project = tmp_path / 'project' + project.mkdir() + with _broker(tmp_path, 'raise AssertionError("no transfer expected")\n') as (path, env): + env = {**env, 'DISPLAY': ':1', 'PYCHARM_GIT_IDENTITY_FROM_HOST': '0', SOCKET_ENV: str(path)} + with pytest.raises(PycharmRunError, match="GitHub SSH"): + build_run_config( + PycharmRunOptions( + project=project, capsule_platform=Platform.LINUX_AMD64, + docker_mode=DockerMode.none, enable_sudo=False, network_mode='bridge', + enable_github_ssh=True, additional_environment={'GIT_SSH_COMMAND': 'ssh'}, + ), + env, + ) diff --git a/devcapsule-src/tests/unit/devcapsule/test_github_proxy.py b/devcapsule-src/tests/unit/devcapsule/test_github_proxy.py new file mode 100644 index 00000000..7a09c104 --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/test_github_proxy.py @@ -0,0 +1,42 @@ +"""Unit tests of ``devcapsule.github_proxy``: pure gating decisions of ``github_ssh_bridge``. + +No sockets are bound and no subprocesses start; every outcome follows from the +explicit arguments passed to ``github_ssh_bridge``. +""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from devcapsule.github_proxy import SOCKET_DESTINATION, SOCKET_ENV, github_ssh_bridge +from devcapsule.github_transport import GitHubTransportError + + +def test_disabled_bridge_yields_none_regardless_of_host_capabilities() -> None: + """When github-ssh is denied, the bridge yields no socket even on unsupported or containerized hosts.""" + with github_ssh_bridge({}, enabled=False, containerized=True, binds_unix_sockets=False) as path: + assert path is None + + +def test_docker_desktop_bridge_is_refused_when_enabled() -> None: + """Enabling github-ssh on a host that cannot bind Unix sockets fails loudly instead of degrading.""" + with pytest.raises(GitHubTransportError, match="Docker Desktop"): + with github_ssh_bridge({}, enabled=True, containerized=False, binds_unix_sockets=False): + pass + + +def test_containerized_bridge_requires_an_inherited_canonical_socket() -> None: + """An inner capsule cannot enable github-ssh without the launcher's canonical socket variable.""" + for environment in ({}, {SOCKET_ENV: "/tmp/forged.sock"}): + with pytest.raises(GitHubTransportError, match="outside launcher"): + with github_ssh_bridge(environment, enabled=True, containerized=True, binds_unix_sockets=True): + pass + assert str(SOCKET_DESTINATION) != "/tmp/forged.sock" + + +def test_host_bridge_requires_openssh_on_path(tmp_path: Path) -> None: + """Enabling github-ssh on the host fails clearly when OpenSSH is absent from an empty PATH directory.""" + with pytest.raises(GitHubTransportError, match="OpenSSH"): + with github_ssh_bridge({"PATH": str(tmp_path)}, enabled=True, containerized=False, binds_unix_sockets=True): + pass diff --git a/devcapsule-src/tests/unit/devcapsule/test_github_transport.py b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py index 695e8997..30560161 100644 --- a/devcapsule-src/tests/unit/devcapsule/test_github_transport.py +++ b/devcapsule-src/tests/unit/devcapsule/test_github_transport.py @@ -6,12 +6,15 @@ from devcapsule.github_transport import GitHubRequest, GitHubTransportError, decode_request, parse_git_ssh -@pytest.mark.parametrize('command,service', [("git-upload-pack 'acme/widgets.git'", 'git-upload-pack'), - ("git-receive-pack '/acme/widgets'", 'git-receive-pack')]) -def test_git_arguments_select_only_the_service_and_repository(command: str, service: str) -> None: - """Git's destination and v2 request admit a GitHub transfer with no SSH option forwarding.""" - request = parse_git_ssh(['-o', 'SendEnv=GIT_PROTOCOL', 'git@github.com', command]) - assert request == GitHubRequest(service, 'acme/widgets.git' if service == 'git-upload-pack' else 'acme/widgets') +@pytest.mark.parametrize('arguments,expected', [ + (['-o', 'SendEnv=GIT_PROTOCOL', 'git@github.com', "git-upload-pack 'acme/widgets.git'"], + GitHubRequest('git-upload-pack', 'acme/widgets.git')), + (['-oSendEnv=GIT_PROTOCOL', '-p', '22', 'github.com', "git-receive-pack '/acme/widgets'"], + GitHubRequest('git-receive-pack', 'acme/widgets')), +]) +def test_git_arguments_select_only_the_service_and_repository(arguments: list[str], expected: GitHubRequest) -> None: + """Git's destination, standard port 22, and v2 request admit a GitHub transfer with no SSH option forwarding.""" + assert parse_git_ssh(arguments) == expected @pytest.mark.parametrize('command', ["sh 'acme/repo'", "git-upload-pack 'acme/repo'; id", "git-upload-pack 'a/../b'", @@ -33,9 +36,13 @@ def test_capsule_ssh_options_cannot_change_host_execution(arguments: list[str]) parse_git_ssh(arguments) -def test_other_git_hosts_are_not_forwarded_to_the_host() -> None: - """A different destination selects the capsule's ordinary SSH fallback.""" - assert parse_git_ssh(['git@example.test', "git-upload-pack 'acme/repo'"]) is None +@pytest.mark.parametrize('arguments', [ + ['git@example.test', "git-upload-pack 'acme/repo'"], + ['-p', '2222', 'git@example.test', "git-upload-pack 'acme/repo'"], +]) +def test_other_git_hosts_are_not_forwarded_to_the_host(arguments: list[str]) -> None: + """A different destination, including with a custom port, selects the capsule's ordinary SSH fallback.""" + assert parse_git_ssh(arguments) is None @pytest.mark.parametrize('document', [{}, {'version': True, 'service': 'git-upload-pack', 'repository': 'a/b'}, @@ -62,6 +69,18 @@ def test_host_command_keeps_verification_and_disables_forwarding() -> None: assert '-oControlPath=none' in command +@pytest.mark.parametrize('payload', [ + b'{', + b'\xff', + b'[]', + json.dumps({'version': 1, 'service': 'git-upload-pack', 'repository': 'a/' + 'b' * 255}).encode(), +]) +def test_wire_validation_rejects_malformed_and_oversized_repository_payloads(payload: bytes) -> None: + """Malformed JSON, non-object frames and overlong repository paths are refused on the wire.""" + with pytest.raises(GitHubTransportError): + decode_request(payload) + + def test_valid_wire_request_roundtrips() -> None: """Encoding a valid request carries exactly the service and repository contract.""" request = GitHubRequest('git-receive-pack', 'acme/widgets.git') From 890fa91d356610165d582b0d021ec018354ffad3 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 06:00:00 +0000 Subject: [PATCH 15/35] Record Hydrogen review follow-up Co-authored-by: GPT-6 Codex --- .../2026-11-16-github-auth/CURRENT-STATUS.md | 1 + .../evidence/round-1.md | 42 +++++++++++++++++++ 2 files changed, 43 insertions(+) create mode 100644 engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 3a5fa54c..9ec99585 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -88,5 +88,6 @@ to hide them. ## Workstream document index - [Review brief](evidence/review-brief.md) +- [Review follow-up 1](evidence/round-1.md) - [Intake](intake/README.md) - [Intake decisions](intake-dispositions.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md new file mode 100644 index 00000000..c2dc01cf --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/round-1.md @@ -0,0 +1,42 @@ +# Review follow-up 1: finish the bounded local review + +Continue the same Gemini Hydrogen review in /home/devcapsule/worktrees/github-auth-review. +Your full build has FINISHED: step 130 returned 20 failed, 2926 passed, 1 skipped, +14 xfailed and 7 xpassed. No pytest or nox process remains. Your CLI stayed idle +after that completion, so the implementor interrupted it and resumed this turn. +Do not wait for the completed job. Do not repeat the full gate: its failures are known. + +The implementor read your diff and accepts its production-code changes and added +coverage in principle. Fix these test issues before committing: +1. test_host_bridge_requires_openssh_on_path passes PATH="". shutil.which searches + the current directory in that case; the test depends on no file named ssh being + present there. Use tmp_path as an explicitly empty PATH directory instead. +2. test_broker_shutdown_cancels_active_ssh_process_group calls stdout.readline with + no timeout. Bound the ready handshake and ensure client cleanup in finally even + if readiness fails. Avoid a test that hangs forever when the product is broken. +3. An orphaned grandchild may be a zombie until the host's reaper runs. The broker + cannot waitpid a grandchild. Verify that no descendant is running, without + asserting timely reaping by an unrelated PID 1. State the test's actual claim. + +Run focused tests INCLUDING your new unit test_github_proxy.py and integration +file and mypy. If new focused tests fail, fix those failures within this scope. +Commit your corrections. Do not merge, reset, rebase or touch the parent checkout. + +Parent updates already committed separately (do not duplicate or cherry-pick): +- f1dbb338 refurbishes tests/configuration/test_nodes.py into its correct mirror, + 20 passing cases; this resolves the new github-ssh name expectation. +- 4ba1725f aligns webconsole/configuration.py SECURE_AUTHORIZATIONS and its contract; + all 458 console tests now pass. +- The remaining 19 failures are reproduced on unchanged upstream/main, by selecting + all 20 failing test IDs in an archive of the base (19 fail, the registry test passes). +- Parent CLI smoke, PEX build/smoke and docs contract pass. Both source and PEX Git + adapters passed read-only authenticated GitHub ls-remote using the capsule's existing + credential on the broker side; not a physical-host launch test. + +Your branch can be pushed to upstream as review preparation, with limitations in +its report. No PR exists because local policy makes creation owner-operated without +its prescribed gh installation. Do not use another GitHub interface or credential. + +Return the findings, corrections, focused results, known full-gate failures and +unimplemented proposals. End with REVIEW_COMPLETE or REVIEW_BLOCKED. Do not begin +another broad investigation; the remaining task is these test corrections and delivery. From 712466ab9547ec7c9153977f725fc6cfe26344f6 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 06:02:40 +0000 Subject: [PATCH 16/35] Record accepted local review and delivery blockers Co-authored-by: GPT-6 Codex --- CURRENT-STATUS.md | 2 +- .../development/review-partner-loop.md | 1 + .../2026-11-16-github-auth/CURRENT-STATUS.md | 33 +++--- .../evidence/delivery.md | 63 +++++++++++ .../evidence/review-assessment.md | 100 ++++++++++++++++++ 5 files changed, 186 insertions(+), 13 deletions(-) create mode 100644 engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md create mode 100644 engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 0d55f14f..877c36e0 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | active; implementation and Gemini Hydrogen Pair Review | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused; implementation/local review prepared; gate, host acceptance and PR delivery pending | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/engineering-docs/development/review-partner-loop.md b/engineering-docs/development/review-partner-loop.md index 09c54df5..32cd4347 100644 --- a/engineering-docs/development/review-partner-loop.md +++ b/engineering-docs/development/review-partner-loop.md @@ -299,6 +299,7 @@ when the workstream pauses, and say so in its open threads. | Date | Workstream | Implementor | Review partner | Slices and reviews | Outcome | |---|---|---|---|---|---| +| 2026-11-16 | github-auth, launcher-owned GitHub SSH proxy | GPT-6 Codex | Antigravity, Gemini 4 Hydrogen High (`gemini-4-hydrogen-high`) | Local preparation; PRs pending | Round 1 accepted locally at `388f62a3`; diagnostics/disclosure and process/authorization coverage corrected; combined candidate 75 focused tests, mypy and conventions passed. Full gate blocked by 19 failures reproduced on unchanged main; physical-host acceptance pending. Reviewer stream stalled after completed full gate; same conversation resumed and completed with verified commit/push. Formal PR-mediated loop remains open because prescribed gh is absent. [Assessment](../wip/2026-11-16-github-auth/evidence/review-assessment.md) records limits and unimplemented proposals. | | 2026-11-08 / 2026-11-09 | maintenance, launcher-delivered web console (`D-0017`, 0.3.1 candidate 2 blocker, stacked on #18) | Antigravity, Gemini 4 Argon (High) pair | Claude Code, Claude Fable 5.1 pair | #16/#17 | accepted at `e23e22c7` without a round, merged at `9919f4ce`, and rebased onto PR #18 (`ws-maintenance/webconsole-legacy-tests`) per owner decision `stacked-delivery-branch = force-push-pr-16`; the reviewer removed the dead per-start `import psutil` subprocess probe (`WEBCONSOLE_RECIPE` 11 settles compatibility on the host) and unified console package precedence in `console_package_path`, fixed `extract_webconsole_bundle` to mask `external_attr >> 16` with `S_IFMT` so typeless regular zip entries (`ZipFile.writestr`) extract, pinned `match=` on every bundle refusal test (99% bundle coverage), tightened `DEVCAPSULE_DEV_WEBCONSOLE` to `1`/`true`/`0`/`false`, and aligned `D-0017` and the bug record; stacked gate 2904 unit tests, 40 integration (3 skipped), 458 webconsole, 332 mypy files, 104 legacy / 44 conforming files with 0 breaches, 249 contract pages, plus live base-Python HTTP check; remaining unimplemented proposals recorded in `maintenance` status; brief committed under `evidence/2026-11-08-webconsole-delivery/review-brief.md` | | 2026-11-09 | maintenance, whole-file legacy test retirement and 4-file refurbishment (`0.3.1` prerequisite) | Antigravity, Gemini 4 Argon (High) pair | Claude Code, Claude Fable 5.1 pair | #18/#19 | accepted at `4aac5dfc` without a round, merged at `a1aad17f`; completed `__all__` in `console.py`, `entrypoint.py` and `materialization.py`, dropped a dead `subprocess` import in `project.py`, stated basename pairing and mirror test deletion refusal across rule 5, the companion, the developer guide and the checker with 4 new checker tests, proved `entrypoint.run` writes session variables only into the supplied `environ`, and restored interpreter-path matching in `test_console_tools.py`; hosted CI on `e301367d` caught `/ide-config` state-slot paths in `test_entrypoint.py` (masked inside the PyCharm capsule), resolved by confining `make_plan` to `tmp_path` and injecting `runtime_root`, `caller_uid`, and `configure_logs` across `filesystem.py`, `identity.py`, and `entrypoint.run` (`test_filesystem.py` added, implementing PR #19's third proposal with zero global state); gate 2842 unit, 36 integration (1 skipped), 458 webconsole, 330 mypy files, 104 legacy / 43 conforming files with 0 breaches, 249 contract pages; two remaining proposals recorded in `maintenance` status; brief committed under `evidence/2026-11-08-webconsole-legacy-tests/review-brief.md` | | 2026-11-08 | maintenance, the web console's Files page (slice 2 of 0.3.1) | Antigravity, Gemini 4 Argon (High) pair | Claude Code, Claude Fable 5.1 pair | #12/#13 | accepted at `49bb2e8e` without a round, merged at `83949a30`; the first loop with the pairs reversed at the owner's word, the reviewer's pair writing the implementor brief from the owner's order; the reviewer removed a raw-route branch that detected its own test and repointed the legacy permission test at the streaming open, and smoked the routes against a standalone console; gate on the merged branch 458 console tests, 2825 unit tests, 0 breaches, 249 contract pages; both briefs committed under the workstream's evidence; the Antigravity stream-json form relayed one event per tool step, proving that form | diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 9ec99585..37dee8c1 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: active; first implementation slice. +State: paused; implementation and local review prepared, gate and PR delivery pending. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -62,28 +62,34 @@ both the source adapter and the built PEX returned GitHub HEAD `1942f396`. That used the current capsule's already-installed SSH credentials on the broker side; it is not physical-host/capsule acceptance evidence. -The branch is pushed to `upstream/ws-github-auth/ssh-proxy`. Gemini Hydrogen High -is reviewing in `/home/devcapsule/worktrees/github-auth-review`. The first invocation -received no brief through redirected stdin. The same conversation resumed with an -explicit path to this committed brief; flags/model stayed unchanged. No credentials -were examined to repair this CLI input issue. +The implementation and review branches are pushed to canonical upstream. +Gemini Hydrogen completed local round 1 at `388f62a3`. Its corrections remain +on the separate review branch for PR delivery. A temporary combined merge passed +75 focused cases, type checking and conventions (zero breaches), then was aborted. +Git connectivity integrity and authenticated fetch/push checks pass. The original +checkout remains untouched. Review evidence and prepared PR text are indexed below. ## Planned next step -Assess Gemini's review diff, incorporate accepted corrections, repeat affected -checks, and prepare owner-operated PR delivery. Full gate remains blocked by the -19 demonstrated mainline/environment failures; no test was disabled or reclassified -to hide them. +Resolve the 19 reproduced baseline/environment failures in an appropriate gate +environment, then complete owner-operated implementation/review PR delivery and +physical-host acceptance. Do not claim a green full gate or mainline integration. +At resume, fetch upstream, check live workflow state and the review SHA, merge +through the prescribed PR process, and validate the actual combined tree. ## Open Threads -- Awaiting the human: none for workstream or first delivery scope; both accepted. +- Awaiting the human: suitable gate environment and owner-operated GitHub PR delivery. + Workstream selection and Git-over-SSH scope are already accepted. - Engineering choices: initial transport must account for Linux/rootless Docker permissions; Docker Desktop cannot bind ordinary host Unix sockets. - Physical-host launch acceptance and Docker Desktop transport remain open. - GitHub CLI is absent at the policy-prescribed path. PR creation/merging remains owner-operated under WORKFLOW-LOCAL.md; prepare reviewed commits and compare links. -- Deliberately not preserved: no credentials or console decision tokens in Git. +- Reviewer worktree removed at pause; pushed `ws-github-auth/ssh-proxy-review` + preserves its commit. Review corrections are not yet merged into implementation. +- Unimplemented review proposals and exact baseline failure IDs are in the assessment. +- Deliberately not preserved: scratch logs/venvs, credentials or console decision tokens in Git. ## Workstream document index @@ -91,3 +97,6 @@ to hide them. - [Review follow-up 1](evidence/round-1.md) - [Intake](intake/README.md) - [Intake decisions](intake-dispositions.md) + +- [Review assessment](evidence/review-assessment.md) +- [Prepared PR delivery](evidence/delivery.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md new file mode 100644 index 00000000..5c4767d6 --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/delivery.md @@ -0,0 +1,63 @@ +# Prepared delivery + +No PR has been opened. WORKFLOW-LOCAL.md, GitHub Integration rules 4–6, +requires a passing unit gate before PR opening/updating, and assigns GitHub +operations to the human when its prescribed gh installation is absent. The +current environment has both blockers. Preserve both branches until the gate +is resolved in a suitable environment and the owner performs PR delivery. + +## Implementation PR draft + +Base: `main`. Head: `ws-github-auth/ssh-proxy`. + +Title: Add opt-in host GitHub SSH authentication for capsules + +Body: + +Git operations currently require installing SSH credentials inside each capsule. +This adds an opt-in `github-ssh` authorization: the outside launcher owns a Unix +socket broker that runs host OpenSSH for validated GitHub upload/receive requests, +while capsule Git uses the shipped PEX adapter. Private keys remain on the host. +Linux/WSL with native Docker is supported first; Docker Desktop is refused. +Host trust and unlocked encrypted keys are prerequisites. The capability grants +Git access to all repositories available to that host identity. + +Validation: combined local candidate passed 75 focused tests, type checking and +convention checks; source/PEX authenticated read-only GitHub probe passed; local +Git clone/fetch/push, smoke/PEX/docs and 458 console tests passed. Full gate remains +blocked by 19 failures also reproduced on unchanged main. Physical-host launch +acceptance is outstanding. See the workstream review assessment for exact limits. + +Pair: GPT-6 Codex with Antigravity Gemini 4 Hydrogen High +(`gemini-4-hydrogen-high`). Local round 1 accepted at `388f62a3`; formal review PR +must be merged into this branch before mainline delivery. + +## Review PR draft + +Base: `ws-github-auth/ssh-proxy`. Head: `ws-github-auth/ssh-proxy-review`. + +Title: Review GitHub SSH proxy diagnostics and lifecycle coverage + +Body: + +Gemini Hydrogen review separates capsule SSH failures from broker failures, +clarifies disclosure and dry-run lifetime, removes an unused frame helper, +declares the public review API and strengthens parser, authorization and process +cleanup coverage. Round 1 addresses test environment independence and cleanup. +The implementor accepted `388f62a3` locally after a conflict-free temporary merge +passed 75 focused cases. The reviewer branch alone lacks the parent's later node +registry refurbishment and console-table fix; validate the merged candidate. +Full gate and owner-operated PR delivery remain outstanding. + +## Resume sequence + +1. Fetch canonical upstream and recheck workflow synchronization and live state. +2. Resolve the known full-gate environment failures without disabling tests; + run the required gate on the combined candidate before PR opening/updating. +3. The owner creates the implementation and review PRs, then merges the review + into the implementation branch. Record the formal assessment at its SHA. +4. Fetch that merged branch, run required validation and physical-host acceptance: + authorize `github-ssh` once, launch with the rebuilt outside PEX, and exercise + GitHub reads plus a push to an explicitly chosen disposable repository. +5. The owner merges the implementation PR into main; fetch and verify main's tree + before recording integration. No release or installation has been performed. diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md new file mode 100644 index 00000000..ecf6b43e --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/review-assessment.md @@ -0,0 +1,100 @@ +# GitHub SSH proxy review assessment + +Date: 2026-11-16. Implementor: GPT-6 Codex. Reviewer: Antigravity, +`gemini-4-hydrogen-high` (Gemini 4 Hydrogen High). + +## Disposition + +Round 1 accepted locally at `388f62a34fe373882201f9118faaf34cec61680b`. +The reviewer branch is pushed as `ws-github-auth/ssh-proxy-review`. +No review PR or implementation PR exists yet. This is local review preparation, +not completion of the PR-mediated Pair Review Loop or mainline integration. + +The implementor inspected the diff and accepted the unused helper removal, +separate capsule-SSH error attribution, public API declaration, launcher dry-run +warning and disclosure layout, deny-help update, and expanded hostile-input, +permission, cancellation, process-group and authorization coverage. +Round 1 replaced an ambient-current-directory PATH check, bounded the readiness +handshake, guaranteed client cleanup, and distinguished dead orphan zombies from +running descendants. The process tests depend on Linux /proc and scheduling +within their explicit timeouts; they do not prove prompt reaping by PID 1. + +The reviewer ran 55 focused cases successfully. A temporary, uncommitted merge of +`388f62a3` with implementation `890fa91d` was conflict-free and passed **75 focused +tests**, mypy on the nine touched modules/test files, and the conventions checker +(103 legacy files, 1380 legacy tests, 50 conforming files, zero breaches). +`git diff --check` and `git fsck --connectivity-only --no-dangling` passed. +The temporary merge was aborted deliberately: reviewer corrections remain on +their separate pushed branch for the owner-operated review PR. + +## Build evidence and limits + +The implementor's full build reached pytest: 20 failed, 2917 passed, 1 skipped, +76 deselected, 14 xfailed, 7 xpassed. The reviewer's independent build found the +same failure set: 20 failed, 2926 passed, 1 skipped, 80 deselected, 14 xfailed, +7 xpassed. The new node-name expectation was repaired on the implementation +branch by refurbishing the complete legacy file into the unit mirror (20 passing +cases); the reviewer was told not to duplicate this change. The other 19 failures +reproduced by selecting all 20 failed IDs on an archive of unchanged canonical +main `1942f396`: 19 failed, 1 passed. They concern inherited capsule trust, +permissions/ownership and recursive-launch environment; they remain unresolved. +A full gate on the combined final tree has not passed. + +CLI smoke, PEX build/smoke and documentation contract passed separately. After +aligning the console authorization mirror, all 458 webconsole tests and its types +passed. Both source and built PEX adapters completed authenticated GitHub +`ls-remote` through the broker, returning mainline HEAD `1942f396`. The broker in +that probe used the current capsule's existing SSH setup. This is **not** evidence +of physical-host Docker launch acceptance. No real GitHub write was tested; local +fixture repositories exercised real Git clone/fetch/push. No containers were +launched and no key files were read by the agent, copied or changed. + +## Remaining proposals + +- Docker Desktop needs a separate transport design; this slice explicitly refuses + hosts unable to bind Unix sockets. Linux/WSL with native Docker is first. +- GitHub aliases, HTTPS/gh, Git LFS authentication, signing, and arbitrary SSH flags + (including Git's -4/-6 switches) are outside this adapter's current contract. +- Additional reference cross-links and a tighter operation-deadline cap inside + the bounded send loop were considered; the existing send timeout is 10 seconds + and cancellation checks remain active. These are recorded follow-ups, not + claims of implemented behavior. +- The inherited canonical socket positive path is not covered by a pure unit + test; physical-host and recursive-launch acceptance remain required. + +## Reviewer execution + +Conversation `6280de13-6b45-4215-8e97-af6b466fbd61` used streaming JSON output. +The first redirected-stdin invocation failed to convey the brief; continuation +with an explicit brief path proceeded. The reviewer's CLI stalled after its full +gate had already returned. The implementor interrupted and continued that same +conversation with the committed round brief and a bounded print timeout. The +final response reported REVIEW_COMPLETE, the final shell marker was +ANTIGRAVITY_EXIT=0, and the commit and remote SHA were independently verified. +The aggregate JSON result retained an ERROR/idle-timeout from the earlier turn; +we therefore rely on observed commits/checks, not that aggregate status alone. +Scratch logs and virtual environments are not durable evidence. Briefs, the +assessment and validation counts are retained here. The reviewer worktree is +removed at pause; its pushed branch preserves all changes. + +## Baseline failure IDs + +- `tests/configuration/test_contract.py::test_second_checkout_cannot_inherit_the_first_checkouts_authority` +- `tests/configuration/test_contract.py::test_config_list_is_data_only_and_show_carries_the_review` +- `tests/test_capability_policy.py::test_cli_check_and_capability_mutation` +- `tests/test_host_daemon.py::test_unmountable_staging_fails_loudly` +- `tests/test_owner_commands.py::test_tool_edit_does_not_change_checkout_answers_when_not_ready[True]` +- `tests/test_recursive_orchestrator.py::test_public_dry_run_is_unique_redacted_and_cleans_owned_staging` +- `tests/test_recursive_orchestrator.py::test_contained_display_dry_run_forwards_no_host_x11` +- `tests/test_run_flow.py::test_a_second_checkout_is_registered_under_its_directory_name` +- `tests/test_run_flow.py::test_interactive_questions_record_answers_where_the_config_family_would` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[False-False]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[False-True]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[True-False]` +- `tests/test_run_flow.py::test_save_with_pending_consent_writes_nothing[True-True]` +- `tests/test_run_flow.py::test_partial_terminal_review_saves_no_answers[1\n]` +- `tests/test_run_flow.py::test_partial_terminal_review_saves_no_answers[1\nn\n]` +- `tests/test_runtime_entrypoint.py::test_entrypoint_rejects_unknown_adapters` +- `tests/test_trust.py::test_a_real_run_launches_on_the_homes_trust_alone` +- `tests/test_trust.py::test_trust_write_is_private_atomic_and_keeps_previous_record_on_failure` +- `tests/test_trust.py::test_terminal_acceptance_stays_staged_if_a_later_answer_is_missing` From f6890bb918c179581c04622004217f5bc0bb590d Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 08:44:41 +0000 Subject: [PATCH 17/35] Prepare local PEX host relaunch and Codex resume handoff Co-authored-by: GPT-6 Codex --- CURRENT-STATUS.md | 2 +- .../2026-11-16-github-auth/CURRENT-STATUS.md | 79 +++++++++++++++++-- 2 files changed, 72 insertions(+), 9 deletions(-) diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 877c36e0..a2a5630a 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused; implementation/local review prepared; gate, host acceptance and PR delivery pending | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused for owner host relaunch; PEX built; recorded recursive authentication E2E next | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 37dee8c1..5e4328d6 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: paused; implementation and local review prepared, gate and PR delivery pending. +State: paused for owner host relaunch; local PEX built, recursive authentication E2E next. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -71,16 +71,79 @@ checkout remains untouched. Review evidence and prepared PR text are indexed bel ## Planned next step -Resolve the 19 reproduced baseline/environment failures in an appropriate gate -environment, then complete owner-operated implementation/review PR delivery and -physical-host acceptance. Do not claim a green full gate or mainline integration. -At resume, fetch upstream, check live workflow state and the review SHA, merge -through the prescribed PR process, and validate the actual combined tree. +Owner chose on 2026-11-16 to exit this capsule, relaunch the same checkout from +the physical host with the new local PEX and github-ssh enabled, then resume Codex. +This supersedes the proposed SSH-to-host harness adaptation. After resumption, +verify the inherited broker and executable bytes, then write and execute the +recorded E2E scenario using a fresh project and explicit github-ssh authorization. +Observe the Git checkout independently and retain WebM/screenshots/action evidence. +Recursive proxy inheritance is still unproved; do not report success until observed. +The 19 reproduced full-gate failures and formal PR delivery remain open separately. + +## Host relaunch handoff + +The owner explicitly chose a development-build self-hosting experiment. The normal +host launcher is not replaced and remains the recovery route; no saved project +configuration is changed by the run-once command. Reuse the same checkout and its +existing persistent state. The original checkout still has only its pre-existing +website gitlink modification. Main remains unchanged; synchronization can wait. + +Build: `.venv/bin/python -m nox -s pex` passed, including shell syntax, packaging +and executable smoke. Built from clean `712466ab9547ec7c9153977f725fc6cfe26344f6`. +The local PEX reports `v0.3.2.dev0-local-linux-x86_64` with revision `unknown` by +local-build policy; the clean Git revision and checksum here identify its inputs +and bytes independently. It contains the committed implementation, including the +node-registry and console-table fixes. Review corrections at `388f62a3` remain on +the separate review branch and are not included; their PR merge is still pending. + +SHA-256: `2f9470c6ca5efeaccd9844bf6feb1494d2e95564d054051a2c005afbc35d78ed`. + +Capsule artifact: `/home/devcapsule/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. +Host artifact (from Docker mount inspection): +`/home/costin/.local/share/devcapsule/projects/by-path/fe326e03c822-devcapsule/home/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. +Checksum and build metadata are saved beside it, outside Git. + +After exiting Codex and stopping the current capsule normally, run on the host: + +```sh +cd /opt/costin3/myProjects/devcapsule +/home/costin/.local/share/devcapsule/projects/by-path/fe326e03c822-devcapsule/home/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex project run \ + --no-update-check \ + --authorize github-ssh true \ + --authorize network host \ + --authorize docker-daemon host-socket +``` + +Keep that outside launcher running throughout the resumed session: it owns the +broker. Host OpenSSH must already authenticate noninteractively to GitHub and +trust its host key. A key without a passphrase needs no SSH agent. + +Inside the relaunched capsule: + +```sh +cd /workspace/301e4208ef81-ChatGPT_Codex +codex resume 01a1e25b-6b76-72d0-b311-614d6adca912 +``` + +The session file exists under ~/.codex/sessions and Docker inspection shows +~/.codex backed by the persistent Codex component home. Installed CLI help +supports explicit session IDs. Conversation/history is resumable; process state +and outstanding tool sessions are not. The feature checkout remains +`/home/devcapsule/worktrees/github-auth`, on `ws-github-auth/ssh-proxy`. + +Resume checks before E2E: read this record, fetch upstream, inspect live workflow +state, claim the slice, check the PEX checksum, verify +`DEVCAPSULE_GITHUB_SOCKET=/run/devcapsule-github.sock` and that it is a socket, +then perform an authenticated GitHub read through the adapter. Fresh E2E homes +must have no copied key; the recording harness must authorize github-ssh for its +fresh project. Do not use this current capsule's manually installed key as proof +of the proxy. No E2E container was launched during handoff preparation. ## Open Threads -- Awaiting the human: suitable gate environment and owner-operated GitHub PR delivery. - Workstream selection and Git-over-SSH scope are already accepted. +- Awaiting the human: exit/relaunch/resume using the command above. + Workstream selection, Git-over-SSH scope and the relaunch approach are accepted. + Full-gate environment and owner-operated GitHub PR delivery also remain open. - Engineering choices: initial transport must account for Linux/rootless Docker permissions; Docker Desktop cannot bind ordinary host Unix sockets. - Physical-host launch acceptance and Docker Desktop transport remain open. From d6fd93e0ba6c18915ace45b9dfef2881068d7b39 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 09:08:56 +0000 Subject: [PATCH 18/35] Propagate rootless Docker identity fix to GitHub proxy branch Apply the code and tests from release-0.3.1 commit a1a54eba3098eb3c80fd78ef3e483d1bb5314466. Adapt CapsuleUser behavior to this branch's foreground_command and prepare_filesystem APIs; retain GitHub authorization and verify both rootless and rootful proxy plans. Release and maintenance records remain with their owner. Co-authored-by: Antigravity Co-authored-by: GPT-6 Codex --- CURRENT-STATUS.md | 2 +- .../assets/docker4pycharm/entrypoint.sh | 4 +- .../devcapsule/assets/pycharm/entrypoint.sh | 4 +- devcapsule-src/devcapsule/commands/project.py | 9 +- .../devcapsule/container_runtime/display.py | 2 +- .../container_runtime/filesystem.py | 2 +- .../devcapsule/container_runtime/identity.py | 5 +- devcapsule-src/devcapsule/host_daemon.py | 42 ++++++++ .../devcapsule/launch/pycharm/_launcher.py | 96 +++++++++++++------ devcapsule-src/devcapsule/logs.py | 4 +- .../devcapsule/recursive_successor.py | 13 ++- devcapsule-src/tests/conftest.py | 2 + .../tests/integration/test_github_proxy.py | 8 +- .../container_runtime/test_filesystem.py | 19 ++++ .../container_runtime/test_identity.py | 32 +++++++ .../launch/pycharm/test_launcher.py | 84 ++++++++++++++++ .../2026-11-16-github-auth/CURRENT-STATUS.md | 30 +++++- 17 files changed, 310 insertions(+), 48 deletions(-) create mode 100644 devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index a2a5630a..66ff89e8 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused for owner host relaunch; PEX built; recorded recursive authentication E2E next | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | active; propagating rootless Docker fix and rebuilding local PEX | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/devcapsule-src/devcapsule/assets/docker4pycharm/entrypoint.sh b/devcapsule-src/devcapsule/assets/docker4pycharm/entrypoint.sh index 06316beb..a3856caf 100755 --- a/devcapsule-src/devcapsule/assets/docker4pycharm/entrypoint.sh +++ b/devcapsule-src/devcapsule/assets/docker4pycharm/entrypoint.sh @@ -17,7 +17,7 @@ export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$IDE_PROJECT_STATE_PATH/home/.cache}" export XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}" RUN_AS_IDE_USER=() -if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ]; then +if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ] && { [ "$IDE_UID" != "0" ] || [ "$IDE_GID" != "0" ]; }; then if [ "$ENABLE_SUDO" = "1" ] && getent passwd "$IDE_USER" >/dev/null 2>&1; then RUN_AS_IDE_USER=(gosu "$IDE_USER") else @@ -38,7 +38,7 @@ install_ide_file() { local dst="$2" local mode="$3" - if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ]; then + if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ] && { [ "$IDE_UID" != "0" ] || [ "$IDE_GID" != "0" ]; }; then install -o "$IDE_UID" -g "$IDE_GID" -m "$mode" "$src" "$dst" else install -m "$mode" "$src" "$dst" diff --git a/devcapsule-src/devcapsule/assets/pycharm/entrypoint.sh b/devcapsule-src/devcapsule/assets/pycharm/entrypoint.sh index d992407c..0dd43273 100644 --- a/devcapsule-src/devcapsule/assets/pycharm/entrypoint.sh +++ b/devcapsule-src/devcapsule/assets/pycharm/entrypoint.sh @@ -17,7 +17,7 @@ export XDG_CACHE_HOME="${XDG_CACHE_HOME:-$IDE_PROJECT_STATE_PATH/home/.cache}" export XDG_DATA_HOME="${XDG_DATA_HOME:-$HOME/.local/share}" RUN_AS_IDE_USER=() -if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ]; then +if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ] && { [ "$IDE_UID" != "0" ] || [ "$IDE_GID" != "0" ]; }; then if [ "$ENABLE_SUDO" = "1" ] && getent passwd "$IDE_USER" >/dev/null 2>&1; then RUN_AS_IDE_USER=(gosu "$IDE_USER") else @@ -38,7 +38,7 @@ install_ide_file() { local dst="$2" local mode="$3" - if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ]; then + if [ "$(id -u)" -eq 0 ] && [ -n "$IDE_UID" ] && [ -n "$IDE_GID" ] && { [ "$IDE_UID" != "0" ] || [ "$IDE_GID" != "0" ]; }; then install -o "$IDE_UID" -g "$IDE_GID" -m "$mode" "$src" "$dst" else install -m "$mode" "$src" "$dst" diff --git a/devcapsule-src/devcapsule/commands/project.py b/devcapsule-src/devcapsule/commands/project.py index 6b0b2541..caf4e162 100644 --- a/devcapsule-src/devcapsule/commands/project.py +++ b/devcapsule-src/devcapsule/commands/project.py @@ -50,6 +50,7 @@ from devcapsule.configuration.approvals import render_software_approvals, select_software_approvals from devcapsule import notifications from devcapsule import version_sets +from devcapsule.host_daemon import daemon_is_rootless from devcapsule import runtime_configuration from devcapsule.runtime_configuration import CapsuleAccess from devcapsule.compat import CliError, DecisionPending @@ -2323,6 +2324,7 @@ def prepare_display(base: ImageDetails) -> None: image_labels: Mapping[str, str] = {} realized = None degraded_download = False + rootless_docker = daemon_is_rootless() if isinstance(lock.get("base"), dict) and isinstance(lock.get("materialization"), dict): while True: try: @@ -2346,7 +2348,11 @@ def prepare_display(base: ImageDetails) -> None: degraded_download = True image = realized.image.reference image_labels = realized.image.labels - checkout_runtime_plan = project_runtime_plan(selected, realized.locked) + checkout_runtime_plan = ( + project_runtime_plan(selected, realized.locked, uid=0, gid=0, user="root") + if rootless_docker + else project_runtime_plan(selected, realized.locked) + ) use_image_process = True action = "Materialized" if realized.created else "Reused" print(f"{action} canonical environment: {image}") @@ -2472,6 +2478,7 @@ def prepare_display(base: ImageDetails) -> None: display_transport=display_transport, web_console=web_console, capsule_platform=lock_platform(lock), + rootless_docker=rootless_docker, ) ) if command_report is not None: diff --git a/devcapsule-src/devcapsule/container_runtime/display.py b/devcapsule-src/devcapsule/container_runtime/display.py index ea53a9aa..7adb14e0 100644 --- a/devcapsule-src/devcapsule/container_runtime/display.py +++ b/devcapsule-src/devcapsule/container_runtime/display.py @@ -313,7 +313,7 @@ def field(value: bytes) -> bytes: def _own(path: Path, plan: RuntimePlan) -> None: - if os.geteuid() == 0: + if os.geteuid() == 0 and (plan.identity.uid != 0 or plan.identity.gid != 0): os.chown(path, plan.identity.uid, plan.identity.gid) diff --git a/devcapsule-src/devcapsule/container_runtime/filesystem.py b/devcapsule-src/devcapsule/container_runtime/filesystem.py index 920cd6d1..38cff2ca 100644 --- a/devcapsule-src/devcapsule/container_runtime/filesystem.py +++ b/devcapsule-src/devcapsule/container_runtime/filesystem.py @@ -53,7 +53,7 @@ def prepare_filesystem( missing.append(cursor) cursor = cursor.parent path.mkdir(parents=True, exist_ok=True) - if effective_uid == 0: + if effective_uid == 0 and (identity.uid != 0 or identity.gid != 0): for created in reversed(missing): apply_chown(created, identity.uid, identity.gid) Path(plan.environment["XDG_RUNTIME_DIR"]).chmod(0o700) diff --git a/devcapsule-src/devcapsule/container_runtime/identity.py b/devcapsule-src/devcapsule/container_runtime/identity.py index b6077bad..fb8c772f 100644 --- a/devcapsule-src/devcapsule/container_runtime/identity.py +++ b/devcapsule-src/devcapsule/container_runtime/identity.py @@ -18,12 +18,13 @@ def foreground_command( """Wrap ``command`` so it runs as the capsule user when the caller is root. Named for its first use; every supervised child, infrastructure included, - is wrapped the same way, so no display or IDE process ever runs as root. + is wrapped the same way. A rootless Docker target of 0:0 already has the + correct identity; invoking gosu there would require dropped capabilities. """ if not command: raise ValueError("foreground command must not be empty") effective_uid = os.geteuid() if caller_uid is None else caller_uid - if effective_uid == 0: + if effective_uid == 0 and (identity.uid != 0 or identity.gid != 0): return ("gosu", f"{identity.uid}:{identity.gid}", *command) return command diff --git a/devcapsule-src/devcapsule/host_daemon.py b/devcapsule-src/devcapsule/host_daemon.py index ff9844e4..afb80343 100644 --- a/devcapsule-src/devcapsule/host_daemon.py +++ b/devcapsule-src/devcapsule/host_daemon.py @@ -231,3 +231,45 @@ def daemon_architecture() -> str: return host_architecture() reported = completed.stdout.strip() return architecture_key(reported) if completed.returncode == 0 and reported else host_architecture() + + +def security_options_indicate_rootless(security_options: Sequence[str] | str) -> bool: + """Return whether Docker daemon ``SecurityOptions`` report rootless mode.""" + + raw_items: Sequence[str] = ( + security_options.splitlines() if isinstance(security_options, str) else security_options + ) + for item in raw_items: + for token in item.split(","): + if token.strip() in {"rootless", "name=rootless"}: + return True + return False + + +def daemon_is_rootless() -> bool: + """Whether the Docker daemon runs in rootless mode. + + Under rootless Docker, host UID/GID are mapped to ``0:0`` inside the + container user namespace, whereas container UID/GID equal to the host IDs + map to subordinate IDs that cannot access ``0600``/``0700`` bind mounts. + Returns ``False`` when the daemon does not answer or runs rootful. + """ + try: + completed = subprocess.run( + ["docker", "info", "--format", "{{json .SecurityOptions}}"], + capture_output=True, text=True, timeout=30, check=False, + ) + except (OSError, subprocess.TimeoutExpired): + return False + reported = completed.stdout.strip() + if completed.returncode != 0 or not reported or reported == "null": + return False + try: + parsed = json.loads(reported) + except ValueError: + return security_options_indicate_rootless(reported) + if not isinstance(parsed, list): + return False + return security_options_indicate_rootless( + [item for item in parsed if isinstance(item, str)] + ) diff --git a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py index 283c5ac6..4be254ee 100644 --- a/devcapsule-src/devcapsule/launch/pycharm/_launcher.py +++ b/devcapsule-src/devcapsule/launch/pycharm/_launcher.py @@ -249,6 +249,8 @@ class PycharmRunOptions: # caller (commands/project.py); without it the launcher announces the # absence once and plans no console: no port, no token, no wait. web_console: WebConsoleAvailability = WebConsoleAvailability(True, None) + # True only when the host Docker daemon reports rootless mode. + rootless_docker: bool = False @dataclass @@ -320,6 +322,7 @@ class PycharmRunConfig: libgl_always_software: str = "1" mesa_loader_driver_override: str = "llvmpipe" libgl_dri3_disable: str = "1" + rootless_docker: bool = False HOST_NETWORK_UNAVAILABLE_ON_MACOS = ( @@ -1006,7 +1009,10 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych # daemon already holds root on its virtual machine, so the group adds # no authority. Elsewhere the host socket file's group applies. socket_gid = HostOs.current().docker_socket_gid - host_docker_gid = socket_gid if socket_gid is not None else host_docker_socket.stat().st_gid + raw_socket_gid = socket_gid if socket_gid is not None else host_docker_socket.stat().st_gid + host_docker_gid = ( + 0 if options.rootless_docker and raw_socket_gid == host_user.gid else raw_socket_gid + ) if enable_sudo: while ide_sudo_gid in {0, host_user.gid, host_docker_gid}: @@ -1238,6 +1244,7 @@ def build_run_config(options: PycharmRunOptions, env: Mapping[str, str]) -> Pych env.get("MESA_LOADER_DRIVER_OVERRIDE", "llvmpipe"), ), libgl_dri3_disable=env.get("PYCHARM_LIBGL_DRI3_DISABLE", env.get("LIBGL_DRI3_DISABLE", "1")), + rootless_docker=options.rootless_docker, ) @@ -1353,6 +1360,10 @@ def build_docker_args( # launcher is not a case this product has; the recursive dogfood holds # its successor's launcher open exactly as a developer holds `project run`. host_user = current_host_user() + rootless = bool(getattr(config, "rootless_docker", False)) + container_uid, container_gid, container_user = ( + (0, 0, "root") if rootless else (host_user.uid, host_user.gid, host_user.name) + ) args = [ "--rm", "-i", @@ -1385,11 +1396,11 @@ def build_docker_args( "--env", "IDE_PROJECT_STATE_PATH=/ide-project-state", "--env", - f"IDE_UID={host_user.uid}", + f"IDE_UID={container_uid}", "--env", - f"IDE_GID={host_user.gid}", + f"IDE_GID={container_gid}", "--env", - f"IDE_USER={host_user.name}", + f"IDE_USER={container_user}", "--env", f"LIBGL_ALWAYS_SOFTWARE={config.libgl_always_software}", "--env", @@ -1620,12 +1631,16 @@ def _display_mount_args(config: PycharmRunConfig, files: TempRuntimeFiles) -> li def append_docker_mode_args(args: list[str], config: PycharmRunConfig, host_user: HostUser) -> None: + rootless = bool(getattr(config, "rootless_docker", False)) + container_uid, container_gid = ( + (0, 0) if rootless else (host_user.uid, host_user.gid) + ) if config.docker_mode == "host": print_host_docker_warning(config) args.extend( [ "--user", - f"{host_user.uid}:{host_user.gid}", + f"{container_uid}:{container_gid}", "--group-add", str(config.host_docker_gid), "--env", @@ -1662,7 +1677,7 @@ def append_docker_mode_args(args: list[str], config: PycharmRunConfig, host_user args.extend( [ "--user", - f"{host_user.uid}:{host_user.gid}", + f"{container_uid}:{container_gid}", "--env", "ENABLE_DIND=0", "--env", @@ -1870,37 +1885,54 @@ def write_xauthority(xauth_file: Path, env: Mapping[str, str]) -> None: def write_user_files(config: PycharmRunConfig, files: TempRuntimeFiles) -> None: - host_user = current_host_user() - files.passwd_file.write_text( - "\n".join( - [ - "root:x:0:0:root:/root:/bin/bash", - f"{host_user.name}:x:{host_user.uid}:{host_user.gid}:PyCharm Docker User:/home/devcapsule:/bin/bash", - "", - ] + if bool(getattr(config, "rootless_docker", False)): + files.passwd_file.write_text( + "root:x:0:0:PyCharm Docker User:/home/devcapsule:/bin/bash\n" ) - ) - group_lines = [ - "root:x:0:", - f"{host_user.group_name}:x:{host_user.gid}:", - ] - if config.host_docker_gid not in (None, 0, host_user.gid): - group_lines.append(f"host-docker:x:{config.host_docker_gid}:") - if config.enable_sudo: - group_lines.append(f"ide-sudo:x:{config.ide_sudo_gid}:{host_user.name}") - files.shadow_file = make_temp(files.group_file.parent, "pycharm-docker-shadow.") - shadow_last_change = int(time.time()) // 86400 - files.shadow_file.write_text( + group_lines = ["root:x:0:"] + if config.host_docker_gid not in (None, 0): + group_lines.append(f"host-docker:x:{config.host_docker_gid}:") + if config.enable_sudo: + group_lines.append(f"ide-sudo:x:{config.ide_sudo_gid}:root") + files.shadow_file = make_temp(files.group_file.parent, "pycharm-docker-shadow.") + shadow_last_change = int(time.time()) // 86400 + files.shadow_file.write_text( + f"root:*:{shadow_last_change}:0:99999:7:::\n" + ) + files.shadow_file.chmod(0o600) + files.group_file.write_text("\n".join([*group_lines, ""])) + else: + host_user = current_host_user() + files.passwd_file.write_text( "\n".join( [ - f"root:*:{shadow_last_change}:0:99999:7:::", - f"{host_user.name}:*:{shadow_last_change}:0:99999:7:::", + "root:x:0:0:root:/root:/bin/bash", + f"{host_user.name}:x:{host_user.uid}:{host_user.gid}:PyCharm Docker User:/home/devcapsule:/bin/bash", "", ] ) ) - files.shadow_file.chmod(0o600) - files.group_file.write_text("\n".join([*group_lines, ""])) + group_lines = [ + "root:x:0:", + f"{host_user.group_name}:x:{host_user.gid}:", + ] + if config.host_docker_gid not in (None, 0, host_user.gid): + group_lines.append(f"host-docker:x:{config.host_docker_gid}:") + if config.enable_sudo: + group_lines.append(f"ide-sudo:x:{config.ide_sudo_gid}:{host_user.name}") + files.shadow_file = make_temp(files.group_file.parent, "pycharm-docker-shadow.") + shadow_last_change = int(time.time()) // 86400 + files.shadow_file.write_text( + "\n".join( + [ + f"root:*:{shadow_last_change}:0:99999:7:::", + f"{host_user.name}:*:{shadow_last_change}:0:99999:7:::", + "", + ] + ) + ) + files.shadow_file.chmod(0o600) + files.group_file.write_text("\n".join([*group_lines, ""])) files.passwd_file.chmod(0o644) files.group_file.chmod(0o644) @@ -1913,6 +1945,10 @@ def ensure_sudoers_policy_ownership( policy = files.sudoers_file if not config.enable_sudo or policy is None: raise PycharmRunError("Development sudo requires a generated sudoers policy.") + if bool(getattr(config, "rootless_docker", False)): + # Rootless Docker maps files owned by the host user directly to 0:0 + # inside the container while host-side st_uid remains the host UID. + return if not HostOs.current().chown_reaches_bind_mounts: files.sudoers_volume = prepare_sudoers_policy_volume(config, policy, env) return diff --git a/devcapsule-src/devcapsule/logs.py b/devcapsule-src/devcapsule/logs.py index ec41310d..af369f3f 100644 --- a/devcapsule-src/devcapsule/logs.py +++ b/devcapsule-src/devcapsule/logs.py @@ -282,9 +282,9 @@ def events_logger(component: str) -> logging.Logger: def own(path: Path, owner: Owner) -> None: - """Give ``path`` to ``owner`` when this process is root; a no-op otherwise.""" + """Give ``path`` to ``owner`` when this process is root and ownership differs.""" - if owner is not None and os.geteuid() == 0: + if owner is not None and os.geteuid() == 0 and owner != (os.geteuid(), os.getegid()): os.chown(path, owner[0], owner[1]) diff --git a/devcapsule-src/devcapsule/recursive_successor.py b/devcapsule-src/devcapsule/recursive_successor.py index b07c9ee3..bfaa0772 100644 --- a/devcapsule-src/devcapsule/recursive_successor.py +++ b/devcapsule-src/devcapsule/recursive_successor.py @@ -13,6 +13,7 @@ from typing import Any, Callable, Mapping, Sequence from devcapsule.build_info import current_build_info +from devcapsule.host_daemon import daemon_is_rootless from devcapsule.launch.pycharm import DockerMode, PycharmRunOptions from devcapsule.launch.pycharm._launcher import ( PycharmRunError, @@ -154,7 +155,12 @@ def launch_successor( selected = fresh_resolved_project(checkout) realized = realize_environment(selected) - runtime_plan = project_runtime_plan(selected, realized.locked) + rootless_docker = daemon_is_rootless() + runtime_plan = ( + project_runtime_plan(selected, realized.locked, uid=0, gid=0, user="root") + if rootless_docker + else project_runtime_plan(selected, realized.locked) + ) source_revision = current_build_info().source_revision name = successor_container_name(run_id) options = _resolved_run_options( @@ -165,6 +171,7 @@ def launch_successor( run_id=run_id, source_revision=source_revision, image_labels=realized.image.labels, + rootless_docker=rootless_docker, ) config = build_run_config(options, env) successor_display_url: str | None = None @@ -189,7 +196,7 @@ def launch_successor( args = build_docker_args(config, files, launch_env) context = _launch_context(report, config.persistent_home, args) host_args = _translate_bind_sources(args, context) - if files.sudoers_file is not None: + if files.sudoers_file is not None and not rootless_docker: _make_root_owned(files.sudoers_file, launch_env) plan = _expected_plan(host_args, files, realized) _write_expected_plan(run_root, plan) @@ -473,6 +480,7 @@ def _resolved_run_options( run_id: str, source_revision: str, image_labels: Mapping[str, str] | None = None, + rootless_docker: bool = False, ) -> PycharmRunOptions: runtime = selected.resolution.get("runtime", {}) state_root = selected.resolution.get("state", {}) @@ -544,6 +552,7 @@ def _resolved_run_options( "--label", f"devcapsule.e2e.role={SUCCESSOR_ROLE}", ], + rootless_docker=rootless_docker, ) diff --git a/devcapsule-src/tests/conftest.py b/devcapsule-src/tests/conftest.py index 1350f921..005cbf82 100644 --- a/devcapsule-src/tests/conftest.py +++ b/devcapsule-src/tests/conftest.py @@ -78,6 +78,8 @@ def amd64_linux_host_by_default(request: pytest.FixtureRequest, monkeypatch: pyt monkeypatch.setattr("devcapsule.platforms.host_architecture", lambda: "amd64") for module in ("environment_realization", "images.build", "launch.pycharm._launcher"): monkeypatch.setattr(f"devcapsule.{module}.daemon_architecture", lambda: "amd64") + for module in ("commands.project", "recursive_successor"): + monkeypatch.setattr(f"devcapsule.{module}.daemon_is_rootless", lambda: False) DOCKER_MARKERS = ("e2e", "base_build_e2e", "recursive_e2e", "contributor_e2e", "ide_smoke", "docker") diff --git a/devcapsule-src/tests/integration/test_github_proxy.py b/devcapsule-src/tests/integration/test_github_proxy.py index a8abf31b..c98f1c07 100644 --- a/devcapsule-src/tests/integration/test_github_proxy.py +++ b/devcapsule-src/tests/integration/test_github_proxy.py @@ -140,8 +140,9 @@ def test_broker_shutdown_closes_incomplete_requests(tmp_path: Path) -> None: @pytest.mark.parametrize('enabled', [False, True]) -def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled: bool) -> None: - """The Docker plan carries the Git adapter and broker mount only after explicit authorization.""" +@pytest.mark.parametrize('rootless', [False, True]) +def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled: bool, rootless: bool) -> None: + """Rootless and rootful plans carry the adapter/socket only with authorization and use the corresponding identity.""" from devcapsule.launch.pycharm._launcher import PycharmRunOptions, DockerMode, TempRuntimeFiles, build_run_config, build_docker_args from devcapsule.platforms import Platform from devcapsule.github_proxy import GIT_SSH_COMMAND, SOCKET_DESTINATION @@ -152,10 +153,11 @@ def test_launcher_exposes_the_socket_only_with_the_grant(tmp_path: Path, enabled env = {**env, 'DISPLAY': ':1', 'PYCHARM_GIT_IDENTITY_FROM_HOST': '0', SOCKET_ENV: str(path)} config = build_run_config(PycharmRunOptions(project=project, capsule_platform=Platform.LINUX_AMD64, docker_mode=DockerMode.none, enable_sudo=False, network_mode='bridge', - enable_github_ssh=enabled), env) + enable_github_ssh=enabled, rootless_docker=rootless), env) files = TempRuntimeFiles(xauth_file=tmp_path / 'xauth', passwd_file=tmp_path / 'passwd', group_file=tmp_path / 'group') args = build_docker_args(config, files, env) assert (f'GIT_SSH_COMMAND={GIT_SSH_COMMAND}' in args) is enabled assert (f'{SOCKET_ENV}={SOCKET_DESTINATION}' in args) is enabled assert (f'type=bind,src={path},dst={SOCKET_DESTINATION},ro' in args) is enabled + assert args[args.index('--user') + 1] == ('0:0' if rootless else f'{os.getuid()}:{os.getgid()}') assert not any('.ssh' in arg or 'SSH_AUTH_SOCK=' in arg for arg in args) diff --git a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py index 3be60b0f..ab1d06af 100644 --- a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py +++ b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_filesystem.py @@ -92,3 +92,22 @@ def test_prepare_filesystem_assigns_ownership_only_to_newly_created_ancestors_wh (tmp_path / "runtime", 1000, 1001), (runtime_dir, 1000, 1001), ] + + +def test_mapped_root_creates_directories_without_redundant_chown(tmp_path: Path) -> None: + """Rootless 0:0 preparation creates private directories without requiring chown capabilities.""" + home = tmp_path / "home" + runtime_dir = tmp_path / "runtime" + plan = FilesystemPlan( + directories=(str(home), str(home / ".ssh"), str(runtime_dir)), + environment={"HOME": str(home), "XDG_RUNTIME_DIR": str(runtime_dir)}, + ) + chowned: list[tuple[Path, int, int]] = [] + prepare_filesystem( + plan, Identity(0, 0, "root"), caller_uid=0, + chown=lambda path, uid, gid: chowned.append((path, uid, gid)), + ) + assert chowned == [] + assert all(Path(directory).is_dir() for directory in plan.directories) + assert stat.S_IMODE((home / ".ssh").stat().st_mode) == 0o700 + assert stat.S_IMODE(runtime_dir.stat().st_mode) == 0o700 diff --git a/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py new file mode 100644 index 00000000..b63a130d --- /dev/null +++ b/devcapsule-src/tests/unit/devcapsule/container_runtime/test_identity.py @@ -0,0 +1,32 @@ +"""Unit tests of container_runtime.identity: privilege switching for explicit identities.""" + +import pytest + +from devcapsule.container_runtime.contract import Identity +from devcapsule.container_runtime.identity import foreground_command + + +@pytest.mark.parametrize("caller_uid", [0, 1000]) +def test_mapped_root_command_needs_no_privilege_switch(caller_uid: int) -> None: + """A target of 0:0 needs no gosu wrapper, including a rootless Docker root caller.""" + command = ("ide", "--wait") + assert foreground_command(command, Identity(0, 0, "root"), caller_uid=caller_uid) == command + + +@pytest.mark.parametrize("uid,gid", [(1000, 1001), (0, 1001), (1000, 0)]) +def test_root_caller_switches_when_either_target_id_differs(uid: int, gid: int) -> None: + """Root still invokes gosu for any target other than 0:0, preserving rootful behavior.""" + assert foreground_command(("ide",), Identity(uid, gid, "developer"), caller_uid=0) == ( + "gosu", f"{uid}:{gid}", "ide", + ) + + +def test_non_root_caller_keeps_the_command() -> None: + """A non-root caller does not invoke a privilege switch.""" + assert foreground_command(("ide",), Identity(1000, 1001, "developer"), caller_uid=1000) == ("ide",) + + +def test_empty_command_is_refused_for_mapped_root() -> None: + """Skipping gosu does not make an empty foreground command valid.""" + with pytest.raises(ValueError, match="must not be empty"): + foreground_command((), Identity(0, 0, "root"), caller_uid=0) diff --git a/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py b/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py index b1581f3c..35d7192c 100644 --- a/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py +++ b/devcapsule-src/tests/unit/devcapsule/launch/pycharm/test_launcher.py @@ -9,6 +9,7 @@ from dataclasses import replace from datetime import datetime, timedelta, timezone import json +import os from pathlib import Path import socket from threading import Event @@ -1883,3 +1884,86 @@ def test_build_run_config_omits_checkout_console_source_under_a_built_launcher_u assert opted_in_config.runtime_plan is not None and opted_in_config.runtime_plan.console is not None assert opted_in_config.runtime_plan.console.source_path == "/workspace/project/devcapsule-webconsole" + +def test_rootless_docker_runs_container_as_mapped_root_while_rootful_keeps_host_uid( + tmp_path: Path, +) -> None: + """Rootless Docker maps host mounts to ``0:0`` and drops ``--user``/``IDE_*`` to ``0:0``/``root``; non-rootless preserves host identity.""" + env = base_env(tmp_path) + env["XDG_RUNTIME_DIR"] = str(tmp_path / "runtime") + rootful_config = contained_config(tmp_path) + rootless_config = contained_config(tmp_path, rootless_docker=True) + assert rootful_config.rootless_docker is False + assert rootless_config.rootless_docker is True + + rootful_files = prepare_temp_runtime_files(rootful_config, env) + rootless_files = prepare_temp_runtime_files(rootless_config, env) + try: + rootful_args = build_docker_args(rootful_config, rootful_files, env) + rootless_args = build_docker_args(rootless_config, rootless_files, env) + + host_uid, host_gid = os.getuid(), os.getgid() + assert rootful_args[rootful_args.index("--user") + 1] == f"{host_uid}:{host_gid}" + assert f"IDE_UID={host_uid}" in rootful_args + assert f"IDE_GID={host_gid}" in rootful_args + + assert rootless_args[rootless_args.index("--user") + 1] == "0:0" + assert "IDE_UID=0" in rootless_args + assert "IDE_GID=0" in rootless_args + assert "IDE_USER=root" in rootless_args + assert rootless_files.passwd_file.read_text(encoding="utf-8") == ( + "root:x:0:0:PyCharm Docker User:/home/devcapsule:/bin/bash\n" + ) + assert rootless_files.group_file.read_text(encoding="utf-8") == "root:x:0:\n" + finally: + cleanup_temp_runtime_files(rootful_files) + cleanup_temp_runtime_files(rootless_files) + + +def test_rootless_host_docker_socket_owned_by_host_user_maps_to_container_root_group( + tmp_path: Path, +) -> None: + """In rootless ``host`` Docker mode, a daemon socket owned by the host user's GID maps to container GID ``0``; rootful keeps its host GID.""" + sock_path = tmp_path / "docker.sock" + listener = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + listener.bind(str(sock_path)) + try: + rootful_config = contained_config( + tmp_path, docker_mode=DockerMode.host, host_docker_socket=sock_path, rootless_docker=False + ) + rootless_config = contained_config( + tmp_path, docker_mode=DockerMode.host, host_docker_socket=sock_path, rootless_docker=True + ) + assert rootful_config.host_docker_gid == os.getgid() + assert rootless_config.host_docker_gid == 0 + + env = base_env(tmp_path) + env["XDG_RUNTIME_DIR"] = str(tmp_path / "runtime") + files = prepare_temp_runtime_files(rootless_config, env) + try: + args = build_docker_args(rootless_config, files, env) + assert args[args.index("--user") + 1] == "0:0" + assert args[args.index("--group-add") + 1] == "0" + assert files.group_file.read_text(encoding="utf-8") == "root:x:0:\n" + finally: + cleanup_temp_runtime_files(files) + finally: + listener.close() + + +def test_rootless_sudoers_policy_ownership_accepts_host_user_file_without_chown( + tmp_path: Path, +) -> None: + """Under rootless Docker, the host-owned sudoers policy is already container-root-owned so preparation skips ``chown``.""" + env = base_env(tmp_path) + env["XDG_RUNTIME_DIR"] = str(tmp_path / "runtime") + config = contained_config(tmp_path, enable_sudo=True, rootless_docker=True) + files = prepare_temp_runtime_files(config, env) + try: + ensure_sudoers_policy_ownership(config, files, env) + assert files.sudoers_volume is None + assert "ide-sudo:x:" in files.group_file.read_text(encoding="utf-8") + assert files.shadow_file is not None + assert files.shadow_file.read_text(encoding="utf-8").startswith("root:*:") + finally: + cleanup_temp_runtime_files(files) diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 5e4328d6..70154149 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: paused for owner host relaunch; local PEX built, recursive authentication E2E next. +State: active; propagating the released rootless Docker fix and rebuilding the host-relaunch PEX. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -80,6 +80,34 @@ Observe the Git checkout independently and retain WebM/screenshots/action eviden Recursive proxy inheritance is still unproved; do not report success until observed. The 19 reproduced full-gate failures and formal PR delivery remain open separately. +## Rootless Docker fix propagation + +The owner's first host relaunch failed with an unwritable PEX cache and permission +denied on `/etc/devcapsule/home/settings.toml`. Fetched canonical `release-0.3.1` +and identified `a1a54eba3098eb3c80fd78ef3e483d1bb5314466`, merged in PR #38 and +recorded as owner-verified on the target host by `8bd56eb3`. Neither current main +nor this feature branch included it. Main has no new commits to synchronize. + +Applied that fix's deliverable/test patch, without release version changes or +another workstream's records, under the owner's explicit propagation request and +WORKFLOW-LOCAL.md's release-fix propagation exception. The branch predates the +release's CapsuleUser refactor: adapted its `command` correction to the existing +`foreground_command`, and its `owner_changes` correction to `prepare_filesystem`. +Rootless launches use container 0:0 (the host user's mapped identity), map the +Docker socket group and home accordingly, and avoid redundant gosu/chown under +dropped capabilities. Rootful behavior is preserved. The available Docker daemon +reports `name=rootless`; no container was launched for this propagation. + +Checks so far: 151 focused tests passed, then all 12 GitHub proxy integration +cases passed after extending socket-grant coverage to both daemon modes. The +full gate's type check passed over 340 files; conventions found zero breaches. +New pure identity tests and filesystem preparation tests cover the adaptation. +Full gate: 19 failed, 2942 passed, 1 skipped, 78 deselected, 14 xfailed, +7 xpassed. All 19 failure IDs match the previously reproduced mainline/environment +baseline exactly; no new failure IDs. Packaging/smoke is run separately because +the full gate stops at pytest. The rebuilt artifact identity follows below. +Mainline integration and the separate Gemini review branch remain pending. + ## Host relaunch handoff The owner explicitly chose a development-build self-hosting experiment. The normal From d679656d8a47ac50adf6ecc122175b504129a04c Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 09:09:55 +0000 Subject: [PATCH 19/35] Record rebuilt rootless-compatible PEX and retry handoff Co-authored-by: GPT-6 Codex --- CURRENT-STATUS.md | 2 +- .../2026-11-16-github-auth/CURRENT-STATUS.md | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 66ff89e8..5cc8e442 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | active; propagating rootless Docker fix and rebuilding local PEX | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused for owner retry; rootless Docker fix included in rebuilt PEX | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 70154149..09b71168 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: active; propagating the released rootless Docker fix and rebuilding the host-relaunch PEX. +State: paused for owner retry; rootless Docker fix propagated and local PEX rebuilt. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -105,7 +105,10 @@ New pure identity tests and filesystem preparation tests cover the adaptation. Full gate: 19 failed, 2942 passed, 1 skipped, 78 deselected, 14 xfailed, 7 xpassed. All 19 failure IDs match the previously reproduced mainline/environment baseline exactly; no new failure IDs. Packaging/smoke is run separately because -the full gate stops at pytest. The rebuilt artifact identity follows below. +the full gate stops at pytest. The separate PEX build and executable smoke passed (14 seconds). The imported +rootless detector reports True against the available daemon. Fix commit: `d6fd93e0`. +The rebuilt artifact identity follows below. Host relaunch itself still needs +the owner to retry; no container or persistent configuration was changed here. Mainline integration and the separate Gemini review branch remain pending. ## Host relaunch handoff @@ -117,21 +120,22 @@ existing persistent state. The original checkout still has only its pre-existing website gitlink modification. Main remains unchanged; synchronization can wait. Build: `.venv/bin/python -m nox -s pex` passed, including shell syntax, packaging -and executable smoke. Built from clean `712466ab9547ec7c9153977f725fc6cfe26344f6`. +and executable smoke. Built from clean `d6fd93e0ba6c18915ace45b9dfef2881068d7b39` with the rootless Docker fix. The local PEX reports `v0.3.2.dev0-local-linux-x86_64` with revision `unknown` by local-build policy; the clean Git revision and checksum here identify its inputs and bytes independently. It contains the committed implementation, including the node-registry and console-table fixes. Review corrections at `388f62a3` remain on the separate review branch and are not included; their PR merge is still pending. -SHA-256: `2f9470c6ca5efeaccd9844bf6feb1494d2e95564d054051a2c005afbc35d78ed`. +SHA-256: `33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b`. Capsule artifact: `/home/devcapsule/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. Host artifact (from Docker mount inspection): `/home/costin/.local/share/devcapsule/projects/by-path/fe326e03c822-devcapsule/home/worktrees/github-auth/devcapsule-src/dist/devcapsule-local.pex`. Checksum and build metadata are saved beside it, outside Git. -After exiting Codex and stopping the current capsule normally, run on the host: +Retry the same command with the replacement file. After exiting Codex and stopping +the current capsule normally, run on the host: ```sh cd /opt/costin3/myProjects/devcapsule @@ -169,7 +173,8 @@ of the proxy. No E2E container was launched during handoff preparation. ## Open Threads -- Awaiting the human: exit/relaunch/resume using the command above. +- Awaiting the human: retry exit/relaunch/resume using the rebuilt PEX above. + The first relaunch failed on the rootless identity defect, now propagated. Workstream selection, Git-over-SSH scope and the relaunch approach are accepted. Full-gate environment and owner-operated GitHub PR delivery also remain open. - Engineering choices: initial transport must account for Linux/rootless Docker From 7ec3c8ce3830315bbc145777e8641a2ef9a7e177 Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 09:18:39 +0000 Subject: [PATCH 20/35] docs: record successful host proxy relaunch preflight --- CURRENT-STATUS.md | 2 +- .../2026-11-16-github-auth/CURRENT-STATUS.md | 51 +++++++++++++++---- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 5cc8e442..20fd4edb 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused for owner retry; rootless Docker fix included in rebuilt PEX | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused after successful host relaunch/proxy preflight; recorded E2E preparation next | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 09b71168..2a8505ed 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: paused for owner retry; rootless Docker fix propagated and local PEX rebuilt. +State: paused after successful host relaunch preflight; recorded E2E preparation is next. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -73,9 +73,9 @@ checkout remains untouched. Review evidence and prepared PR text are indexed bel Owner chose on 2026-11-16 to exit this capsule, relaunch the same checkout from the physical host with the new local PEX and github-ssh enabled, then resume Codex. -This supersedes the proposed SSH-to-host harness adaptation. After resumption, -verify the inherited broker and executable bytes, then write and execute the -recorded E2E scenario using a fresh project and explicit github-ssh authorization. +This supersedes the proposed SSH-to-host harness adaptation. The resumed-session +preflight below verified the broker and executable bytes. Next, write and execute +the recorded E2E scenario using a fresh project and explicit github-ssh authorization. Observe the Git checkout independently and retain WebM/screenshots/action evidence. Recursive proxy inheritance is still unproved; do not report success until observed. The 19 reproduced full-gate failures and formal PR delivery remain open separately. @@ -107,8 +107,8 @@ Full gate: 19 failed, 2942 passed, 1 skipped, 78 deselected, 14 xfailed, baseline exactly; no new failure IDs. Packaging/smoke is run separately because the full gate stops at pytest. The separate PEX build and executable smoke passed (14 seconds). The imported rootless detector reports True against the available daemon. Fix commit: `d6fd93e0`. -The rebuilt artifact identity follows below. Host relaunch itself still needs -the owner to retry; no container or persistent configuration was changed here. +The rebuilt artifact identity follows below. The owner subsequently relaunched +successfully; the resumed-session observations are recorded below. Mainline integration and the separate Gemini review branch remain pending. ## Host relaunch handoff @@ -171,15 +171,44 @@ must have no copied key; the recording harness must authorize github-ssh for its fresh project. Do not use this current capsule's manually installed key as proof of the proxy. No E2E container was launched during handoff preparation. +## Resumed-session readiness, 2026-11-16 + +The owner resumed inside the rebuilt capsule. Running `/opt/devcapsule/bin/devcapsule.pex` +and the saved local PEX both match the SHA-256 above. Runtime identity is 0:0 and +Docker reports `name=rootless`. The canonical proxy socket exists, and Git's SSH +command points to the shipped `github-ssh` adapter. + +A read-only `git ls-remote git@github.com:mycodespace-ai/devcapsule.git HEAD` +returned `1942f396dd11ffabb64784eb7279f283181e32dd`. Repeating it from a fresh empty +HOME with system/global Git configuration disabled and the adapter explicitly +selected also passed. Pointing that invocation at a missing socket failed with +exit 128. This demonstrates proxy authentication independently of the manually +installed capsule key; it does not yet prove a new child capsule's inheritance. + +Docker mount inspection resolves the inherited socket to the outside launcher's +host socket, and resolves both the persistent E2E workspace root and PEX directory +to host-backed paths. Docker access, Git, pytest and nox are available. No new +container was started for this preflight. Canonical main was fetched: this branch +is ten commits ahead and zero behind, so no synchronization is needed. Mail and +intake remain empty. + +Recording preparation remains: author the GitHub showcase scenario and its Markdown +procedure, explicitly grant github-ssh in its fresh project, and prepare Playwright. +This capsule has neither local Playwright nor FFmpeg nor `/opt/playwright`; the +existing recorder supports Chromium/FFmpeg in a child with `browser-automation`. +The scenario must request that component, verify a real SSH Git checkout from +outside the recorded UI, retain video/screenshots/action evidence, and clean up. +No nested E2E or recording has run yet. No additional host credentials or restart +are indicated by the preflight. The outside launcher must stay running. + ## Open Threads -- Awaiting the human: retry exit/relaunch/resume using the rebuilt PEX above. - The first relaunch failed on the rootless identity defect, now propagated. - Workstream selection, Git-over-SSH scope and the relaunch approach are accepted. - Full-gate environment and owner-operated GitHub PR delivery also remain open. +- Host relaunch and authenticated proxy preflight passed. The next task is the + fresh-project recorded E2E; recursive inheritance and recording remain unproved. + Full-gate environment and owner-operated GitHub PR delivery remain open. - Engineering choices: initial transport must account for Linux/rootless Docker permissions; Docker Desktop cannot bind ordinary host Unix sockets. -- Physical-host launch acceptance and Docker Desktop transport remain open. +- Physical-host relaunch is observed working; Docker Desktop transport remains deferred. - GitHub CLI is absent at the policy-prescribed path. PR creation/merging remains owner-operated under WORKFLOW-LOCAL.md; prepare reviewed commits and compare links. - Reviewer worktree removed at pause; pushed `ws-github-auth/ssh-proxy-review` From e45866b2d420cd8f71cd6d1c311d02cdaaa9d7df Mon Sep 17 00:00:00 2001 From: Costin Cozianu Date: Mon, 16 Nov 2026 09:31:34 +0000 Subject: [PATCH 21/35] test: record GitHub proxy clone in a fresh capsule Allow scenario-specific launch needs and consents in the interactive recorder, bound its interactive phase, and retain the procedure and successful rootless proxy E2E evidence. Co-authored-by: GPT-6 Astra Codex --- CURRENT-STATUS.md | 2 +- devcapsule-src/tests/e2e/scenario_recorder.py | 12 +- .../2026-11-16-github-auth/CURRENT-STATUS.md | 51 ++++-- .../evidence/showcase.md | 147 ++++++++++++++++++ 4 files changed, 199 insertions(+), 13 deletions(-) create mode 100644 engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md diff --git a/CURRENT-STATUS.md b/CURRENT-STATUS.md index 20fd4edb..36cc4715 100644 --- a/CURRENT-STATUS.md +++ b/CURRENT-STATUS.md @@ -9,7 +9,7 @@ coordination branch. Detailed state belongs in each linked status file. | Name | Started | Goal | State | Branch association | Status file | |---|---|---|---|---|---| -| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused after successful host relaunch/proxy preflight; recorded E2E preparation next | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | +| `github-auth` | 2026-11-16 | GitHub SSH authentication through a launcher-owned host proxy | paused after successful recorded proxy E2E; review integration and baseline gate disposition pending | `ws-github-auth/ssh-proxy` | [workstream status](engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md) | | `workflow-improvements` | 2026-08-09 | Improve the multiple-stream workflow from concrete dogfood findings during the recursive E2E cycle | active 2026-10-30; Pair Review Loop naming implemented and packaging verified for 0.3.0; mainline and release integration pending | `ws-workflow-improvements/v1` | [workstream status](engineering-docs/wip/2026-08-09-workflow-improvements/CURRENT-STATUS.md) | | `project-management` | 2026-08-09 | Maintain project-wide priorities, sequencing, dependencies, and lifecycle coordination | paused 2026-10-11 at owner direction; consent and forced configuration before RC work; remaining diagnostics/image scope and release cut decisions retained | `ws-project-management/coordination` | [workstream status](engineering-docs/wip/2026-08-09-project-management/CURRENT-STATUS.md) | | `maintenance` | 2026-09-18 | Own the defects no open workstream covers, on `main` and on maintained release lines, and drive maintenance releases | active; releasing 0.3.1; cut 2026-11-07 from `main` at `31e8fec7`, `main` reopened at `0.3.2.dev0` (#8); candidate 2 (`v0.3.1-rc2` on `be0140ca`, propagated to `main` via PR #22 `aabace3c`) published and verified: Eclipse libsoup (#10, closed on rc1), launcher-delivered `devcapsule_webconsole` (`D-0017`, #18/#16, closed on rc2), Codex npm abbreviated metadata (#20, closed on rc2 + `component-status` `0bc31c64`), and Codex `0.167.0` + Claude Code `2.1.321` pins (`embedded-33`, #21); rulings open: executables for the Mac, the compatibility feed's home | `release-0.3.1`, the workstream's selection from the cut to the final tag; `release-0.3.0` closed and retained; earlier slice branches merged and closed | [workstream status](engineering-docs/wip/2026-09-18-maintenance/CURRENT-STATUS.md) | diff --git a/devcapsule-src/tests/e2e/scenario_recorder.py b/devcapsule-src/tests/e2e/scenario_recorder.py index a8dd8352..c1df19ef 100644 --- a/devcapsule-src/tests/e2e/scenario_recorder.py +++ b/devcapsule-src/tests/e2e/scenario_recorder.py @@ -599,18 +599,25 @@ def serve_interactive_controller( executable: Path, spec: ScenarioSpec, evidence: Path, + *, + surface: IdeSurface | None = None, + timeout: float = 1800.0, ) -> None: """Start an interactive E2E IDE session and HTTP controller for live agent takeover. ``controller.json`` under ``evidence`` names the port once the IDE is ready; a stale copy from an earlier run is removed first, so a waiting ``ctl`` never dials a dead port. + A caller may supply scenario-specific needs and consents through ``surface``. + The interactive phase has an overall deadline, including time between commands. """ from playwright.sync_api import sync_playwright # type: ignore[import-not-found,unused-ignore] evidence.mkdir(parents=True, exist_ok=True) (evidence / "controller.json").unlink(missing_ok=True) os.environ["DEVCAPSULE_SMOKE_COMPONENT_BROWSER"] = "1" - surface: IdeSurface = next(s for s in SURFACES if s.name == spec.surface) + surface = surface or next(s for s in SURFACES if s.name == spec.surface) + if surface.name != spec.surface: + raise DriverError("Scenario and launch surface must agree") recorder = BurstFrameRecorder(evidence / "video-frames") action_log: list[dict[str, Any]] = [] step_counter = 0 @@ -657,8 +664,11 @@ def serve_interactive_controller( print(f"CONTROLLER_READY port={port} project_path={session.project_path}", flush=True) stopped = False + deadline = time.monotonic() + timeout try: while not stopped: + if time.monotonic() >= deadline: + raise DriverError("Interactive scenario deadline expired") item = queue.take() if item is None: time.sleep(0.05) diff --git a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md index 2a8505ed..b7d2ee11 100644 --- a/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md +++ b/engineering-docs/wip/2026-11-16-github-auth/CURRENT-STATUS.md @@ -4,7 +4,7 @@ Mnemonic: `github-auth` Start date: 2026-11-16 -State: paused after successful host relaunch preflight; recorded E2E preparation is next. +State: paused after successful recorded GitHub SSH proxy E2E; review integration and baseline gate disposition remain open. Definition read: WORKFLOW.md@89de30622697, WORKFLOW-LOCAL.md@d92bdcbb0a01 @@ -71,14 +71,12 @@ checkout remains untouched. Review evidence and prepared PR text are indexed bel ## Planned next step -Owner chose on 2026-11-16 to exit this capsule, relaunch the same checkout from -the physical host with the new local PEX and github-ssh enabled, then resume Codex. -This supersedes the proposed SSH-to-host harness adaptation. The resumed-session -preflight below verified the broker and executable bytes. Next, write and execute -the recorded E2E scenario using a fresh project and explicit github-ssh authorization. -Observe the Git checkout independently and retain WebM/screenshots/action evidence. -Recursive proxy inheritance is still unproved; do not report success until observed. -The 19 reproduced full-gate failures and formal PR delivery remain open separately. +The owner-authorized host relaunch and recorded fresh-capsule E2E are complete. +The next delivery step is to resolve the known gate baseline and have the owner +merge the separate Gemini review corrections through the prepared PR process. +Validate the combined branch before integration; the recorded PEX does not contain +those corrections. The engineering showcase and procedure are indexed below. +No further restart or host credential setup is needed for this completed test. ## Rootless Docker fix propagation @@ -201,10 +199,40 @@ outside the recorded UI, retain video/screenshots/action evidence, and clean up. No nested E2E or recording has run yet. No additional host credentials or restart are indicated by the preflight. The outside launcher must stay running. +## Recorded E2E acceptance, 2026-11-16 + +One fresh VSCodium capsule, with browser-automation and explicit github-ssh +consent, was launched by the saved PEX from this capsule. The outer physical-host +launcher remained alive. Child and parent Docker inspection showed the same host +broker source, mounted read-only at the canonical socket. The child had no SSH +files or agent; its delivered runtime checksum matched the tested PEX. + +Through the actual recorded terminal, Git cloned the canonical repository over +SSH. Independent Git inspection of the host-backed checkout confirmed expected +HEAD `1942f396dd11ffabb64784eb7279f283181e32dd`, the SSH origin, a clean tree and +the README. A missing-socket read failed with exit 128. Ordinary proxy access +succeeded again afterward. This proves recursive proxy inheritance on this +rootless Linux host. No GitHub write was made. + +Evidence: `devcapsule-src/dist/e2e-evidence/github-auth/20261116T092248Z-33c3c7bc/`. +The 38.17-second WebM, full desktop capture, poster, action trace, `run.json`, +`verification.json` and `result.json` are retained outside Git. The test-owned +container, fresh workspace and checkout records are independently confirmed gone. +The controller gained optional per-scenario surface needs/consents and a bounded +interactive phase; the product PEX was not changed. All 62 existing recorder tests +passed. See the procedure for the on-screen empty-directory check correction and +a minor final command/output overlap, both retained in the original recording. + +Full gate for the recorder change: 19 failed, 2942 passed, 1 skipped, 78 deselected, +14 xfailed and 7 xpassed. All 19 failure IDs exactly match the recorded baseline. +Types passed over 340 files; the corrected convention check found zero breaches. The initial attempt +used the stale default origin/main comparison and stopped on three unrelated +convention differences. The corrected run used DEVCAPSULE_DIFF_BASE=upstream/main. Its log is retained at +`~/.cache/devcapsule-gate/github-auth-showcase-build-upstream.log`. + ## Open Threads -- Host relaunch and authenticated proxy preflight passed. The next task is the - fresh-project recorded E2E; recursive inheritance and recording remain unproved. +- Host relaunch, recursive inheritance and recorded fresh-project E2E passed. Full-gate environment and owner-operated GitHub PR delivery remain open. - Engineering choices: initial transport must account for Linux/rootless Docker permissions; Docker Desktop cannot bind ordinary host Unix sockets. @@ -218,6 +246,7 @@ are indicated by the preflight. The outside launcher must stay running. ## Workstream document index +- [Recorded E2E procedure](evidence/showcase.md) - [Review brief](evidence/review-brief.md) - [Review follow-up 1](evidence/round-1.md) - [Intake](intake/README.md) diff --git a/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md b/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md new file mode 100644 index 00000000..49fff04b --- /dev/null +++ b/engineering-docs/wip/2026-11-16-github-auth/evidence/showcase.md @@ -0,0 +1,147 @@ +# GitHub SSH proxy: recorded fresh-capsule clone + +## Purpose and acceptance + +Show a user cloning a GitHub repository over SSH inside a fresh capsule. The +outside launcher uses the host identity. No SSH key or agent is supplied to the +child. Public GitHub repositories also require authentication on their SSH URL. +This scenario reads the canonical DevCapsule repository; it does not push. + +Pass requires a real clone, the expected independently read commit, the unchanged +SSH origin, a clean working tree, no child SSH files or agent, and the canonical +broker socket mounted from the same host source as the parent. A second Git read +with an absent broker socket must fail. Keep the recording and independent +observations, and remove the fresh container, workspace and checkout state. +A video alone does not pass the test. + +## Preconditions + +Keep the outside launcher alive with `github-ssh` enabled. The host must already +trust GitHub and authenticate noninteractively. An unencrypted default SSH key +needs no host agent. The parent needs host Docker and host networking. + +From `devcapsule-src`, use the development virtualenv and the exact PEX under test. +The recorded run uses SHA-256 +`33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b`, built from +`d6fd93e0`. Do not rebuild between the launch and the evidence capture. + +```sh +.venv/bin/python -m pip install 'playwright==1.63.0' +export DEVCAPSULE_EXPECTED_SHA256=33c3c7bc7fe6ff6772eedee06424c95afcf4aebddca20ab99f9a31a7c103d35b +``` + +## Start the recording controller + +Save the following as a temporary Python script and run it with +`PYTHONPATH=. .venv/bin/python