diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 0000000..07aa457 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,49 @@ +FROM mcr.microsoft.com/devcontainers/base:trixie + +# Version pins — change these ARGs and rebuild to switch versions. +ARG PYTHON_VERSION=3.13 +ARG NODE_VERSION=lts + +ENV PYENV_ROOT=/usr/local/pyenv \ + NVM_DIR=/usr/local/nvm + +# The padawan-fw runc shim bind-mounts a mode-0700 tmpdir over /etc/ssl/certs in every +# container it starts. Each RUN step opens with chmod 755 /etc/ssl/certs so that both +# root and the non-root vscode user can reach the cert bundle for TLS. + +# Build dependencies needed by pyenv to compile Python from source. +RUN chmod 755 /etc/ssl/certs \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ + libsqlite3-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \ + libxmlsec1-dev libffi-dev liblzma-dev \ + && rm -rf /var/lib/apt/lists/* + +# Install pyenv to /usr/local/pyenv and compile the pinned Python version. +# Direct git clone avoids the pyenv-installer script's Launchpad/GitHub API calls +# and makes the install reproducible. +RUN chmod 755 /etc/ssl/certs \ + && git clone https://github.com/pyenv/pyenv.git ${PYENV_ROOT} \ + && ${PYENV_ROOT}/bin/pyenv install ${PYTHON_VERSION} \ + && ${PYENV_ROOT}/bin/pyenv global ${PYTHON_VERSION} \ + && chown -R root:root ${PYENV_ROOT} \ + && chmod -R a+rX ${PYENV_ROOT} \ + && ln -sf "$(${PYENV_ROOT}/bin/pyenv which python3)" /usr/local/bin/python3 \ + && ln -sf "$(${PYENV_ROOT}/bin/pyenv which pip3)" /usr/local/bin/pip3 \ + && ln -sf "$(${PYENV_ROOT}/bin/pyenv which pip3)" /usr/local/bin/pip + +# Install nvm to /usr/local/nvm and install the pinned Node version. +RUN chmod 755 /etc/ssl/certs \ + && mkdir -p "${NVM_DIR}" \ + && curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.40.3/install.sh \ + | NVM_DIR="${NVM_DIR}" bash \ + && . "${NVM_DIR}/nvm.sh" \ + && nvm install "${NODE_VERSION}" \ + && nvm alias default "${NODE_VERSION}" \ + && NODE_BIN="$(nvm which default)" \ + && chown -R root:root "${NVM_DIR}" \ + && chmod -R a+rX "${NVM_DIR}" \ + && ln -sf "${NODE_BIN}" /usr/local/bin/node \ + && ln -sf "$(dirname "${NODE_BIN}")/npm" /usr/local/bin/npm \ + && ln -sf "$(dirname "${NODE_BIN}")/npx" /usr/local/bin/npx diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index d21bba3..324d17d 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -3,14 +3,8 @@ "dockerComposeFile": "docker-compose.yml", "service": "app", "workspaceFolder": "/workspaces/repro", - "features": { - "ghcr.io/devcontainers/features/node:1": { - "version": "22" - }, - "ghcr.io/devcontainers/features/python:1": { - "version": "3.13" - } - }, "onCreateCommand": "./.devcontainer/on-create.sh", - "updateContentCommand": "./.devcontainer/update-content.sh" + "updateContentCommand": "./.devcontainer/update-content.sh", + "postCreateCommand": "./.devcontainer/post-create.sh", + "postStartCommand": "./.devcontainer/post-start.sh" } diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml index 1a2d269..eee9924 100644 --- a/.devcontainer/docker-compose.yml +++ b/.devcontainer/docker-compose.yml @@ -1,12 +1,8 @@ services: app: - image: mcr.microsoft.com/devcontainers/base:debian-13 + build: + context: .. + dockerfile: .devcontainer/Dockerfile command: sleep infinity volumes: - ..:/workspaces/repro - - npm-cache:/home/vscode/.cache/npm - - pip-cache:/home/vscode/.cache/pip - -volumes: - npm-cache: - pip-cache: diff --git a/.devcontainer/on-create.sh b/.devcontainer/on-create.sh index c74a536..b965956 100755 --- a/.devcontainer/on-create.sh +++ b/.devcontainer/on-create.sh @@ -1,44 +1,44 @@ #!/usr/bin/env bash set -euxo pipefail -mkdir -p "$HOME/.cache/npm" "$HOME/.cache/pip" -sudo chown "$(id -u):$(id -g)" "$HOME/.cache" "$HOME/.cache/npm" "$HOME/.cache/pip" || true +# The padawan-fw runc shim bind-mounts a mode-0700 tmpdir over /etc/ssl/certs when this +# container was started. Repair the permissions so non-root users can traverse the +# directory and use TLS normally for the lifetime of this container. +sudo chmod 755 /etc/ssl/certs -echo "=== user ===" +echo "=== [onCreateCommand] user ===" whoami id -echo "=== env ===" -env | sort -echo "=== filtered env ===" -env | grep -E 'SSL|REQUESTS|CURL|NODE|NPM|PIP|PYTHON' || true +echo "=== [onCreateCommand] python version (root) ===" +sudo python3 --version -echo "=== ssl bundle ===" -ls -l /etc/ssl/certs || true -ls -l /etc/ssl/certs/ca-certificates.crt || true +echo "=== [onCreateCommand] python version (vscode) ===" +python3 --version + +echo "=== [onCreateCommand] node version (root) ===" +sudo "$(which node)" --version -echo "=== python ssl defaults ===" -python3 - <<'PY' -import ssl -print(ssl.get_default_verify_paths()) -PY +echo "=== [onCreateCommand] node version (vscode) ===" +node --version + +echo "=== [onCreateCommand] ssl bundle ===" +ls -l /etc/ssl/certs/ca-certificates.crt || true -echo "=== root curl ===" +echo "=== [onCreateCommand] root curl ===" sudo bash -lc ' set -euxo pipefail - whoami - id - env | grep -E "SSL|REQUESTS|CURL|NODE|NPM|PIP|PYTHON" || true - curl -Ivs https://github.com >/tmp/root-curl.txt 2>&1 || { - cat /tmp/root-curl.txt + echo "root curl user: $(whoami)" + curl -Ivs https://github.com >/tmp/oncreate-root-curl.txt 2>&1 || { + cat /tmp/oncreate-root-curl.txt exit 1 } - cat /tmp/root-curl.txt + cat /tmp/oncreate-root-curl.txt ' -echo "=== user curl ===" -curl -Ivs https://github.com >/tmp/user-curl.txt 2>&1 || { - cat /tmp/user-curl.txt +echo "=== [onCreateCommand] user curl ===" +curl -Ivs https://github.com >/tmp/oncreate-user-curl.txt 2>&1 || { + cat /tmp/oncreate-user-curl.txt exit 1 } -cat /tmp/user-curl.txt +cat /tmp/oncreate-user-curl.txt diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh new file mode 100755 index 0000000..5b221a2 --- /dev/null +++ b/.devcontainer/post-create.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# The padawan-fw runc shim bind-mounts a mode-0700 tmpdir over /etc/ssl/certs when this +# container was started. Repair the permissions so non-root users can traverse the +# directory and use TLS normally for the lifetime of this container. +sudo chmod 755 /etc/ssl/certs + +echo "=== [postCreateCommand] user ===" +whoami +id + +echo "=== [postCreateCommand] python version (root) ===" +sudo python3 --version + +echo "=== [postCreateCommand] python version (vscode) ===" +python3 --version + +echo "=== [postCreateCommand] node version (root) ===" +sudo "$(which node)" --version + +echo "=== [postCreateCommand] node version (vscode) ===" +node --version + +echo "=== [postCreateCommand] ssl bundle ===" +ls -l /etc/ssl/certs/ca-certificates.crt || true + +echo "=== [postCreateCommand] root curl ===" +sudo bash -lc ' + set -euxo pipefail + echo "root curl user: $(whoami)" + curl -Ivs https://github.com >/tmp/postcreate-root-curl.txt 2>&1 || { + cat /tmp/postcreate-root-curl.txt + exit 1 + } + cat /tmp/postcreate-root-curl.txt +' + +echo "=== [postCreateCommand] user curl ===" +curl -Ivs https://github.com >/tmp/postcreate-user-curl.txt 2>&1 || { + cat /tmp/postcreate-user-curl.txt + exit 1 +} +cat /tmp/postcreate-user-curl.txt + +echo "=== [postCreateCommand] pip install (non-root) ===" +pip install --user cowsay + +echo "=== [postCreateCommand] npm install (non-root) ===" +mkdir -p /tmp/npm-test +cd /tmp/npm-test +npm install cowsay diff --git a/.devcontainer/post-start.sh b/.devcontainer/post-start.sh new file mode 100755 index 0000000..5fe07c4 --- /dev/null +++ b/.devcontainer/post-start.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# The padawan-fw runc shim bind-mounts a mode-0700 tmpdir over /etc/ssl/certs each time +# this container starts. Repair the permissions so non-root users can traverse the +# directory and use TLS normally for the lifetime of this container session. +sudo chmod 755 /etc/ssl/certs + +echo "=== [postStartCommand] user ===" +whoami +id + +echo "=== [postStartCommand] python version (root) ===" +sudo python3 --version + +echo "=== [postStartCommand] python version (vscode) ===" +python3 --version + +echo "=== [postStartCommand] node version (root) ===" +sudo "$(which node)" --version + +echo "=== [postStartCommand] node version (vscode) ===" +node --version + +echo "=== [postStartCommand] ssl bundle ===" +ls -l /etc/ssl/certs/ca-certificates.crt || true + +echo "=== [postStartCommand] root curl ===" +sudo bash -lc ' + set -euxo pipefail + echo "root curl user: $(whoami)" + curl -Ivs https://github.com >/tmp/poststart-root-curl.txt 2>&1 || { + cat /tmp/poststart-root-curl.txt + exit 1 + } + cat /tmp/poststart-root-curl.txt +' + +echo "=== [postStartCommand] user curl ===" +curl -Ivs https://github.com >/tmp/poststart-user-curl.txt 2>&1 || { + cat /tmp/poststart-user-curl.txt + exit 1 +} +cat /tmp/poststart-user-curl.txt diff --git a/.devcontainer/update-content.sh b/.devcontainer/update-content.sh index b3676bb..00754a2 100755 --- a/.devcontainer/update-content.sh +++ b/.devcontainer/update-content.sh @@ -1,27 +1,44 @@ #!/usr/bin/env bash set -euxo pipefail -echo "=== updateContent user ===" +# The padawan-fw runc shim bind-mounts a mode-0700 tmpdir over /etc/ssl/certs when this +# container was started. Repair the permissions so non-root users can traverse the +# directory and use TLS normally for the lifetime of this container. +sudo chmod 755 /etc/ssl/certs + +echo "=== [updateContentCommand] user ===" whoami id -echo "=== filtered env ===" -env | grep -E 'SSL|REQUESTS|CURL|NODE|NPM|PIP|PYTHON' || true +echo "=== [updateContentCommand] python version (root) ===" +sudo python3 --version -echo "=== tool versions ===" -node --version -npm --version +echo "=== [updateContentCommand] python version (vscode) ===" python3 --version -python3 -m pip --version -echo "=== npm https test ===" -npm view lodash version +echo "=== [updateContentCommand] node version (root) ===" +sudo "$(which node)" --version + +echo "=== [updateContentCommand] node version (vscode) ===" +node --version + +echo "=== [updateContentCommand] ssl bundle ===" +ls -l /etc/ssl/certs/ca-certificates.crt || true -echo "=== pip https test ===" -python3 -m pip install --user requests +echo "=== [updateContentCommand] root curl ===" +sudo bash -lc ' + set -euxo pipefail + echo "root curl user: $(whoami)" + curl -Ivs https://github.com >/tmp/updatecontent-root-curl.txt 2>&1 || { + cat /tmp/updatecontent-root-curl.txt + exit 1 + } + cat /tmp/updatecontent-root-curl.txt +' -echo "=== python ssl defaults ===" -python3 - <<'PY' -import ssl -print(ssl.get_default_verify_paths()) -PY +echo "=== [updateContentCommand] user curl ===" +curl -Ivs https://github.com >/tmp/updatecontent-user-curl.txt 2>&1 || { + cat /tmp/updatecontent-user-curl.txt + exit 1 +} +cat /tmp/updatecontent-user-curl.txt diff --git a/README.md b/README.md index b7cf87b..77381b4 100644 --- a/README.md +++ b/README.md @@ -1,67 +1,188 @@ # Copilot cloud agent devcontainer non-root SSL repro -This repository is a minimal reproduction for a suspected issue in the GitHub Copilot cloud agent when using a devcontainer. +Minimal reproduction for the padawan-fw runc shim SSL bug in the GitHub Copilot cloud +agent, and an active workbench for testing the devcontainer setup. -## Summary +--- -The goal is to test whether HTTPS/network operations behave differently for: +## Root cause -- root/setup-time commands, versus -- the non-root devcontainer user (`vscode`) +The Copilot cloud agent environment runs **`padawan-fw`**, a network firewall that +performs TLS interception (MITM) on all outbound HTTPS. It installs a Bash shim at +`/usr/bin/runc` that intercepts every `runc create` call and injects bind-mounts into +the container's OCI `config.json` before the real `runc` runs. -inside the Copilot cloud agent environment. +The relevant part of the shim: -This repo intentionally avoids: +```bash +CERT_TMP_DIR=$(mktemp -d) # drwx------ (mode 0700) +cp "$CERT_PATH" "$CERT_TMP_DIR/ca-certificates.crt" +chmod 644 "$CERT_TMP_DIR/ca-certificates.crt" # fixes the FILE, not the DIR +# missing: chmod 755 "$CERT_TMP_DIR" +``` -- application code -- private registries -- custom CA certificates -- custom SSL configuration -- database setup +Inside every container the result is: -## What this repo does +``` +drwx------ 2 root root /etc/ssl/certs ← 0700 +-rw-r--r-- 1 root root /etc/ssl/certs/ca-certificates.crt ← 0644 +``` -The devcontainer: +`root` can traverse the 0700 dir (via `CAP_DAC_OVERRIDE`). Non-root users cannot, so +`curl` as `vscode` fails immediately with exit 77 (`CURLE_SSL_CACERT_BADFILE`). -- uses `mcr.microsoft.com/devcontainers/base:debian-13` -- installs standard Node and Python devcontainer features -- runs an `onCreateCommand` that: - - prints user and environment information - - checks the system CA bundle - - performs an HTTPS `curl` test as `root` - - performs the same HTTPS `curl` test as the non-root user -- runs an `updateContentCommand` that: - - prints tool versions - - performs HTTPS-backed package-manager operations as the non-root user: - - `npm view lodash version` - - `python3 -m pip install --user requests` +The shim runs at the OCI layer and affects every container regardless of the base image. -## Expected result +**Real fix:** add `chmod 755 "$CERT_TMP_DIR"` to the padawan-fw shim right after +`mktemp -d`. -All HTTPS operations should succeed: +--- -- root `curl` -- user `curl` -- `npm view lodash version` -- `python3 -m pip install --user requests` +## Current devcontainer approach -## Actual result in Copilot cloud agent +### Base image -If the suspected issue is present, one or more non-root operations fail with SSL/certificate/network errors, while root operations succeed. +`mcr.microsoft.com/devcontainers/base:trixie` (Debian 13 / Trixie). -## Local comparison +Chosen because: +- Debian is the standard base for devcontainers +- Trixie ships Python 3.13 in its default repos (convenient for the test, though we + install via pyenv anyway for version-pinning reasons — see below) -This repo is also intended to be run locally in a normal Dev Containers environment. +### Python and Node: version-pinnable, not system-level -If it works locally but fails in Copilot cloud agent, that suggests a cloud-agent/platform issue rather than a repo-specific configuration problem. +Devcontainer `features` were dropped in favour of tools installed directly in the +Dockerfile, giving full control over TLS calls during the build: -## Files +| Tool | Installed via | Location | Version pin | +|------|--------------|----------|-------------| +| Python | **pyenv** (compiled from source) | `/usr/local/pyenv` | `ARG PYTHON_VERSION=3.13` | +| Node | **nvm** | `/usr/local/nvm` | `ARG NODE_VERSION=lts` | -- `.devcontainer/devcontainer.json` -- `.devcontainer/docker-compose.yml` -- `.devcontainer/on-create.sh` -- `.devcontainer/update-content.sh` +Both are installed as root to world-readable system-wide paths. Symlinks in +`/usr/local/bin/` make `python3`, `pip`, `pip3`, `node`, `npm`, `npx` available to +all users and `sudo`. -## Notes +To pin a different version, rebuild with a build-arg: -This reproduction does not use any custom CA certificates or certificate overrides. +```bash +docker build --build-arg PYTHON_VERSION=3.12 --build-arg NODE_VERSION=20 \ + -f .devcontainer/Dockerfile . +``` + +### SSL workaround: `chmod 755 /etc/ssl/certs` + +Moving to a local Dockerfile means all Dockerfile `RUN` steps execute as root and can +traverse the 0700-mounted `/etc/ssl/certs` without any cert redirection. So the +previous workaround machinery was dropped entirely: + +- ❌ removed: CA bundle copy to `/usr/local/share/ca-certificates.crt` +- ❌ removed: `~/.curlrc` cacert overrides +- ❌ removed: `git config --system http.sslCAInfo` +- ❌ removed: `/etc/environment` and `/etc/profile.d/ca-bundle-workaround.sh` +- ❌ removed: `ENV CURL_CA_BUNDLE / GIT_SSL_CAINFO / SSL_CERT_FILE / …` block +- ❌ removed: `features` block in `devcontainer.json` + +Each Dockerfile `RUN` step that makes network calls opens with +`chmod 755 /etc/ssl/certs` — harmless when the dir is already 755, essential when the +shim has mounted a 0700 dir. + +Each lifecycle hook script opens with `sudo chmod 755 /etc/ssl/certs` for the same +reason: the shim applies a fresh bind-mount every time a container starts, so +`postStartCommand` in particular must re-apply the fix on every boot. + +--- + +## Repository structure + +``` +.devcontainer/ + Dockerfile # trixie base + pyenv Python + nvm Node + devcontainer.json # no features; references docker-compose.yml + hooks + docker-compose.yml # builds Dockerfile; mounts workspace + on-create.sh # onCreateCommand — chmod fix + version/curl checks + update-content.sh # updateContentCommand — chmod fix + version/curl checks + post-create.sh # postCreateCommand — chmod fix + version/curl/pip/npm checks + post-start.sh # postStartCommand — chmod fix + version/curl checks +``` + +### What each hook script does + +1. `sudo chmod 755 /etc/ssl/certs` — shim workaround +2. `whoami` / `id` +3. Python and Node version checks (root and non-root) +4. `ls -l /etc/ssl/certs/ca-certificates.crt` +5. `curl -Ivs https://github.com` as root and as `vscode` +6. (`post-create.sh` only) `pip install --user cowsay` and `npm install cowsay` + +--- + +## Current status / what the next agent needs to do + +The Dockerfile was written but **the image build has not yet been completed or +verified**. The build was started and then stopped because it requires outbound access +to domains that may be firewall-blocked in the runner environment. + +### Domains that must be reachable during `docker build` + +| Step | Domain(s) needed | +|------|-----------------| +| `apt-get update` | `deb.debian.org`, `security.debian.org` | +| pyenv git clone | `github.com` | +| pyenv Python build | `www.python.org` (source tarball download) | +| nvm install script | `raw.githubusercontent.com` | +| nvm Node download | `nodejs.org` | + +### Steps to complete + +1. **Verify the domains above are reachable** from a `docker build` container. If any + are blocked, ask the user to lift the firewall restriction before proceeding. + +2. **Build the image:** + ```bash + cd /home/runner/work/devcontainer-ssl-problem/devcontainer-ssl-problem + docker build --no-cache -t repro-test -f .devcontainer/Dockerfile . + ``` + +3. **Start a container and run the hook scripts manually** (simulating what the + devcontainer lifecycle would do): + ```bash + docker run -d --name repro-test-ctr repro-test sleep infinity + docker exec -u vscode repro-test-ctr bash /workspaces/repro/.devcontainer/on-create.sh + docker exec -u vscode repro-test-ctr bash /workspaces/repro/.devcontainer/update-content.sh + docker exec -u vscode repro-test-ctr bash /workspaces/repro/.devcontainer/post-create.sh + docker exec -u vscode repro-test-ctr bash /workspaces/repro/.devcontainer/post-start.sh + ``` + +4. **Run manual version and install checks inside the running container:** + ```bash + docker exec repro-test-ctr python3 --version + docker exec repro-test-ctr pip3 --version + docker exec repro-test-ctr node --version + docker exec repro-test-ctr npm --version + docker exec -u vscode repro-test-ctr python3 --version + docker exec -u vscode repro-test-ctr node --version + docker exec -u vscode repro-test-ctr pip install --user cowsay + docker exec -u vscode repro-test-ctr bash -c 'cd /tmp && npm install cowsay' + ``` + +5. **Summarise hook output and manual check results** and update the + [Test results](#test-results) section below. + +6. **Commit and push.** + +--- + +## Test results + +> **Not yet recorded.** The image build has not been completed. See +> [Current status](#current-status--what-the-next-agent-needs-to-do) above. + +### Expected results (after build succeeds) + +| Hook | User | curl | python3 --version | node --version | +|------|------|------|-------------------|----------------| +| `onCreateCommand` | vscode | ✅ HTTP 200 | ✅ 3.13.x | ✅ LTS | +| `updateContentCommand` | vscode | ✅ HTTP 200 | ✅ 3.13.x | ✅ LTS | +| `postCreateCommand` | vscode | ✅ HTTP 200 | ✅ 3.13.x | ✅ LTS | +| `postStartCommand` | vscode | ✅ HTTP 200 | ✅ 3.13.x | ✅ LTS |