Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
125 changes: 117 additions & 8 deletions .github/workflows/release-pex.yml
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,73 @@ concurrency:
cancel-in-progress: false

jobs:
# The Linux x86-64 executable, devcapsule.pex, is built and proven below.
# These are the other platforms' executables of the same tag, each built on
# its own architecture: the install script picks the one for the computer,
# and on a Mac also takes both Linux ones as the runtimes its capsules run.
build-other-platforms:
name: Build ${{ matrix.asset }}
runs-on: ${{ matrix.runner }}
strategy:
matrix:
include:
- runner: ubuntu-24.04-arm
scie-platform: linux-aarch64
asset: devcapsule-linux-arm64
- runner: macos-15
scie-platform: macos-aarch64
asset: devcapsule-macos-arm64
env:
RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }}
defaults:
run:
working-directory: devcapsule-src

steps:
- name: Check out the release tag
uses: actions/checkout@v4
with:
ref: ${{ env.RELEASE_TAG }}
fetch-depth: 0

- name: Set up Python 3.12 for the backend build
uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
cache-dependency-path: devcapsule-src/dev-requirements.txt

- name: Install locked build dependencies
run: |
python -m pip install --upgrade pip
python -m pip install -r dev-requirements.txt
python -m pip install -e . --no-deps

- name: Build and smoke the self-contained executable
env:
DEVCAPSULE_SOURCE_REPOSITORY: ${{ github.server_url }}/${{ github.repository }}
DEVCAPSULE_SCIE_PLATFORM: ${{ matrix.scie-platform }}
ASSET: ${{ matrix.asset }}
run: |
scripts/build-pex.sh \
--source-revision "$(git rev-parse HEAD)" \
--release-mnemonic "$RELEASE_TAG" \
--output "dist/$ASSET"
"dist/$ASSET" --help >/dev/null
"dist/$ASSET" version --json | tee "$RUNNER_TEMP/version.json"
test "$(python -c 'import json, sys; print(json.load(open(sys.argv[1]))["build_mnemonic"])' "$RUNNER_TEMP/version.json")" = "$RELEASE_TAG"

- name: Hand the executable to the publishing job
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.asset }}
path: devcapsule-src/dist/${{ matrix.asset }}
if-no-files-found: error
retention-days: 30

build-publish-and-prove:
name: Build, publish, and prove
needs: build-other-platforms
runs-on: ubuntu-24.04
env:
RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }}
Expand Down Expand Up @@ -91,10 +156,13 @@ jobs:
if grep -Fxq "$RELEASE_TAG" "$RUNNER_TEMP/releases"; then
gh release download "$RELEASE_TAG" --dir dist \
--pattern devcapsule.pex --pattern devcapsule.pex.sha256 \
--pattern devcapsule-linux-arm64 --pattern devcapsule-macos-arm64 \
--pattern SHA256SUMS --pattern install.sh \
--pattern release-manifest.json
(cd dist && sha256sum --check devcapsule.pex.sha256)
chmod 0755 dist/devcapsule.pex
manifest_args=(--gate "$RUNNER_TEMP/release-gate.json")
(cd dist && sha256sum --check devcapsule.pex.sha256 && sha256sum --check SHA256SUMS)
chmod 0755 dist/devcapsule.pex dist/devcapsule-linux-arm64 dist/devcapsule-macos-arm64
manifest_args=(--gate "$RUNNER_TEMP/release-gate.json"
--artifact dist/devcapsule-linux-arm64 --artifact dist/devcapsule-macos-arm64)
if [ "$RELEASE_PRERELEASE" = false ]; then
manifest_args+=(--candidate "$RUNNER_TEMP/accepted-candidate/release-manifest.json")
fi
Expand All @@ -103,6 +171,14 @@ jobs:
echo 'exists=true' >> "$GITHUB_OUTPUT"
fi

- name: Take the other platforms' executables
if: steps.recover.outputs.exists != 'true'
uses: actions/download-artifact@v4
with:
pattern: devcapsule-*-arm64
merge-multiple: true
path: devcapsule-src/dist

- name: Build the self-contained release PEX
if: steps.recover.outputs.exists != 'true'
env:
Expand All @@ -114,13 +190,28 @@ jobs:
dist/devcapsule.pex --help >/dev/null
dist/devcapsule.pex version --json | tee dist/devcapsule-version.json
test "$(python -c 'import json; print(json.load(open("dist/devcapsule-version.json"))["build_mnemonic"])')" = "$RELEASE_TAG"
chmod 0755 dist/devcapsule-linux-arm64 dist/devcapsule-macos-arm64
(cd dist && sha256sum devcapsule.pex > devcapsule.pex.sha256)
manifest_args=(--gate "$RUNNER_TEMP/release-gate.json")
(cd dist && sha256sum devcapsule.pex devcapsule-linux-arm64 devcapsule-macos-arm64 > SHA256SUMS)
manifest_args=(--gate "$RUNNER_TEMP/release-gate.json"
--artifact dist/devcapsule-linux-arm64 --artifact dist/devcapsule-macos-arm64)
if [ "$RELEASE_PRERELEASE" = false ]; then
manifest_args+=(--candidate "$RUNNER_TEMP/accepted-candidate/release-manifest.json")
fi
python scripts/release-manifest.py "$RELEASE_TAG" "$(git rev-parse HEAD)" "${manifest_args[@]}"

- name: Write the install script for this release
run: |
release_url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/releases/download/$RELEASE_TAG"
sed "s|@RELEASE_URL@|$release_url|" scripts/install.sh > "$RUNNER_TEMP/install.sh"
if grep -q '@RELEASE_URL@' "$RUNNER_TEMP/install.sh"; then exit 1; fi
sh -n "$RUNNER_TEMP/install.sh"
if [ "${{ steps.recover.outputs.exists }}" = true ]; then
cmp "$RUNNER_TEMP/install.sh" dist/install.sh
else
install -m 0755 "$RUNNER_TEMP/install.sh" dist/install.sh
fi

- name: Run packaging integration tests against the release PEX
env:
DEVCAPSULE_PEX_UNDER_TEST: ${{ github.workspace }}/devcapsule-src/dist/devcapsule.pex
Expand All @@ -142,6 +233,10 @@ jobs:
path: |
devcapsule-src/dist/devcapsule.pex
devcapsule-src/dist/devcapsule.pex.sha256
devcapsule-src/dist/devcapsule-linux-arm64
devcapsule-src/dist/devcapsule-macos-arm64
devcapsule-src/dist/SHA256SUMS
devcapsule-src/dist/install.sh
devcapsule-src/dist/release-manifest.json
if-no-files-found: error
retention-days: 30
Expand All @@ -150,8 +245,12 @@ jobs:
if: steps.recover.outputs.exists != 'true' && env.RELEASE_PRERELEASE == 'true'
run: |
gh release create "$RELEASE_TAG" \
"dist/install.sh#Installer for Linux and Apple Silicon Macs" \
"dist/devcapsule.pex#Self-contained Linux x86-64 CLI" \
"dist/devcapsule-linux-arm64#Self-contained Linux ARM64 CLI (experimental)" \
"dist/devcapsule-macos-arm64#Self-contained macOS Apple Silicon CLI (experimental)" \
"dist/devcapsule.pex.sha256#SHA-256 checksum" \
"dist/SHA256SUMS#SHA-256 checksums of every executable" \
"dist/release-manifest.json#Release inputs and artifact checksums" \
--verify-tag --generate-notes --draft --prerelease="$RELEASE_PRERELEASE" --latest=false

Expand All @@ -160,8 +259,12 @@ jobs:
run: |
python scripts/release-docs.py notes "${RELEASE_TAG#v}" > "$RUNNER_TEMP/release-notes.md"
gh release create "$RELEASE_TAG" \
"dist/install.sh#Installer for Linux and Apple Silicon Macs" \
"dist/devcapsule.pex#Self-contained Linux x86-64 CLI" \
"dist/devcapsule-linux-arm64#Self-contained Linux ARM64 CLI (experimental)" \
"dist/devcapsule-macos-arm64#Self-contained macOS Apple Silicon CLI (experimental)" \
"dist/devcapsule.pex.sha256#SHA-256 checksum" \
"dist/SHA256SUMS#SHA-256 checksums of every executable" \
"dist/release-manifest.json#Release inputs and artifact checksums" \
--verify-tag --notes-file "$RUNNER_TEMP/release-notes.md" --draft --prerelease="$RELEASE_PRERELEASE" --latest=false

Expand All @@ -172,11 +275,16 @@ jobs:
gh release download "$RELEASE_TAG" \
--pattern devcapsule.pex \
--pattern devcapsule.pex.sha256 \
--pattern devcapsule-linux-arm64 \
--pattern devcapsule-macos-arm64 \
--pattern SHA256SUMS \
--pattern install.sh \
--pattern release-manifest.json \
--dir "$published"
(cd "$published" && sha256sum --check devcapsule.pex.sha256)
cmp dist/devcapsule.pex "$published/devcapsule.pex"
cmp dist/release-manifest.json "$published/release-manifest.json"
(cd "$published" && sha256sum --check devcapsule.pex.sha256 && sha256sum --check SHA256SUMS)
for asset in devcapsule.pex devcapsule-linux-arm64 devcapsule-macos-arm64 SHA256SUMS install.sh release-manifest.json; do
cmp "dist/$asset" "$published/$asset"
done
chmod 0755 "$published/devcapsule.pex"

- name: Publish the complete release
Expand Down Expand Up @@ -216,6 +324,7 @@ jobs:
echo
echo "- Tag: \`$RELEASE_TAG\`"
echo "- Revision: \`$(git rev-parse HEAD)\`"
echo "- $(cat dist/devcapsule.pex.sha256)"
echo "- Executables:"
sed 's/^/ - /' dist/SHA256SUMS
echo "- Built and retested after downloading from GitHub Releases"
} >> "$GITHUB_STEP_SUMMARY"
2 changes: 1 addition & 1 deletion CURRENT-STATUS.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ coordination branch. Detailed state belongs in each linked status file.
| `capsule-webconsole` | 2026-10-09 | Serve a token-gated web console from every capsule: configuration, versions, info, live resources, later records and workflow state | active; work order accepted 2026-10-09; the first iteration, deliverables 1 and 2, is three stacked pull requests, #184, #186 and the runtime-and-base slice, each reviewed by the Codex pair; the owner merges | `ws-capsule-webconsole/first-slice` | [workstream status](engineering-docs/wip/2026-10-09-capsule-webconsole/CURRENT-STATUS.md) |
| `wrapped-projects` | 2026-11-02 | Wrap a foreign repository as a submodule of a small wrapper project and work on it in a capsule with three commands; FRC learning project first | active; PR1 (#333) accepted by the owner 2026-11-03, awaiting merge; the project-capabilities design handed to workflow-improvements | `ws-wrapped-projects/wrap` | [workstream status](engineering-docs/wip/2026-11-02-wrapped-projects/CURRENT-STATUS.md) |
| `macos-launcher` | 2026-11-03 | Run the DevCapsule launcher natively on Apple Silicon Macs against Docker Desktop, with capsules as `linux-amd64` under emulation | active 2026-11-03; slices 1 to 4 done, the PyCharm smoke passing on an Apple Silicon Mac; waits on publication and the owner's rulings | `ws-macos-launcher/host-port` | [workstream status](engineering-docs/wip/2026-11-03-macos-launcher/CURRENT-STATUS.md) |
| `linux-arm64` | 2026-11-04 | Run capsules as `linux-arm64` as well as `linux-amd64`: natively on Apple Silicon Macs and on ARM Linux hosts | active 2026-11-04; slices 1 to 4 done, six IDEs running natively on Apple Silicon; waits on publication, the owner's arm64 base and rulings | `ws-linux-arm64/capsules` | [workstream status](engineering-docs/wip/2026-11-04-linux-arm64/CURRENT-STATUS.md) |
| `linux-arm64` | 2026-11-04 | Run capsules as `linux-arm64` as well as `linux-amd64`: natively on Apple Silicon Macs and on ARM Linux hosts | active 2026-11-04; slices 1 to 4 merged in #338; the one-line install (per-platform release executables, `install.sh`) awaits its pull request; the owner's arm64 base and rulings follow | `ws-linux-arm64/one-line-install` | [workstream status](engineering-docs/wip/2026-11-04-linux-arm64/CURRENT-STATUS.md) |
| `website` | 2026-09-16 | Deliver the website autonomy experiment: repository-owned landing/docs/blog content, independently developable website submodule, local preview and publishing mechanism | paused 2026-09-20 at owner request to switch to maintenance; experiment accepted, A−; blog integrated and published; content/parent-integration follow-ups remain | `ws-website/initial-cut`; renamed from `website/initial-cut`; preparation and backlog-handoff exceptions in status | [workstream status](engineering-docs/wip/2026-09-16-website/CURRENT-STATUS.md) |

Paused and blocked workstreams remain open until they conclude successfully or
Expand Down
27 changes: 22 additions & 5 deletions devcapsule-src/devcapsule/runtime_artifact.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,16 @@
from devcapsule.compat import CliError
from devcapsule.platforms import HostOs, Platform

__all__ = ["DEFAULT_RUNTIME_DIRECTORY", "RUNTIME_PEX_ENV", "executable_platform", "runtime_artifact"]

RUNTIME_PEX_ENV = "DEVCAPSULE_RUNTIME_PEX"

DEFAULT_RUNTIME_DIRECTORY = "~/.local/lib/devcapsule"
"""Where a packaged launcher looks for its runtimes when ``DEVCAPSULE_RUNTIME_PEX`` is unset.

The release's install script puts a Mac's Linux runtimes here.
"""

_MACH_O_MAGICS = (b"\xcf\xfa\xed\xfe", b"\xce\xfa\xed\xfe", b"\xca\xfe\xba\xbe")
_UNKNOWN_REVISION = "unknown"

Expand All @@ -23,10 +31,11 @@ def runtime_artifact(capsule_platform: Platform) -> Path:
(D-0009). A launcher that cannot, such as a macOS executable launching
Linux capsules, delivers the runtime ``DEVCAPSULE_RUNTIME_PEX`` names: one
executable, or a directory holding one per capsule platform as
``devcapsule-<platform>``. A source-form launcher names its built runtime
the same way. A delivered runtime that is not the launcher's own bytes
must be the launcher's build: the same version, and the same source
revision wherever both record one (R-RUNTIME-001).
``devcapsule-<platform>``. Unset, it delivers the ``devcapsule-<platform>``
in ``DEFAULT_RUNTIME_DIRECTORY`` when there is one. A source-form launcher
names its built runtime with the variable. A delivered runtime that is not
the launcher's own bytes must be the launcher's build: the same version,
and the same source revision wherever both record one (R-RUNTIME-001).
"""

# scie preserves the executable's path in SCIE; PEX points to the original
Expand All @@ -38,6 +47,8 @@ def runtime_artifact(capsule_platform: Platform) -> Path:
delivered = own
elif named:
delivered = _named_runtime(Path(named).expanduser(), capsule_platform)
elif own is not None and (installed := _installed_runtime(capsule_platform)) is not None:
delivered = installed
else:
delivered = own or _resolved(sys.argv[0])
try:
Expand All @@ -52,7 +63,8 @@ def runtime_artifact(capsule_platform: Platform) -> Path:
raise CliError(
f"{delivered} is {_describe(executable_platform(delivered))} and cannot run inside "
f"{capsule_platform} capsules. Set {RUNTIME_PEX_ENV} to a {capsule_platform} DevCapsule "
f"executable of the same version, or to a directory holding it as devcapsule-{capsule_platform}."
f"executable of the same version, or to a directory holding it as devcapsule-{capsule_platform}, "
f"or put it in {DEFAULT_RUNTIME_DIRECTORY} as devcapsule-{capsule_platform}."
)
if delivered != own:
_require_same_build(delivered, delivered_build, current_build_info())
Expand Down Expand Up @@ -94,6 +106,11 @@ def _runs_in(path: Path, capsule_platform: Platform) -> bool:
return runs_on == capsule_platform


def _installed_runtime(capsule_platform: Platform) -> Path | None:
candidate = Path(DEFAULT_RUNTIME_DIRECTORY).expanduser() / f"devcapsule-{capsule_platform}"
return _resolved(str(candidate)) if candidate.is_file() else None


def _named_runtime(named: Path, capsule_platform: Platform) -> Path:
if not named.is_dir():
return _resolved(str(named))
Expand Down
102 changes: 102 additions & 0 deletions devcapsule-src/scripts/install.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
#!/bin/sh
# Install DevCapsule from one GitHub release:
#
# curl -fsSL <release URL>/install.sh | sh
#
# It picks the executable for this computer and, on a Mac, also the Linux
# runtimes its capsules run, which the launcher finds in ~/.local/lib/devcapsule.
# Every file is checked against the release's SHA256SUMS before anything is
# installed. The release workflow writes its own URL over @RELEASE_URL@;
# DEVCAPSULE_RELEASE_URL overrides it, for a mirror or a test. Everything runs
# in main, called on the last line, so a download cut short runs nothing.
set -eu

RELEASE_URL="@RELEASE_URL@"

main() {
release_url="${DEVCAPSULE_RELEASE_URL:-$RELEASE_URL}"
case "$release_url" in
https://* | http://* | file://*) ;;
*) die "this install.sh names no release; download it from a DevCapsule release" ;;
esac
bin_dir="$HOME/.local/bin"
runtime_dir="$HOME/.local/lib/devcapsule"

os="$(uname -s)"
machine="$(uname -m)"
# A shell under Rosetta reports x86_64 on an Apple Silicon Mac.
if [ "$os" = Darwin ] && [ "$(sysctl -n hw.optional.arm64 2>/dev/null || true)" = 1 ]; then
machine=arm64
fi
case "$os-$machine" in
Linux-x86_64 | Linux-amd64) launcher=devcapsule.pex ;;
Linux-aarch64 | Linux-arm64) launcher=devcapsule-linux-arm64 ;;
Darwin-arm64) launcher=devcapsule-macos-arm64 ;;
Darwin-*) die "DevCapsule runs on Apple Silicon Macs only" ;;
*) die "DevCapsule has no executable for $os $machine" ;;
esac
files="$launcher"
if [ "$os" = Darwin ]; then
files="$files devcapsule.pex devcapsule-linux-arm64"
fi

command -v curl >/dev/null 2>&1 || die "curl is required"
if command -v sha256sum >/dev/null 2>&1; then
sha256="sha256sum"
elif command -v shasum >/dev/null 2>&1; then
sha256="shasum -a 256"
else
die "sha256sum or shasum is required to verify the download"
fi

work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT
trap 'exit 1' INT TERM
cd "$work"

fetch SHA256SUMS
for file in $files; do
echo "Downloading $file"
fetch "$file"
verify "$file"
done

if [ "$os" = Darwin ]; then
mkdir -p "$runtime_dir"
install -m 0755 devcapsule.pex "$runtime_dir/devcapsule-linux-amd64"
install -m 0755 devcapsule-linux-arm64 "$runtime_dir/devcapsule-linux-arm64"
echo "Installed the Linux runtimes in $runtime_dir"
fi
mkdir -p "$bin_dir"
install -m 0755 "$launcher" "$bin_dir/devcapsule"
echo "Installed $bin_dir/devcapsule"
"$bin_dir/devcapsule" version

if [ "$os" = Darwin ] && [ -n "${DEVCAPSULE_RUNTIME_PEX:-}" ]; then
echo "DEVCAPSULE_RUNTIME_PEX is set to $DEVCAPSULE_RUNTIME_PEX, so the launcher uses that instead of"
echo "$runtime_dir. Remove it from your shell profile to use the runtimes just installed."
fi
case ":$PATH:" in
*":$bin_dir:"*) ;;
*) echo "Add $bin_dir to PATH in your shell profile to run devcapsule by name." ;;
esac
}

die() {
echo "devcapsule install: $*" >&2
exit 1
}

fetch() {
curl --fail --silent --show-error --location --output "$1" "$release_url/$1" \
|| die "could not download $release_url/$1"
}

verify() {
expected="$(awk -v name="$1" '$2 == name || $2 == "*" name { print $1; exit }' SHA256SUMS)"
[ -n "$expected" ] || die "the release's SHA256SUMS lists no $1"
actual="$($sha256 "$1" | awk '{ print $1 }')"
[ "$actual" = "$expected" ] || die "$1 does not match its SHA-256 checksum; nothing was installed"
}

main "$@"
Loading
Loading