Skip to content

Commit 1d08616

Browse files
mdhellermdheller
committed
merge main (keep PR docs) — land #265 
2 parents 9075c54 + f98d1f3 commit 1d08616

153 files changed

Lines changed: 11069 additions & 262 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/actions/setup-nix/action.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
name: setup-nix
2+
description: >
3+
Install Nix (Determinate) and, if the SourceOS GCS binary cache is configured,
4+
add it as a substituter. Replaces the sunset magic-nix-cache-action. Graceful:
5+
with no cache vars set it just installs Nix (falls back to cache.nixos.org).
6+
inputs:
7+
cache-url:
8+
description: "Public HTTPS base of the GCS Nix cache, e.g. https://storage.googleapis.com/<bucket>/nix-cache"
9+
required: false
10+
default: ""
11+
cache-pubkey:
12+
description: "Trusted public key for the cache (form name:base64). Pair with the push secret."
13+
required: false
14+
default: ""
15+
runs:
16+
using: composite
17+
steps:
18+
- name: Compose nix extra-conf (cache substituter, if configured)
19+
id: conf
20+
shell: bash
21+
run: |
22+
if [ -n "${{ inputs.cache-url }}" ] && [ -n "${{ inputs.cache-pubkey }}" ]; then
23+
{
24+
echo 'extra-conf<<NIXEOF'
25+
echo "extra-substituters = ${{ inputs.cache-url }}"
26+
echo "extra-trusted-public-keys = ${{ inputs.cache-pubkey }}"
27+
echo 'NIXEOF'
28+
} >> "$GITHUB_OUTPUT"
29+
echo "[setup-nix] GCS cache substituter enabled"
30+
else
31+
echo 'extra-conf=' >> "$GITHUB_OUTPUT"
32+
echo "[setup-nix] no cache vars — using cache.nixos.org only"
33+
fi
34+
- uses: DeterminateSystems/nix-installer-action@v14
35+
with:
36+
extra-conf: ${{ steps.conf.outputs.extra-conf }}

‎.github/workflows/build-custom.yml‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# build-custom — build a user-customized SourceOS image from a spec.
2+
# Dispatched by the socioprophet backend (self-serve image builder, free tier).
3+
# Builds the ISO on GitHub's runners (x86_64 or native ARM) and uploads it +
4+
# a status marker to the per-user GCS prefix the backend polls.
5+
name: build-custom
6+
7+
on:
8+
workflow_dispatch:
9+
inputs:
10+
spec:
11+
description: 'Build spec JSON (edition, arch, hostname, packages, ...)'
12+
required: true
13+
type: string
14+
uid:
15+
description: 'Owning user id'
16+
required: true
17+
type: string
18+
build_id:
19+
description: 'Build id (Firestore doc id)'
20+
required: true
21+
type: string
22+
target:
23+
description: 'Artifact target'
24+
required: false
25+
default: 'iso'
26+
type: choice
27+
options: [iso, netboot]
28+
29+
env:
30+
GCS_BUCKET: sourceos-artifacts-socioprophet
31+
32+
jobs:
33+
build:
34+
# Native runner per requested arch (free on public repo).
35+
runs-on: ${{ fromJSON(github.event.inputs.spec).arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-latest' }}
36+
timeout-minutes: 120
37+
permissions: { contents: read, id-token: write }
38+
steps:
39+
- uses: actions/checkout@v4
40+
- uses: ./.github/actions/setup-nix
41+
with:
42+
cache-url: ${{ vars.NIX_CACHE_URL }}
43+
cache-pubkey: ${{ vars.NIX_CACHE_PUBKEY }}
44+
- name: Free disk space
45+
run: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android || true; df -h
46+
47+
- name: Authenticate to GCP
48+
uses: google-github-actions/auth@v2
49+
with:
50+
workload_identity_provider: ${{ secrets.GCP_WORKLOAD_IDENTITY_PROVIDER }}
51+
service_account: ${{ secrets.GCP_SERVICE_ACCOUNT }}
52+
- uses: google-github-actions/setup-gcloud@v2
53+
54+
- name: Mark building
55+
run: |
56+
PREFIX="gs://${GCS_BUCKET}/user-builds/${{ github.event.inputs.uid }}/${{ github.event.inputs.build_id }}"
57+
echo '{"status":"building","run":"${{ github.run_id }}"}' | gsutil cp - "$PREFIX/status.json"
58+
59+
- name: Build custom image
60+
id: build
61+
env:
62+
NIX_CACHE_BUCKET: ${{ vars.NIX_CACHE_BUCKET }}
63+
NIX_CACHE_SECRET_KEY: ${{ secrets.NIX_CACHE_SECRET_KEY }}
64+
# Self-managed signing key (minisign, no third-party authority). If
65+
# the secret is unset the build still produces unsigned provenance.
66+
SOURCEOS_SIGN_SECRET_KEY: ${{ secrets.SOURCEOS_SIGN_SECRET_KEY }}
67+
SOURCEOS_SIGN_PUBKEY: ${{ vars.SOURCEOS_SIGN_PUBKEY }}
68+
run: |
69+
PREFIX="gs://${GCS_BUCKET}/user-builds/${{ github.event.inputs.uid }}/${{ github.event.inputs.build_id }}"
70+
printf '%s' '${{ github.event.inputs.spec }}' > /tmp/spec.json
71+
SPEC_FILE=/tmp/spec.json OUT=out GCS_PREFIX="$PREFIX" \
72+
TARGET="${{ github.event.inputs.target || 'iso' }}" \
73+
BUILD_ID="${{ github.event.inputs.build_id }}" BUILD_UID="${{ github.event.inputs.uid }}" \
74+
VERSION="${{ vars.SOURCEOS_VERSION || '26.11' }}" SOURCE_REV="${GITHUB_SHA}" \
75+
bash scripts/build-custom-image.sh
76+
echo "artifact=$(cat out/artifact-url.txt)" >> "$GITHUB_OUTPUT"
77+
78+
- name: Mark done
79+
if: success()
80+
run: |
81+
PREFIX="gs://${GCS_BUCKET}/user-builds/${{ github.event.inputs.uid }}/${{ github.event.inputs.build_id }}"
82+
printf '{"status":"complete","artifact":"%s"}' "${{ steps.build.outputs.artifact }}" \
83+
| gsutil cp - "$PREFIX/status.json"
84+
85+
- name: Mark failed
86+
if: failure()
87+
run: |
88+
PREFIX="gs://${GCS_BUCKET}/user-builds/${{ github.event.inputs.uid }}/${{ github.event.inputs.build_id }}"
89+
echo '{"status":"error"}' | gsutil cp - "$PREFIX/status.json" || true

‎.github/workflows/image-tests.yml‎

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# Image tests — validate that SourceOS editions actually boot.
2+
# Layer 1 (always): deterministic nixosTests in QEMU/KVM on GitHub runners.
3+
# Layer 2 (opt-in): Agent-S GUI test — needs a grounding endpoint + key, so
4+
# it's gated and expects those to be provided.
5+
name: image-tests
6+
7+
on:
8+
workflow_dispatch:
9+
inputs:
10+
run_agent_s:
11+
description: 'Also run the Agent-S GUI test (needs AS_GROUND_URL + ANTHROPIC_API_KEY)'
12+
required: false
13+
default: false
14+
type: boolean
15+
run_arm_boot:
16+
description: 'Also run aarch64 boot tests (needs a KVM-capable self-hosted ARM runner)'
17+
required: false
18+
default: false
19+
type: boolean
20+
run_install:
21+
description: 'Also run the install-to-disk test (real installer partitions + installs + boots from disk)'
22+
required: false
23+
default: false
24+
type: boolean
25+
pull_request:
26+
paths:
27+
- 'profiles/**'
28+
- 'flake.nix'
29+
- 'tests/editions/**'
30+
- '.github/workflows/image-tests.yml'
31+
push:
32+
branches: [main]
33+
paths:
34+
- 'profiles/**'
35+
- 'tests/editions/**'
36+
37+
jobs:
38+
# ── Layer 1: deterministic boot tests (free, KVM on GitHub) ──────────────────
39+
# x86_64 only: nixosTests require the `kvm` feature, and GitHub's free
40+
# ubuntu-24.04-arm runners do NOT expose /dev/kvm. aarch64 boot tests run on a
41+
# KVM-capable ARM host (GCP arm VM / self-hosted arm runner) — see boot-tests-arm.
42+
boot-tests:
43+
name: boot ${{ matrix.check }} (x86_64)
44+
strategy:
45+
fail-fast: false
46+
matrix:
47+
check: [edition-desktop-boot, edition-server-boot, edition-edge-boot]
48+
runs-on: ubuntu-latest
49+
env: { arch: x86_64 }
50+
timeout-minutes: 90
51+
steps:
52+
- uses: actions/checkout@v4
53+
- name: Enable KVM
54+
run: |
55+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
56+
| sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null
57+
sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm || true
58+
ls -l /dev/kvm || echo "no /dev/kvm (test will use slower TCG)"
59+
- uses: ./.github/actions/setup-nix
60+
with:
61+
cache-url: ${{ vars.NIX_CACHE_URL }}
62+
cache-pubkey: ${{ vars.NIX_CACHE_PUBKEY }}
63+
- name: Free disk space
64+
run: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android || true; df -h
65+
- name: Run boot test
66+
run: |
67+
nix build ".#checks.x86_64-linux.${{ matrix.check }}" \
68+
--print-build-logs --show-trace -L
69+
70+
# ── Install-to-disk test (opt-in) ────────────────────────────────────────────
71+
# Heavy: the real installer partitions a blank disk, installs the server
72+
# edition, then the disk is rebooted and asserted to come up on its own
73+
# bootloader. Opt-in via run_install so it never gates ordinary PRs.
74+
install-test:
75+
name: install-to-disk (x86_64)
76+
if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_install == 'true'
77+
runs-on: ubuntu-latest
78+
timeout-minutes: 120
79+
steps:
80+
- uses: actions/checkout@v4
81+
- name: Enable KVM
82+
run: |
83+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
84+
| sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null
85+
sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm || true
86+
ls -l /dev/kvm || echo "no /dev/kvm (test will use slower TCG)"
87+
- uses: ./.github/actions/setup-nix
88+
with:
89+
cache-url: ${{ vars.NIX_CACHE_URL }}
90+
cache-pubkey: ${{ vars.NIX_CACHE_PUBKEY }}
91+
- name: Free disk space
92+
run: sudo rm -rf /usr/share/dotnet /opt/ghc /usr/local/lib/android || true; df -h
93+
- name: Run install-to-disk test
94+
run: |
95+
nix build ".#checks.x86_64-linux.edition-server-install" \
96+
--print-build-logs --show-trace -L
97+
98+
# ── aarch64 boot tests — need KVM on ARM (your GCP arm VM / self-hosted runner) ──
99+
# Opt-in: GitHub's free ARM runners have no /dev/kvm, so these run on a
100+
# KVM-capable self-hosted aarch64 runner when you enable run_arm_boot.
101+
boot-tests-arm:
102+
name: boot ${{ matrix.check }} (aarch64)
103+
if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_arm_boot == 'true'
104+
strategy:
105+
fail-fast: false
106+
matrix:
107+
check: [edition-server-boot, edition-edge-boot]
108+
runs-on: [self-hosted, aarch64-linux, Linux]
109+
timeout-minutes: 120
110+
steps:
111+
- uses: actions/checkout@v4
112+
- name: Run boot test
113+
run: |
114+
. /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh || true
115+
nix build ".#checks.aarch64-linux.${{ matrix.check }}" \
116+
--print-build-logs --show-trace -L
117+
118+
# ── Layer 2: Agent-S GUI test (opt-in, needs grounding endpoint + key) ───────
119+
agent-s:
120+
name: Agent-S GUI test
121+
if: github.event_name == 'workflow_dispatch' && github.event.inputs.run_agent_s == 'true'
122+
runs-on: ubuntu-latest
123+
timeout-minutes: 90
124+
steps:
125+
- uses: actions/checkout@v4
126+
- uses: ./.github/actions/setup-nix
127+
with:
128+
cache-url: ${{ vars.NIX_CACHE_URL }}
129+
cache-pubkey: ${{ vars.NIX_CACHE_PUBKEY }}
130+
- name: Enable KVM
131+
run: |
132+
echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' \
133+
| sudo tee /etc/udev/rules.d/99-kvm4all.rules >/dev/null
134+
sudo udevadm control --reload-rules && sudo udevadm trigger --name-match=kvm || true
135+
- name: Build desktop image + harness deps
136+
run: |
137+
nix build .#packages.x86_64-linux.sourceos-image-qcow2-desktop --print-build-logs
138+
sudo apt-get update -qq
139+
sudo apt-get install -y -qq xvfb x11vnc tigervnc-viewer qemu-system-x86 python3-pip
140+
pip install -r tests/agent-s/requirements.txt
141+
- name: Run Agent-S GUI test
142+
env:
143+
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
144+
AS_GROUND_URL: ${{ secrets.AS_GROUND_URL }} # external UI-TARS endpoint
145+
run: |
146+
IMG=$(find -L result -name '*.qcow2' | head -1) \
147+
bash tests/agent-s/harness.sh
148+
- uses: actions/upload-artifact@v4
149+
if: always()
150+
with: { name: agent-s-artifacts, path: tests/agent-s/artifacts/*, if-no-files-found: ignore }

0 commit comments

Comments
 (0)