diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index aba51e22e..c287f0769 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -46,7 +46,7 @@ jobs: - name: Packages the tests use run: | pacman -Syu --noconfirm --needed git base-devel e2fsprogs zstd openssl \ - python python-dbus python-gobject dbus nodejs dtc cpio file jq busybox \ + python python-dbus python-gobject dbus nodejs dtc cpio file jq busybox rsync \ xz gzip qt6-declarative shellcheck # Arch's nodejs can lag a simdjson soname bump: on 2026-10-29 # simdjson 5.0.8 dropped the libsimdjson.so.34 that nodejs still diff --git a/tests/all b/tests/all index 4afae7bc4..e8759c7a5 100755 --- a/tests/all +++ b/tests/all @@ -9,8 +9,9 @@ # and flash against a fake fastboot, omamobile-firstboot's own tests, and the # smoke test of the newest root image in out/ if there is one, the felix, # lynx and panther device packages' own tests, tools/alarm-wake-test's -# check, and a contributor's local builds (tools/local-key-init, -# tools/fetch-published, --local). +# check, a contributor's local builds (tools/local-key-init, +# tools/fetch-published, --local), and tools/repo-publish --from-main's copy +# of the prebuilt binaries into a fixture's publish tree. set -euo pipefail cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." for f in tools/install-phone tools/phone-provision tools/build-rootfs tools/build-packages tools/make-bundle \ @@ -24,7 +25,7 @@ sh -n device/flame/initramfs/init sh -n device/flame/initramfs/build bash -n tools/build-flame-probe echo "ok - scripts parse" -for t in tests/pr-check tests/kernel-pr-check tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/initramfs-grow tests/images-layout tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/make-bundle-keys tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package; do +for t in tests/pr-check tests/kernel-pr-check tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/initramfs-grow tests/images-layout tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/make-bundle-keys tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package tests/repo-publish; do echo "# $t" "$t" done diff --git a/tests/repo-publish b/tests/repo-publish new file mode 100755 index 000000000..976ed30a4 --- /dev/null +++ b/tests/repo-publish @@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# tools/repo-publish --from-main's copy of the gitignored prebuilt binaries +# (packages/*/…/prebuilt/) into out/publish-tree, in a fixture repository +# (no network, no keys, nothing built or published). The real git lists the +# inventory and the real rsync copies it; a stand-in git answers everything +# that would touch origin or create the worktree, git and rsync fail on +# request, and a stand-in publisher in the fixture's out/publish-tree +# records what it was handed. An empty inventory is a no-op, names with +# spaces and newlines are copied, and a failing inventory or copy stops +# before the publisher runs. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +fail() { printf 'not ok - %s\n' "$1" >&2; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +command -v rsync >/dev/null || { pass "# skip: no rsync"; exit 0; } +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null # never the user's git config +export REAL_GIT REAL_RSYNC STUB_LOG=$t/git.log +REAL_GIT=$(command -v git) REAL_RSYNC=$(command -v rsync) + +R=$t/repo wt=$t/repo/out/publish-tree +mkdir -p "$R/tools" "$R/packages/a" "$R/packages/b/bionic" "$t/bin" "$wt/.git" "$wt/tools" +cp tools/repo-publish "$R/tools/" +echo /out > "$R/.gitignore" +echo 'prebuilt/' > "$R/packages/a/.gitignore" +printf '*.log\nbionic/prebuilt/\n' > "$R/packages/b/.gitignore" +git -C "$R" init -q +git -C "$R" add -A +git -C "$R" -c user.name=test -c user.email=test@invalid commit -qm fixture + +# Only the inventory reaches the real git; fetch, checkout, clean and the +# worktree are stood in for, so nothing outside the fixture is touched. +cat > "$t/bin/git" <<'EOF' +#!/bin/bash +if [[ " $* " == *" ls-files "* ]]; then + # FAIL_GIT_MIDWAY: one good name, then a failure, as a read error would + [[ -z ${FAIL_GIT_MIDWAY-} ]] || { printf 'packages/a/prebuilt/with space.bin\0'; exit 128; } + exec "$REAL_GIT" "$@" +fi +echo "git $*" >> "$STUB_LOG" +[[ " $* " != *" rev-parse "* ]] || echo 0000000 +EOF +# The copy fails on request with rsync's status, after reading its list, +# so the failure isn't a SIGPIPE upstream. +cat > "$t/bin/rsync" <<'EOF' +#!/bin/bash +[[ -z ${FAIL_RSYNC-} ]] || { cat > /dev/null; echo "rsync: (stand-in) No space left on device" >&2; exit 23; } +exec "$REAL_RSYNC" "$@" +EOF +printf '#!/bin/sh\nexit 0\n' > "$t/bin/bsdtar" # checked for, never run on this path +cat > "$wt/tools/repo-publish" < "$t/exec.args" +EOF +chmod +x "$t/bin/"* "$wt/tools/repo-publish" + +run() { # run [ENV=…]: tools/repo-publish --from-main --dry-run in the fixture + rm -f "$t/exec.args" + (cd "$R" && env PATH="$t/bin:$PATH" "$@" tools/repo-publish --from-main --dry-run) > "$t/log" 2>&1 +} +published() { # published WHAT [ENV=…]: the publisher ran, given all but --from-main + run "${@:2}" || { cat "$t/log"; fail "$1: the publisher runs"; } + [[ $(cat "$t/exec.args") == --dry-run ]] || fail "$1: the publisher gets the arguments but --from-main" +} +stopped() { # stopped WHAT [ENV=…]: the run failed before the publisher ran + if run "${@:2}"; then fail "$1: the run stops"; fi + [[ ! -e $t/exec.args ]] || fail "$1: the publisher doesn't run" +} + +# --- an empty inventory is a no-op --- +published "nothing ignored" +grep -c '^git fetch ' "$STUB_LOG" > /dev/null || fail "the fetch went to the stand-in git" +if grep -c ' worktree ' "$STUB_LOG" > /dev/null; then fail "the fixture's publish tree is used, no new worktree"; fi +echo "build output" > "$R/packages/b/build.log" +published "ignored files, none under prebuilt/" +[[ ! -e $wt/packages ]] || fail "an empty inventory copies nothing" +pass "an empty prebuilt inventory is a no-op, and the clean tree is published" + +# --- a populated inventory is copied, names with spaces and newlines too --- +mkdir -p "$R/packages/a/prebuilt" "$R/packages/b/bionic/prebuilt" +echo one > "$R/packages/a/prebuilt/with space.bin" +echo two > "$R/packages/b/bionic/prebuilt/new"$'\n'"line" +published "prebuilt files" +[[ $(cat "$wt/packages/a/prebuilt/with space.bin") == one ]] || fail "a name with a space is copied" +[[ $(cat "$wt/packages/b/bionic/prebuilt/new"$'\n'"line") == two ]] || fail "a name with a newline is copied" +[[ ! -e $wt/packages/b/build.log ]] || fail "ignored files outside prebuilt/ stay behind" +pass "prebuilt files are copied by NUL-separated name, and nothing else" + +# --- failures stop before the publisher runs --- +echo "not an index" > "$t/bad-index" # the real git ls-files fails on it +stopped "a failing git ls-files" GIT_INDEX_FILE="$t/bad-index" +grep -c 'fatal:' "$t/log" > /dev/null || { cat "$t/log"; fail "git's own error is shown"; } +stopped "a git ls-files that fails after some names" FAIL_GIT_MIDWAY=1 +pass "a failing inventory stops before publishing, also after some names" + +stopped "a failing rsync" FAIL_RSYNC=1 +pass "a failing copy stops before publishing" diff --git a/tools/repo-publish b/tools/repo-publish index 9c944269a..cf45a332c 100755 --- a/tools/repo-publish +++ b/tools/repo-publish @@ -75,7 +75,9 @@ if ((from_main)); then git -C "$wt" clean -qfdx -e /keys -e /out ln -sfn "$ROOT/keys" "$wt/keys" ln -sfn "$ROOT/out" "$wt/out" - git ls-files -z --others --ignored --exclude-standard -- packages | grep -z '/prebuilt/' | + # git picks them by pathspec: a checkout with none gets an empty list, + # which copies nothing. + git ls-files -z --others --ignored --exclude-standard -- ':(glob)packages/**/prebuilt/**' | rsync -a --from0 --files-from=- ./ "$wt/" say "Publishing from origin/main at $(git -C "$wt" rev-parse --short HEAD)" exec "$wt/tools/repo-publish" "${args[@]}"