From fcee4a4fc95bf09199d0218473d00bc109aa7f0f Mon Sep 17 00:00:00 2001 From: Victor Vogel Date: Sun, 1 Nov 2026 22:22:13 +0000 Subject: [PATCH] tools: repo-publish --from-main publishes a tree with no prebuilt files --from-main copies the gitignored prebuilt binaries into out/publish-tree with git ls-files | grep -z /prebuilt/ | rsync. In a checkout that has none (a fresh clone, or one where nothing has been built), grep finds no match and exits 1; under set -euo pipefail that ends the script with no message before it hands over to the clean tree, so nothing is built or published. git now selects the files itself with the pathspec ':(glob)packages/**/prebuilt/**', which picks the same names the grep did and exits 0 when there are none; rsync then gets an empty list and copies nothing. A failing git ls-files or rsync still stops the script before the publisher runs. tests/repo-publish runs the block in a fixture repository: the real git lists the inventory and the real rsync copies it, a stand-in git answers fetch, checkout and clean, and a stand-in publisher in the fixture's out/publish-tree records its arguments. It covers no ignored files, ignored files with none under prebuilt/, prebuilt names with a space and a newline at two depths, a git ls-files that fails at once (a corrupt index) or after some names, and a failing rsync. CI's Arch container gets rsync so the test runs there. Tested on the laptop (no phone; the script never runs on one): tests/repo-publish passes on this commit and fails at its first case on the parent; mutants that swallow the git or rsync status each fail it; the pathspec and the old grep select the same files from a tree of awkward names. tools/pr-check and tests/all pass. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/checks.yml | 2 +- tests/all | 7 +-- tests/repo-publish | 99 ++++++++++++++++++++++++++++++++++++ tools/repo-publish | 4 +- 4 files changed, 107 insertions(+), 5 deletions(-) create mode 100755 tests/repo-publish diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index aba51e22e..c287f0769 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -46,7 +46,7 @@ jobs: - name: Packages the tests use run: | pacman -Syu --noconfirm --needed git base-devel e2fsprogs zstd openssl \ - python python-dbus python-gobject dbus nodejs dtc cpio file jq busybox \ + python python-dbus python-gobject dbus nodejs dtc cpio file jq busybox rsync \ xz gzip qt6-declarative shellcheck # Arch's nodejs can lag a simdjson soname bump: on 2026-10-29 # simdjson 5.0.8 dropped the libsimdjson.so.34 that nodejs still diff --git a/tests/all b/tests/all index 4afae7bc4..e8759c7a5 100755 --- a/tests/all +++ b/tests/all @@ -9,8 +9,9 @@ # and flash against a fake fastboot, omamobile-firstboot's own tests, and the # smoke test of the newest root image in out/ if there is one, the felix, # lynx and panther device packages' own tests, tools/alarm-wake-test's -# check, and a contributor's local builds (tools/local-key-init, -# tools/fetch-published, --local). +# check, a contributor's local builds (tools/local-key-init, +# tools/fetch-published, --local), and tools/repo-publish --from-main's copy +# of the prebuilt binaries into a fixture's publish tree. set -euo pipefail cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." for f in tools/install-phone tools/phone-provision tools/build-rootfs tools/build-packages tools/make-bundle \ @@ -24,7 +25,7 @@ sh -n device/flame/initramfs/init sh -n device/flame/initramfs/build bash -n tools/build-flame-probe echo "ok - scripts parse" -for t in tests/pr-check tests/kernel-pr-check tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/initramfs-grow tests/images-layout tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/make-bundle-keys tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package; do +for t in tests/pr-check tests/kernel-pr-check tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/initramfs-grow tests/images-layout tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/make-bundle-keys tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package tests/repo-publish; do echo "# $t" "$t" done diff --git a/tests/repo-publish b/tests/repo-publish new file mode 100755 index 000000000..976ed30a4 --- /dev/null +++ b/tests/repo-publish @@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# tools/repo-publish --from-main's copy of the gitignored prebuilt binaries +# (packages/*/…/prebuilt/) into out/publish-tree, in a fixture repository +# (no network, no keys, nothing built or published). The real git lists the +# inventory and the real rsync copies it; a stand-in git answers everything +# that would touch origin or create the worktree, git and rsync fail on +# request, and a stand-in publisher in the fixture's out/publish-tree +# records what it was handed. An empty inventory is a no-op, names with +# spaces and newlines are copied, and a failing inventory or copy stops +# before the publisher runs. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +fail() { printf 'not ok - %s\n' "$1" >&2; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +command -v rsync >/dev/null || { pass "# skip: no rsync"; exit 0; } +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null # never the user's git config +export REAL_GIT REAL_RSYNC STUB_LOG=$t/git.log +REAL_GIT=$(command -v git) REAL_RSYNC=$(command -v rsync) + +R=$t/repo wt=$t/repo/out/publish-tree +mkdir -p "$R/tools" "$R/packages/a" "$R/packages/b/bionic" "$t/bin" "$wt/.git" "$wt/tools" +cp tools/repo-publish "$R/tools/" +echo /out > "$R/.gitignore" +echo 'prebuilt/' > "$R/packages/a/.gitignore" +printf '*.log\nbionic/prebuilt/\n' > "$R/packages/b/.gitignore" +git -C "$R" init -q +git -C "$R" add -A +git -C "$R" -c user.name=test -c user.email=test@invalid commit -qm fixture + +# Only the inventory reaches the real git; fetch, checkout, clean and the +# worktree are stood in for, so nothing outside the fixture is touched. +cat > "$t/bin/git" <<'EOF' +#!/bin/bash +if [[ " $* " == *" ls-files "* ]]; then + # FAIL_GIT_MIDWAY: one good name, then a failure, as a read error would + [[ -z ${FAIL_GIT_MIDWAY-} ]] || { printf 'packages/a/prebuilt/with space.bin\0'; exit 128; } + exec "$REAL_GIT" "$@" +fi +echo "git $*" >> "$STUB_LOG" +[[ " $* " != *" rev-parse "* ]] || echo 0000000 +EOF +# The copy fails on request with rsync's status, after reading its list, +# so the failure isn't a SIGPIPE upstream. +cat > "$t/bin/rsync" <<'EOF' +#!/bin/bash +[[ -z ${FAIL_RSYNC-} ]] || { cat > /dev/null; echo "rsync: (stand-in) No space left on device" >&2; exit 23; } +exec "$REAL_RSYNC" "$@" +EOF +printf '#!/bin/sh\nexit 0\n' > "$t/bin/bsdtar" # checked for, never run on this path +cat > "$wt/tools/repo-publish" < "$t/exec.args" +EOF +chmod +x "$t/bin/"* "$wt/tools/repo-publish" + +run() { # run [ENV=…]: tools/repo-publish --from-main --dry-run in the fixture + rm -f "$t/exec.args" + (cd "$R" && env PATH="$t/bin:$PATH" "$@" tools/repo-publish --from-main --dry-run) > "$t/log" 2>&1 +} +published() { # published WHAT [ENV=…]: the publisher ran, given all but --from-main + run "${@:2}" || { cat "$t/log"; fail "$1: the publisher runs"; } + [[ $(cat "$t/exec.args") == --dry-run ]] || fail "$1: the publisher gets the arguments but --from-main" +} +stopped() { # stopped WHAT [ENV=…]: the run failed before the publisher ran + if run "${@:2}"; then fail "$1: the run stops"; fi + [[ ! -e $t/exec.args ]] || fail "$1: the publisher doesn't run" +} + +# --- an empty inventory is a no-op --- +published "nothing ignored" +grep -c '^git fetch ' "$STUB_LOG" > /dev/null || fail "the fetch went to the stand-in git" +if grep -c ' worktree ' "$STUB_LOG" > /dev/null; then fail "the fixture's publish tree is used, no new worktree"; fi +echo "build output" > "$R/packages/b/build.log" +published "ignored files, none under prebuilt/" +[[ ! -e $wt/packages ]] || fail "an empty inventory copies nothing" +pass "an empty prebuilt inventory is a no-op, and the clean tree is published" + +# --- a populated inventory is copied, names with spaces and newlines too --- +mkdir -p "$R/packages/a/prebuilt" "$R/packages/b/bionic/prebuilt" +echo one > "$R/packages/a/prebuilt/with space.bin" +echo two > "$R/packages/b/bionic/prebuilt/new"$'\n'"line" +published "prebuilt files" +[[ $(cat "$wt/packages/a/prebuilt/with space.bin") == one ]] || fail "a name with a space is copied" +[[ $(cat "$wt/packages/b/bionic/prebuilt/new"$'\n'"line") == two ]] || fail "a name with a newline is copied" +[[ ! -e $wt/packages/b/build.log ]] || fail "ignored files outside prebuilt/ stay behind" +pass "prebuilt files are copied by NUL-separated name, and nothing else" + +# --- failures stop before the publisher runs --- +echo "not an index" > "$t/bad-index" # the real git ls-files fails on it +stopped "a failing git ls-files" GIT_INDEX_FILE="$t/bad-index" +grep -c 'fatal:' "$t/log" > /dev/null || { cat "$t/log"; fail "git's own error is shown"; } +stopped "a git ls-files that fails after some names" FAIL_GIT_MIDWAY=1 +pass "a failing inventory stops before publishing, also after some names" + +stopped "a failing rsync" FAIL_RSYNC=1 +pass "a failing copy stops before publishing" diff --git a/tools/repo-publish b/tools/repo-publish index 9c944269a..cf45a332c 100755 --- a/tools/repo-publish +++ b/tools/repo-publish @@ -75,7 +75,9 @@ if ((from_main)); then git -C "$wt" clean -qfdx -e /keys -e /out ln -sfn "$ROOT/keys" "$wt/keys" ln -sfn "$ROOT/out" "$wt/out" - git ls-files -z --others --ignored --exclude-standard -- packages | grep -z '/prebuilt/' | + # git picks them by pathspec: a checkout with none gets an empty list, + # which copies nothing. + git ls-files -z --others --ignored --exclude-standard -- ':(glob)packages/**/prebuilt/**' | rsync -a --from0 --files-from=- ./ "$wt/" say "Publishing from origin/main at $(git -C "$wt" rev-parse --short HEAD)" exec "$wt/tools/repo-publish" "${args[@]}"