diff --git a/tests/all b/tests/all index 9805323af..825765444 100755 --- a/tests/all +++ b/tests/all @@ -19,8 +19,9 @@ # tools/fetch-toolchain against local fixtures, tools/release-upload against # a scratch server, tools/extract-camera-data's adrp decoding, # tools/phone-smoke's dmesg error classification, the OWL conformance -# report's differences on the host, and bin/lineage-ref sync's branch -# handling against stand-ins for docker and repo. +# report's differences on the host, bin/lineage-ref sync's branch handling +# against stand-ins for docker and repo, and tools/repo-publish --from-main's +# copy of the prebuilt binaries into a fixture's publish tree. set -euo pipefail cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." for f in tools/install-phone tools/phone-provision tools/nv-backup tools/build-rootfs tools/build-packages tools/make-bundle \ @@ -37,7 +38,7 @@ sh -n device/flame/initramfs/build bash -n tools/build-flame-probe echo "ok - scripts parse" python3 tests/bt-uart-gpio.py -for t in tests/pr-check tests/kernel-pr-check tests/phone-smoke tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/nv-backup tests/initramfs-grow tests/images-layout tests/boot-update-parity tests/stage-boot-files tests/initramfs-reproducible tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/modules-push tests/make-bundle-keys tests/make-bundle tests/release-signing tests/pinned-packages tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package tests/publish-kernel tests/fetch-toolchain tests/release-upload tests/extract-camera-data tests/release tests/lineage-ref; do +for t in tests/pr-check tests/kernel-pr-check tests/phone-smoke tests/alarm-wake-test tests/install-phone tests/build-rootfs tests/phone-provision tests/nv-backup tests/initramfs-grow tests/images-layout tests/boot-update-parity tests/stage-boot-files tests/initramfs-reproducible tests/device-overlays tests/simplefb-trigger tests/initramfs-steps tests/storage-guard tests/usb-product tests/boot-method tests/modules-push tests/make-bundle-keys tests/make-bundle tests/release-signing tests/pinned-packages tests/flame-probe packages/omamobile-firstboot/test/all packages/omamobile-device-felix/test/all packages/omamobile-device-lynx/test/all packages/omamobile-device-panther/test/all tests/local-builds tests/phone-install-package tests/publish-kernel tests/fetch-toolchain tests/release-upload tests/extract-camera-data tests/release tests/lineage-ref tests/repo-publish; do echo "# $t" "$t" done diff --git a/tests/repo-publish b/tests/repo-publish new file mode 100755 index 000000000..976ed30a4 --- /dev/null +++ b/tests/repo-publish @@ -0,0 +1,99 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# tools/repo-publish --from-main's copy of the gitignored prebuilt binaries +# (packages/*/…/prebuilt/) into out/publish-tree, in a fixture repository +# (no network, no keys, nothing built or published). The real git lists the +# inventory and the real rsync copies it; a stand-in git answers everything +# that would touch origin or create the worktree, git and rsync fail on +# request, and a stand-in publisher in the fixture's out/publish-tree +# records what it was handed. An empty inventory is a no-op, names with +# spaces and newlines are copied, and a failing inventory or copy stops +# before the publisher runs. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +fail() { printf 'not ok - %s\n' "$1" >&2; exit 1; } +pass() { printf 'ok - %s\n' "$1"; } +command -v rsync >/dev/null || { pass "# skip: no rsync"; exit 0; } +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT +export GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null # never the user's git config +export REAL_GIT REAL_RSYNC STUB_LOG=$t/git.log +REAL_GIT=$(command -v git) REAL_RSYNC=$(command -v rsync) + +R=$t/repo wt=$t/repo/out/publish-tree +mkdir -p "$R/tools" "$R/packages/a" "$R/packages/b/bionic" "$t/bin" "$wt/.git" "$wt/tools" +cp tools/repo-publish "$R/tools/" +echo /out > "$R/.gitignore" +echo 'prebuilt/' > "$R/packages/a/.gitignore" +printf '*.log\nbionic/prebuilt/\n' > "$R/packages/b/.gitignore" +git -C "$R" init -q +git -C "$R" add -A +git -C "$R" -c user.name=test -c user.email=test@invalid commit -qm fixture + +# Only the inventory reaches the real git; fetch, checkout, clean and the +# worktree are stood in for, so nothing outside the fixture is touched. +cat > "$t/bin/git" <<'EOF' +#!/bin/bash +if [[ " $* " == *" ls-files "* ]]; then + # FAIL_GIT_MIDWAY: one good name, then a failure, as a read error would + [[ -z ${FAIL_GIT_MIDWAY-} ]] || { printf 'packages/a/prebuilt/with space.bin\0'; exit 128; } + exec "$REAL_GIT" "$@" +fi +echo "git $*" >> "$STUB_LOG" +[[ " $* " != *" rev-parse "* ]] || echo 0000000 +EOF +# The copy fails on request with rsync's status, after reading its list, +# so the failure isn't a SIGPIPE upstream. +cat > "$t/bin/rsync" <<'EOF' +#!/bin/bash +[[ -z ${FAIL_RSYNC-} ]] || { cat > /dev/null; echo "rsync: (stand-in) No space left on device" >&2; exit 23; } +exec "$REAL_RSYNC" "$@" +EOF +printf '#!/bin/sh\nexit 0\n' > "$t/bin/bsdtar" # checked for, never run on this path +cat > "$wt/tools/repo-publish" < "$t/exec.args" +EOF +chmod +x "$t/bin/"* "$wt/tools/repo-publish" + +run() { # run [ENV=…]: tools/repo-publish --from-main --dry-run in the fixture + rm -f "$t/exec.args" + (cd "$R" && env PATH="$t/bin:$PATH" "$@" tools/repo-publish --from-main --dry-run) > "$t/log" 2>&1 +} +published() { # published WHAT [ENV=…]: the publisher ran, given all but --from-main + run "${@:2}" || { cat "$t/log"; fail "$1: the publisher runs"; } + [[ $(cat "$t/exec.args") == --dry-run ]] || fail "$1: the publisher gets the arguments but --from-main" +} +stopped() { # stopped WHAT [ENV=…]: the run failed before the publisher ran + if run "${@:2}"; then fail "$1: the run stops"; fi + [[ ! -e $t/exec.args ]] || fail "$1: the publisher doesn't run" +} + +# --- an empty inventory is a no-op --- +published "nothing ignored" +grep -c '^git fetch ' "$STUB_LOG" > /dev/null || fail "the fetch went to the stand-in git" +if grep -c ' worktree ' "$STUB_LOG" > /dev/null; then fail "the fixture's publish tree is used, no new worktree"; fi +echo "build output" > "$R/packages/b/build.log" +published "ignored files, none under prebuilt/" +[[ ! -e $wt/packages ]] || fail "an empty inventory copies nothing" +pass "an empty prebuilt inventory is a no-op, and the clean tree is published" + +# --- a populated inventory is copied, names with spaces and newlines too --- +mkdir -p "$R/packages/a/prebuilt" "$R/packages/b/bionic/prebuilt" +echo one > "$R/packages/a/prebuilt/with space.bin" +echo two > "$R/packages/b/bionic/prebuilt/new"$'\n'"line" +published "prebuilt files" +[[ $(cat "$wt/packages/a/prebuilt/with space.bin") == one ]] || fail "a name with a space is copied" +[[ $(cat "$wt/packages/b/bionic/prebuilt/new"$'\n'"line") == two ]] || fail "a name with a newline is copied" +[[ ! -e $wt/packages/b/build.log ]] || fail "ignored files outside prebuilt/ stay behind" +pass "prebuilt files are copied by NUL-separated name, and nothing else" + +# --- failures stop before the publisher runs --- +echo "not an index" > "$t/bad-index" # the real git ls-files fails on it +stopped "a failing git ls-files" GIT_INDEX_FILE="$t/bad-index" +grep -c 'fatal:' "$t/log" > /dev/null || { cat "$t/log"; fail "git's own error is shown"; } +stopped "a git ls-files that fails after some names" FAIL_GIT_MIDWAY=1 +pass "a failing inventory stops before publishing, also after some names" + +stopped "a failing rsync" FAIL_RSYNC=1 +pass "a failing copy stops before publishing" diff --git a/tests/stage-boot-files b/tests/stage-boot-files index 928040800..6d40affd8 100755 --- a/tests/stage-boot-files +++ b/tests/stage-boot-files @@ -41,7 +41,7 @@ rm -rf "$bf" refuses "no boot files at all" "no boot files in" pass "--check refuses stale, tampered, SOURCES-less or missing boot files" -grep -qE "grep -zE '/\(prebuilt\|boot-files\)/'" tools/repo-publish || fail "repo-publish --from-main carries boot-files/ into its clean tree" +grep -qF "':(glob)packages/**/boot-files/**'" tools/repo-publish || fail "repo-publish --from-main carries boot-files/ into its clean tree" grep -qF 'tools/stage-boot-files --check "$d/boot-files"' tools/build-packages || fail "build-packages checks the boot files before building" grep -qF '"$d/stage" --verify "$d/prebuilt"' tools/build-packages || fail "build-packages verifies the staged kernel before building" grep -qE '^NAMED_ONLY=\(linux-omamobile\)$' tools/build-packages || fail "linux-omamobile is built only when named" diff --git a/tools/repo-publish b/tools/repo-publish index 3c467cd04..283f4e38d 100755 --- a/tools/repo-publish +++ b/tools/repo-publish @@ -98,7 +98,9 @@ if ((from_main)); then git -C "$wt" clean -qfdx -e /keys -e /out ln -sfn "$ROOT/keys" "$wt/keys" ln -sfn "$ROOT/out" "$wt/out" - git ls-files -z --others --ignored --exclude-standard -- packages | grep -zE '/(prebuilt|boot-files)/' | + # git picks them by pathspec: a checkout with none gets an empty list, + # which copies nothing (a grep over an empty list fails under pipefail). + git ls-files -z --others --ignored --exclude-standard -- ':(glob)packages/**/prebuilt/**' ':(glob)packages/**/boot-files/**' | rsync -a --from0 --files-from=- ./ "$wt/" say "Publishing from origin/main at $(git -C "$wt" rev-parse --short HEAD)" exec "$wt/tools/repo-publish" "${args[@]}"