diff --git a/packages/omamobile-gnss/bionic/Makefile b/packages/omamobile-gnss/bionic/Makefile index 0b22a991b..e8526b9a2 100644 --- a/packages/omamobile-gnss/bionic/Makefile +++ b/packages/omamobile-gnss/bionic/Makefile @@ -6,7 +6,10 @@ # prebuilt/stubs/*.so stand-ins and preloads for the gnss profile # prebuilt/omamobile-gnss-client the Pixel 9a GNSS HAL client (gnss-slsi profile) # -# make NDK=/opt/android-ndk (Android NDK r27 or later) +# make NDK=/opt/android-ndk (Android NDK r27 or later; the +# pinned one: tools/fetch-ndk +# --accept-google-terms, then +# make NDK="$(tools/fetch-ndk --print)") # make CLANG=... BIONIC=... CHROOT=... # (an AOSP/LineageOS tree's prebuilt # clang and bionic headers, linked diff --git a/tests/all b/tests/all index be2267308..f9356d2d6 100755 --- a/tests/all +++ b/tests/all @@ -100,3 +100,5 @@ echo "# tests/build-packages-deps" # a package's dependencies into the build c tests/build-packages-deps echo "# tests/build-packages-cache" # a forced same-version rebuild drops the stale cached package tests/build-packages-cache +echo "# tests/fetch-ndk" # the pinned Android NDK's fetch, against a local archive and stand-ins +tests/fetch-ndk diff --git a/tests/fetch-ndk b/tests/fetch-ndk new file mode 100755 index 000000000..e1077d594 --- /dev/null +++ b/tests/fetch-ndk @@ -0,0 +1,108 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# tools/fetch-ndk against a local fixture (a file:// URL, no network): an +# archive that matches its pin is unpacked with its executables kept and the +# archive removed, an NDK already there is used without a download, and a +# wrong size, a wrong SHA-256, an archive without the pinned revision, a +# different NDK in the way and a lock that isn't one are refused, leaving +# nothing behind. --print never downloads, a host other than Google's and a +# computer that isn't x86_64 are refused before curl is called (a stand-in +# curl records any call), and the tool holds curl to HTTPS. Nothing is +# downloaded without --accept-google-terms; an NDK already there needs none. +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." +bash -n tools/fetch-ndk +for tool in curl python3 sha256sum; do + command -v "$tool" >/dev/null || { echo "ok - # skip: no $tool"; exit 0; } +done +command -v unzip >/dev/null || command -v bsdtar >/dev/null || { echo "ok - # skip: no unzip or bsdtar"; exit 0; } +t=$(mktemp -d) +trap 'rm -rf "$t"' EXIT +fail() { echo "not ok - $*" >&2; [[ -f $t/log ]] && cat "$t/log" >&2; exit 1; } +fetch() { tools/fetch-ndk --dir "$t/cache" --lock "$t/lock" --any-host --accept-google-terms "$@" > "$t/out" 2> "$t/log"; } +print() { tools/fetch-ndk --dir "$t/cache" --lock "$t/lock" --print > "$t/out" 2> "$t/log"; } +# Stand-ins that record a call: curl fails after recording one, uname says aarch64 +mkdir -p "$t/fake" +printf '#!/bin/sh\necho "$@" >> "%s/curl-called"\nexit 7\n' "$t" > "$t/fake/curl" +printf '#!/bin/sh\necho aarch64\n' > "$t/fake/uname" +chmod 755 "$t/fake/curl" "$t/fake/uname" +fake_fetch() { PATH="$t/fake:$PATH" tools/fetch-ndk --dir "$t/cache" --lock "$t/lock" "$@" > "$t/out" 2> "$t/log"; } + +# The fixture: an NDK-shaped tree with an executable compiler, zipped as Google zips it +mkdir -p "$t/src/android-ndk-r9z/toolchains/llvm/prebuilt/linux-x86_64/bin" +printf 'Pkg.Desc = Android NDK\nPkg.Revision = 9.0.1234\n' > "$t/src/android-ndk-r9z/source.properties" +printf '#!/bin/sh\necho clang\n' > "$t/src/android-ndk-r9z/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" +chmod 755 "$t/src/android-ndk-r9z/toolchains/llvm/prebuilt/linux-x86_64/bin/clang" +python3 -I - "$t/src" "$t/ndk.zip" <<'PY' +import os, sys, zipfile +root, out = sys.argv[1], sys.argv[2] +with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: + for d, _, files in os.walk(root): + for f in files: + p = os.path.join(d, f) + z.write(p, os.path.relpath(p, root)) +PY +sum=$(sha256sum "$t/ndk.zip" | cut -d' ' -f1); size=$(stat -c %s "$t/ndk.zip") +lock() { printf '# a comment\n\n%s\n' "$1" > "$t/lock"; } + +lock "r9z 9.0.1234 $sum $size file://$t/ndk.zip" +if print; then fail "--print without the NDK fails"; fi +grep -c 'fetch it first with tools/fetch-ndk --accept-google-terms' "$t/log" >/dev/null || fail "--print without the NDK says how to fetch it: $(cat "$t/log")" +[[ ! -e $t/cache ]] || fail "--print without the NDK creates nothing: $(ls -A "$t/cache")" +echo "ok - --print never downloads: without the NDK it says how to fetch it" + +fetch || fail "a matching archive is fetched" +[[ $(cat "$t/out") == "$t/cache/android-ndk-r9z" ]] || fail "the fetch prints the NDK's path: $(cat "$t/out")" +[[ -x $t/cache/android-ndk-r9z/toolchains/llvm/prebuilt/linux-x86_64/bin/clang ]] || fail "the compiler stays executable" +[[ -z $(find "$t/cache" -maxdepth 1 -name '.*' -newer "$t/lock") ]] || fail "no archive or work directory is left: $(ls -A "$t/cache")" +print || fail "--print finds the NDK once it is fetched" +[[ $(cat "$t/out") == "$t/cache/android-ndk-r9z" ]] || fail "--print prints the NDK's path: $(cat "$t/out")" +[[ ! -s $t/log ]] || fail "--print says nothing else" +echo "ok - a matching archive is unpacked, its compiler executable, the archive removed; --print then prints its path" + +mv "$t/ndk.zip" "$t/ndk.zip.away" +tools/fetch-ndk --dir "$t/cache" --lock "$t/lock" > "$t/out" 2> "$t/log" || fail "an NDK already there needs no download, and no option" +grep -c 'is in' "$t/log" >/dev/null && [[ $(cat "$t/out") == "$t/cache/android-ndk-r9z" ]] || fail "it says where the NDK is" +mv "$t/ndk.zip.away" "$t/ndk.zip" +echo "ok - an NDK already there is used as it is" + +rm -rf "$t/cache" +lock "r9z 9.0.1234 $sum $((size + 1)) file://$t/ndk.zip" +if fetch; then fail "a wrong size is refused"; fi +grep -c 'not the pinned' "$t/log" >/dev/null || fail "the size refusal says why" +lock "r9z 9.0.1234 $(printf '%064d' 0) $size file://$t/ndk.zip" +if fetch; then fail "a wrong SHA-256 is refused"; fi +grep -c "SHA-256 is $sum" "$t/log" >/dev/null || fail "the checksum refusal says why" +[[ -z $(ls -A "$t/cache") ]] || fail "a refused archive leaves nothing: $(ls -A "$t/cache")" +lock "r9z 9.9.9 $sum $size file://$t/ndk.zip" +if fetch; then fail "an archive without the pinned revision is refused"; fi +[[ ! -e $t/cache/android-ndk-r9z ]] || fail "an archive without the pinned revision isn't put in place" +echo "ok - a wrong size, a wrong SHA-256 or another revision is refused, leaving nothing" + +mkdir -p "$t/cache/android-ndk-r9z"; printf 'Pkg.Revision = 8.0.1\n' > "$t/cache/android-ndk-r9z/source.properties" +lock "r9z 9.0.1234 $sum $size file://$t/ndk.zip" +if fetch; then fail "another NDK in the way is refused"; fi +grep -c 'remove it first' "$t/log" >/dev/null || fail "it says to remove the other NDK" +rm -rf "$t/cache" "$t/curl-called" +lock "r9z 9.0.1234 $sum $size https://example.org/android-ndk-r9z-linux.zip" +if fake_fetch --any-host; then fail "a download from anywhere but Google is refused"; fi +grep -c 'the NDK comes from https://dl.google.com/android/repository/' "$t/log" >/dev/null || fail "the host refusal says why: $(cat "$t/log")" +[[ ! -e $t/curl-called ]] || fail "another host is refused before curl is called: $(cat "$t/curl-called")" +lock "r9z 9.0.1234 $sum" +if fetch; then fail "a short lock line is refused"; fi +echo "ok - another NDK in the way, another host (no request made) and a broken lock are refused" + +rm -rf "$t/cache" "$t/curl-called" +lock "r9z 9.0.1234 $sum $size https://dl.google.com/android/repository/android-ndk-r9z-linux.zip" +if fake_fetch; then fail "a computer that isn't x86_64 is refused"; fi +grep -c 'x86_64 Linux only (--any-host' "$t/log" >/dev/null || fail "the host refusal names --any-host: $(cat "$t/log")" +[[ ! -e $t/curl-called ]] || fail "a computer that isn't x86_64 is refused before curl is called" +if fake_fetch --any-host; then fail "without --accept-google-terms nothing is downloaded"; fi +grep -c 'https://developer.android.com/ndk/downloads' "$t/log" >/dev/null && grep -c -- '--accept-google-terms' "$t/log" >/dev/null \ + || fail "the stop names Google's page and the option: $(cat "$t/log")" +[[ ! -e $t/curl-called ]] || fail "without --accept-google-terms curl isn't called" +[[ -z $(ls -A "$t/cache") ]] || fail "without --accept-google-terms nothing is left: $(ls -A "$t/cache")" +if fake_fetch --any-host --accept-google-terms; then fail "the stand-in curl fails the download"; fi +grep -c -- "--proto =https,file --proto-redir =https" "$t/curl-called" >/dev/null || fail "curl is held to HTTPS: $(cat "$t/curl-called")" +[[ -z $(ls -A "$t/cache") ]] || fail "a failed download leaves nothing: $(ls -A "$t/cache")" +echo "ok - a computer that isn't x86_64 is refused unless --any-host; no download without --accept-google-terms; curl is held to HTTPS and a failed download leaves nothing" diff --git a/tools/build-packages b/tools/build-packages index 0a436b739..6573fc39a 100755 --- a/tools/build-packages +++ b/tools/build-packages @@ -461,7 +461,7 @@ for dir in "${build[@]}"; do # then fails at its start. Its install script says so in the middle of the # build output; say it again where a publish run is read. [[ $name == omamobile-gnss && ! -f $dir/bionic/prebuilt/omamobile-gnss-client ]] && - warned+=("omamobile-gnss was built without the Pixel 9a's HAL client: no Location on a 9a from this package (make -C packages/omamobile-gnss/bionic, then build it again)") + warned+=("omamobile-gnss was built without the Pixel 9a's HAL client: no Location on a 9a from this package (tools/fetch-ndk --accept-google-terms, then make -C packages/omamobile-gnss/bionic NDK=\"\$(tools/fetch-ndk --print)\", then build it again)") echo " built in $(( ($(date +%s) - start) / 60 )) min" # Index after each package, so the next one can depend on it sign_and_index diff --git a/tools/fetch-ndk b/tools/fetch-ndk new file mode 100755 index 000000000..97142c605 --- /dev/null +++ b/tools/fetch-ndk @@ -0,0 +1,115 @@ +#!/bin/bash +# SPDX-License-Identifier: MIT +# Fetch the Android NDK the Pixel 9a's GNSS HAL client is built with +# (packages/omamobile-gnss/bionic), at the release tools/ndk.lock pins, from +# Google's download host, checked against the SHA-256 and the size recorded +# there before anything is unpacked. +# +# tools/fetch-ndk --accept-google-terms [--dir DIR] [--lock FILE] [--any-host] +# tools/fetch-ndk --print [--dir DIR] [--lock FILE] +# --dir where the NDK is kept (default $OMAMOBILE_NDK_DIR, else +# ~/.cache/omamobile/ndk: outside the repository) +# --lock the pin (default tools/ndk.lock): RELEASE REVISION SHA256 SIZE URL +# --print print only the path of the NDK already fetched, for +# make -C packages/omamobile-gnss/bionic NDK="$(tools/fetch-ndk --print)"; +# never downloads: without the NDK it stops and says how to +# fetch it +# --any-host fetch on a computer that isn't x86_64 all the same +# --accept-google-terms +# you have read the terms and conditions Google asks everyone +# to agree to before downloading the NDK, on its download page +# (https://developer.android.com/ndk/downloads), and accept +# them. A download needs it: without it the fetch stops, +# naming the page and the option. An NDK already fetched needs +# no option, and neither does --print. +# +# So the two steps are: tools/fetch-ndk --accept-google-terms, then the make +# with --print. +# +# The archive is about 740 MB and the NDK about 2.5 GB unpacked; an NDK that +# is already there, with the pinned revision in its source.properties, is +# used as it is. The archive is removed once it is unpacked. Google builds +# the NDK's compilers for x86_64 Linux (and macOS), so the make runs on an +# x86_64 build host, and a fetch elsewhere stops unless asked for. The +# download is held to HTTPS, redirects included. Nothing is built or +# installed here. +# +# Licence: the NDK is Google's (Apache-2.0 and others; its NOTICE files are +# in the tree). The project redistributes none of it. +set -euo pipefail +self=$(readlink -f "${BASH_SOURCE[0]}") +ROOT=$(cd -- "$(dirname -- "$self")/.." && pwd) +die() { printf '\e[31merror:\e[0m %s\n' "$*" >&2; exit 1; } +say() { ((quiet)) || printf '\e[1m==> %s\e[0m\n' "$*" >&2; } + +dir=${OMAMOBILE_NDK_DIR:-${XDG_CACHE_HOME:-$HOME/.cache}/omamobile/ndk} +lock=$ROOT/tools/ndk.lock +quiet=0 +any_host=0 +accept=0 +terms=https://developer.android.com/ndk/downloads +while (($#)); do + case $1 in + --dir) dir=${2:?--dir needs a directory}; shift ;; + --lock) lock=${2:?--lock needs a file}; shift ;; + --print) quiet=1 ;; + --any-host) any_host=1 ;; + --accept-google-terms) accept=1 ;; + -h|--help) source "$ROOT/tools/lib/usage.sh"; usage_from_header "$self"; exit 0 ;; + *) die "unknown argument $1" ;; + esac + shift +done + +[[ -f $lock ]] || die "no lock file $lock" +read -r release revision sum size url < <(grep -v '^[[:space:]]*\(#\|$\)' "$lock" | head -1) || true +[[ $release =~ ^r[0-9]+[a-z]?$ && $revision =~ ^[0-9]+(\.[0-9]+)+$ && $sum =~ ^[0-9a-f]{64}$ && $size =~ ^[0-9]+$ ]] || + die "$lock: one line of RELEASE REVISION SHA256 SIZE URL" +[[ $url == https://dl.google.com/android/repository/* || $url == file://* ]] || + die "$lock: the NDK comes from https://dl.google.com/android/repository/" + +((quiet)) || mkdir -p "$dir" +[[ -d $dir ]] && dir=$(cd -- "$dir" && pwd) +ndk=$dir/android-ndk-$release +has_revision() { [[ $(sed -n 's/^Pkg\.Revision *= *//p' "$1/source.properties" 2>/dev/null | tr -d '\r') == "$revision" ]]; } + +if has_revision "$ndk"; then + say "the NDK $release ($revision) is in $ndk" + printf '%s\n' "$ndk" + exit 0 +fi +[[ -e $ndk ]] && die "$ndk is there but isn't the NDK $release ($revision): remove it first" +((quiet)) && die "the NDK $release isn't in $dir yet: fetch it first with tools/fetch-ndk --accept-google-terms (Google's terms for it: $terms)" +[[ $(uname -m) == x86_64 ]] || ((any_host)) || + die "this computer is $(uname -m): the NDK's compilers run on x86_64 Linux only (--any-host to fetch it all the same)" +# Google's terms, as for the factory images: nothing of Google's is downloaded +# until the builder has said they accept them +((accept)) || die "the Android NDK is Google's, and Google asks everyone who downloads it to agree to its terms and conditions first. Read them on its download page, $terms, and if you accept them, run: tools/fetch-ndk --accept-google-terms" + +command -v curl >/dev/null || die "curl is needed" +if command -v unzip >/dev/null; then unpack() { unzip -q "$1" -d "$2"; } +elif command -v bsdtar >/dev/null; then unpack() { bsdtar -xf "$1" -C "$2"; } +else die "unzip or bsdtar is needed (they keep the compilers executable)"; fi + +# Each run downloads and unpacks under names of its own, so two runs at once +# can't mix their files; the second to finish finds the NDK in place. +part=$dir/.android-ndk-$release.zip.$$.part +work=$dir/.unpack-$$ +trap 'rm -rf "$work" "$part"' EXIT +say "downloading the NDK $release ($((size / 1000000)) MB) from ${url%%/android-ndk-*}" +curl -fL --retry 3 --progress-bar --proto '=https,file' --proto-redir '=https' -o "$part" "$url" || + die "the download failed" +got_size=$(stat -c %s "$part") +[[ $got_size == "$size" ]] || die "the download is $got_size bytes, not the pinned $size: nothing unpacked" +got=$(sha256sum "$part" | cut -d' ' -f1) +[[ $got == "$sum" ]] || die "the download's SHA-256 is $got, not the pinned $sum: nothing unpacked" + +mkdir -p "$work" +unpack "$part" "$work" || die "unpacking failed" +[[ -d $work/android-ndk-$release ]] && has_revision "$work/android-ndk-$release" || + die "the archive holds no android-ndk-$release with revision $revision" +if ! mv -T "$work/android-ndk-$release" "$ndk" 2>/dev/null; then + has_revision "$ndk" || die "could not put the NDK in $ndk" +fi +say "the NDK $release ($revision) is in $ndk" +printf '%s\n' "$ndk" diff --git a/tools/ndk.lock b/tools/ndk.lock new file mode 100644 index 000000000..6c1d0c122 --- /dev/null +++ b/tools/ndk.lock @@ -0,0 +1,7 @@ +# SPDX-License-Identifier: MIT +# The Android NDK tools/fetch-ndk fetches, for the bionic programs built on +# the computer (packages/omamobile-gnss/bionic: the Pixel 9a's GNSS HAL +# client). RELEASE REVISION SHA256 SIZE URL, one line. The SHA-256 and the +# size are of the archive as dl.google.com served it on 2026-11-18; the +# revision is the one in its source.properties (Pkg.Revision). +r30 30.0.16248370 753611f410d002cfcd3f3dc2ef49aad532089d3180b436c060a90bf0fcb64df2 738633529 https://dl.google.com/android/repository/android-ndk-r30-linux.zip