diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d14cbea..2114ad8 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,14 +99,14 @@ jobs: - name: Point Gradle at the SDK run: echo "sdk.dir=$ANDROID_SDK_ROOT" > local.properties - # Both caches are keyed on the hash of build-android.sh: the dep URLs, + # Both caches are keyed on the hash of build-android.sh and native/patches: the dep URLs, # sha256 sums and build flags all live there, so any change invalidates # them. - name: Cache dependency tarballs uses: actions/cache@v4.2.0 with: path: native/build/dl - key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }} + key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} # Cache the built static libs. This is the ~10 min saver: on a hit the # deps step below is skipped and only QEMU is rebuilt. Safe because the @@ -118,7 +118,7 @@ jobs: uses: actions/cache@v4.2.0 with: path: native/build/sysroot - key: deps-sysroot-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh') }} + key: deps-sysroot-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} # Sets up Gradle and caches ~/.gradle (caches + wrapper dists) automatically. - name: Set up Gradle @@ -218,14 +218,14 @@ jobs: - name: Point Gradle at the SDK run: echo "sdk.dir=$ANDROID_SDK_ROOT" > local.properties - # Both caches are keyed on the hash of build-android.sh: the dep URLs, + # Both caches are keyed on the hash of build-android.sh and native/patches: the dep URLs, # sha256 sums and build flags all live there, so any change invalidates # them. - name: Cache dependency tarballs uses: actions/cache@v4.2.0 with: path: native/build/dl - key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }} + key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} # Cache the built static libs. This is the ~10 min saver: on a hit the # deps step below is skipped and only QEMU is rebuilt. Safe because the @@ -237,7 +237,7 @@ jobs: uses: actions/cache@v4.2.0 with: path: native/build/sysroot - key: deps-sysroot-baseline-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh') }} + key: deps-sysroot-baseline-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} # Sets up Gradle and caches ~/.gradle (caches + wrapper dists) automatically. - name: Set up Gradle @@ -341,7 +341,7 @@ jobs: uses: actions/cache@v4.2.0 with: path: native/build/dl - key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }} + key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} - name: Set up Gradle uses: gradle/actions/setup-gradle@v4.2.2 diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 31928af..2cccddd 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -94,14 +94,14 @@ jobs: uses: actions/cache@v4.2.0 with: path: native/build/dl - key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }} + key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} - name: Cache static sysroot id: sysroot-cache uses: actions/cache@v4.2.0 with: path: native/build/sysroot - key: deps-sysroot-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }} + key: deps-sysroot-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }} - name: Set up Gradle uses: gradle/actions/setup-gradle@v4.2.2 diff --git a/docs/NETWORKING.md b/docs/NETWORKING.md index 626cb7f..481701f 100644 --- a/docs/NETWORKING.md +++ b/docs/NETWORKING.md @@ -105,8 +105,9 @@ ideally iOS shows that as Wi-Fi. * `virtualhere` listens on TCP; * `inferno` connects to a unix socket. The app uses this since #76. Nothing listens on the socket during a normal boot. -* **The app has no INTERNET permission** (#67, #76), and the Android QEMU - build has `-Dslirp=disabled`. +* **The app has no INTERNET permission** (#67, #76). The Android QEMU build + has libslirp since #82 (`-netdev user` exists), with the DNS change described + under "libslirp on Android" below. ## The real hardware, and what the drivers expect @@ -274,11 +275,64 @@ without the permission (about 1 day), or a helper APK that holds INTERNET and gets frames through a socketpair fd passed over Binder (about 1 week). **Android-specific gotcha (#82):** libslirp finds the upstream resolver by -reading `/etc/resolv.conf`, which Android doesn't have. The app has to pass the -resolver from `LinkProperties`, or DNS has to go through a proxy in the app. +reading `/etc/resolv.conf`, which Android doesn't have. The fix is below. `-netdev user,restrict=on` (DHCP and guest-to-host only) needs no permission, which is handy for testing. +## libslirp on Android (#82) + +`native/build-android.sh` builds libslirp (a pinned release tarball, like the +other deps) and configures QEMU with `-Dslirp=enabled`, for both the +executable and the in-process library. It applies one small patch, +`native/patches/libslirp-android.patch`. + +**DNS: the interface between QEMU and the app.** The patch makes libslirp read +its resolvers from the file named by the environment variable +**`SLIRP_RESOLV_CONF`** instead of `/etc/resolv.conf`. Nothing else changes, +and nothing changes on the QEMU command line. When the variable is unset or +empty, libslirp reads `/etc/resolv.conf` as upstream does. + +* **Who sets it:** the app, before QEMU starts. For the exec path, in the + `ProcessBuilder` environment. For the in-process path (`:vm`), with + `Os.setenv("SLIRP_RESOLV_CONF", path, true)` before `qemu_android_main`. + libslirp reads the variable once, at the first DNS lookup, so it must not + change while QEMU runs. +* **The file:** resolv.conf format, one line per entry of + `LinkProperties.getDnsServers()` (needs `ACCESS_NETWORK_STATE`): + `nameserver `. IPv4 and IPv6 both work. A link-local IPv6 + address needs its interface: `nameserver fe80::1%wlan0` + (`LinkProperties.getInterfaceName()`). libslirp uses the first valid IPv4 + entry for the guest's IPv4 queries and the first IPv6 entry for IPv6 ones. + The file lives in the app's private storage, for example + `/net/resolv.conf`. +* **Network changes:** write a temporary file in the same directory and + `rename()` it over the old one. libslirp checks the file at most once a + second and re-reads it when its inode, size or mtime changed, so no restart + is needed. +* **No resolvers:** delete the file. Don't write one without `nameserver` + lines: libslirp would then fall back to 127.0.0.1, and Android has no + resolver there. With the file missing, the guest's DNS queries are dropped + until it comes back; nothing else breaks. +* **The guest** keeps using the virtual nameserver 10.0.2.3 that slirp's DHCP + hands out; slirp forwards its UDP port 53 traffic to the address from the + file. Android's Private DNS (DNS over TLS) isn't used: queries go as plain + UDP to the network's resolvers. + +We chose this over a DNS proxy in the app (more code, and another hop for +every query) and over a new `-netdev user` option (a change to QEMU's net +code in Inferno, and the exec path would need a restart on every network +change). + +**Without the INTERNET permission**, Android refuses `socket()` for +`AF_INET`/`AF_INET6` with `EACCES`. libslirp treats that like any other socket +error: the guest's TCP connection gets a reset and the UDP packet is dropped. +The patch also turns one `g_error()` (an abort) on a failed socket into a +warning; it was on the path that forwards to a host unix socket. A +`hostfwd` rule can't open its listening socket without the permission either, +so QEMU then stops at start with "could not set up host forwarding rule". +That's an error message, not a crash, but the app shouldn't pass `hostfwd` +without the permission. + ## Measured (M1, desktop) Inferno [#21](https://github.com/Scion-Project/pyre/pull/21) adds @@ -316,7 +370,7 @@ first-boot progress bar, though networking was already up. review. The remaining work there: batch frames per NTB for throughput. **M2: on the Pixel (#82, #83). About 1–2 weeks.** libslirp in the Android -build with the DNS fix, and the Network setting that passes +build with the DNS fix (#82: in the build, see "libslirp on Android"), and the Network setting that passes `-netdev user,id=n0` and `usb-uplink-type=net,usb-uplink-addr=n0`. **M3: fake Wi-Fi spike (#84). 2–3 weeks, time-boxed.** A BCM4378 PCIe endpoint diff --git a/native/build-android.sh b/native/build-android.sh index caa0476..1b7852c 100755 --- a/native/build-android.sh +++ b/native/build-android.sh @@ -280,6 +280,16 @@ build_glib() { -Dforce_fallback_for=pcre2,libffi,proxy-libintl } +# User-mode networking (-netdev user). Only needs glib. Android has no +# /etc/resolv.conf, so native/patches/libslirp-android.patch lets the app name +# a resolv.conf-format file in SLIRP_RESOLV_CONF (see docs/NETWORKING.md). +build_libslirp() { + local s; s="$(source_dir https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v4.9.7/libslirp-v4.9.7.tar.gz \ + d2567fa06a6357c7755d61c67293b2c0bf069b3f2173bde6b76cf768529a1938 libslirp)" || exit 1 + patch -d "$s" -p1 --forward < "$NATIVE/patches/libslirp-android.patch" || exit 1 + meson_dep "$s" +} + build_pixman() { local s; s="$(source_dir https://www.cairographics.org/releases/pixman-0.46.4.tar.xz \ a098c33924754ad43f981b740f6d576c70f9ed1006e12221b1845431ebce1239 pixman)" || exit 1 @@ -294,6 +304,7 @@ build_deps() { build_lz4 build_lzfse build_glib + build_libslirp build_pixman } @@ -324,7 +335,7 @@ configure_qemu() { # builddir [extra meson args...] -Dkvm=disabled -Dhvf=disabled -Dwhpx=disabled -Dtcg=enabled \ -Dvnc=enabled -Dvnc_jpeg=disabled -Dpng=disabled -Dvnc_sasl=disabled \ -Dgnutls=disabled -Dsdl=disabled -Dsdl_image=disabled -Dgtk=disabled -Dvte=disabled \ - -Dcurses=disabled -Dcocoa=disabled -Dslirp=disabled -Dlibssh=disabled \ + -Dcurses=disabled -Dcocoa=disabled -Dslirp=enabled -Dlibssh=disabled \ -Dzstd=disabled -Dlzo=disabled -Dsnappy=disabled -Dbzip2=disabled -Dcurl=disabled \ -Dlinux_aio=disabled -Dlinux_io_uring=disabled -Dlibudev=disabled -Dlibusb=disabled \ -Dusb_redir=disabled -Dseccomp=disabled -Dcap_ng=disabled -Dxkbcommon=disabled \ @@ -421,6 +432,6 @@ case "${1:-all}" in qemu-lib) build_qemu_lib ;; pgo-merge) shift; pgo_merge "$@" ;; all) build_deps; build_qemu ;; - gmp|nettle|tasn1|lz4|lzfse|glib|pixman) "build_$1" ;; - *) echo "usage: $0 [deps|qemu|qemu-lib|all|pgo-merge|gmp|nettle|tasn1|lz4|lzfse|glib|pixman]" >&2; exit 2 ;; + gmp|nettle|tasn1|lz4|lzfse|glib|libslirp|pixman) "build_$1" ;; + *) echo "usage: $0 [deps|qemu|qemu-lib|all|pgo-merge|gmp|nettle|tasn1|lz4|lzfse|glib|libslirp|pixman]" >&2; exit 2 ;; esac diff --git a/native/patches/libslirp-android.patch b/native/patches/libslirp-android.patch new file mode 100644 index 0000000..3ffb259 --- /dev/null +++ b/native/patches/libslirp-android.patch @@ -0,0 +1,54 @@ +--- a/src/slirp.c ++++ b/src/slirp.c +@@ -386,11 +386,27 @@ + #else // !defined(_WIN32) && !defined(__APPLE__) + + #if defined(__HAIKU__) +-#define RESOLV_CONF_PATH "/boot/system/settings/network/resolv.conf" ++#define RESOLV_CONF_DEFAULT "/boot/system/settings/network/resolv.conf" + #else +-#define RESOLV_CONF_PATH "/etc/resolv.conf" ++#define RESOLV_CONF_DEFAULT "/etc/resolv.conf" + #endif + ++/* ++ * Scion: Android has no /etc/resolv.conf. The app writes the network's ++ * resolvers to a file in resolv.conf format and names it in ++ * SLIRP_RESOLV_CONF before QEMU starts. The variable is read once. ++ */ ++static const char *resolv_conf_path(void) ++{ ++ static const char *path; ++ if (!path) { ++ const char *env = g_getenv("SLIRP_RESOLV_CONF"); ++ path = (env && env[0]) ? g_strdup(env) : RESOLV_CONF_DEFAULT; ++ } ++ return path; ++} ++#define RESOLV_CONF_PATH resolv_conf_path() ++ + static int get_dns_addr_cached(void *pdns_addr, void *cached_addr, + socklen_t addrlen, struct stat *cached_stat, + unsigned *cached_time) +--- a/src/socket.c ++++ b/src/socket.c +@@ -1176,7 +1176,8 @@ + break; + } + if (not_valid_socket(s)) { +- g_error("Ephemeral slirp_socket() allocation failed"); ++ /* Scion: no abort when sockets are refused (no INTERNET). */ ++ g_warning("Ephemeral slirp_socket() allocation failed"); + goto unix2inet_cont; + } + memset(&in_addr, 0, sizeof(in_addr)); +@@ -1219,7 +1220,8 @@ + break; + } + if (not_valid_socket(s)) { +- g_error("Ephemeral slirp_socket() allocation failed"); ++ /* Scion: no abort when sockets are refused (no INTERNET). */ ++ g_warning("Ephemeral slirp_socket() allocation failed"); + goto unix2inet6_cont; + } + memset(&in6_addr, 0, sizeof(in6_addr));