Android build: libslirp (-netdev user) and DNS from the app

.github/workflows/ci.yaml

@@ -99,14 +99,14 @@ jobs:
       - name: Point Gradle at the SDK
         run: echo "sdk.dir=$ANDROID_SDK_ROOT" > local.properties
 
-      # Both caches are keyed on the hash of build-android.sh: the dep URLs,
+      # Both caches are keyed on the hash of build-android.sh and native/patches: the dep URLs,
       # sha256 sums and build flags all live there, so any change invalidates
       # them.
       - name: Cache dependency tarballs
         uses: actions/cache@v4.2.0
         with:
           path: native/build/dl
-          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       # Cache the built static libs. This is the ~10 min saver: on a hit the
       # deps step below is skipped and only QEMU is rebuilt. Safe because the
@@ -118,7 +118,7 @@ jobs:
         uses: actions/cache@v4.2.0
         with:
           path: native/build/sysroot
-          key: deps-sysroot-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-sysroot-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       # Sets up Gradle and caches ~/.gradle (caches + wrapper dists) automatically.
       - name: Set up Gradle
@@ -218,14 +218,14 @@ jobs:
       - name: Point Gradle at the SDK
         run: echo "sdk.dir=$ANDROID_SDK_ROOT" > local.properties
 
-      # Both caches are keyed on the hash of build-android.sh: the dep URLs,
+      # Both caches are keyed on the hash of build-android.sh and native/patches: the dep URLs,
       # sha256 sums and build flags all live there, so any change invalidates
       # them.
       - name: Cache dependency tarballs
         uses: actions/cache@v4.2.0
         with:
           path: native/build/dl
-          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       # Cache the built static libs. This is the ~10 min saver: on a hit the
       # deps step below is skipped and only QEMU is rebuilt. Safe because the
@@ -237,7 +237,7 @@ jobs:
         uses: actions/cache@v4.2.0
         with:
           path: native/build/sysroot
-          key: deps-sysroot-baseline-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-sysroot-baseline-${{ runner.os }}-${{ github.event.repository.name }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       # Sets up Gradle and caches ~/.gradle (caches + wrapper dists) automatically.
       - name: Set up Gradle
@@ -341,7 +341,7 @@ jobs:
         uses: actions/cache@v4.2.0
         with:
           path: native/build/dl
-          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4.2.2

.github/workflows/release.yaml

@@ -94,14 +94,14 @@ jobs:
         uses: actions/cache@v4.2.0
         with:
           path: native/build/dl
-          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-dl-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       - name: Cache static sysroot
         id: sysroot-cache
         uses: actions/cache@v4.2.0
         with:
           path: native/build/sysroot
-          key: deps-sysroot-${{ runner.os }}-${{ hashFiles('native/build-android.sh') }}
+          key: deps-sysroot-${{ runner.os }}-${{ hashFiles('native/build-android.sh', 'native/patches/**') }}
 
       - name: Set up Gradle
         uses: gradle/actions/setup-gradle@v4.2.2

docs/NETWORKING.md

@@ -105,8 +105,9 @@ ideally iOS shows that as Wi-Fi.
   * `virtualhere` listens on TCP;
   * `inferno` connects to a unix socket. The app uses this since #76. Nothing
     listens on the socket during a normal boot.
-* **The app has no INTERNET permission** (#67, #76), and the Android QEMU
-  build has `-Dslirp=disabled`.
+* **The app has no INTERNET permission** (#67, #76). The Android QEMU build
+  has libslirp since #82 (`-netdev user` exists), with the DNS change described
+  under "libslirp on Android" below.
 
 ## The real hardware, and what the drivers expect
 
@@ -274,11 +275,64 @@ without the permission (about 1 day), or a helper APK that holds INTERNET and
 gets frames through a socketpair fd passed over Binder (about 1 week).
 
 **Android-specific gotcha (#82):** libslirp finds the upstream resolver by
-reading `/etc/resolv.conf`, which Android doesn't have. The app has to pass the
-resolver from `LinkProperties`, or DNS has to go through a proxy in the app.
+reading `/etc/resolv.conf`, which Android doesn't have. The fix is below.
 `-netdev user,restrict=on` (DHCP and guest-to-host only) needs no permission,
 which is handy for testing.
 
+## libslirp on Android (#82)
+
+`native/build-android.sh` builds libslirp (a pinned release tarball, like the
+other deps) and configures QEMU with `-Dslirp=enabled`, for both the
+executable and the in-process library. It applies one small patch,
+`native/patches/libslirp-android.patch`.
+
+**DNS: the interface between QEMU and the app.** The patch makes libslirp read
+its resolvers from the file named by the environment variable
+**`SLIRP_RESOLV_CONF`** instead of `/etc/resolv.conf`. Nothing else changes,
+and nothing changes on the QEMU command line. When the variable is unset or
+empty, libslirp reads `/etc/resolv.conf` as upstream does.
+
+* **Who sets it:** the app, before QEMU starts. For the exec path, in the
+  `ProcessBuilder` environment. For the in-process path (`:vm`), with
+  `Os.setenv("SLIRP_RESOLV_CONF", path, true)` before `qemu_android_main`.
+  libslirp reads the variable once, at the first DNS lookup, so it must not
+  change while QEMU runs.
+* **The file:** resolv.conf format, one line per entry of
+  `LinkProperties.getDnsServers()` (needs `ACCESS_NETWORK_STATE`):
+  `nameserver <numeric address>`. IPv4 and IPv6 both work. A link-local IPv6
+  address needs its interface: `nameserver fe80::1%wlan0`
+  (`LinkProperties.getInterfaceName()`). libslirp uses the first valid IPv4
+  entry for the guest's IPv4 queries and the first IPv6 entry for IPv6 ones.
+  The file lives in the app's private storage, for example
+  `<filesDir>/net/resolv.conf`.
+* **Network changes:** write a temporary file in the same directory and
+  `rename()` it over the old one. libslirp checks the file at most once a
+  second and re-reads it when its inode, size or mtime changed, so no restart
+  is needed.
+* **No resolvers:** delete the file. Don't write one without `nameserver`
+  lines: libslirp would then fall back to 127.0.0.1, and Android has no
+  resolver there. With the file missing, the guest's DNS queries are dropped
+  until it comes back; nothing else breaks.
+* **The guest** keeps using the virtual nameserver 10.0.2.3 that slirp's DHCP
+  hands out; slirp forwards its UDP port 53 traffic to the address from the
+  file. Android's Private DNS (DNS over TLS) isn't used: queries go as plain
+  UDP to the network's resolvers.
+
+We chose this over a DNS proxy in the app (more code, and another hop for
+every query) and over a new `-netdev user` option (a change to QEMU's net
+code in Inferno, and the exec path would need a restart on every network
+change).
+
+**Without the INTERNET permission**, Android refuses `socket()` for
+`AF_INET`/`AF_INET6` with `EACCES`. libslirp treats that like any other socket
+error: the guest's TCP connection gets a reset and the UDP packet is dropped.
+The patch also turns one `g_error()` (an abort) on a failed socket into a
+warning; it was on the path that forwards to a host unix socket. A
+`hostfwd` rule can't open its listening socket without the permission either,
+so QEMU then stops at start with "could not set up host forwarding rule".
+That's an error message, not a crash, but the app shouldn't pass `hostfwd`
+without the permission.
+
 ## Measured (M1, desktop)
 
 Inferno [#21](https://github.com/Scion-Project/pyre/pull/21) adds
@@ -316,7 +370,7 @@ first-boot progress bar, though networking was already up.
 review. The remaining work there: batch frames per NTB for throughput.
 
 **M2: on the Pixel (#82, #83). About 1–2 weeks.** libslirp in the Android
-build with the DNS fix, and the Network setting that passes
+build with the DNS fix (#82: in the build, see "libslirp on Android"), and the Network setting that passes
 `-netdev user,id=n0` and `usb-uplink-type=net,usb-uplink-addr=n0`.
 
 **M3: fake Wi-Fi spike (#84). 2–3 weeks, time-boxed.** A BCM4378 PCIe endpoint

native/build-android.sh

@@ -280,6 +280,16 @@ build_glib() {
     -Dforce_fallback_for=pcre2,libffi,proxy-libintl
 }
 
+# User-mode networking (-netdev user). Only needs glib. Android has no
+# /etc/resolv.conf, so native/patches/libslirp-android.patch lets the app name
+# a resolv.conf-format file in SLIRP_RESOLV_CONF (see docs/NETWORKING.md).
+build_libslirp() {
+  local s; s="$(source_dir https://gitlab.freedesktop.org/slirp/libslirp/-/archive/v4.9.7/libslirp-v4.9.7.tar.gz \
+    d2567fa06a6357c7755d61c67293b2c0bf069b3f2173bde6b76cf768529a1938 libslirp)" || exit 1
+  patch -d "$s" -p1 --forward < "$NATIVE/patches/libslirp-android.patch" || exit 1
+  meson_dep "$s"
+}
+
 build_pixman() {
   local s; s="$(source_dir https://www.cairographics.org/releases/pixman-0.46.4.tar.xz \
     a098c33924754ad43f981b740f6d576c70f9ed1006e12221b1845431ebce1239 pixman)" || exit 1
@@ -294,6 +304,7 @@ build_deps() {
   build_lz4
   build_lzfse
   build_glib
+  build_libslirp
   build_pixman
 }
 
@@ -324,7 +335,7 @@ configure_qemu() { # builddir [extra meson args...]
     -Dkvm=disabled -Dhvf=disabled -Dwhpx=disabled -Dtcg=enabled \
     -Dvnc=enabled -Dvnc_jpeg=disabled -Dpng=disabled -Dvnc_sasl=disabled \
     -Dgnutls=disabled -Dsdl=disabled -Dsdl_image=disabled -Dgtk=disabled -Dvte=disabled \
-    -Dcurses=disabled -Dcocoa=disabled -Dslirp=disabled -Dlibssh=disabled \
+    -Dcurses=disabled -Dcocoa=disabled -Dslirp=enabled -Dlibssh=disabled \
     -Dzstd=disabled -Dlzo=disabled -Dsnappy=disabled -Dbzip2=disabled -Dcurl=disabled \
     -Dlinux_aio=disabled -Dlinux_io_uring=disabled -Dlibudev=disabled -Dlibusb=disabled \
     -Dusb_redir=disabled -Dseccomp=disabled -Dcap_ng=disabled -Dxkbcommon=disabled \
@@ -421,6 +432,6 @@ case "${1:-all}" in
   qemu-lib) build_qemu_lib ;;
   pgo-merge) shift; pgo_merge "$@" ;;
   all) build_deps; build_qemu ;;
-  gmp|nettle|tasn1|lz4|lzfse|glib|pixman) "build_$1" ;;
-  *) echo "usage: $0 [deps|qemu|qemu-lib|all|pgo-merge|gmp|nettle|tasn1|lz4|lzfse|glib|pixman]" >&2; exit 2 ;;
+  gmp|nettle|tasn1|lz4|lzfse|glib|libslirp|pixman) "build_$1" ;;
+  *) echo "usage: $0 [deps|qemu|qemu-lib|all|pgo-merge|gmp|nettle|tasn1|lz4|lzfse|glib|libslirp|pixman]" >&2; exit 2 ;;
 esac

native/patches/libslirp-android.patch

@@ -0,0 +1,54 @@
+--- a/src/slirp.c
++++ b/src/slirp.c
+@@ -386,11 +386,27 @@
+ #else // !defined(_WIN32) && !defined(__APPLE__)
+
+ #if defined(__HAIKU__)
+-#define RESOLV_CONF_PATH "/boot/system/settings/network/resolv.conf"
++#define RESOLV_CONF_DEFAULT "/boot/system/settings/network/resolv.conf"
+ #else
+-#define RESOLV_CONF_PATH "/etc/resolv.conf"
++#define RESOLV_CONF_DEFAULT "/etc/resolv.conf"
+ #endif
+
++/*
++ * Scion: Android has no /etc/resolv.conf. The app writes the network's
++ * resolvers to a file in resolv.conf format and names it in
++ * SLIRP_RESOLV_CONF before QEMU starts. The variable is read once.
++ */
++static const char *resolv_conf_path(void)
++{
++    static const char *path;
++    if (!path) {
++        const char *env = g_getenv("SLIRP_RESOLV_CONF");
++        path = (env && env[0]) ? g_strdup(env) : RESOLV_CONF_DEFAULT;
++    }
++    return path;
++}
++#define RESOLV_CONF_PATH resolv_conf_path()
++
+ static int get_dns_addr_cached(void *pdns_addr, void *cached_addr,
+                                socklen_t addrlen, struct stat *cached_stat,
+                                unsigned *cached_time)
+--- a/src/socket.c
++++ b/src/socket.c
+@@ -1176,7 +1176,8 @@
+                 break;
+             }
+             if (not_valid_socket(s)) {
+-                g_error("Ephemeral slirp_socket() allocation failed");
++                /* Scion: no abort when sockets are refused (no INTERNET). */
++                g_warning("Ephemeral slirp_socket() allocation failed");
+                 goto unix2inet_cont;
+             }
+             memset(&in_addr, 0, sizeof(in_addr));
+@@ -1219,7 +1220,8 @@
+                 break;
+             }
+             if (not_valid_socket(s)) {
+-                g_error("Ephemeral slirp_socket() allocation failed");
++                /* Scion: no abort when sockets are refused (no INTERNET). */
++                g_warning("Ephemeral slirp_socket() allocation failed");
+                 goto unix2inet6_cont;
+             }
+             memset(&in6_addr, 0, sizeof(in6_addr));