diff --git a/Makefile b/Makefile index 43e57260..364bd77c 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ BIN := bin/mobium VERSION := $(shell cat VERSION 2>/dev/null || echo dev) -.PHONY: all build dist mcpb test fmt fmt-check vet lint clients java crosscompile api api-check flags flags-check quickstart license-check docs-check ci clean +.PHONY: all build dist mcpb test fmt fmt-check vet lint clients java crosscompile api api-check flags flags-check quickstart license-check docs-check ci clean reader all: build test @@ -10,6 +10,22 @@ build: go build -ldflags "-X main.version=$(VERSION)" -o $(BIN) ./cmd/mobium @echo "built $(BIN)" +# reader rebuilds internal/mobiumdriver/reader/reader.dex, the screen reader- +# safe hierarchy reader, from Main.java. Only for changing the reader: the +# dex is checked in and pinned by checksum, so a build of Mobium needs neither +# a JDK nor the Android SDK. Needs javac, d8 (9.3.16 builds the pinned bytes) +# and an android.jar, found under ANDROID_HOME. +READER := internal/mobiumdriver/reader +ANDROID_JAR ?= $(lastword $(sort $(wildcard $(ANDROID_HOME)/platforms/android-*/android.jar))) + +reader: + @test -n "$(ANDROID_JAR)" || { echo "no android.jar — set ANDROID_HOME or ANDROID_JAR"; exit 1; } + @tmp=$$(mktemp -d) && \ + javac --release 8 -Xlint:-options -nowarn -cp "$(ANDROID_JAR)" -d $$tmp $(READER)/Main.java && \ + d8 --release --min-api 24 --output $$tmp $$tmp/dev/mobium/reader/*.class && \ + cp $$tmp/classes.dex $(READER)/reader.dex && rm -rf $$tmp && \ + echo "built $(READER)/reader.dex — set reader.SHA256 to:" && shasum -a 256 $(READER)/reader.dex + # The Go client is a separate module, so ./... does not reach it and it has to # be named. Forgetting that would mean it never runs in CI. MODULES := . clients/go diff --git a/docs/CHALLENGES.md b/docs/CHALLENGES.md index 54015f9d..502b1019 100644 --- a/docs/CHALLENGES.md +++ b/docs/CHALLENGES.md @@ -7,7 +7,7 @@ the source. The pattern across them is the reason the document exists: **almost nothing here was found by reading code or by a test written from imagination.** Of -207 defects, 171 were found only by running against a real device. The other +208 defects, 172 were found only by running against a real device. The other thirty-six — 4, 5, 6, 14, 23, 24, 32, 35, 36, 39, 44, 50, 53, 54, 57, 66, 89, 99, 100, 121, 123, 129, 130, 133, 134, 139, 140, 141, 142, 144, 150, 158, 162, 164, 165 and 172 — came from reading code, the compiler, a test, a linter, @@ -4623,6 +4623,39 @@ app with no second install. Mobium does not answer the dialog itself: which answer to give is the phone owner's, and "Always send" changes a setting for every app. +### 208. Every read silenced the device's screen reader + +**Found by:** reading Prime Video on a Fire TV, which publishes nothing but +one surface to `map`, to see whether it publishes more while Amazon's screen +reader, VoiceView, is on. Every read of the screen — `uiautomator dump` and +the UiAutomator2 server alike — unbound VoiceView for as long as it ran: +`dumpsys accessibility` listed it before the read and not during it, and +touch exploration went off. Android's test-automation connection suppresses +every other accessibility service unless it is opened with +`FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES`, and neither opens it so: the dump +command has no option for it, and the UiAutomator2 server accepted +`disableSuppressAccessibilityService`, as a setting and as a capability, and +went on unbinding VoiceView — the setting did not even read back. So anybody +who relies on TalkBack or VoiceView lost it for each read, without a word, +and an app that publishes its contents only to a screen reader stopped +publishing them before the read could see them. The same held on a Pixel 8 +Pro with TalkBack. + +The dump backend now reads through a reader of Mobium's own when any +accessibility service is enabled: a small dex, pinned by checksum, pushed into +a folder of its own in `/data/local/tmp`, run with `app_process`, and deleted +with the runtime's compiled copy of it after every read. It writes what +`uiautomator dump` writes, over a connection opened with that flag. On the +Fire TV (Android 9) VoiceView stayed bound through every read, and Prime Video +gave 14 nodes, 3 of them labeled, where the dump gave 12 and none; on the +Pixel 8 Pro (Android 17) `uiautomator dump` unbound TalkBack in both reads and +the reader in neither, the same 13 nodes either way. The first build died on +Android 17 within 150 ms: newer accessibility code builds a Handler on the +main looper, which a bare `app_process` does not have, and Android 9 had never +asked for one. With no service enabled, reads go through `uiautomator dump` +as before. The default driver, UiAutomator2, still silences a screen reader +(ROADMAP). + ## Findings that were not defects Worth recording because each one closed off an approach that looked obvious. diff --git a/docs/RELEASE-CHECKLIST.md b/docs/RELEASE-CHECKLIST.md index 83b8ebf4..82431dd7 100644 --- a/docs/RELEASE-CHECKLIST.md +++ b/docs/RELEASE-CHECKLIST.md @@ -4,7 +4,7 @@ [CI workflow](../.github/workflows/ci.yml) runs on every push — formatting, vet, both modules' tests, the four non-Go clients' own tests, and six cross-compile targets. This file is the rest, and the rest is where most -serious defects in this project have come from: 171 of 207 were found only by +serious defects in this project have come from: 172 of 208 were found only by running against a real device. Work through it before tagging a release, on three substrates: an Android diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index cbc27e29..9c884d47 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -90,6 +90,15 @@ this is what is not. now where the Never button is, and every later login of a package gets Never (six of six). `alert` read it, refused accept and dismiss, and a rule `--when "Save password" --press "Not now"` answered it in the way of a tap. +- **Reading without silencing a screen reader, on UiAutomator2 too.** The + dump backend reads through a reader of Mobium's own whenever an + accessibility service is enabled, and TalkBack and VoiceView stay running + through it, measured on a Pixel 8 Pro and a Fire TV (CHALLENGES 208). The + UiAutomator2 server, the default driver, still unbinds them for as long as + a session is open: it accepted `disableSuppressAccessibilityService` and + went on unbinding. Left: the same for UiAutomator2 — its connection opened + with the flag, or its reads done by Mobium's reader — and saying so when a + session opens on a device with a screen reader on, until then. - **Windows.** Everything that needs no device passes on a GitHub-hosted Windows runner, every run: both modules' tests, the named-pipe daemon transport's acceptance tests five times over, and the built `mobium.exe` — diff --git a/docs/SETUP.md b/docs/SETUP.md index ece64275..cfed9046 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -317,6 +317,25 @@ One more, rarely: if debugging is on and still nothing appears, pull down the notification shade, tap the USB notification and change the mode from charging to **File transfer**. On Android 12+ this is usually unnecessary. +### With a screen reader on + +Android's test-automation connection silences every other accessibility +service while it is open, unless it is opened not to. So on a device with +TalkBack — or VoiceView, on a Fire TV — on, a read of the screen takes the +screen reader away for as long as it runs, and an app that publishes its +contents only to a screen reader stops publishing them (CHALLENGES 208). + +`--driver uiautomator` reads through a reader of Mobium's own whenever an +accessibility service is enabled, and that one leaves the screen reader +running. The default driver, UiAutomator2, does not yet: with it, the screen +reader is silent while a session is open and comes back when the session +ends. Nothing in the device's settings changes either way. + +`MOBIUM_DUMP_READER=mobium` uses Mobium's reader for every read, and +`=uiautomator` uses `uiautomator dump` for every read. **It is read by the +daemon**, so it takes effect only if it is set when the daemon starts — set +on a later command, it does nothing. Run `mobium daemon stop` first. + ### Over Wi-Fi Android 11+. The phone and the Mac must be on the same network. @@ -750,7 +769,9 @@ session — is [SHUTDOWN.md](SHUTDOWN.md). Nothing is left in the device's filesystem. The dump backend writes a hierarchy file to `/data/local/tmp` and deletes it after every read — that file is a serialization of whatever is on screen, so on a real phone it is -your data, and it is not left behind. Verify with: +your data, and it is not left behind. With a screen reader on, it pushes its +own reader to `/data/local/tmp/mobium-reader` instead, and deletes the folder, +with the copy Android compiles into it, after every read. Verify with: ```sh adb -s shell ls /data/local/tmp diff --git a/internal/mobiumdriver/android.go b/internal/mobiumdriver/android.go index f6b9ef6e..8ffd9f7d 100644 --- a/internal/mobiumdriver/android.go +++ b/internal/mobiumdriver/android.go @@ -5,10 +5,12 @@ import ( "context" "fmt" "github.com/mobiumdev/mobium/internal/mobiumerr" + "os" "strings" "time" "github.com/mobiumdev/mobium/internal/device" + "github.com/mobiumdev/mobium/internal/mobiumdriver/reader" "github.com/mobiumdev/mobium/internal/uitree" ) @@ -76,6 +78,13 @@ func (a *Android) dump(ctx context.Context) ([]byte, error) { } func (a *Android) dumpOnce(ctx context.Context) ([]byte, error) { + use, err := a.useReader(ctx) + if err != nil { + return nil, err + } + if use { + return a.readOnce(ctx) + } out, diag, err := a.adb.ShellDiagnostics(ctx, "uiautomator", "dump", dumpPath) if err != nil { return nil, err @@ -123,6 +132,80 @@ func (a *Android) dumpOnce(ctx context.Context) ([]byte, error) { return xml, nil } +// useReader says whether a read goes through Mobium's own reader rather than +// `uiautomator dump`. uiautomator connects in a way that suppresses every +// other accessibility service while it reads — a screen reader goes quiet, +// and an app that publishes its contents only to one stops publishing — so +// the reader is used whenever an accessibility service is enabled +// (CHALLENGES 208). +// MOBIUM_DUMP_READER=mobium or =uiautomator chooses one regardless. +func (a *Android) useReader(ctx context.Context) (bool, error) { + switch v := os.Getenv("MOBIUM_DUMP_READER"); v { + case "mobium": + return true, nil + case "uiautomator": + return false, nil + case "": + default: + return false, mobiumerr.New(mobiumerr.InvalidArgument, + "MOBIUM_DUMP_READER is %q — it takes mobium or uiautomator, or unset to decide by the device", v) + } + out, err := a.adb.Shell(ctx, "settings get secure accessibility_enabled; settings get secure enabled_accessibility_services") + if err != nil { + return false, err + } + lines := strings.Split(strings.TrimSpace(string(out)), "\n") + if len(lines) < 2 { + return false, nil + } + services := strings.TrimSpace(lines[1]) + return strings.TrimSpace(lines[0]) == "1" && services != "" && services != "null", nil +} + +// readOnce reads the screen with Mobium's own reader: pushed into a folder of +// its own, run with app_process, and the folder deleted after, with the +// runtime's compiled copy of it inside — whether or not the read worked. +func (a *Android) readOnce(ctx context.Context) ([]byte, error) { + dex, err := reader.Dex() + if err != nil { + return nil, err + } + f, err := os.CreateTemp("", "mobium-reader-*.dex") + if err != nil { + return nil, mobiumerr.New(mobiumerr.Internal, "stage the reader: %w", err) + } + defer os.Remove(f.Name()) + if _, err := f.Write(dex); err != nil { + f.Close() + return nil, mobiumerr.New(mobiumerr.Internal, "stage the reader: %w", err) + } + if err := f.Close(); err != nil { + return nil, mobiumerr.New(mobiumerr.Internal, "stage the reader: %w", err) + } + + defer func() { + cctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second) + defer cancel() + _, _ = a.adb.Shell(cctx, "rm", "-rf", reader.Dir) + }() + if _, err := a.adb.Shell(ctx, "mkdir", "-p", reader.Dir); err != nil { + return nil, err + } + remote := reader.Dir + "/reader.dex" + if _, err := a.adb.Run(ctx, "push", f.Name(), remote); err != nil { + return nil, err + } + out, diag, err := a.adb.ShellDiagnostics(ctx, "CLASSPATH="+remote, "app_process", "/system/bin", reader.Class) + if err != nil { + return nil, err + } + xml, err := reader.Hierarchy(out) + if err != nil && len(bytes.TrimSpace(diag)) > 0 { + return nil, fmt.Errorf("%w (stderr: %s)", err, firstLine(strings.TrimSpace(string(diag)))) + } + return xml, err +} + // Screenshot captures the framebuffer as PNG. func (a *Android) Screenshot(ctx context.Context) ([]byte, error) { png, err := a.adb.ExecOut(ctx, "screencap", "-p") diff --git a/internal/mobiumdriver/dumpreader_test.go b/internal/mobiumdriver/dumpreader_test.go new file mode 100644 index 00000000..d0ee2066 --- /dev/null +++ b/internal/mobiumdriver/dumpreader_test.go @@ -0,0 +1,93 @@ +package mobiumdriver + +import ( + "context" + "strings" + "testing" + + "github.com/mobiumdev/mobium/internal/mobiumdriver/reader" + "github.com/mobiumdev/mobium/internal/uitree" +) + +// readerADB is an adb on a device whose accessibility settings are given, and +// whose app_process prints what the reader would: a vendor line first, then +// the hierarchy between the markers (or the error line, when fail is set). +func readerADB(t *testing.T, a11yEnabled, services string, fail bool) (*Android, string) { + t.Helper() + readerOut := `echo "Init wrapper sys mutex successful. Pid:1029"; echo MOBIUM-READER-BEGIN; echo '` + miniDump + `'; echo MOBIUM-READER-END` + if fail { + readerOut = `echo "MOBIUM-READER-ERROR: IllegalStateException: no active window from UiAutomation in 10s"` + } + adb, log := fakeADB(t, `case "$*" in + *"settings get secure accessibility_enabled"*) echo `+a11yEnabled+`; echo "`+services+`" ;; + *app_process*) `+readerOut+` ;; + *"uiautomator dump"*) echo "UI hierchary dumped to: /data/local/tmp/mobium-dump.xml" ;; + *"cat /data/local/tmp/mobium-dump.xml"*) echo '`+miniDump+`' ;; +esac`) + return NewAndroid(adb), log +} + +// With a screen reader enabled, the read goes through Mobium's reader, and the +// tree is the same one uiautomator's dump would have given. +func TestSnapshotUsesTheReaderWhenAScreenReaderIsOn(t *testing.T) { + t.Setenv("MOBIUM_DUMP_READER", "") + a, log := readerADB(t, "1", "com.amazon.logan/com.amazon.logan.LoganAccessibilityService", false) + tree, err := a.Snapshot(context.Background()) + if err != nil { + t.Fatal(err) + } + found := false + tree.Walk(func(n *uitree.Node) bool { + found = found || n.Text == "Hello" + return true + }) + if !found { + t.Fatal("the reader's hierarchy did not parse into the tree") + } + got := calls(t, log) + for _, want := range []string{"push", reader.Dir + "/reader.dex", "app_process /system/bin " + reader.Class, "rm -rf " + reader.Dir} { + if !strings.Contains(got, want) { + t.Errorf("no %q in the calls:\n%s", want, got) + } + } + if strings.Contains(got, "uiautomator dump") { + t.Errorf("uiautomator dump ran, which silences the screen reader:\n%s", got) + } +} + +// The positive control: with no accessibility service on, nothing changes. +func TestSnapshotUsesUiautomatorWithNoServiceOn(t *testing.T) { + t.Setenv("MOBIUM_DUMP_READER", "") + for _, c := range []struct{ enabled, services string }{{"0", "null"}, {"1", "null"}, {"1", ""}, {"0", "x/y"}} { + a, log := readerADB(t, c.enabled, c.services, false) + if _, err := a.Snapshot(context.Background()); err != nil { + t.Fatal(err) + } + got := calls(t, log) + if strings.Contains(got, "app_process") || !strings.Contains(got, "uiautomator dump") { + t.Errorf("enabled=%s services=%q: want uiautomator dump, got:\n%s", c.enabled, c.services, got) + } + } +} + +// A failed read still deletes what it put on the device, and says why. +func TestReaderCleansUpAfterAFailedRead(t *testing.T) { + t.Setenv("MOBIUM_DUMP_READER", "mobium") + a, log := readerADB(t, "0", "null", true) + _, err := a.Snapshot(context.Background()) + if err == nil || !strings.Contains(err.Error(), "no active window") { + t.Fatalf("want the reader's reason, got %v", err) + } + got := calls(t, log) + if strings.Count(got, "app_process") != snapshotRetries || strings.Count(got, "rm -rf "+reader.Dir) != snapshotRetries { + t.Errorf("want %d reads each followed by a cleanup:\n%s", snapshotRetries, got) + } +} + +func TestReaderOverrideIsChecked(t *testing.T) { + t.Setenv("MOBIUM_DUMP_READER", "talkback") + a, _ := readerADB(t, "0", "null", false) + if _, err := a.Snapshot(context.Background()); err == nil || !strings.Contains(err.Error(), "MOBIUM_DUMP_READER") { + t.Fatalf("an unknown value was accepted: %v", err) + } +} diff --git a/internal/mobiumdriver/reader/Main.java b/internal/mobiumdriver/reader/Main.java new file mode 100644 index 00000000..bca9d9fd --- /dev/null +++ b/internal/mobiumdriver/reader/Main.java @@ -0,0 +1,253 @@ +package dev.mobium.reader; + +import android.accessibilityservice.AccessibilityServiceInfo; +import android.app.UiAutomation; +import android.graphics.Point; +import android.graphics.Rect; +import android.os.Build; +import android.os.HandlerThread; +import android.os.Looper; +import android.view.Display; +import android.view.accessibility.AccessibilityNodeInfo; +import java.io.PrintStream; +import java.lang.reflect.Constructor; +import java.lang.reflect.Method; + +// Mobium's own reader of the screen: what `uiautomator dump` writes, read over +// a UiAutomation connection opened with FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES. +// +// `uiautomator dump` connects with no flags, and a connection with no flags +// suppresses every other accessibility service for as long as it is open: on a +// Fire TV, VoiceView was unbound for each read, and an app that publishes its +// contents only to a screen reader stopped publishing them. Somebody who +// relies on TalkBack or VoiceView loses it while the read runs. This reader +// leaves them running. +// +// Run from the shell, as uiautomator is: +// +// CLASSPATH=/reader.dex app_process /system/bin dev.mobium.reader.Main +// +// It prints the hierarchy between BEGIN and END marker lines, so a vendor +// library that writes to stdout first — a Fire TV's does — cannot corrupt it, +// and reports a failure on a line starting ERROR_MARK. The format is +// AccessibilityNodeInfoDumper's: the same elements and attributes, nodes not +// visible to the user left out, bounds clipped to the display. +public final class Main { + static final String BEGIN = "MOBIUM-READER-BEGIN"; + static final String END = "MOBIUM-READER-END"; + static final String ERROR_MARK = "MOBIUM-READER-ERROR: "; + + // UiAutomation.FLAG_DONT_SUPPRESS_ACCESSIBILITY_SERVICES, API 24. + static final int DONT_SUPPRESS = 1; + + // main gives the process a main looper and runs the read beside it. Since + // Android 14 or so, AccessibilityInteractionClient builds a Handler on the + // main looper when the connection comes up, and a bare app_process has + // none: on a Pixel 8 Pro on Android 17 the reader died with a + // NullPointerException in Handler. 150ms in. A Fire TV on Android 9 + // never asked for it. + public static void main(String[] args) { + Looper.prepareMainLooper(); + Thread worker = new Thread(new Runnable() { + @Override + public void run() { + read(); + } + }, "mobium-reader-main"); + worker.start(); + Looper.loop(); + } + + static void read() { + PrintStream out; + try { + out = new PrintStream(System.out, false, "UTF-8"); + } catch (Exception e) { + out = System.out; + } + HandlerThread thread = new HandlerThread("mobium-reader"); + thread.start(); + UiAutomation ua = null; + try { + if (Build.VERSION.SDK_INT < 24) { + throw new IllegalStateException("API " + Build.VERSION.SDK_INT + + " cannot open a connection that leaves screen readers running (needs 24)"); + } + ua = connect(thread.getLooper()); + AccessibilityServiceInfo info = ua.getServiceInfo(); + info.flags |= AccessibilityServiceInfo.FLAG_INCLUDE_NOT_IMPORTANT_VIEWS + | AccessibilityServiceInfo.FLAG_REPORT_VIEW_IDS + | AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS; + ua.setServiceInfo(info); + boolean idle = true; + try { + ua.waitForIdle(500, 3000); + } catch (Exception e) { + idle = false; + } + // A fresh connection has no window state until the system sends it, + // so the first asks can return null: on a Fire TV a cold read + // found no active window for five seconds and the next one at + // once. The window list is asked as well, and whichever answers + // first is used; "via" says which, and after how many tries. + AccessibilityNodeInfo root = null; + String via = ""; + for (int i = 0; i < 50 && root == null; i++) { + root = ua.getRootInActiveWindow(); + via = "active-window/" + (i + 1); + if (root == null) { + root = activeWindowRoot(ua); + via = "window-list/" + (i + 1); + } + if (root == null) { + Thread.sleep(200); + } + } + if (root == null) { + throw new IllegalStateException("no active window from UiAutomation in 10s"); + } + Point size = new Point(); + int rotation = displayInfo(size); + StringBuilder xml = new StringBuilder(64 * 1024); + xml.append(""); + xml.append("'); + dump(xml, root, 0, size.x, size.y); + xml.append(""); + out.println(BEGIN); + out.println(xml); + out.println(END); + } catch (Throwable t) { + Throwable cause = t.getCause() != null ? t.getCause() : t; + out.println(ERROR_MARK + cause.getClass().getSimpleName() + ": " + cause.getMessage()); + } finally { + if (ua != null) { + try { + Method disconnect = UiAutomation.class.getDeclaredMethod("disconnect"); + disconnect.setAccessible(true); + disconnect.invoke(ua); + } catch (Throwable ignored) { + // The process is about to exit, which closes the connection anyway. + } + } + thread.quitSafely(); + out.flush(); + } + System.exit(0); + } + + // connect opens the connection `uiautomator` itself opens, with the flag it + // does not pass. Both the constructor and connect(int) are hidden API, + // reachable from a shell process. + static UiAutomation connect(Looper looper) throws Exception { + Class connCls = Class.forName("android.app.UiAutomationConnection"); + Object conn = connCls.getDeclaredConstructor().newInstance(); + Class iconn = Class.forName("android.app.IUiAutomationConnection"); + Constructor c = UiAutomation.class.getDeclaredConstructor(Looper.class, iconn); + c.setAccessible(true); + UiAutomation ua = c.newInstance(looper, conn); + Method connect = UiAutomation.class.getDeclaredMethod("connect", int.class); + connect.setAccessible(true); + connect.invoke(ua, DONT_SUPPRESS); + return ua; + } + + // activeWindowRoot is the root of the window the system calls active, from + // the window list rather than the active-window shortcut. + static AccessibilityNodeInfo activeWindowRoot(UiAutomation ua) { + for (android.view.accessibility.AccessibilityWindowInfo w : ua.getWindows()) { + if (w.isActive()) { + return w.getRoot(); + } + } + return null; + } + + // displayInfo fills in the default display's real size and returns its + // rotation, through DisplayManagerGlobal: a shell process has no Context. + static int displayInfo(Point size) throws Exception { + Class dmg = Class.forName("android.hardware.display.DisplayManagerGlobal"); + Object global = dmg.getMethod("getInstance").invoke(null); + Display display = (Display) dmg.getMethod("getRealDisplay", int.class).invoke(global, Display.DEFAULT_DISPLAY); + display.getRealSize(size); + return display.getRotation(); + } + + static void dump(StringBuilder xml, AccessibilityNodeInfo node, int index, int width, int height) { + Rect bounds = new Rect(); + node.getBoundsInScreen(bounds); + if (!bounds.intersect(0, 0, width, height)) { + bounds.setEmpty(); + } + xml.append("= 26) { + attr(xml, "hint", node.getHintText()); + bool(xml, "showing-hint", node.isShowingHintText()); + } + if (Build.VERSION.SDK_INT >= 24) { + xml.append(" drawing-order=\"").append(node.getDrawingOrder()).append('"'); + } + xml.append(" bounds=\"[").append(bounds.left).append(',').append(bounds.top).append("][") + .append(bounds.right).append(',').append(bounds.bottom).append("]\">"); + int n = node.getChildCount(); + for (int i = 0; i < n; i++) { + AccessibilityNodeInfo child = node.getChild(i); + if (child == null) { + continue; + } + if (child.isVisibleToUser()) { + dump(xml, child, i, width, height); + } + child.recycle(); + } + xml.append(""); + } + + static void bool(StringBuilder xml, String name, boolean v) { + xml.append(' ').append(name).append("=\"").append(v).append('"'); + } + + // attr writes an attribute, escaped, with characters XML cannot carry + // dropped, as the platform dumper does. + static void attr(StringBuilder xml, String name, CharSequence v) { + xml.append(' ').append(name).append("=\""); + if (v != null) { + for (int i = 0; i < v.length(); i++) { + char ch = v.charAt(i); + switch (ch) { + case '&': xml.append("&"); break; + case '<': xml.append("<"); break; + case '>': xml.append(">"); break; + case '"': xml.append("""); break; + case '\n': xml.append(" "); break; + case '\r': xml.append(" "); break; + case '\t': xml.append(" "); break; + default: + if (ch >= 0x20 && ch != 0xFFFE && ch != 0xFFFF) { + xml.append(ch); + } + } + } + } + xml.append('"'); + } +} diff --git a/internal/mobiumdriver/reader/reader.dex b/internal/mobiumdriver/reader/reader.dex new file mode 100644 index 00000000..1ed626c8 Binary files /dev/null and b/internal/mobiumdriver/reader/reader.dex differ diff --git a/internal/mobiumdriver/reader/reader.go b/internal/mobiumdriver/reader/reader.go new file mode 100644 index 00000000..4e4a4f3c --- /dev/null +++ b/internal/mobiumdriver/reader/reader.go @@ -0,0 +1,84 @@ +// Package reader holds Mobium's own reader of an Android screen: a small dex, +// run with app_process, that writes what `uiautomator dump` writes over a +// UiAutomation connection that leaves other accessibility services running. +// +// `uiautomator dump` suppresses every other accessibility service while it +// reads, so on a device with TalkBack or VoiceView on, each read silenced the +// screen reader, and an app that publishes its contents only to a screen +// reader stopped publishing them before the read could see them (measured on +// a Fire TV, CHALLENGES 208). Main.java is the source; reader.dex is built +// from it by `make reader` and pinned here by checksum, as every device-side +// artifact is. +package reader + +import ( + "bytes" + "crypto/sha256" + _ "embed" + "encoding/hex" + + "github.com/mobiumdev/mobium/internal/mobiumerr" +) + +//go:embed reader.dex +var dex []byte + +// SHA256 is reader.dex's checksum. `make reader` rebuilds the dex and prints +// the new one; d8 9.3.16 builds the same bytes from the same source. +const SHA256 = "8d7ff8b46994cdfb88226e282cfe5328a9ffb076aa73923311ebb167ba61aa97" + +// Class is the entry point app_process runs. +const Class = "dev.mobium.reader.Main" + +// Dir is where a read puts the dex on the device. A folder of its own, because +// the runtime compiles the dex into an oat/ folder beside it, and deleting the +// folder takes both. +const Dir = "/data/local/tmp/mobium-reader" + +const ( + begin = "MOBIUM-READER-BEGIN" + end = "MOBIUM-READER-END" + errorMark = "MOBIUM-READER-ERROR: " +) + +// Dex is the reader, checked against its pinned checksum. +func Dex() ([]byte, error) { + sum := sha256.Sum256(dex) + if got := hex.EncodeToString(sum[:]); got != SHA256 { + return nil, mobiumerr.New(mobiumerr.Internal, "the embedded screen reader-safe reader has checksum %s, "+ + "want %s — rebuild it with `make reader`", got, SHA256) + } + return dex, nil +} + +// Hierarchy takes the hierarchy out of what the reader printed. Anything a +// vendor library wrote first is ignored: a Fire TV prints a line of its own +// before a screencap's PNG, and app_process runs the same libraries. +func Hierarchy(out []byte) ([]byte, error) { + if i := bytes.Index(out, []byte(errorMark)); i >= 0 { + msg := out[i+len(errorMark):] + if j := bytes.IndexByte(msg, '\n'); j >= 0 { + msg = msg[:j] + } + return nil, mobiumerr.New(mobiumerr.DeviceServer, "the reader could not read the screen: %s", + bytes.TrimSpace(msg)) + } + b := bytes.Index(out, []byte(begin)) + e := bytes.LastIndex(out, []byte(end)) + if b < 0 || e < b { + return nil, mobiumerr.New(mobiumerr.DeviceServer, "the reader printed no hierarchy: %q", + firstLine(bytes.TrimSpace(out))) + } + xml := bytes.TrimSpace(out[b+len(begin) : e]) + if len(xml) == 0 { + return nil, mobiumerr.New(mobiumerr.DeviceServer, "the reader printed an empty hierarchy") + } + return xml, nil +} + +func firstLine(b []byte) []byte { + if i := bytes.IndexByte(b, '\n'); i >= 0 { + return b[:i] + } + return b +} diff --git a/internal/mobiumdriver/reader/reader_test.go b/internal/mobiumdriver/reader/reader_test.go new file mode 100644 index 00000000..ae4f226c --- /dev/null +++ b/internal/mobiumdriver/reader/reader_test.go @@ -0,0 +1,50 @@ +package reader + +import ( + "errors" + "strings" + "testing" + + "github.com/mobiumdev/mobium/internal/mobiumerr" +) + +func TestDexMatchesItsPinnedChecksum(t *testing.T) { + b, err := Dex() + if err != nil { + t.Fatal(err) + } + if len(b) < 1000 || string(b[:4]) != "dex\n" { + t.Fatalf("reader.dex is not a dex file: %d bytes, starts %q", len(b), b[:4]) + } +} + +// A Fire TV's libraries print a line on stdout before anything else. +func TestHierarchyIgnoresWhatCameBefore(t *testing.T) { + out := "Init wrapper sys mutex successful. Pid:1029\n" + begin + "\n\n" + end + "\n" + got, err := Hierarchy([]byte(out)) + if err != nil { + t.Fatal(err) + } + if string(got) != `` { + t.Fatalf("got %q", got) + } +} + +func TestHierarchyReportsTheReadersError(t *testing.T) { + _, err := Hierarchy([]byte(errorMark + "IllegalStateException: no active window from UiAutomation in 10s\n")) + var me *mobiumerr.Error + if !errors.As(err, &me) || me.Code != mobiumerr.DeviceServer { + t.Fatalf("want device_server, got %v", err) + } + if !strings.Contains(err.Error(), "no active window") { + t.Fatalf("the reader's reason is lost: %v", err) + } +} + +func TestHierarchyRefusesOutputWithoutMarkers(t *testing.T) { + for _, out := range []string{"", "Segmentation fault\n", begin + "\n", begin + "\n\n" + end} { + if _, err := Hierarchy([]byte(out)); err == nil { + t.Errorf("%q was taken as a hierarchy", out) + } + } +}