Skip to content

Commit 4294736

Browse files
committed
app_speech: listening, in every client (mobiumdev/mobium#255)
Two scenarios: listen start before an action and listen stop after it, in all five clients — Java and .NET as listenStart and listenStop. Captured on the Android 15 emulator around a tap on the Screen Reader Demo's Add aloud, whose count is a live region: the stop heard "Live cart: 1" as a change on screen. TalkBack's log output level was set to VERBOSE by hand before and back to ERROR after, and TalkBack is off again. check_examples: 118 scenarios, 0 problems in every client, .NET included.
1 parent 4b6ab1e commit 4294736

8 files changed

Lines changed: 65 additions & 0 deletions

File tree

‎data/runs.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -802,5 +802,21 @@
802802
"verbatim": true,
803803
"exit": 0,
804804
"output": "TalkBack says:\n 1. \"not checked. Team. Radio button\" — Team (radio)\n 2. \"OFF. Dark mode. Switch\" — Dark mode (switch)\n 3. \"Dark mode, for the whole app\" — Dark mode, for the whole app, not in map\n 4. \"Read-only\" — Read-only, not in map\n 5. \"This field cannot be edited. Edit box. disabled\"\n 6. \"notify=false terms=false plan=free theme=light\" — notify=false terms=false plan=free theme=light, not in map\n 7. (nothing)\n 8. \"Back. Button\" — Back (button)\n 9. \"Form\" — Form, not in map\n 10. \"not checked. Email me. Check box\" — Email me (checkbox)\n 11. \"not checked. Accept terms. Check box\" — Accept terms (checkbox)\n 12. \"checked. Free. Radio button\" — Free (radio)\n 13. \"not checked. Pro. Radio button\" — Pro (radio)\nended: the cursor came back round to the first element"
805+
},
806+
"app_speech[3]": {
807+
"tool": "app_speech",
808+
"index": 3,
809+
"command": "mobium speech --listen start",
810+
"verbatim": true,
811+
"exit": 0,
812+
"output": "listening to TalkBack — act, then stop it with listen stop"
813+
},
814+
"app_speech[4]": {
815+
"tool": "app_speech",
816+
"index": 4,
817+
"command": "mobium speech --listen stop",
818+
"verbatim": true,
819+
"exit": 0,
820+
"output": "TalkBack said, since listening started: \"Live cart: 1\" — Live cart: 1, not in map [a change on screen]"
805821
}
806822
}

‎examples/canonical.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -937,6 +937,20 @@
937937
"args": {
938938
"target": "label=Pro"
939939
}
940+
},
941+
{
942+
"title": "Start listening to the screen reader, before an action",
943+
"cli": "mobium speech --listen start",
944+
"args": {
945+
"listen": "start"
946+
}
947+
},
948+
{
949+
"title": "Stop listening, and hear what the action made it say",
950+
"cli": "mobium speech --listen stop",
951+
"args": {
952+
"listen": "stop"
953+
}
940954
}
941955
],
942956
"app_hit_test": [

‎examples/dotnet.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,12 @@
468468
},
469469
{
470470
"code": "var said = device.Speech(\"label=Pro\");\nConsole.WriteLine(said[\"steps\"]);"
471+
},
472+
{
473+
"code": "device.ListenStart();\ndevice.Tap(\"testid=addLive\");"
474+
},
475+
{
476+
"code": "var heard = device.ListenStop();\nConsole.WriteLine(heard[\"steps\"]);"
471477
}
472478
],
473479
"app_hit_test": [

‎examples/go.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,12 @@
468468
},
469469
{
470470
"code": "said, err := dev.Speech(ctx, mobium.SpeechOptions{Target: \"label=Pro\"})\nif err != nil {\n\treturn err\n}\nfmt.Println(said.Steps[0].Said)"
471+
},
472+
{
473+
"code": "if _, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"start\"}); err != nil {\n\treturn err\n}\nif err := dev.Tap(ctx, \"testid=addLive\"); err != nil {\n\treturn err\n}"
474+
},
475+
{
476+
"code": "heard, err := dev.Speech(ctx, mobium.SpeechOptions{Listen: \"stop\"})\nif err != nil {\n\treturn err\n}\nfor _, step := range heard.Steps {\n\tfmt.Println(step.Said, step.Cause)\n}"
471477
}
472478
],
473479
"app_hit_test": [

‎examples/java.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,12 @@
468468
},
469469
{
470470
"code": "Map<String, Object> said = device.speech(\"label=Pro\", false, 0);\nSystem.out.println(said.get(\"steps\"));"
471+
},
472+
{
473+
"code": "device.listenStart();\ndevice.tap(\"testid=addLive\");"
474+
},
475+
{
476+
"code": "Map<String, Object> heard = device.listenStop();\nSystem.out.println(heard.get(\"steps\"));"
471477
}
472478
],
473479
"app_hit_test": [

‎examples/javascript.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,12 @@
468468
},
469469
{
470470
"code": "const said = await device.speech({ target: 'label=Pro' })\nconsole.log(said.steps)"
471+
},
472+
{
473+
"code": "await device.speech({ listen: 'start' })\nawait device.tap('testid=addLive')"
474+
},
475+
{
476+
"code": "const heard = await device.speech({ listen: 'stop' })\nconsole.log(heard.steps)"
471477
}
472478
],
473479
"app_hit_test": [

‎examples/python.json‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -468,6 +468,12 @@
468468
},
469469
{
470470
"code": "said = device.speech(\"label=Pro\")\nprint(said[\"steps\"])"
471+
},
472+
{
473+
"code": "device.speech(listen=\"start\")\ndevice.tap(\"testid=addLive\")"
474+
},
475+
{
476+
"code": "heard = device.speech(listen=\"stop\")\nfor step in heard[\"steps\"]:\n print(step[\"said\"], step.get(\"cause\"))"
471477
}
472478
],
473479
"app_hit_test": [

‎scripts/capture.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -166,6 +166,11 @@ def app_screen(name):
166166
"$sleep 3"], after=True, pause=1.0),
167167
dict(tool="app_speech", i=0),
168168
dict(tool="app_speech", i=1, restore=["accessibility screen_reader off"]),
169+
# Listening around a tap on the Screen Reader Demo's Add aloud, whose
170+
# count is a live region: TalkBack says the new count.
171+
dict(tool="app_speech", i=3, setup=app_screen("Screen Reader Demo") + ["wait testid=addLive",
172+
"accessibility screen_reader on", "$sleep 4"]),
173+
dict(tool="app_speech", i=4, setup=["tap testid=addLive"], restore=["accessibility screen_reader off"]),
169174
dict(tool="app_list_apps", i=0),
170175
dict(tool="app_contexts", i=0, setup=app_screen("WebViews") + ["tap 'text=Plain page'", "wait text=Back"],
171176
after=True),

0 commit comments

Comments
 (0)