Feature/eval framework

.github/workflows/android.yml

@@ -71,9 +71,23 @@ jobs:
       - name: Assemble debug
         run: ./gradlew assembleDebug --stacktrace
 
-      - name: Unit tests
+      # This also runs the deterministic eval suite (com.screensaathi.evals.*).
+      # OFFLINE mode replays recorded responses through the real parser, so it
+      # needs no API key and costs nothing — safe on every PR. The golden suite
+      # fails the build on a critical safety or schema regression; the
+      # adversarial suite reports known gaps without gating. LIVE evals run
+      # separately in evals-live.yml.
+      - name: Unit tests + deterministic evals
         run: ./gradlew testDebugUnitTest --stacktrace
 
+      - name: Upload eval report
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-report
+          path: eval-results/
+          if-no-files-found: ignore
+
       - name: Lint
         run: ./gradlew lintDebug --stacktrace
         continue-on-error: true

.github/workflows/evals-live.yml

@@ -0,0 +1,61 @@
+# LIVE evaluation suite — calls Sarvam for real and therefore costs money.
+#
+# Deliberately NOT on push/pull_request. The deterministic suite already gates
+# every PR for free in android.yml; running paid model calls on every commit
+# would cost more than it tells us. This is manual, plus a weekly drift check.
+name: Evals (LIVE)
+
+on:
+  workflow_dispatch:
+  schedule:
+    # Weekly, Monday 03:00 UTC — catches model-side drift without a daily bill.
+    - cron: '0 3 * * 1'
+
+permissions:
+  contents: read
+
+jobs:
+  live-evals:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: '21'
+
+      - uses: gradle/actions/setup-gradle@v4
+      - uses: android-actions/setup-android@v3
+
+      - name: Install SDK packages
+        run: sdkmanager --install "platforms;android-36.1" "build-tools;36.1.0"
+
+      # Skips rather than fails when the secret is absent, so a fork or a repo
+      # without the key does not show a permanently red scheduled job.
+      - name: Check for key
+        id: key
+        run: |
+          if [ -z "${{ secrets.SARVAM_API_KEY }}" ]; then
+            echo "::warning::SARVAM_API_KEY not set — skipping LIVE evals."
+            echo "present=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "present=true" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Run LIVE evals
+        if: steps.key.outputs.present == 'true'
+        env:
+          SCREENSAATHI_EVAL_LIVE: '1'
+          SARVAM_API_KEY: ${{ secrets.SARVAM_API_KEY }}
+        run: ./gradlew :app:testDebugUnitTest --tests "com.screensaathi.evals.*" --rerun-tasks --stacktrace
+
+      - name: Upload LIVE eval report
+        if: always() && steps.key.outputs.present == 'true'
+        uses: actions/upload-artifact@v4
+        with:
+          name: eval-report-live
+          path: eval-results/
+          if-no-files-found: ignore

.gitignore

@@ -2,6 +2,8 @@
 # local.properties holds the Sarvam API key and the SDK path. It must NEVER be
 # committed — the key reaches the app through BuildConfig at build time instead.
 local.properties
+*.keystore
+*.jks
 
 # Build output
 build/
@@ -20,3 +22,7 @@ scripts/*.wav
 .idea/
 *.iml
 .DS_Store
+
+# Generated eval reports. The committed artefact is the baseline in
+# evals/baselines/, not each run's output.
+eval-results/

app/build.gradle.kts

@@ -56,6 +56,16 @@ android {
             // The classes under test call android.util.Log. Without this every
             // Log line throws "not mocked" and the test fails for no reason.
             isReturnDefaultValues = true
+
+            all {
+                // The eval suite prints its scorecard and each failing case to
+                // stdout. Without this Gradle swallows it and a red build says
+                // nothing about WHICH case regressed.
+                it.testLogging {
+                    showStandardStreams = true
+                    events("passed", "failed", "skipped")
+                }
+            }
         }
     }
 }
@@ -65,6 +75,7 @@ dependencies {
     implementation(libs.androidx.appcompat)
     implementation(libs.material)
     implementation(libs.androidx.constraintlayout)
+    implementation(libs.androidx.recyclerview)
     implementation(libs.okhttp)
 
     testImplementation(libs.junit)

app/src/main/AndroidManifest.xml

@@ -8,22 +8,31 @@
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
     <uses-permission android:name="android.permission.FOREGROUND_SERVICE_MICROPHONE" />
 
-    <!-- Package visibility (Android 11+): without these, getLaunchIntentForPackage
-         returns null for every one of them and the app reports that no taxi app
-         is installed on a phone that plainly has three. Named explicitly rather
-         than QUERY_ALL_PACKAGES — we only ever offer this list. Keep in sync
-         with RideApps.KNOWN. -->
+    <!--
+      Universal app discovery (Android 11+ package visibility).
+
+      This replaced a seven-package allow-list, which made the assistant
+      structurally incapable of its own core function: any app outside that list
+      was invisible, so "open WhatsApp" could only ever fail, and DeviceContext
+      correctly but uselessly reported UNKNOWN for nearly everything installed.
+
+      QUERY_ALL_PACKAGES is a Play-policy restricted permission. The
+      justification here is that app discovery IS the user-facing feature — the
+      user names an app and the assistant must determine whether it exists — and
+      a Play submission would have to declare exactly that. Nothing leaves the
+      device: the inventory is read on-device and only a compact, capped list of
+      labels ever reaches the model (see DeviceContext.toPromptText).
+    -->
+    <uses-permission android:name="android.permission.QUERY_ALL_PACKAGES" />
+
     <queries>
-        <package android:name="com.ubercab" />
-        <package android:name="com.olacabs.customer" />
-        <package android:name="com.rapido.passenger" />
-        <package android:name="in.indrive.client" />
-        <package android:name="com.blusmart.rider" />
-        <package android:name="app.nammayatri.passenger" />
-        <package android:name="com.meru.mgt" />
+        <intent>
+            <action android:name="android.intent.action.MAIN" />
+            <category android:name="android.intent.category.LAUNCHER" />
+        </intent>
     </queries>
 
     <application
         android:name=".SaathiApp"
         android:allowBackup="false"
         android:label="@string/app_name"
@@ -42,6 +51,33 @@
             </intent-filter>
         </activity>
 
+        <!--
+          ScreenSaathi as the home screen.
+
+          Opt-in by construction: declaring CATEGORY_HOME only makes the app
+          *eligible*; Android routes HOME here solely if the user selects
+          ScreenSaathi as their default launcher. A user who never does is
+          unaffected, and MainActivity remains the normal entry point.
+
+          stateUnchanged/adjustPan and the excludeFromRecents/singleTask combo
+          are the standard launcher posture — a home screen should not stack
+          instances or appear in Recents as if it were an ordinary app.
+        -->
+        <activity
+            android:name=".launcher.LauncherActivity"
+            android:exported="true"
+            android:launchMode="singleTask"
+            android:excludeFromRecents="true"
+            android:stateNotNeeded="true"
+            android:windowSoftInputMode="stateUnchanged|adjustPan"
+            android:theme="@style/Theme.ScreenSaathi">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.HOME" />
+                <category android:name="android.intent.category.DEFAULT" />
+            </intent-filter>
+        </activity>
+
         <!-- The controlled screen the assistant guides the user through. Real
              Android views with real resource ids, read live over accessibility. -->
         <activity
@@ -50,6 +86,21 @@
             android:label="@string/demo_title"
             android:windowSoftInputMode="adjustResize" />
 
+        <!--
+          Trigger a highlight without changing the foreground app. Exported so
+          it can be driven from adb during development and demos; it carries no
+          data and only starts our own already-running service.
+        -->
+        <receiver
+            android:name=".HighlightReceiver"
+            android:exported="true">
+            <intent-filter>
+                <action android:name="com.screensaathi.HIGHLIGHT" />
+                <action android:name="com.screensaathi.CLEAR_HIGHLIGHT" />
+                <action android:name="com.screensaathi.DUMP_SCREEN" />
+            </intent-filter>
+        </receiver>
+
         <service
             android:name=".OverlayService"
             android:exported="false"

app/src/main/assets/tasks/pay_bill.json

@@ -41,6 +41,7 @@
         "hi-IN": "बिल भरने के लिए यह बटन दबाइए।"
       },
       "expects_value": false,
+      "irreversible": true,
       "highlight": { "shape": "rect", "pulse": true }
     }
   ]

app/src/main/java/com/screensaathi/AppLauncher.kt

@@ -4,27 +4,56 @@ import android.content.Context
 import android.content.Intent
 
 object AppLauncher {
+
     /**
-     * Resolves an app name to a package and launches it natively.
+     * Resolves a spoken app name to a package name, without launching it.
+     *
+     * Split from [launchApp] so the caller can pin the resolved package (for
+     * subsequent screen matching) and run it past the safety guard BEFORE
+     * anything starts.
+     *
+     * Exact label match wins over a substring one: "Uber" must not resolve to
+     * "Uber Eats" just because that happened to be enumerated first.
      */
-    fun launchApp(context: Context, appName: String): Boolean {
-        if (appName.isBlank()) return false
+    fun resolvePackageName(context: Context, appName: String): String? {
+        if (appName.isBlank()) return null
         val pm = context.packageManager
         val mainIntent = Intent(Intent.ACTION_MAIN, null)
-        mainIntent.addCategory(Intent.CATEGORY_LAUNCHER)
-
-        val apps = pm.queryIntentActivities(mainIntent, 0)
-        for (info in apps) {
-            val label = info.loadLabel(pm).toString()
-            if (label.equals(appName, ignoreCase = true) || label.contains(appName, ignoreCase = true)) {
-                val intent = pm.getLaunchIntentForPackage(info.activityInfo.packageName)
-                if (intent != null) {
-                    intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
-                    context.startActivity(intent)
-                    return true
-                }
-            }
+            .addCategory(Intent.CATEGORY_LAUNCHER)
+
+        val apps = try {
+            pm.queryIntentActivities(mainIntent, 0)
+        } catch (e: Exception) {
+            return null
+        }
+
+        val labelled = apps.mapNotNull { info ->
+            val pkg = info.activityInfo?.packageName ?: return@mapNotNull null
+            val label = runCatching { info.loadLabel(pm).toString() }.getOrNull() ?: return@mapNotNull null
+            label to pkg
+        }
+
+        labelled.firstOrNull { (label, _) -> label.equals(appName, ignoreCase = true) }
+            ?.let { return it.second }
+
+        return labelled.firstOrNull { (label, _) -> label.contains(appName, ignoreCase = true) }?.second
+    }
+
+    /**
+     * Brings the named app to the front. Returns false when it could not be
+     * resolved or started — the caller must not report success on false.
+     */
+    fun launchApp(context: Context, appName: String): Boolean {
+        val pkg = resolvePackageName(context, appName) ?: return false
+        val intent = context.packageManager.getLaunchIntentForPackage(pkg) ?: return false
+        // Resume an existing task rather than starting a cold copy: the user may
+        // already be signed in and part-way through.
+        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK or Intent.FLAG_ACTIVITY_RESET_TASK_IF_NEEDED)
+        return try {
+            context.startActivity(intent)
+            true
+        } catch (e: Exception) {
+            false
         }
-        return false
     }
 }

app/src/main/java/com/screensaathi/HighlightReceiver.kt

@@ -0,0 +1,63 @@
+package com.screensaathi
+
+import android.content.BroadcastReceiver
+import android.content.Context
+import android.content.Intent
+import android.util.Log
+
+/**
+ * Triggers a highlight without disturbing whatever is on screen.
+ *
+ * An Activity-based trigger cannot work for this: starting an Activity brings
+ * it to the foreground, so by the time the resolver reads the accessibility
+ * tree the target app is no longer in front and it resolves against the wrong
+ * screen. A broadcast changes nothing about the foreground task.
+ *
+ *   adb shell am broadcast -a com.screensaathi.HIGHLIGHT --es query "Wi-Fi"
+ *
+ * This is the same resolver and overlay the voice path uses — only speech
+ * recognition is skipped — so exercising it is a real test of the pipeline.
+ */
+class HighlightReceiver : BroadcastReceiver() {
+
+    override fun onReceive(context: Context, intent: Intent) {
+        when (intent.action) {
+            ACTION_HIGHLIGHT -> {
+                val query = intent.getStringExtra("query").orEmpty()
+                if (query.isBlank()) return
+                Log.d(TAG, "HIGHLIGHT_REQUEST query='$query'")
+                OverlayService.highlight(context, query)
+            }
+            // get_current_screen(): dump the live ScreenModel exactly as the
+            // resolver sees it. This is the tool the agent will call, and it is
+            // also the only honest way to tell "the node is missing from our
+            // snapshot" apart from "the node is there but scored too low".
+            ACTION_DUMP -> {
+                val snap = ScreenReaderService.instance?.snapshot()
+                if (snap == null) {
+                    Log.w(TAG, "SCREEN_DUMP unavailable: accessibility service not bound")
+                    return
+                }
+                Log.w(TAG, "SCREEN_DUMP pkg=${snap.packageName} settled=${snap.settled} " +
+                    "elements=${snap.elements.size}")
+                val filter = intent.getStringExtra("filter")?.lowercase().orEmpty()
+                snap.elements.forEachIndexed { i, e ->
+                    val line = "[$i] id='${e.resourceId}' text='${e.text}' cls=${e.className} " +
+                        "click=${e.clickable} edit=${e.editable} bounds=${e.bounds.toShortString()}"
+                    if (filter.isEmpty() || line.lowercase().contains(filter)) Log.w(TAG, "  $line")
+                }
+            }
+            ACTION_CLEAR -> {
+                Log.d(TAG, "HIGHLIGHT_CLEAR")
+                OverlayService.clearHighlight(context)
+            }
+        }
+    }
+
+    companion object {
+        const val ACTION_HIGHLIGHT = "com.screensaathi.HIGHLIGHT"
+        const val ACTION_CLEAR = "com.screensaathi.CLEAR_HIGHLIGHT"
+        const val ACTION_DUMP = "com.screensaathi.DUMP_SCREEN"
+        private const val TAG = "HighlightReceiver"
+    }
+}

app/src/main/java/com/screensaathi/MainActivity.kt

@@ -113,6 +113,11 @@ class MainActivity : AppCompatActivity() {
                 OverlayService.runTask(this, task, language)
             }
             "choose" -> OverlayService.choose(this, intent.getIntExtra("choice", 0))
+            // adb: am start -n com.screensaathi/.MainActivity --es saathi_action highlight --es query "Wi-Fi"
+            "highlight" -> {
+                val q = intent.getStringExtra("query").orEmpty()
+                if (q.isNotBlank()) OverlayService.highlight(this, q)
+            }
         }
         intent.removeExtra(EXTRA_ACTION)
         finish()