From 4776f2bceb2a3f0431a38ada5c16476268918875 Mon Sep 17 00:00:00 2001 From: wenpei shao Date: Wed, 5 Aug 2026 03:34:47 -0500 Subject: [PATCH 1/9] Security hardening, diagnostics, and upload stress tooling - Pin TLS certificates for mindpulse.ssc.wisc.edu (InCommon/USERTrust CAs, fail-open 2027-07-08) in network_security_config - Add SecureStore: enrollment token moves from plaintext SharedPreferences to AndroidKeyStore-backed AES-256-GCM, with legacy migration and tests - Sweep crash-orphaned plaintext temp files (Logger.sweepStaleTempFiles) - Expand DeviceStateCollector battery/runtime diagnostics - Make legacy Encryptor.encryptFile throw instead of silently no-op: callers delete the plaintext source on "success", so a no-op destroyed video recordings while logging success - Add 100k-file upload backlog stress-test scripts and runbook --- STRESS_TEST_UPLOAD.md | 57 +++++ .../screenomics/ExampleInstrumentedTest.java | 4 +- .../com/screenomics/SecurityFixesTest.java | 126 +++++++++++ app/src/main/java/com/screenomics/Batch.java | 2 +- .../com/screenomics/DeviceStateCollector.java | 25 +++ .../main/java/com/screenomics/Encryptor.java | 14 ++ app/src/main/java/com/screenomics/Logger.java | 113 ++++++++++ .../com/screenomics/RegisterActivity.java | 10 +- .../java/com/screenomics/SecureStore.java | 112 ++++++++++ .../main/res/xml/network_security_config.xml | 22 ++ tools/stress/cleanup_stress_files.ps1 | 55 +++++ tools/stress/run_upload_stress.ps1 | 204 ++++++++++++++++++ 12 files changed, 736 insertions(+), 8 deletions(-) create mode 100644 STRESS_TEST_UPLOAD.md create mode 100644 app/src/androidTest/java/com/screenomics/SecurityFixesTest.java create mode 100644 app/src/main/java/com/screenomics/SecureStore.java create mode 100644 tools/stress/cleanup_stress_files.ps1 create mode 100644 tools/stress/run_upload_stress.ps1 diff --git a/STRESS_TEST_UPLOAD.md b/STRESS_TEST_UPLOAD.md new file mode 100644 index 0000000..ad33f91 --- /dev/null +++ b/STRESS_TEST_UPLOAD.md @@ -0,0 +1,57 @@ +# Upload Stress Test (100,000 files) + +This procedure creates a large synthetic backlog on a connected Android device and verifies upload drain behavior. + +## Prerequisites + +- ADB installed (`platform-tools`) and device visible in `adb devices` +- App installed and enrolled on device +- Device on charger and stable network +- Recommended: run against staging backend first + +## 1) Generate Backlog + Trigger Upload + Monitor + +From repo root: + +```powershell +powershell -ExecutionPolicy Bypass -File .\tools\stress\run_upload_stress.ps1 ` + -PackageName "edu.wisc.chm.screenomics" ` + -Count 100000 ` + -ChunkSize 2000 ` + -FilePrefix "stress" ` + -TriggerUpload ` + -ContinueWithoutWifi $true ` + -MonitorMinutes 180 ` + -PollSeconds 60 +``` + +Notes: + +- Files are created in `/sdcard/Android/data//files/encrypt` +- Generated filenames are `stress__image.jpg` so they flow through current upload path +- Script retriggers `UploadService` automatically if drain progress stalls + +## 2) Observe Upload Logs + +```powershell +adb logcat -d -s SCREENOMICS_UPLOAD A11yCaptureService +``` + +Look for: + +- `Backlog drain mode enabled` +- `Progress: ...` +- Decreasing remaining synthetic file count in script output + +## 3) Cleanup Synthetic Files + +```powershell +powershell -ExecutionPolicy Bypass -File .\tools\stress\cleanup_stress_files.ps1 ` + -PackageName "edu.wisc.chm.screenomics" ` + -FilePrefix "stress" +``` + +## Optional Parameters + +- `-AdbPath "C:\Android\platform-tools\adb.exe"` if `adb` is not in PATH +- `-DeviceSerial ""` if multiple devices are connected diff --git a/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java b/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java index 263fd0a..a217cfd 100644 --- a/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java +++ b/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java @@ -22,6 +22,8 @@ public void useAppContext() { // Context of the app under test. Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); - assertEquals("com.screenomics", appContext.getPackageName()); + // applicationId is edu.wisc.chm.screenomics (+ ".mindpulse" for the mindpulseDev flavor), + // not the "com.screenomics" namespace — assert flavor-agnostically. + assertTrue(appContext.getPackageName().startsWith("edu.wisc.chm.screenomics")); } } diff --git a/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java b/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java new file mode 100644 index 0000000..c9dd9cd --- /dev/null +++ b/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java @@ -0,0 +1,126 @@ +package com.screenomics; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import android.content.Context; +import android.content.SharedPreferences; + +import androidx.preference.PreferenceManager; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.platform.app.InstrumentationRegistry; + +import org.junit.After; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.FileOutputStream; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.Response; + +/** + * On-device runtime verification of the security fixes: + * #3 SecureStore (enrollment-token encryption at rest, round-trip, no plaintext, migration) + * #5 Logger.sweepStaleTempFiles (orphan temp cleanup) + * #1 TLS certificate pinning (real pinned handshake to the SSCC server) + */ +@RunWith(AndroidJUnit4.class) +public class SecurityFixesTest { + + private Context ctx() { + return InstrumentationRegistry.getInstrumentation().getTargetContext(); + } + + @After + public void cleanup() { + SecureStore.removeSecret(ctx(), "enrollment_token_test"); + } + + // ---- #3 SecureStore ---- + + @Test + public void secureStore_roundtrip_and_no_plaintext() { + Context c = ctx(); + String secret = "tok-" + System.nanoTime(); + SecureStore.putSecret(c, "enrollment_token_test", secret); + + // round-trips correctly + assertEquals(secret, SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT")); + + SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(c); + // no plaintext copy left behind + assertNull("plaintext token must not exist", p.getString("enrollment_token_test", null)); + // an encrypted blob exists and is NOT the plaintext + String enc = p.getString("enrollment_token_test_enc", null); + assertNotNull("encrypted blob must exist", enc); + assertNotEquals("stored value must be ciphertext, not plaintext", secret, enc); + + // removal clears everything + SecureStore.removeSecret(c, "enrollment_token_test"); + assertEquals("DEFAULT", SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT")); + } + + @Test + public void secureStore_migrates_legacy_plaintext() { + Context c = ctx(); + SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(c); + // simulate an already-enrolled device with a legacy plaintext token + p.edit().putString("enrollment_token_test", "legacy-123").remove("enrollment_token_test_enc").apply(); + + // read migrates it to encrypted storage and returns the value + assertEquals("legacy-123", SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT")); + assertNull("legacy plaintext must be removed after migration", + p.getString("enrollment_token_test", null)); + assertNotNull("migrated encrypted blob must exist", + p.getString("enrollment_token_test_enc", null)); + } + + // ---- #5 temp-file sweep ---- + + @Test + public void sweep_deletes_stale_but_keeps_fresh() throws Exception { + Context c = ctx(); + File dir = c.getExternalFilesDir(null); + assertNotNull(dir); + File stale = new File(dir, "tmp_test_stale.jpg"); + File fresh = new File(dir, "tmp_test_fresh.jpg"); + try (FileOutputStream f = new FileOutputStream(stale)) { f.write(new byte[]{1, 2, 3}); } + try (FileOutputStream f = new FileOutputStream(fresh)) { f.write(new byte[]{4}); } + assertTrue(stale.setLastModified(System.currentTimeMillis() - 10 * 60 * 1000L)); // 10 min old + + Logger.sweepStaleTempFiles(c); + + assertFalse("stale temp file should be swept", stale.exists()); + assertTrue("fresh temp file should be kept", fresh.exists()); + //noinspection ResultOfMethodCallIgnored + fresh.delete(); + } + + // ---- #1 TLS certificate pinning ---- + + @Test + public void tlsPinning_allows_real_server() { + // Runs in the app process => the app's network_security_config (cert pinning) applies. + // A successful HTTPS response proves the pinned TLS handshake to the real server works. + OkHttpClient client = new OkHttpClient(); + Request req = new Request.Builder() + .url("https://mindpulse.ssc.wisc.edu/api/v1/health").get().build(); + try (Response resp = client.newCall(req).execute()) { + assertTrue("pinned TLS handshake succeeded, HTTP code=" + resp.code(), resp.code() > 0); + System.out.println("PINNING TEST: reached server through pinned TLS, HTTP " + resp.code()); + } catch (javax.net.ssl.SSLPeerUnverifiedException e) { + fail("TLS pinning REJECTED the real server (pins are wrong): " + e.getMessage()); + } catch (java.io.IOException e) { + // DNS/timeout/offline — inconclusive for pinning, not a pinning failure. + System.out.println("PINNING TEST inconclusive (network issue, not pinning): " + e); + } + } +} diff --git a/app/src/main/java/com/screenomics/Batch.java b/app/src/main/java/com/screenomics/Batch.java index 7a37f2e..82b4b1f 100644 --- a/app/src/main/java/com/screenomics/Batch.java +++ b/app/src/main/java/com/screenomics/Batch.java @@ -86,7 +86,7 @@ public String[] sendFiles() { String baseUrl = prefs.getString("base_url", Constants.BASE_URL); String pptId = prefs.getString("ppt_id", ""); String studyId = prefs.getString("study_id", ""); - String bearerToken = prefs.getString("enrollment_token", ""); + String bearerToken = SecureStore.getSecret(context, "enrollment_token", ""); String imagePubPem = prefs.getString("image_public_key", ""); if (studyId.isEmpty() || pptId.isEmpty()) { diff --git a/app/src/main/java/com/screenomics/DeviceStateCollector.java b/app/src/main/java/com/screenomics/DeviceStateCollector.java index 0b3c50c..dcbd71f 100644 --- a/app/src/main/java/com/screenomics/DeviceStateCollector.java +++ b/app/src/main/java/com/screenomics/DeviceStateCollector.java @@ -71,6 +71,31 @@ private static JSONObject collectBattery(Context context) throws Exception { default: plugType = "none"; break; } obj.put("battery_plugged_type", plugType); + int temp = batteryStatus.getIntExtra(BatteryManager.EXTRA_TEMPERATURE, -1); + if (temp >= 0) { + obj.put("battery_temp_c", temp / 10.0); + } + int voltage = batteryStatus.getIntExtra(BatteryManager.EXTRA_VOLTAGE, -1); + if (voltage > 0) { + obj.put("battery_voltage_mv", voltage); + } + int health = batteryStatus.getIntExtra(BatteryManager.EXTRA_HEALTH, -1); + obj.put("battery_health", health); + } + BatteryManager bm = (BatteryManager) context.getSystemService(Context.BATTERY_SERVICE); + if (bm != null) { + int currentNow = bm.getIntProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_NOW); + if (currentNow != Integer.MIN_VALUE) { + obj.put("battery_current_ua", currentNow); + } + long energyCounter = bm.getLongProperty(BatteryManager.BATTERY_PROPERTY_ENERGY_COUNTER); + if (energyCounter != Long.MIN_VALUE && energyCounter > 0) { + obj.put("battery_energy_nwh", energyCounter); + } + int avgCurrent = bm.getIntProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_AVERAGE); + if (avgCurrent != Integer.MIN_VALUE) { + obj.put("battery_current_avg_ua", avgCurrent); + } } PowerManager pm = (PowerManager) context.getSystemService(Context.POWER_SERVICE); if (pm != null) { diff --git a/app/src/main/java/com/screenomics/Encryptor.java b/app/src/main/java/com/screenomics/Encryptor.java index e0202f3..6ba5ccd 100644 --- a/app/src/main/java/com/screenomics/Encryptor.java +++ b/app/src/main/java/com/screenomics/Encryptor.java @@ -93,6 +93,20 @@ public static Result encryptFileToEnc(File in, File outEnc, String serverImagePu return new Result(outEnc, aesKeyEncB64, nonceB64); } + /** + * Legacy entry point kept only so the disabled mindpulseDev video-capture code + * compiles. It must never silently succeed: callers delete the plaintext source + * after this returns, so a no-op here would destroy the recording while logging + * success. Throwing keeps the original file intact (callers catch and log). + * + * @deprecated Port callers to {@link #encryptFileToEnc} before re-enabling video. + */ + @Deprecated + public static void encryptFile(byte[] key, String inputPath, String outputPath, byte[] iv) { + throw new UnsupportedOperationException( + "Legacy encryptFile is not implemented; port to encryptFileToEnc before use"); + } + // ---- helpers ---- private static SecretKey genAesKey(int bits) throws Exception { diff --git a/app/src/main/java/com/screenomics/Logger.java b/app/src/main/java/com/screenomics/Logger.java index 7afbc23..5f91bdf 100644 --- a/app/src/main/java/com/screenomics/Logger.java +++ b/app/src/main/java/com/screenomics/Logger.java @@ -130,6 +130,31 @@ public static boolean queueDiagnosticsForUpload(Context context) { } } + /** + * Defense-in-depth: delete stale plaintext temp files (tmp_*.jpg / tmp_*.log) left in the + * files/ root if the process was killed between writing and deleting during encryption. + * Normally these are deleted in a finally block within milliseconds; this only catches + * crash-orphaned files. Call on collection-service startup. + */ + public static void sweepStaleTempFiles(Context context) { + try { + File extDir = context.getApplicationContext().getExternalFilesDir(null); + if (extDir == null) return; + File[] files = extDir.listFiles((dir, name) -> + name.startsWith("tmp_") && (name.endsWith(".jpg") || name.endsWith(".log"))); + if (files == null) return; + long cutoff = System.currentTimeMillis() - 5 * 60 * 1000L; // older than 5 minutes + for (File f : files) { + if (f.lastModified() < cutoff) { + //noinspection ResultOfMethodCallIgnored + f.delete(); + } + } + } catch (Exception ignored) { + // best-effort cleanup; never fail the caller + } + } + static boolean queueTextForUpload( Context context, String content, @@ -195,6 +220,94 @@ static boolean queueTextForUpload( } } + /** + * Binary sibling of {@link #queueTextForUpload}: encrypt a JPEG (e.g. an OMI + * Glass photo received over BLE) into the same /encrypt upload queue used by + * screenshots. Tagged meta type="image" so the Receiver ingests it like a + * screenshot, plus source/orientation for provenance. The existing + * UploadService/Batch pipeline carries it with no changes. + * + * @param descriptor filename component (e.g. "glass") + * @param source provenance tag written to meta (e.g. "omi_glass") + * @param orientation image orientation byte from the device, or -1 if unknown + * @return true if an .enc + .meta pair was queued + */ + public static boolean queueImageForUpload( + Context context, + byte[] jpeg, + String descriptor, + String source, + int orientation + ) { + SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); + + String pubKeyPem = prefs.getString("image_public_key", ""); + if (pubKeyPem == null || pubKeyPem.trim().isEmpty()) { + Log.w(TAG, "No image_public_key available, skipping " + descriptor + " image upload"); + return false; + } + if (jpeg == null || jpeg.length == 0) { + Log.w(TAG, "Empty jpeg, skipping " + descriptor + " image upload"); + return false; + } + + File extDir = context.getApplicationContext().getExternalFilesDir(null); + if (extDir == null) { + Log.e(TAG, "getExternalFilesDir returned null, cannot queue " + descriptor + " image"); + return false; + } + + File encryptDir = new File(extDir, "encrypt"); + if (!encryptDir.exists() && !encryptDir.mkdirs()) { + Log.e(TAG, "Failed to create encrypt directory for " + descriptor + " image"); + return false; + } + + String hashFull = prefs.getString("hash", "00000000"); + String hash = hashFull.substring(0, Math.min(8, hashFull.length())); + String baseName = hash + "_" + System.currentTimeMillis() + "_" + descriptor; + + File tempFile = new File(extDir, "tmp_" + UUID.randomUUID() + ".jpg"); + try { + try (java.io.FileOutputStream fos = new java.io.FileOutputStream(tempFile, false)) { + fos.write(jpeg); + } + + File encFile = new File(encryptDir, baseName + ".enc"); + Encryptor.Result result = Encryptor.encryptFileToEnc(tempFile, encFile, pubKeyPem); + + JSONObject metaObj = new JSONObject(); + metaObj.put("aes_key_encrypted_b64", result.aesKeyEncB64); + metaObj.put("tag_len_bits", result.tagLenBits); + metaObj.put("mime", "image/jpeg"); + metaObj.put("type", "image"); + metaObj.put("captured_at", utcIsoMillis().format(new Date())); + metaObj.put("epoch_ms", System.currentTimeMillis()); + if (source != null && !source.isEmpty()) { + metaObj.put("source", source); + } + if (orientation >= 0) { + metaObj.put("orientation", orientation); + } + + File metaFile = new File(encryptDir, baseName + ".meta"); + try (FileWriter fw = new FileWriter(metaFile, false)) { + fw.write(metaObj.toString()); + } + + Log.i(TAG, "Queued encrypted " + descriptor + " image: " + encFile.getName()); + return true; + } catch (Exception e) { + Log.e(TAG, "Failed to queue " + descriptor + " image for upload", e); + return false; + } finally { + if (tempFile.exists()) { + //noinspection ResultOfMethodCallIgnored + tempFile.delete(); + } + } + } + static String captureOwnProcessLogcat() { String pidArg = "--pid=" + android.os.Process.myPid(); List primaryCmd = Arrays.asList("logcat", "-d", "-t", "800", pidArg, "*:V"); diff --git a/app/src/main/java/com/screenomics/RegisterActivity.java b/app/src/main/java/com/screenomics/RegisterActivity.java index 7b3b735..3a400ba 100644 --- a/app/src/main/java/com/screenomics/RegisterActivity.java +++ b/app/src/main/java/com/screenomics/RegisterActivity.java @@ -276,14 +276,12 @@ private void enrollWithReceiver(String code) { ed.putString("key", key); ed.putString("hash", hash); - // enrollment_token stored in plain SharedPreferences -- accepted risk. - // EncryptedSharedPreferences not in project deps; token is also sent as - // Authorization header on every upload, so SharedPreferences is not the - // weakest link. + // enrollment_token is stored encrypted at rest via SecureStore + // (AndroidKeyStore AES-256-GCM); never written to plaintext prefs. if (longToken) { - ed.putString("enrollment_token", code); + SecureStore.putSecret(getApplicationContext(), "enrollment_token", code); } else { - ed.remove("enrollment_token"); + SecureStore.removeSecret(getApplicationContext(), "enrollment_token"); } // remove any legacy key name diff --git a/app/src/main/java/com/screenomics/SecureStore.java b/app/src/main/java/com/screenomics/SecureStore.java new file mode 100644 index 0000000..4fab599 --- /dev/null +++ b/app/src/main/java/com/screenomics/SecureStore.java @@ -0,0 +1,112 @@ +package com.screenomics; + +import android.content.Context; +import android.content.SharedPreferences; +import android.security.keystore.KeyGenParameterSpec; +import android.security.keystore.KeyProperties; +import android.util.Base64; +import android.util.Log; + +import androidx.preference.PreferenceManager; + +import java.security.KeyStore; + +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.SecretKey; +import javax.crypto.spec.GCMParameterSpec; + +/** + * Stores small secrets (e.g. the enrollment bearer token) encrypted at rest using an + * AndroidKeyStore-backed AES-256-GCM key. No external dependencies. + * + * The ciphertext is kept in the app's default SharedPreferences under "<name>_enc"; + * the key material lives only in the hardware-backed AndroidKeyStore and is never exported. + * + * Fail-safe by design: if key access or crypto fails for any reason, it transparently + * falls back to the previous plaintext SharedPreferences behavior, so authentication can + * never break. Reading a legacy plaintext value also migrates it to encrypted storage. + */ +public final class SecureStore { + private static final String TAG = "SecureStore"; + private static final String KEYSTORE = "AndroidKeyStore"; + private static final String KEY_ALIAS = "mindpulse_secure_store_key"; + private static final String ENC_SUFFIX = "_enc"; + private static final int GCM_TAG_BITS = 128; + private static final int GCM_IV_LEN = 12; + + private SecureStore() {} + + private static SecretKey getOrCreateKey() throws Exception { + KeyStore ks = KeyStore.getInstance(KEYSTORE); + ks.load(null); + KeyStore.Entry entry = ks.getEntry(KEY_ALIAS, null); + if (entry instanceof KeyStore.SecretKeyEntry) { + return ((KeyStore.SecretKeyEntry) entry).getSecretKey(); + } + KeyGenerator kg = KeyGenerator.getInstance(KeyProperties.KEY_ALGORITHM_AES, KEYSTORE); + kg.init(new KeyGenParameterSpec.Builder(KEY_ALIAS, + KeyProperties.PURPOSE_ENCRYPT | KeyProperties.PURPOSE_DECRYPT) + .setBlockModes(KeyProperties.BLOCK_MODE_GCM) + .setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE) + .setKeySize(256) + .build()); + return kg.generateKey(); + } + + /** Encrypt and store {@code value} under {@code name}; removes any plaintext copy. */ + public static void putSecret(Context ctx, String name, String value) { + SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(ctx); + try { + Cipher c = Cipher.getInstance("AES/GCM/NoPadding"); + c.init(Cipher.ENCRYPT_MODE, getOrCreateKey()); + byte[] iv = c.getIV(); + byte[] ct = c.doFinal(value.getBytes("UTF-8")); + byte[] blob = new byte[iv.length + ct.length]; + System.arraycopy(iv, 0, blob, 0, iv.length); + System.arraycopy(ct, 0, blob, iv.length, ct.length); + p.edit() + .putString(name + ENC_SUFFIX, Base64.encodeToString(blob, Base64.NO_WRAP)) + .remove(name) // never leave a plaintext copy behind + .apply(); + } catch (Exception e) { + Log.w(TAG, "putSecret encryption failed; falling back to plaintext for " + name, e); + p.edit().putString(name, value).apply(); + } + } + + /** Remove both the encrypted and any legacy plaintext value for {@code name}. */ + public static void removeSecret(Context ctx, String name) { + PreferenceManager.getDefaultSharedPreferences(ctx).edit() + .remove(name + ENC_SUFFIX).remove(name).apply(); + } + + /** + * Read the secret. Prefers the encrypted value; if only a legacy plaintext value exists + * it is returned and transparently migrated to encrypted storage. + */ + public static String getSecret(Context ctx, String name, String def) { + SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(ctx); + String enc = p.getString(name + ENC_SUFFIX, null); + if (enc != null) { + try { + byte[] blob = Base64.decode(enc, Base64.NO_WRAP); + byte[] iv = new byte[GCM_IV_LEN]; + System.arraycopy(blob, 0, iv, 0, GCM_IV_LEN); + byte[] ct = new byte[blob.length - GCM_IV_LEN]; + System.arraycopy(blob, GCM_IV_LEN, ct, 0, ct.length); + Cipher c = Cipher.getInstance("AES/GCM/NoPadding"); + c.init(Cipher.DECRYPT_MODE, getOrCreateKey(), new GCMParameterSpec(GCM_TAG_BITS, iv)); + return new String(c.doFinal(ct), "UTF-8"); + } catch (Exception e) { + Log.w(TAG, "getSecret decryption failed for " + name + "; trying plaintext", e); + } + } + String legacy = p.getString(name, null); + if (legacy != null) { + putSecret(ctx, name, legacy); // one-time migration plaintext -> encrypted + return legacy; + } + return def; + } +} diff --git a/app/src/main/res/xml/network_security_config.xml b/app/src/main/res/xml/network_security_config.xml index ad85c04..c9d474a 100644 --- a/app/src/main/res/xml/network_security_config.xml +++ b/app/src/main/res/xml/network_security_config.xml @@ -6,6 +6,28 @@ 10.0.2.2 + + + mindpulse.ssc.wisc.edu + + + + + nIUvrOVzCyKOqY+U4sofEeIMk94Dt/WuMgaesi8NITk= + x4QzPSC810K5/cMjb05Qm4k3Bw5zBn4lTdO/nEW/Td4= + + + diff --git a/tools/stress/cleanup_stress_files.ps1 b/tools/stress/cleanup_stress_files.ps1 new file mode 100644 index 0000000..03bd6c0 --- /dev/null +++ b/tools/stress/cleanup_stress_files.ps1 @@ -0,0 +1,55 @@ +param( + [string]$PackageName = "edu.wisc.chm.screenomics", + [string]$FilePrefix = "stress", + [string]$AdbPath = "", + [string]$DeviceSerial = "" +) + +$ErrorActionPreference = "Stop" + +function Resolve-Adb { + param([string]$RequestedPath) + if ($RequestedPath -and (Test-Path $RequestedPath)) { + return (Resolve-Path $RequestedPath).Path + } + $adbCmd = Get-Command adb -ErrorAction SilentlyContinue + if ($adbCmd) { + return $adbCmd.Source + } + if ($env:ANDROID_HOME) { + $candidate = Join-Path $env:ANDROID_HOME "platform-tools\adb.exe" + if (Test-Path $candidate) { return (Resolve-Path $candidate).Path } + } + if ($env:ANDROID_SDK_ROOT) { + $candidate = Join-Path $env:ANDROID_SDK_ROOT "platform-tools\adb.exe" + if (Test-Path $candidate) { return (Resolve-Path $candidate).Path } + } + throw "adb not found. Install Android platform-tools or pass -AdbPath." +} + +function Invoke-Adb { + param([string]$Adb, [string[]]$CommandArgs) + $full = @() + if ($DeviceSerial) { + $full += "-s" + $full += $DeviceSerial + } + $full += $CommandArgs + $output = & $Adb @full + if ($LASTEXITCODE -ne 0) { + throw "adb command failed: $($full -join ' ')" + } + return $output +} + +function Invoke-AdbShell { + param([string]$Adb, [string]$ShellCommand) + return Invoke-Adb -Adb $Adb -CommandArgs @("shell", $ShellCommand) +} + +$adb = Resolve-Adb -RequestedPath $AdbPath +$encryptDir = "/sdcard/Android/data/$PackageName/files/encrypt" +$cmd = "if [ -d '$encryptDir' ]; then find '$encryptDir' -maxdepth 1 -type f -name '${FilePrefix}_*_image.jpg' -delete; rm -f '$encryptDir/.stress_template.jpg'; fi" +Invoke-AdbShell -Adb $adb -ShellCommand $cmd | Out-Null + +Write-Host "Removed stress files from $encryptDir" diff --git a/tools/stress/run_upload_stress.ps1 b/tools/stress/run_upload_stress.ps1 new file mode 100644 index 0000000..3e2771b --- /dev/null +++ b/tools/stress/run_upload_stress.ps1 @@ -0,0 +1,204 @@ +param( + [string]$PackageName = "edu.wisc.chm.screenomics", + [int]$Count = 100000, + [int]$ChunkSize = 2000, + [string]$FilePrefix = "stress", + [string]$AdbPath = "", + [string]$DeviceSerial = "", + [switch]$TriggerUpload = $true, + [bool]$ContinueWithoutWifi = $true, + [int]$MonitorMinutes = 120, + [int]$PollSeconds = 60 +) + +$ErrorActionPreference = "Stop" + +function Resolve-Adb { + param([string]$RequestedPath) + if ($RequestedPath -and (Test-Path $RequestedPath)) { + return (Resolve-Path $RequestedPath).Path + } + + $adbCmd = Get-Command adb -ErrorAction SilentlyContinue + if ($adbCmd) { + return $adbCmd.Source + } + + $candidates = @() + if ($env:ANDROID_HOME) { + $candidates += (Join-Path $env:ANDROID_HOME "platform-tools\adb.exe") + } + if ($env:ANDROID_SDK_ROOT) { + $candidates += (Join-Path $env:ANDROID_SDK_ROOT "platform-tools\adb.exe") + } + + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { + return (Resolve-Path $candidate).Path + } + } + + throw "adb not found. Install Android platform-tools or pass -AdbPath." +} + +function Invoke-Adb { + param( + [string]$Adb, + [string[]]$CommandArgs + ) + $full = @() + if ($DeviceSerial) { + $full += "-s" + $full += $DeviceSerial + } + $full += $CommandArgs + $output = & $Adb @full + $exitCode = $LASTEXITCODE + if ($exitCode -ne 0) { + $cmdText = ($full -join " ") + throw "adb command failed (exit $exitCode): $cmdText" + } + return $output +} + +function Invoke-AdbShell { + param( + [string]$Adb, + [string]$ShellCommand + ) + return Invoke-Adb -Adb $Adb -CommandArgs @("shell", $ShellCommand) +} + +function Get-RemoteFileCount { + param( + [string]$Adb, + [string]$RemoteDir, + [string]$Prefix + ) + $countCmd = "if [ -d '$RemoteDir' ]; then find '$RemoteDir' -maxdepth 1 -type f -name '${Prefix}_*_image.jpg' | wc -l; else echo 0; fi" + $raw = Invoke-AdbShell -Adb $Adb -ShellCommand $countCmd + $n = 0 + [void][int]::TryParse(($raw | Out-String).Trim(), [ref]$n) + return $n +} + +$adb = Resolve-Adb -RequestedPath $AdbPath +Write-Host "Using adb: $adb" + +$devices = Invoke-Adb -Adb $adb -CommandArgs @("devices") +$deviceLines = ($devices | Select-String "device$").Count +if ($deviceLines -lt 1) { + throw "No device connected. Connect a phone/emulator and enable USB debugging." +} + +$encryptDir = "/sdcard/Android/data/$PackageName/files/encrypt" +$remoteScript = "/data/local/tmp/ndscreenomics_stress_gen.sh" +$localScript = Join-Path $env:TEMP "ndscreenomics_stress_gen.sh" + +$scriptBody = @( + "#!/system/bin/sh", + 'DIR="$1"', + 'START="$2"', + 'END="$3"', + 'PREFIX="$4"', + 'if [ ! -d "$DIR" ]; then', + ' mkdir -p "$DIR"', + 'fi', + 'TPL="$DIR/.stress_template.jpg"', + 'if [ ! -f "$TPL" ]; then', + ' printf "\\377\\330\\377\\331" > "$TPL"', + 'fi', + 'i="$START"', + 'while [ "$i" -le "$END" ]; do', + ' ts=$(printf "%013d" "$i")', + ' cp "$TPL" "$DIR/${PREFIX}_${ts}_image.jpg"', + ' i=$((i + 1))', + 'done' +) -join "`n" + +[System.IO.File]::WriteAllText($localScript, $scriptBody, [System.Text.Encoding]::ASCII) +Invoke-Adb -Adb $adb -CommandArgs @("push", $localScript, $remoteScript) | Out-Null +Invoke-Adb -Adb $adb -CommandArgs @("shell", "chmod", "755", $remoteScript) | Out-Null +Invoke-AdbShell -Adb $adb -ShellCommand "mkdir -p '$encryptDir'; find '$encryptDir' -maxdepth 1 -type f -name '${FilePrefix}_*_image.jpg' -delete; rm -f '$encryptDir/.stress_template.jpg'" | Out-Null + +Write-Host "Generating $Count synthetic backlog files in $encryptDir" +$start = 1 +while ($start -le $Count) { + $end = [Math]::Min($start + $ChunkSize - 1, $Count) + $genCmd = "sh '$remoteScript' '$encryptDir' $start $end '$FilePrefix'" + Invoke-AdbShell -Adb $adb -ShellCommand $genCmd | Out-Null + Write-Host "Generated files: $end / $Count" + $start = $end + 1 +} + +$created = Get-RemoteFileCount -Adb $adb -RemoteDir $encryptDir -Prefix $FilePrefix +Write-Host "Synthetic files present: $created" +if ($created -lt $Count) { + throw "Generation incomplete: expected $Count files, found $created." +} + +if ($TriggerUpload) { + $wifiFlag = if ($ContinueWithoutWifi) { "true" } else { "false" } + Write-Host "Triggering foreground upload service..." + try { + Invoke-Adb -Adb $adb -CommandArgs @( + "shell", "am", "start-foreground-service", + "-n", "$PackageName/com.screenomics.UploadService", + "--es", "dirPath", $encryptDir, + "--ez", "continueWithoutWifi", $wifiFlag + ) | Out-Null + } catch { + Write-Warning "Direct UploadService start is blocked (service not exported). Launching app as fallback." + Invoke-Adb -Adb $adb -CommandArgs @( + "shell", "am", "start", + "-n", "$PackageName/com.screenomics.MainActivity" + ) | Out-Null + Write-Warning "Open the app and tap Upload (or wait for auto-upload)." + } +} + +if ($MonitorMinutes -gt 0) { + Write-Host "Monitoring drain for up to $MonitorMinutes minute(s)..." + $deadline = (Get-Date).AddMinutes($MonitorMinutes) + $last = Get-RemoteFileCount -Adb $adb -RemoteDir $encryptDir -Prefix $FilePrefix + $stagnant = 0 + while ((Get-Date) -lt $deadline) { + Start-Sleep -Seconds $PollSeconds + $current = Get-RemoteFileCount -Adb $adb -RemoteDir $encryptDir -Prefix $FilePrefix + $delta = $last - $current + Write-Host ("[{0}] remaining={1} drained={2}" -f (Get-Date -Format "HH:mm:ss"), $current, $delta) + + if ($current -le 0) { + Write-Host "Backlog drained successfully." + break + } + + if ($delta -le 0) { + $stagnant++ + } else { + $stagnant = 0 + } + + if ($TriggerUpload -and $stagnant -ge 3) { + Write-Host "No drain progress for 3 polls; retriggering upload service." + $wifiFlag = if ($ContinueWithoutWifi) { "true" } else { "false" } + try { + Invoke-Adb -Adb $adb -CommandArgs @( + "shell", "am", "start-foreground-service", + "-n", "$PackageName/com.screenomics.UploadService", + "--es", "dirPath", $encryptDir, + "--ez", "continueWithoutWifi", $wifiFlag + ) | Out-Null + } catch { + Write-Warning "UploadService cannot be started directly from adb on this build. Use in-app Upload button." + } + $stagnant = 0 + } + + $last = $current + } +} + +Write-Host "Done." +Write-Host "Tip: inspect logs with:" +Write-Host "$adb logcat -d -s SCREENOMICS_UPLOAD A11yCaptureService" From 8c5a91c26c2c72a270fca8f3efb5c1eaec0af6ff Mon Sep 17 00:00:00 2001 From: wenpei shao Date: Wed, 5 Aug 2026 03:35:06 -0500 Subject: [PATCH 2/9] Add OMI Glass BLE + on-device VLM research stack, isolated to mindpulseDev Research features live only in the mindpulseDev flavor; the production standard flavor gets no-op stubs, no BLE permissions, and no native code. Flavor separation: - GlassBleService/GlassPhotoAssembler/VlmBridge/VlmBenchmark/OcrProcessor/ BehaviorContext moved to src/mindpulseDev; GlassesFeature facade + VlmBenchmark stub in src/standard - BLE permissions and service declaration in mindpulseDev manifest overlay - Native build gated by -DVLM_ENABLED=ON (dev flavor only); standard builds a placeholder excluded from packaging, restoring all-ABI production APKs with zero .so files; llama.cpp vendored as a pinned git submodule - Five-tap DevTools entry gated to debug builds / mindpulseDev Reliability and privacy fixes baked in: - GlassBleService: scan filter + balanced mode + exponential backoff (results now arrive with screen off), connect-fallback recovers from remembered-MAC changes, BT adapter state receiver, GATT status checks (a failed CCCD write no longer starts glasses capture), assembler reset on reconnect + 2MB cap (no cross-connection JPEG merging), stop-capture write grace before close, upload I/O off the BLE binder thread, removed the indefinite wake lock, fine-location gate on API 29-30 - VLM: OCR posted to worker thread (was silently returning "" on the accessibility path), native mutex + double-load guard (was leaking ~700MB per reload), n_batch follows n_ctx (prompts >512 tokens failed), UTF-8-safe JNI string returns, loadLibrary guard, -march i8mm removed (SIGILL on armv8.2 cores), DevTools listener leak fixed - Screen-derived text (VLM narrative/captions) and BLE MAC no longer written to logcat, which is uploaded as diagnostics --- .gitmodules | 3 + app/build.gradle | 41 +- app/src/main/AndroidManifest.xml | 1 + app/src/main/cpp/CMakeLists.txt | 116 ++++ app/src/main/cpp/llama.cpp | 1 + app/src/main/cpp/noop.cpp | 5 + app/src/main/cpp/vlm_bridge.cpp | 547 +++++++++++++++++ .../AccessibilityCaptureService.java | 12 + .../java/com/screenomics/BootReceiver.java | 4 + .../java/com/screenomics/CaptureService.java | 19 +- .../com/screenomics/DevToolsActivity.java | 171 ++++++ .../java/com/screenomics/MainActivity.java | 50 +- .../com/screenomics/ScreenLifeFragment.java | 38 ++ app/src/main/res/layout/dev_tools.xml | 150 +++++ .../main/res/layout/fragment_screenlife.xml | 56 ++ app/src/mindpulseDev/AndroidManifest.xml | 22 + .../java/com/screenomics/BehaviorContext.java | 333 +++++++++++ .../java/com/screenomics/GlassBleService.java | 559 ++++++++++++++++++ .../com/screenomics/GlassPhotoAssembler.java | 101 ++++ .../java/com/screenomics/GlassesFeature.java | 95 +++ .../java/com/screenomics/OcrProcessor.java | 42 ++ .../java/com/screenomics/VlmBenchmark.java | 558 +++++++++++++++++ .../java/com/screenomics/VlmBridge.java | 84 +++ .../java/com/screenomics/GlassesFeature.java | 32 + .../java/com/screenomics/VlmBenchmark.java | 50 ++ 25 files changed, 3085 insertions(+), 5 deletions(-) create mode 100644 .gitmodules create mode 100644 app/src/main/cpp/CMakeLists.txt create mode 160000 app/src/main/cpp/llama.cpp create mode 100644 app/src/main/cpp/noop.cpp create mode 100644 app/src/main/cpp/vlm_bridge.cpp create mode 100644 app/src/mindpulseDev/AndroidManifest.xml create mode 100644 app/src/mindpulseDev/java/com/screenomics/BehaviorContext.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/GlassBleService.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/GlassPhotoAssembler.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/GlassesFeature.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/OcrProcessor.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/VlmBenchmark.java create mode 100644 app/src/mindpulseDev/java/com/screenomics/VlmBridge.java create mode 100644 app/src/standard/java/com/screenomics/GlassesFeature.java create mode 100644 app/src/standard/java/com/screenomics/VlmBenchmark.java diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..0d9dd19 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "app/src/main/cpp/llama.cpp"] + path = app/src/main/cpp/llama.cpp + url = https://github.com/ggml-org/llama.cpp.git diff --git a/app/build.gradle b/app/build.gradle index 603fc45..70d8b86 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -23,6 +23,9 @@ android { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } + + ndkVersion "27.2.12479018" + defaultConfig { applicationId "edu.wisc.chm.screenomics" minSdkVersion 29 @@ -32,15 +35,43 @@ android { testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" buildConfigField "boolean", "MINDPULSE_ENABLED", "false" } + + externalNativeBuild { + cmake { + path "src/main/cpp/CMakeLists.txt" + version "3.22.1" + } + } productFlavors { standard { dimension "mindpulse" + // Production flavor: no VLM native stack. CMake only builds the tiny + // vlm_noop placeholder (excluded from packaging below), keeping the + // llama.cpp checkout and NDK toolchain out of the release APK. + externalNativeBuild { + cmake { + targets "vlm_noop" + } + } } mindpulseDev { dimension "mindpulse" applicationIdSuffix ".mindpulse" versionNameSuffix "-mindpulse" buildConfigField "boolean", "MINDPULSE_ENABLED", "true" + // On-device VLM (llama.cpp) is dev-only; arm64 test devices only. + // NOTE: -march is forced at compile time (bypasses ggml runtime CPU + // dispatch). i8mm (ARMv8.6) removed — it SIGILLs on armv8.2 cores. + ndk { + abiFilters 'arm64-v8a' + } + externalNativeBuild { + cmake { + targets "vlm_bridge" + cppFlags "-std=c++17 -O3 -march=armv8.2-a+dotprod+fp16" + arguments "-DANDROID_STL=c++_shared", "-DVLM_ENABLED=ON" + } + } } } signingConfigs { @@ -54,6 +85,9 @@ android { } } buildTypes { + debug { + signingConfig signingConfigs.debug + } release { minifyEnabled true shrinkResources true @@ -70,6 +104,8 @@ android { packaging { jniLibs { useLegacyPackaging = false + // Placeholder target built by the standard flavor; never ship it. + excludes += "**/libvlm_noop.so" } } @@ -109,6 +145,9 @@ dependencies { implementation 'androidx.activity:activity:1.8.2' implementation 'com.google.android.play:app-update:2.1.0' + // ML Kit OCR (on-device text recognition) - mindpulseDev only + mindpulseDevImplementation 'com.google.mlkit:text-recognition:16.0.1' + // Firebase temporarily disabled - uncomment after getting new google-services.json // implementation platform('com.google.firebase:firebase-bom:30.1.0') // implementation 'com.google.firebase:firebase-analytics' @@ -182,7 +221,7 @@ def zipalignOutputsTask = tasks.register("zipalignArtifacts") { } tasks.configureEach { task -> - if (task.name.startsWith("assemble") || task.name.startsWith("bundle")) { + if ((task.name.startsWith("assemble") || task.name.startsWith("bundle")) && !task.name.contains("Debug")) { task.finalizedBy(zipalignOutputsTask) } } diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 0182ffb..e37fd00 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -87,6 +87,7 @@ diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt new file mode 100644 index 0000000..a3f7423 --- /dev/null +++ b/app/src/main/cpp/CMakeLists.txt @@ -0,0 +1,116 @@ +cmake_minimum_required(VERSION 3.22.1) +project("screenomics-vlm" C CXX) + +# The VLM stack (llama.cpp + mtmd + JNI bridge) is only built for the +# mindpulseDev flavor, which passes -DVLM_ENABLED=ON. The standard flavor +# builds only the vlm_noop placeholder so it never needs the llama.cpp +# checkout or the heavy native toolchain. +option(VLM_ENABLED "Build the on-device VLM native stack" OFF) + +if(NOT VLM_ENABLED) + add_library(vlm_noop SHARED noop.cpp) + return() +endif() + +if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/CMakeLists.txt) + message(FATAL_ERROR "VLM_ENABLED=ON but the llama.cpp submodule is missing at " + "${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp — run " + "'git submodule update --init' before building the mindpulseDev flavor.") +endif() + +# Disable everything we don't need from llama.cpp +set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE) +set(LLAMA_BUILD_COMMON OFF CACHE BOOL "" FORCE) +set(LLAMA_CURL OFF CACHE BOOL "" FORCE) +set(LLAMA_OPENSSL OFF CACHE BOOL "" FORCE) +set(GGML_OPENMP OFF CACHE BOOL "" FORCE) +set(GGML_METAL OFF CACHE BOOL "" FORCE) +set(GGML_CUDA OFF CACHE BOOL "" FORCE) +# To enable Mali GPU (Vulkan) offload of the ViT: flip this to ON, set use_gpu=true +# (already toggleable via nativeSetUseGpu), AND provide a HOST C++ compiler so the +# vulkan-shaders-gen tool can be built (the NDK clang defaults to x86_64-w64-windows-gnu +# but the machine has no MinGW/MSVC sysroot, so linking a host .exe fails). glslc itself +# ships in the NDK at ${CMAKE_ANDROID_NDK}/shader-tools/windows-x86_64/glslc.exe. +set(GGML_VULKAN OFF CACHE BOOL "" FORCE) +set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE) +set(LLAMA_TOOLS_INSTALL OFF CACHE BOOL "" FORCE) + +# ARM NEON + dotprod optimizations (critical for performance). +# i8mm removed: it is an ARMv8.6 extension and SIGILLs on armv8.2 cores +# (Cortex-A55/A75 era); this forced -march bypasses ggml's runtime dispatch. +set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE) + +# Set version variable that mtmd expects +if (NOT DEFINED LLAMA_INSTALL_VERSION) + set(LLAMA_INSTALL_VERSION "0.0.0") +endif() + +# Build core llama.cpp (ggml + llama libraries) +add_subdirectory(llama.cpp) + +# Build mtmd (multimodal) library manually +set(MTMD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/tools/mtmd) +add_library(mtmd STATIC + ${MTMD_DIR}/mtmd.cpp + ${MTMD_DIR}/mtmd-audio.cpp + ${MTMD_DIR}/mtmd-image.cpp + ${MTMD_DIR}/mtmd-helper.cpp + ${MTMD_DIR}/clip.cpp + ${MTMD_DIR}/models/cogvlm.cpp + ${MTMD_DIR}/models/conformer.cpp + ${MTMD_DIR}/models/dotsocr.cpp + ${MTMD_DIR}/models/gemma4v.cpp + ${MTMD_DIR}/models/glm4v.cpp + ${MTMD_DIR}/models/hunyuanocr.cpp + ${MTMD_DIR}/models/internvl.cpp + ${MTMD_DIR}/models/kimivl.cpp + ${MTMD_DIR}/models/kimik25.cpp + ${MTMD_DIR}/models/nemotron-v2-vl.cpp + ${MTMD_DIR}/models/llama4.cpp + ${MTMD_DIR}/models/llava.cpp + ${MTMD_DIR}/models/minicpmv.cpp + ${MTMD_DIR}/models/paddleocr.cpp + ${MTMD_DIR}/models/pixtral.cpp + ${MTMD_DIR}/models/qwen2vl.cpp + ${MTMD_DIR}/models/qwen3vl.cpp + ${MTMD_DIR}/models/step3vl.cpp + ${MTMD_DIR}/models/siglip.cpp + ${MTMD_DIR}/models/whisper-enc.cpp + ${MTMD_DIR}/models/deepseekocr.cpp + ${MTMD_DIR}/models/mobilenetv5.cpp + ${MTMD_DIR}/models/youtuvl.cpp +) + +target_link_libraries(mtmd PUBLIC ggml llama) +target_include_directories(mtmd PUBLIC ${MTMD_DIR}) +target_include_directories(mtmd PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/vendor +) +target_compile_features(mtmd PRIVATE cxx_std_17) + +find_package(Threads REQUIRED) +target_link_libraries(mtmd PRIVATE Threads::Threads) + +# JNI bridge library +add_library(vlm_bridge SHARED vlm_bridge.cpp) + +target_link_libraries(vlm_bridge + PRIVATE + llama + ggml + mtmd + android + log +) + +target_include_directories(vlm_bridge PRIVATE + llama.cpp/include + llama.cpp/ggml/include + ${MTMD_DIR} +) + +target_compile_features(vlm_bridge PRIVATE cxx_std_17) diff --git a/app/src/main/cpp/llama.cpp b/app/src/main/cpp/llama.cpp new file mode 160000 index 0000000..d6f3030 --- /dev/null +++ b/app/src/main/cpp/llama.cpp @@ -0,0 +1 @@ +Subproject commit d6f3030047f85a98b009189e76f441fe818ea44d diff --git a/app/src/main/cpp/noop.cpp b/app/src/main/cpp/noop.cpp new file mode 100644 index 0000000..57d45ac --- /dev/null +++ b/app/src/main/cpp/noop.cpp @@ -0,0 +1,5 @@ +// Placeholder translation unit for the standard (production) flavor. +// The real VLM native stack (vlm_bridge + llama.cpp) is only built when +// -DVLM_ENABLED=ON is passed by the mindpulseDev flavor. The resulting +// libvlm_noop.so is excluded from packaging in build.gradle. +extern "C" int screenomics_vlm_noop(void) { return 0; } diff --git a/app/src/main/cpp/vlm_bridge.cpp b/app/src/main/cpp/vlm_bridge.cpp new file mode 100644 index 0000000..4d8b2ab --- /dev/null +++ b/app/src/main/cpp/vlm_bridge.cpp @@ -0,0 +1,547 @@ +#include +#include +#include +#include +#include + +#include "llama.h" +#include "mtmd.h" +#include "mtmd-helper.h" + +#define LOG_TAG "VLM_BRIDGE" +#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__) +#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__) + +// Global state — single model loaded at a time. All access is serialized by +// g_mutex: load/unload/inference can be called from different Java threads +// (worker thread restarts, headless benchmark threads) and must never overlap. +static std::mutex g_mutex; +static llama_model * g_model = nullptr; +static llama_context * g_ctx = nullptr; +static mtmd_context * g_mtmd = nullptr; +static llama_sampler * g_sampler = nullptr; +static bool g_use_gpu = false; // toggled via nativeSetUseGpu(); offloads the ViT (mtmd) to a GPU backend when a GPU backend is compiled in (GGML_VULKAN) +static bool g_think = false; // toggled via nativeSetThinking(); when false, prefill an empty so the model skips reasoning + +// Timing stats +static double g_last_load_ms = 0.0; +static double g_last_encode_ms = 0.0; // vision encoder +static double g_last_decode_ms = 0.0; // text generation +static double g_last_total_ms = 0.0; +static int g_last_n_tokens = 0; +static int g_last_n_embd = 0; +static long g_peak_mem_bytes = 0; + +static double get_time_ms() { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6; +} + +// Free all model state. Caller must hold g_mutex. +static void unload_locked() { + if (g_sampler) { llama_sampler_free(g_sampler); g_sampler = nullptr; } + if (g_mtmd) { mtmd_free(g_mtmd); g_mtmd = nullptr; } + if (g_ctx) { llama_free(g_ctx); g_ctx = nullptr; } + if (g_model) { llama_model_free(g_model); g_model = nullptr; } + llama_backend_free(); +} + +// NewStringUTF requires *modified* UTF-8 and aborts under CheckJNI on 4-byte +// sequences (emoji are common in screen-derived model output). Build the +// String from raw bytes + charset instead. +static jstring make_jstring_utf8(JNIEnv *env, const std::string &s) { + jbyteArray bytes = env->NewByteArray((jsize)s.size()); + if (!bytes) return nullptr; + env->SetByteArrayRegion(bytes, 0, (jsize)s.size(), (const jbyte *)s.data()); + jclass cls = env->FindClass("java/lang/String"); + jmethodID ctor = env->GetMethodID(cls, "", "([BLjava/lang/String;)V"); + jstring charset = env->NewStringUTF("UTF-8"); + jstring out = (jstring)env->NewObject(cls, ctor, bytes, charset); + env->DeleteLocalRef(bytes); + env->DeleteLocalRef(charset); + env->DeleteLocalRef(cls); + return out; +} + +extern "C" { + +JNIEXPORT jboolean JNICALL +Java_com_screenomics_VlmBridge_nativeLoadModel( + JNIEnv *env, jclass clazz, + jstring model_path_j, jstring mmproj_path_j, + jint n_threads, jint n_ctx) +{ + std::lock_guard lock(g_mutex); + + const char *model_path = model_path_j ? env->GetStringUTFChars(model_path_j, nullptr) : nullptr; + if (!model_path) { LOGE("nativeLoadModel: null model path"); return JNI_FALSE; } + const char *mmproj_path = mmproj_path_j ? env->GetStringUTFChars(mmproj_path_j, nullptr) : nullptr; + + LOGI("Loading model: %s", model_path); + LOGI("mmproj: %s", mmproj_path ? mmproj_path : "(unified/none)"); + LOGI("threads=%d ctx=%d", n_threads, n_ctx); + + // A second load without an unload must not leak the previous model + // (hundreds of MB of native memory). + if (g_model || g_ctx || g_mtmd || g_sampler) { + LOGI("Model already loaded; unloading previous instance first"); + unload_locked(); + } + + double t0 = get_time_ms(); + + // Initialize llama backend + llama_backend_init(); + + // Load model + llama_model_params model_params = llama_model_default_params(); + model_params.n_gpu_layers = 0; // CPU only on Android + g_model = llama_model_load_from_file(model_path, model_params); + if (!g_model) { + LOGE("Failed to load model from %s", model_path); + env->ReleaseStringUTFChars(model_path_j, model_path); + if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path); + return JNI_FALSE; + } + + // Create context + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = n_ctx > 0 ? n_ctx : 2048; + ctx_params.n_threads = n_threads > 0 ? n_threads : 4; + // n_batch must cover the largest single prompt submission: nativeInferText + // decodes the whole prompt in one llama_batch, so a batch smaller than + // n_ctx makes prompts over that size fail outright. + ctx_params.n_batch = ctx_params.n_ctx; + g_ctx = llama_init_from_model(g_model, ctx_params); + if (!g_ctx) { + LOGE("Failed to create llama context"); + llama_model_free(g_model); + g_model = nullptr; + env->ReleaseStringUTFChars(model_path_j, model_path); + if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path); + return JNI_FALSE; + } + + // Initialize multimodal context (for vision) + if (mmproj_path) { + mtmd_context_params mtmd_params = mtmd_context_params_default(); + mtmd_params.use_gpu = g_use_gpu; // ViT on GPU when true + a GPU backend is compiled in + mtmd_params.n_threads = n_threads > 0 ? n_threads : 4; + mtmd_params.warmup = false; // skip warmup to speed up load + g_mtmd = mtmd_init_from_file(mmproj_path, g_model, mtmd_params); + if (!g_mtmd) { + LOGE("Failed to create mtmd context from %s", mmproj_path); + // Continue without multimodal — text-only mode + } else { + LOGI("Multimodal context created, vision=%d", mtmd_support_vision(g_mtmd)); + } + } + + // Create sampler (greedy for benchmark — we just want any output) + g_sampler = llama_sampler_chain_init(llama_sampler_chain_default_params()); + llama_sampler_chain_add(g_sampler, llama_sampler_init_greedy()); + + double t1 = get_time_ms(); + g_last_load_ms = t1 - t0; + LOGI("Model loaded in %.1f ms", g_last_load_ms); + + env->ReleaseStringUTFChars(model_path_j, model_path); + if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path); + return JNI_TRUE; +} + +JNIEXPORT jstring JNICALL +Java_com_screenomics_VlmBridge_nativeInferImage( + JNIEnv *env, jclass clazz, + jbyteArray image_data_j, jint width, jint height, + jstring prompt_j, jint max_tokens) +{ + std::lock_guard lock(g_mutex); + + if (!g_model || !g_ctx) { + LOGE("Model not loaded"); + return env->NewStringUTF(""); + } + + double t_start = get_time_ms(); + + const char *prompt_str = prompt_j ? env->GetStringUTFChars(prompt_j, nullptr) : nullptr; + if (!prompt_str) { LOGE("nativeInferImage: null prompt"); return env->NewStringUTF(""); } + jbyte *image_bytes = image_data_j ? env->GetByteArrayElements(image_data_j, nullptr) : nullptr; + jsize image_len = image_data_j ? env->GetArrayLength(image_data_j) : 0; + + std::string result_text; + double t_encode = 0, t_decode = 0; + + // Clear KV cache for fresh inference + llama_memory_clear(llama_get_memory(g_ctx), true); + + if (g_mtmd && image_bytes && image_len > 0) { + // --- Multimodal path: image + text --- + double t0 = get_time_ms(); + + // Create bitmap from raw RGB data (width * height * 3 bytes) + // If image_data is JPEG/PNG bytes, use helper instead + mtmd_bitmap *bitmap = mtmd_helper_bitmap_init_from_buf( + g_mtmd, (const unsigned char *)image_bytes, image_len); + + if (!bitmap) { + LOGE("Failed to create bitmap from image data (%d bytes)", image_len); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return env->NewStringUTF(""); + } + + // Build prompt with the Qwen-VL chat template (marker = where the image goes). + // Without the chat template the instruct model emits EOS immediately (0 tokens). + // When g_think is false, prefill an empty so the (Qwen3) model + // skips reasoning and answers directly (saves decode tokens/time). When true, let + // it reason first (may help on hard/low-quality frames, at higher token cost). + std::string assistant_prefix = g_think + ? "<|im_start|>assistant\n" + : "<|im_start|>assistant\n\n\n\n\n"; + std::string full_prompt = + "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n" + "<|im_start|>user\n" + std::string(mtmd_default_marker()) + prompt_str + "<|im_end|>\n" + + assistant_prefix; + + // Tokenize text + image + mtmd_input_chunks *chunks = mtmd_input_chunks_init(); + mtmd_input_text input_text; + input_text.text = full_prompt.c_str(); + input_text.add_special = true; + input_text.parse_special = true; + + const mtmd_bitmap *bitmap_ptr = bitmap; + int32_t tok_result = mtmd_tokenize(g_mtmd, chunks, &input_text, &bitmap_ptr, 1); + if (tok_result != 0) { + LOGE("mtmd_tokenize failed: %d", tok_result); + mtmd_bitmap_free(bitmap); + mtmd_input_chunks_free(chunks); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return env->NewStringUTF(""); + } + + double t1 = get_time_ms(); + + // Eval all chunks (vision encode + text tokens) + llama_pos n_past = 0; + int32_t eval_result = mtmd_helper_eval_chunks( + g_mtmd, g_ctx, chunks, n_past, 0, 512, true, &n_past); + + double t2 = get_time_ms(); + t_encode = t2 - t0; + + mtmd_bitmap_free(bitmap); + mtmd_input_chunks_free(chunks); + + if (eval_result != 0) { + LOGE("mtmd_helper_eval_chunks failed: %d", eval_result); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return env->NewStringUTF(""); + } + + // Generate text tokens + double t_dec_start = get_time_ms(); + int n_generated = 0; + const llama_vocab *vocab = llama_model_get_vocab(g_model); + + for (int i = 0; i < max_tokens; i++) { + llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1); + + if (llama_vocab_is_eog(vocab, token)) break; + + char buf[256]; + int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true); + if (n > 0) { + result_text.append(buf, n); + } + n_generated++; + + // Prepare next decode + llama_batch batch = llama_batch_get_one(&token, 1); + if (llama_decode(g_ctx, batch) != 0) { + LOGE("llama_decode failed at token %d", i); + break; + } + } + t_decode = get_time_ms() - t_dec_start; + g_last_n_tokens = n_generated; + + } else { + // --- Text-only path (fallback if no mmproj) --- + double t0 = get_time_ms(); + + const llama_vocab *vocab = llama_model_get_vocab(g_model); + std::string full_prompt = std::string(prompt_str); + + // Tokenize + std::vector tokens(full_prompt.size() + 64); + int n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(), + tokens.data(), tokens.size(), true, true); + if (n_tokens < 0) { + tokens.resize(-n_tokens); + n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(), + tokens.data(), tokens.size(), true, true); + } + tokens.resize(n_tokens); + + // Eval prompt + llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size()); + if (llama_decode(g_ctx, batch) != 0) { + LOGE("llama_decode (prompt) failed"); + if (image_bytes) env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return env->NewStringUTF(""); + } + + t_encode = get_time_ms() - t0; + + // Generate + double t_dec_start = get_time_ms(); + int n_generated = 0; + for (int i = 0; i < max_tokens; i++) { + llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1); + if (llama_vocab_is_eog(vocab, token)) break; + + char buf[256]; + int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true); + if (n > 0) result_text.append(buf, n); + n_generated++; + + llama_batch next_batch = llama_batch_get_one(&token, 1); + if (llama_decode(g_ctx, next_batch) != 0) break; + } + t_decode = get_time_ms() - t_dec_start; + g_last_n_tokens = n_generated; + } + + double t_total = get_time_ms() - t_start; + + g_last_encode_ms = t_encode; + g_last_decode_ms = t_decode; + g_last_total_ms = t_total; + + LOGI("Inference done: encode=%.0fms decode=%.0fms total=%.0fms tokens=%d (%.1f tok/s)", + t_encode, t_decode, t_total, g_last_n_tokens, + g_last_n_tokens > 0 ? g_last_n_tokens / (t_decode / 1000.0) : 0); + + if (image_bytes) env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + + return make_jstring_utf8(env, result_text); +} + +// Vision-encoder-only path: runs the image through the mmproj (ViT + projector) +// and returns the projected embedding tokens WITHOUT any LLM decode. This is the +// "extract embedding on device" cost the caller cares about. If return_embd is +// false, we skip the (potentially multi-MB) float copy and only record timing — +// used for the sustained power benchmark loop. +JNIEXPORT jfloatArray JNICALL +Java_com_screenomics_VlmBridge_nativeEncodeImageToEmbedding( + JNIEnv *env, jclass clazz, jbyteArray image_data_j, jboolean return_embd) +{ + std::lock_guard lock(g_mutex); + + if (!g_mtmd || !g_model) { + LOGE("nativeEncodeImageToEmbedding: mtmd/model not loaded"); + return nullptr; + } + if (!image_data_j) { LOGE("nativeEncodeImageToEmbedding: null image"); return nullptr; } + + jbyte *image_bytes = env->GetByteArrayElements(image_data_j, nullptr); + if (!image_bytes) { LOGE("nativeEncodeImageToEmbedding: pin failed"); return nullptr; } + jsize image_len = env->GetArrayLength(image_data_j); + + double t0 = get_time_ms(); + + mtmd_bitmap *bitmap = mtmd_helper_bitmap_init_from_buf( + g_mtmd, (const unsigned char *)image_bytes, image_len); + if (!bitmap) { + LOGE("nativeEncodeImageToEmbedding: bitmap init failed (%d bytes)", image_len); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + return nullptr; + } + + std::string marker = std::string(mtmd_default_marker()); + mtmd_input_chunks *chunks = mtmd_input_chunks_init(); + mtmd_input_text input_text; + input_text.text = marker.c_str(); + input_text.add_special = false; + input_text.parse_special = true; + + const mtmd_bitmap *bp = bitmap; + int32_t tr = mtmd_tokenize(g_mtmd, chunks, &input_text, &bp, 1); + if (tr != 0) { + LOGE("nativeEncodeImageToEmbedding: mtmd_tokenize failed: %d", tr); + mtmd_bitmap_free(bitmap); + mtmd_input_chunks_free(chunks); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + return nullptr; + } + + double t_enc0 = get_time_ms(); // after jpeg-decode + preprocess + tokenize + jfloatArray result = nullptr; + size_t n_chunks = mtmd_input_chunks_size(chunks); + for (size_t i = 0; i < n_chunks; i++) { + const mtmd_input_chunk *ch = mtmd_input_chunks_get(chunks, i); + if (mtmd_input_chunk_get_type(ch) != MTMD_INPUT_CHUNK_TYPE_IMAGE) continue; + + int32_t enc = mtmd_encode_chunk(g_mtmd, ch); // <-- the ViT forward pass + if (enc != 0) { LOGE("mtmd_encode_chunk failed: %d", enc); continue; } + + size_t n_tokens = mtmd_input_chunk_get_n_tokens(ch); + int n_embd = llama_model_n_embd_inp(g_model); + g_last_n_tokens = (int)n_tokens; + g_last_n_embd = n_embd; + + if (return_embd) { + float *embd = mtmd_get_output_embd(g_mtmd); + size_t total = n_tokens * (size_t)n_embd; + if (result) env->DeleteLocalRef(result); // multi-chunk: drop the previous array + result = env->NewFloatArray((jsize)total); + if (!result) { + // allocation failed (pending OutOfMemoryError) — clear it and + // return null instead of continuing with an exception pending + env->ExceptionClear(); + LOGE("nativeEncodeImageToEmbedding: NewFloatArray(%zu) failed", total); + continue; + } + if (embd) env->SetFloatArrayRegion(result, 0, (jsize)total, embd); + } + } + double t1 = get_time_ms(); + g_last_encode_ms = t1 - t_enc0; // pure vision-encode (ViT+projector) time + g_last_total_ms = t1 - t0; // incl. jpeg decode + preprocess + tokenize + + mtmd_bitmap_free(bitmap); + mtmd_input_chunks_free(chunks); + env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT); + + LOGI("encode-only: n_tokens=%d n_embd=%d encode=%.1fms total(incl.preproc)=%.1fms", + g_last_n_tokens, g_last_n_embd, g_last_encode_ms, g_last_total_ms); + return result; +} + +JNIEXPORT jdoubleArray JNICALL +Java_com_screenomics_VlmBridge_nativeGetStats(JNIEnv *env, jclass clazz) { + // Returns: [load_ms, encode_ms, decode_ms, total_ms, n_tokens, tok_per_sec] + jdoubleArray arr = env->NewDoubleArray(6); + double vals[6] = { + g_last_load_ms, + g_last_encode_ms, + g_last_decode_ms, + g_last_total_ms, + (double)g_last_n_tokens, + g_last_n_tokens > 0 ? g_last_n_tokens / (g_last_decode_ms / 1000.0) : 0 + }; + env->SetDoubleArrayRegion(arr, 0, 6, vals); + return arr; +} + +JNIEXPORT void JNICALL +Java_com_screenomics_VlmBridge_nativeUnloadModel(JNIEnv *env, jclass clazz) { + std::lock_guard lock(g_mutex); + LOGI("Unloading model..."); + unload_locked(); + LOGI("Model unloaded"); +} + +JNIEXPORT jboolean JNICALL +Java_com_screenomics_VlmBridge_nativeIsLoaded(JNIEnv *env, jclass clazz) { + std::lock_guard lock(g_mutex); + return (g_model != nullptr && g_ctx != nullptr) ? JNI_TRUE : JNI_FALSE; +} + +JNIEXPORT void JNICALL +Java_com_screenomics_VlmBridge_nativeSetUseGpu(JNIEnv *env, jclass clazz, jboolean use_gpu) { + g_use_gpu = (use_gpu == JNI_TRUE); + LOGI("nativeSetUseGpu: %d", (int)g_use_gpu); +} + +JNIEXPORT void JNICALL +Java_com_screenomics_VlmBridge_nativeSetThinking(JNIEnv *env, jclass clazz, jboolean think) { + g_think = (think == JNI_TRUE); + LOGI("nativeSetThinking: %d", (int)g_think); +} + +JNIEXPORT jstring JNICALL +Java_com_screenomics_VlmBridge_nativeInferText( + JNIEnv *env, jclass clazz, + jstring prompt_j, jint max_tokens) +{ + std::lock_guard lock(g_mutex); + + if (!g_model || !g_ctx) { + LOGE("Model not loaded"); + return env->NewStringUTF(""); + } + + double t_start = get_time_ms(); + const char *prompt_str = prompt_j ? env->GetStringUTFChars(prompt_j, nullptr) : nullptr; + if (!prompt_str) { LOGE("nativeInferText: null prompt"); return env->NewStringUTF(""); } + + // Clear KV cache + llama_memory_clear(llama_get_memory(g_ctx), true); + + const llama_vocab *vocab = llama_model_get_vocab(g_model); + std::string full_prompt(prompt_str); + + // Tokenize + std::vector tokens(full_prompt.size() + 128); + int n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(), + tokens.data(), tokens.size(), true, true); + if (n_tokens < 0) { + tokens.resize(-n_tokens); + n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(), + tokens.data(), tokens.size(), true, true); + } + tokens.resize(n_tokens); + + double t_tok = get_time_ms(); + LOGI("Tokenized %d tokens in %.0fms", n_tokens, t_tok - t_start); + + // Eval prompt + llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size()); + if (llama_decode(g_ctx, batch) != 0) { + LOGE("llama_decode (prompt) failed"); + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return env->NewStringUTF(""); + } + + double t_prompt = get_time_ms(); + g_last_encode_ms = t_prompt - t_start; + + // Generate + std::string result_text; + int n_generated = 0; + for (int i = 0; i < max_tokens; i++) { + llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1); + if (llama_vocab_is_eog(vocab, token)) break; + + char buf[256]; + int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true); + if (n > 0) result_text.append(buf, n); + n_generated++; + + llama_batch next_batch = llama_batch_get_one(&token, 1); + if (llama_decode(g_ctx, next_batch) != 0) break; + } + + double t_end = get_time_ms(); + g_last_decode_ms = t_end - t_prompt; + g_last_total_ms = t_end - t_start; + g_last_n_tokens = n_generated; + + LOGI("Text inference done: prompt=%.0fms decode=%.0fms total=%.0fms tokens=%d (%.1f tok/s)", + g_last_encode_ms, g_last_decode_ms, g_last_total_ms, n_generated, + n_generated > 0 ? n_generated / (g_last_decode_ms / 1000.0) : 0); + + env->ReleaseStringUTFChars(prompt_j, prompt_str); + return make_jstring_utf8(env, result_text); +} + +} // extern "C" diff --git a/app/src/main/java/com/screenomics/AccessibilityCaptureService.java b/app/src/main/java/com/screenomics/AccessibilityCaptureService.java index e02f118..c22e884 100644 --- a/app/src/main/java/com/screenomics/AccessibilityCaptureService.java +++ b/app/src/main/java/com/screenomics/AccessibilityCaptureService.java @@ -62,6 +62,10 @@ public class AccessibilityCaptureService extends AccessibilityService { private static volatile boolean serviceConnected = false; private static volatile boolean captureActive = false; + // VLM benchmark (dev only) — static because AccessibilityService can't be bound + private static volatile VlmBenchmark sVlmBenchmark; + public static void setVlmBenchmark(VlmBenchmark benchmark) { sVlmBenchmark = benchmark; } + private final Handler handler = new Handler(Looper.getMainLooper()); private final ExecutorService ioExecutor = Executors.newSingleThreadExecutor(); @@ -128,6 +132,14 @@ public void onSuccess(@NonNull ScreenshotResult screenshotResult) { Bitmap finalBitmap = bitmap; String foregroundApp = getForegroundApp(); Log.d(TAG, "Screenshot captured | foreground_app=" + foregroundApp); + + // VLM benchmark: submit a copy with context + VlmBenchmark vlm = sVlmBenchmark; + if (vlm != null && vlm.isRunning()) { + Bitmap copy = finalBitmap.copy(Bitmap.Config.ARGB_8888, false); + if (copy != null) vlm.submitFrame(copy, foregroundApp, 0, 0); + } + ioExecutor.execute(() -> { try { boolean imageSaved = encryptImage(finalBitmap, "image", foregroundApp); diff --git a/app/src/main/java/com/screenomics/BootReceiver.java b/app/src/main/java/com/screenomics/BootReceiver.java index 610adff..276f71f 100644 --- a/app/src/main/java/com/screenomics/BootReceiver.java +++ b/app/src/main/java/com/screenomics/BootReceiver.java @@ -44,6 +44,10 @@ public void onReceive(Context context, Intent intent) { SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); boolean wasRecording = prefs.getBoolean("recordingState", false); + // OMI Glass collection (mindpulseDev only) runs independently of screen + // recording. The facade checks the pref and BLE permissions before starting. + GlassesFeature.ensureStartedIfEnabled(context); + if (!wasRecording) { Log.i(TAG, "Recording was not active before reboot, nothing to restart"); return; diff --git a/app/src/main/java/com/screenomics/CaptureService.java b/app/src/main/java/com/screenomics/CaptureService.java index eca0d7e..da0be42 100644 --- a/app/src/main/java/com/screenomics/CaptureService.java +++ b/app/src/main/java/com/screenomics/CaptureService.java @@ -54,6 +54,7 @@ import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.TimeUnit; +import android.graphics.Bitmap.Config; public class CaptureService extends Service { @@ -90,6 +91,13 @@ public class CaptureService extends Service { private static final String WAKE_LOCK_TAG = "MindPulse:CaptureWakeLock"; private long lastLowStorageLogMs = 0L; + // VLM benchmark (dev only) + private VlmBenchmark mVlmBenchmark; + + public void setVlmBenchmark(VlmBenchmark benchmark) { + mVlmBenchmark = benchmark; + } + private class ImageAvailableListener implements ImageReader.OnImageAvailableListener { @Override public void onImageAvailable(ImageReader reader) { @@ -225,6 +233,15 @@ private void processCapturedImage(Image image) { displayWidth + rowPadding / pixelStride, displayHeight, Bitmap.Config.ARGB_8888); bitmap.copyPixelsFromBuffer(buffer); + + // VLM benchmark: submit a copy before bitmap is recycled by encryptImage + if (mVlmBenchmark != null && mVlmBenchmark.isRunning()) { + Bitmap copy = bitmap.copy(Bitmap.Config.ARGB_8888, false); + if (copy != null) { + mVlmBenchmark.submitFrame(copy); + } + } + encryptImage(bitmap, "image", foregroundApp); } @@ -527,7 +544,7 @@ private void stopCapturing() { if (mBackgroundThread != null) { // Post pause image before quitSafely - quitSafely processes pending messages - mBackgroundHandler.post(insertPauseImage); + if (mBackgroundHandler != null) mBackgroundHandler.post(insertPauseImage); mBackgroundThread.quitSafely(); try { mBackgroundThread.join(); diff --git a/app/src/main/java/com/screenomics/DevToolsActivity.java b/app/src/main/java/com/screenomics/DevToolsActivity.java index 8c525d9..b52ab91 100644 --- a/app/src/main/java/com/screenomics/DevToolsActivity.java +++ b/app/src/main/java/com/screenomics/DevToolsActivity.java @@ -1,10 +1,14 @@ package com.screenomics; import android.app.Activity; +import android.content.ComponentName; +import android.content.Context; import android.content.Intent; +import android.content.ServiceConnection; import android.content.SharedPreferences; import android.os.AsyncTask; import android.os.Bundle; +import android.os.IBinder; import androidx.preference.PreferenceManager; import android.text.Editable; import android.text.TextWatcher; @@ -23,16 +27,39 @@ import java.io.File; import java.io.IOException; import java.net.URL; +import java.util.Locale; import java.util.concurrent.TimeUnit; public class DevToolsActivity extends Activity { String imagesPath; + private VlmBenchmark vlmBenchmark; + private CaptureService captureService; + private boolean serviceBound = false; + + private final ServiceConnection captureConnection = new ServiceConnection() { + @Override + public void onServiceConnected(ComponentName name, IBinder binder) { + captureService = ((CaptureService.LocalBinder) binder).getService(); + serviceBound = true; + } + @Override + public void onServiceDisconnected(ComponentName name) { + captureService = null; + serviceBound = false; + } + }; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); + // Defense in depth: even if the isDev pref was set before this gate + // existed, the screen must not open in a production release build. + if (!MainActivity.isDevToolsAllowed()) { + finish(); + return; + } setContentView(R.layout.dev_tools); imagesPath = getApplicationContext().getExternalFilesDir(null).getAbsolutePath() + File.separator + "encrypt"; Button resetButton = findViewById(R.id.clearPrefs); @@ -166,6 +193,136 @@ public void afterTextChanged(Editable edt) { editor.apply(); finish(); }); + + // --- VLM Benchmark (mindpulseDev only; card hidden in standard) --- + // Everything below this guard is dev-flavor-only setup. + if (!VlmBenchmark.AVAILABLE) { + findViewById(R.id.vlmCard).setVisibility(android.view.View.GONE); + return; + } + Switch vlmSwitch = findViewById(R.id.vlmBenchmarkSwitch); + EditText vlmModelPath = findViewById(R.id.vlmModelPathInput); + EditText vlmMmprojPath = findViewById(R.id.vlmMmprojPathInput); + EditText vlmThreadsInput = findViewById(R.id.vlmThreadsInput); + TextView vlmStatus = findViewById(R.id.vlmStatusText); + TextView vlmMetrics = findViewById(R.id.vlmMetricsText); + Button vlmExportLog = findViewById(R.id.vlmExportLogButton); + + // Restore saved model path + vlmModelPath.setText(prefs.getString("vlmModelPath", "")); + vlmMmprojPath.setText(prefs.getString("vlmMmprojPath", "")); + + // Bind to CaptureService to inject VLM benchmark + Intent captureIntent = new Intent(this, CaptureService.class); + bindService(captureIntent, captureConnection, Context.BIND_AUTO_CREATE); + + vlmSwitch.setOnCheckedChangeListener((buttonView, isChecked) -> { + if (isChecked) { + String modelPath = vlmModelPath.getText().toString().trim(); + if (modelPath.isEmpty()) { + Toast.makeText(this, "Set model path first", Toast.LENGTH_SHORT).show(); + vlmSwitch.setChecked(false); + return; + } + if (!new File(modelPath).exists()) { + Toast.makeText(this, "Model file not found: " + modelPath, Toast.LENGTH_LONG).show(); + vlmSwitch.setChecked(false); + return; + } + + // Save paths + prefs.edit() + .putString("vlmModelPath", modelPath) + .putString("vlmMmprojPath", vlmMmprojPath.getText().toString().trim()) + .apply(); + + String mmproj = vlmMmprojPath.getText().toString().trim(); + int threads = 4; + try { threads = Integer.parseInt(vlmThreadsInput.getText().toString()); } catch (Exception ignored) {} + + vlmBenchmark = new VlmBenchmark(DevToolsActivity.this); + vlmBenchmark.setStatusListener(new VlmBenchmark.StatusListener() { + @Override + public void onStatusUpdate(String status) { + vlmStatus.setText(status); + } + @Override + public void onMetricsUpdate(double lastMs, double avgMs, int total, int skipped, float batteryDrain) { + vlmMetrics.setText(String.format(Locale.US, + "Last: %.0fms Avg: %.0fms Total: %d Skip: %d Bat: -%.1f%%", + lastMs, avgMs, total, skipped, batteryDrain)); + } + }); + + vlmBenchmark.start(modelPath, mmproj.isEmpty() ? null : mmproj, threads); + + // Inject into CaptureService and AccessibilityCaptureService + if (serviceBound && captureService != null) { + captureService.setVlmBenchmark(vlmBenchmark); + } + AccessibilityCaptureService.setVlmBenchmark(vlmBenchmark); + + Toast.makeText(this, "VLM benchmark started", Toast.LENGTH_SHORT).show(); + } else { + if (vlmBenchmark != null) { + vlmBenchmark.stop(); + if (serviceBound && captureService != null) { + captureService.setVlmBenchmark(null); + } + AccessibilityCaptureService.setVlmBenchmark(null); + vlmBenchmark = null; + } + vlmStatus.setText("Stopped"); + Toast.makeText(this, "VLM benchmark stopped", Toast.LENGTH_SHORT).show(); + } + }); + + vlmExportLog.setOnClickListener(v -> { + if (vlmBenchmark != null && vlmBenchmark.getLogFile() != null) { + String path = vlmBenchmark.getLogFile().getAbsolutePath(); + Toast.makeText(this, "Log: " + path, Toast.LENGTH_LONG).show(); + Log.i("VLM_DEVTOOLS", "Benchmark CSV: " + path); + } else { + Toast.makeText(this, "No active benchmark log", Toast.LENGTH_SHORT).show(); + } + }); + + // Headless embedding-only benchmark trigger (dev/testing via adb am start). + // Example: + // adb shell am start -n edu.wisc.chm.screenomics.mindpulse/com.screenomics.DevToolsActivity \ + // --ez run_embed_bench true \ + // --es model /sdcard/Android/data/.../files/models/Qwen3.5-0.8B-Q4_K_M.gguf \ + // --es mmproj /sdcard/Android/data/.../files/models/mmproj-Qwen3.5-0.8B-f16.gguf \ + // --es image /sdcard/Android/data/.../files/models/testimg.jpg \ + // --ei threads 6 --ei dur_ms 360000 + Intent launchIntent = getIntent(); + if (launchIntent != null && launchIntent.getBooleanExtra("run_embed_bench", false)) { + String m = launchIntent.getStringExtra("model"); + String mm = launchIntent.getStringExtra("mmproj"); + String img = launchIntent.getStringExtra("image"); + int th = launchIntent.getIntExtra("threads", 4); + long dur = launchIntent.getIntExtra("dur_ms", 360000); + boolean useGpu = launchIntent.getBooleanExtra("use_gpu", false); + Log.i("VLM_DEVTOOLS", "Embed bench: model=" + m + " mmproj=" + mm + " image=" + img + + " threads=" + th + " dur_ms=" + dur + " use_gpu=" + useGpu); + Toast.makeText(this, "Embedding benchmark started (see logcat EMBED_BENCH)", Toast.LENGTH_LONG).show(); + VlmBenchmark.runEmbeddingBenchmark(getApplicationContext(), m, mm, img, th, dur, useGpu); + } + + if (launchIntent != null && launchIntent.getBooleanExtra("run_caption_bench", false)) { + String m = launchIntent.getStringExtra("model"); + String mm = launchIntent.getStringExtra("mmproj"); + String img = launchIntent.getStringExtra("image"); + int th = launchIntent.getIntExtra("threads", 4); + long dur = launchIntent.getIntExtra("dur_ms", 30000); + int maxTok = launchIntent.getIntExtra("max_tokens", 40); + String prompt = launchIntent.getStringExtra("prompt"); + if (prompt == null) prompt = "Describe what you see in one short sentence."; + boolean think = launchIntent.getBooleanExtra("think", false); + Log.i("VLM_DEVTOOLS", "Caption bench: model=" + m + " image=" + img + " maxTok=" + maxTok + " think=" + think); + Toast.makeText(this, "Caption benchmark started (see logcat CAP_BENCH)", Toast.LENGTH_LONG).show(); + VlmBenchmark.runCaptionBenchmark(getApplicationContext(), m, mm, img, th, dur, prompt, maxTok, think); + } } @@ -178,6 +335,20 @@ protected Void doInBackground(Void... params){ } }*/ + @Override + protected void onDestroy() { + super.onDestroy(); + if (serviceBound) { + unbindService(captureConnection); + serviceBound = false; + } + // The benchmark may keep running in the capture services after this screen + // closes, but the listener holds this Activity and its views — detach it. + if (vlmBenchmark != null) { + vlmBenchmark.setStatusListener(null); + } + } + public Response canReachEndpoint(){ OkHttpClient client = new OkHttpClient(); Request request = new Request.Builder().url(Constants.HEALTHCHECK_ADDRESS).get().build(); diff --git a/app/src/main/java/com/screenomics/MainActivity.java b/app/src/main/java/com/screenomics/MainActivity.java index 7603d4e..7530418 100644 --- a/app/src/main/java/com/screenomics/MainActivity.java +++ b/app/src/main/java/com/screenomics/MainActivity.java @@ -81,6 +81,7 @@ public class MainActivity extends AppCompatActivity { private AlertDialog activeUpdateDialog; private AppUpdateManager appUpdateManager; private ActivityResultLauncher locationPermissionRequest; + private ActivityResultLauncher blePermissionRequest; private final InstallStateUpdatedListener installStateUpdatedListener = state -> { if (state.installStatus() == InstallStatus.DOWNLOADED && appUpdateManager != null) { @@ -131,6 +132,22 @@ protected void onCreate(Bundle savedInstanceState) { } }); + // Register BLE permission launcher for OMI Glass collection (mindpulseDev only) + blePermissionRequest = registerForActivityResult( + new ActivityResultContracts.RequestMultiplePermissions(), result -> { + boolean allGranted = !result.isEmpty(); + for (Boolean granted : result.values()) { + if (!Boolean.TRUE.equals(granted)) allGranted = false; + } + if (allGranted) { + GlassesFeature.start(this); + } else { + Log.w(TAG, "BLE permission denied; glasses collection unavailable"); + Toast.makeText(this, "Bluetooth permission is required to connect glasses", + Toast.LENGTH_LONG).show(); + } + }); + maybeEnableAccessibilityModeForAndroid15(prefs); initInAppUpdate(); @@ -156,7 +173,9 @@ protected void onCreate(Bundle savedInstanceState) { infoButton.setOnClickListener(v -> { infoOpenCount++; - if (infoOpenCount == 5) { + // Five-tap dev entry works only in internal builds (any debug build or + // the mindpulseDev flavor) — never in the production Play release. + if (infoOpenCount == 5 && isDevToolsAllowed()) { editor.putBoolean("isDev", true); editor.apply(); devButton.setVisibility(View.VISIBLE); @@ -306,8 +325,8 @@ protected void onResume() { } SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this); - boolean isDev = prefs.getBoolean("isDev", false); - if (!isDev) devButton.setVisibility(View.GONE); + boolean isDev = prefs.getBoolean("isDev", false) && isDevToolsAllowed(); + devButton.setVisibility(isDev ? View.VISIBLE : View.GONE); Intent launchIntent = getIntent(); if (launchIntent != null && launchIntent.getBooleanExtra("start_video_recording", false)) { @@ -317,6 +336,10 @@ protected void onResume() { } } + // OMI Glass (mindpulseDev only): restart the service if it should be running + // but isn't. No-op when already running, so a live connection is untouched. + GlassesFeature.ensureStartedIfEnabled(this); + checkForAppUpdate(true); maybeResumeInProgressUpdate(); } @@ -406,6 +429,27 @@ public void stopLocationService(){ stopService(intent); } + /** Dev tools are for internal builds only: any debug build, or the mindpulseDev flavor. */ + static boolean isDevToolsAllowed() { + return BuildConfig.DEBUG || BuildConfig.MINDPULSE_ENABLED; + } + + // ---- OMI Glass BLE collection (mindpulseDev only; no-op in standard) ---- + + /** Start OMI Glass BLE collection, requesting BLE permissions first if needed. */ + public void startGlassesService() { + if (!GlassesFeature.AVAILABLE) return; + if (!GlassesFeature.hasBlePermissions(this)) { + blePermissionRequest.launch(GlassesFeature.requiredRuntimePermissions()); + return; + } + GlassesFeature.start(this); + } + + public void stopGlassesService() { + GlassesFeature.stop(this); + } + public void startMediaProjectionRequest() { // Android 11+ (R): always prefer accessibility capture -- it survives reboots // and does not require a one-time MediaProjection token. diff --git a/app/src/main/java/com/screenomics/ScreenLifeFragment.java b/app/src/main/java/com/screenomics/ScreenLifeFragment.java index 6fe4b54..2f2cac4 100644 --- a/app/src/main/java/com/screenomics/ScreenLifeFragment.java +++ b/app/src/main/java/com/screenomics/ScreenLifeFragment.java @@ -61,6 +61,10 @@ public class ScreenLifeFragment extends Fragment { private Timer numImageRefreshTimer; private UploadService uploadService; private TextView accessibilityModeStatus; + private View glassCard; + private View glassStatusDot; + private TextView glassStatus; + private Button glassConnectButton; private final ExecutorService statsExecutor = Executors.newSingleThreadExecutor(); private final AtomicBoolean statsRefreshInFlight = new AtomicBoolean(false); @@ -121,6 +125,13 @@ private void initializeViews(View view) { // statsSettingsButton removed - each permission row opens its own settings accessibilityCaptureButton = view.findViewById(R.id.accessibilityCaptureButton); accessibilityModeStatus = view.findViewById(R.id.accessibilityModeStatus); + glassCard = view.findViewById(R.id.glassCard); + glassStatusDot = view.findViewById(R.id.glassStatusDot); + glassStatus = view.findViewById(R.id.glassStatus); + glassConnectButton = view.findViewById(R.id.glassConnectButton); + if (!GlassesFeature.AVAILABLE && glassCard != null) { + glassCard.setVisibility(View.GONE); + } // Permission status dots cameraPermissionDot = view.findViewById(R.id.cameraPermissionDot); @@ -238,6 +249,13 @@ private void setupListeners() { accessibilityCaptureButton.setOnClickListener(view -> handleAccessibilityCaptureSelection()); + glassConnectButton.setOnClickListener(v -> { + MainActivity act = (MainActivity) requireActivity(); + if (GlassesFeature.isEnabled(requireContext())) act.stopGlassesService(); + else act.startGlassesService(); + v.postDelayed(this::updateGlassUi, 800); + }); + uploadButton.setOnClickListener(v -> { if (!InternetConnection.checkWiFiConnection(requireContext())) { AlertDialog alertDialog = new AlertDialog.Builder(requireContext()).create(); @@ -336,6 +354,7 @@ public void onResume() { startImageRefreshTimer(); updatePermissionStatus(); updateAccessibilityCaptureUi(); + updateGlassUi(); // If user toggled capture ON but accessibility wasn't enabled yet, check now if (pendingCaptureStart) { @@ -432,6 +451,7 @@ public void run() { updatePermissionStatus(); updateAccessibilityCaptureUi(); + updateGlassUi(); }); } catch (Exception e) { Log.w(TAG, "Failed to refresh file stats", e); @@ -814,6 +834,24 @@ private void updateAccessibilityCaptureUi() { } } + private void updateGlassUi() { + if (!GlassesFeature.AVAILABLE) return; + if (getActivity() == null || glassStatus == null) return; + boolean enabled = GlassesFeature.isEnabled(requireContext()); + boolean connected = GlassesFeature.isConnected(); + updatePermissionDot(glassStatusDot, connected); + if (!enabled) { + glassStatus.setText("Off"); + glassConnectButton.setText("Connect OMI Glass"); + } else { + int batteryPct = GlassesFeature.batteryPct(); + String batt = batteryPct >= 0 ? (batteryPct + "%") : "—"; + glassStatus.setText((connected ? "Connected" : "Scanning…") + + " · photos " + GlassesFeature.photoCount() + " · battery " + batt); + glassConnectButton.setText("Disconnect OMI Glass"); + } + } + private void openAppSettings() { try { Intent intent = new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS); diff --git a/app/src/main/res/layout/dev_tools.xml b/app/src/main/res/layout/dev_tools.xml index 9fb4f4f..063bd3f 100644 --- a/app/src/main/res/layout/dev_tools.xml +++ b/app/src/main/res/layout/dev_tools.xml @@ -343,6 +343,156 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +