diff --git a/.claude/settings.local.json b/.claude/settings.local.json
deleted file mode 100644
index 1109631..0000000
--- a/.claude/settings.local.json
+++ /dev/null
@@ -1,9 +0,0 @@
-{
- "permissions": {
- "allow": [
- "Read(//z/mindpulse_test/9ec6137a/2025-10-03/**)"
- ],
- "deny": [],
- "ask": []
- }
-}
\ No newline at end of file
diff --git a/.github/workflows/android-ci.yml b/.github/workflows/android-ci.yml
new file mode 100644
index 0000000..89e5c5e
--- /dev/null
+++ b/.github/workflows/android-ci.yml
@@ -0,0 +1,52 @@
+name: Android CI
+
+# Compile both flavors, run lint, and run the JVM unit test suite on every
+# push and pull request. Deliberately skips the native (llama.cpp) build and
+# instrumented tests: the standard flavor never needs the submodule, and the
+# mindpulseDev Java compile doesn't trigger CMake.
+on:
+ push:
+ pull_request:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref }}
+ cancel-in-progress: true
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v4
+ # submodules intentionally NOT checked out (llama.cpp is dev-native only)
+
+ - name: Set up JDK 17
+ uses: actions/setup-java@v4
+ with:
+ distribution: temurin
+ java-version: '17'
+
+ - name: Set up Gradle (with cache)
+ uses: gradle/actions/setup-gradle@v3
+
+ - name: Compile production (standard) flavor
+ run: ./gradlew :app:compileStandardReleaseJavaWithJavac --console=plain
+
+ - name: Compile mindpulseDev flavor (Java only)
+ run: ./gradlew :app:compileMindpulseDevDebugJavaWithJavac --console=plain
+
+ - name: Lint (standard)
+ run: ./gradlew :app:lintStandardDebug --console=plain
+
+ - name: Unit tests (both flavors)
+ run: ./gradlew :app:testStandardDebugUnitTest :app:testMindpulseDevDebugUnitTest --console=plain
+
+ - name: Upload lint and test reports
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: reports
+ path: |
+ app/build/reports/lint-results-*.txt
+ app/build/reports/tests/
+ retention-days: 14
diff --git a/.gitignore b/.gitignore
index 1c7728c..27c78e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,3 +22,21 @@ output.json
/gradlew
/app/release
.idea/
+
+# Local machine settings (contains local paths / participant hashes)
+.claude/settings.local.json
+
+# On-device VLM model weights (hundreds of MB; side-loaded to the device)
+models/
+*.gguf
+
+# Decrypted participant data / device pulls — NEVER commit (PII)
+recentlog/
+Screenshot_*.png
+
+# Device-state artifacts and local dev prefs
+tools/prefs.xml
+nul
+
+# Local correspondence / scratch notes
+email_to_garrett_dashboard.md
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000..0d9dd19
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "app/src/main/cpp/llama.cpp"]
+ path = app/src/main/cpp/llama.cpp
+ url = https://github.com/ggml-org/llama.cpp.git
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 35eb1dd..521e2a4 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -2,5 +2,6 @@
+
\ No newline at end of file
diff --git a/STRESS_TEST_UPLOAD.md b/STRESS_TEST_UPLOAD.md
new file mode 100644
index 0000000..ad33f91
--- /dev/null
+++ b/STRESS_TEST_UPLOAD.md
@@ -0,0 +1,57 @@
+# Upload Stress Test (100,000 files)
+
+This procedure creates a large synthetic backlog on a connected Android device and verifies upload drain behavior.
+
+## Prerequisites
+
+- ADB installed (`platform-tools`) and device visible in `adb devices`
+- App installed and enrolled on device
+- Device on charger and stable network
+- Recommended: run against staging backend first
+
+## 1) Generate Backlog + Trigger Upload + Monitor
+
+From repo root:
+
+```powershell
+powershell -ExecutionPolicy Bypass -File .\tools\stress\run_upload_stress.ps1 `
+ -PackageName "edu.wisc.chm.screenomics" `
+ -Count 100000 `
+ -ChunkSize 2000 `
+ -FilePrefix "stress" `
+ -TriggerUpload `
+ -ContinueWithoutWifi $true `
+ -MonitorMinutes 180 `
+ -PollSeconds 60
+```
+
+Notes:
+
+- Files are created in `/sdcard/Android/data//files/encrypt`
+- Generated filenames are `stress__image.jpg` so they flow through current upload path
+- Script retriggers `UploadService` automatically if drain progress stalls
+
+## 2) Observe Upload Logs
+
+```powershell
+adb logcat -d -s SCREENOMICS_UPLOAD A11yCaptureService
+```
+
+Look for:
+
+- `Backlog drain mode enabled`
+- `Progress: ...`
+- Decreasing remaining synthetic file count in script output
+
+## 3) Cleanup Synthetic Files
+
+```powershell
+powershell -ExecutionPolicy Bypass -File .\tools\stress\cleanup_stress_files.ps1 `
+ -PackageName "edu.wisc.chm.screenomics" `
+ -FilePrefix "stress"
+```
+
+## Optional Parameters
+
+- `-AdbPath "C:\Android\platform-tools\adb.exe"` if `adb` is not in PATH
+- `-DeviceSerial ""` if multiple devices are connected
diff --git a/app/build.gradle b/app/build.gradle
index 603fc45..935d3ca 100644
--- a/app/build.gradle
+++ b/app/build.gradle
@@ -20,27 +20,58 @@ android {
flavorDimensions "mindpulse"
compileOptions {
- sourceCompatibility JavaVersion.VERSION_1_8
- targetCompatibility JavaVersion.VERSION_1_8
+ sourceCompatibility JavaVersion.VERSION_17
+ targetCompatibility JavaVersion.VERSION_17
}
+
+ ndkVersion "27.2.12479018"
+
defaultConfig {
applicationId "edu.wisc.chm.screenomics"
minSdkVersion 29
targetSdkVersion 35
- versionCode 27
- versionName "1.21"
+ versionCode 28
+ versionName "1.22"
testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
buildConfigField "boolean", "MINDPULSE_ENABLED", "false"
}
+
+ externalNativeBuild {
+ cmake {
+ path "src/main/cpp/CMakeLists.txt"
+ version "3.22.1"
+ }
+ }
productFlavors {
standard {
dimension "mindpulse"
+ // Production flavor: no VLM native stack. CMake only builds the tiny
+ // vlm_noop placeholder (excluded from packaging below), keeping the
+ // llama.cpp checkout and NDK toolchain out of the release APK.
+ externalNativeBuild {
+ cmake {
+ targets "vlm_noop"
+ }
+ }
}
mindpulseDev {
dimension "mindpulse"
applicationIdSuffix ".mindpulse"
versionNameSuffix "-mindpulse"
buildConfigField "boolean", "MINDPULSE_ENABLED", "true"
+ // On-device VLM (llama.cpp) is dev-only; arm64 test devices only.
+ // NOTE: -march is forced at compile time (bypasses ggml runtime CPU
+ // dispatch). i8mm (ARMv8.6) removed — it SIGILLs on armv8.2 cores.
+ ndk {
+ abiFilters 'arm64-v8a'
+ }
+ externalNativeBuild {
+ cmake {
+ targets "vlm_bridge"
+ cppFlags "-std=c++17 -O3 -march=armv8.2-a+dotprod+fp16"
+ arguments "-DANDROID_STL=c++_shared", "-DVLM_ENABLED=ON"
+ }
+ }
}
}
signingConfigs {
@@ -54,6 +85,9 @@ android {
}
}
buildTypes {
+ debug {
+ signingConfig signingConfigs.debug
+ }
release {
minifyEnabled true
shrinkResources true
@@ -64,12 +98,23 @@ android {
}
}
kotlinOptions {
- jvmTarget = '1.8'
+ jvmTarget = '17'
}
packaging {
jniLibs {
useLegacyPackaging = false
+ // Placeholder target built by the standard flavor; never ship it.
+ excludes += "**/libvlm_noop.so"
+ }
+ }
+
+ testOptions {
+ unitTests {
+ // Robolectric (Encryptor/Logger tests) needs resources; plain JUnit
+ // tests (GlassPhotoAssembler) need android.util.Log to no-op.
+ includeAndroidResources = true
+ returnDefaultValues = true
}
}
@@ -91,6 +136,7 @@ dependencies {
implementation 'com.squareup.okhttp3:okhttp:4.11.0'
implementation 'com.journeyapps:zxing-android-embedded:4.3.0'
testImplementation 'junit:junit:4.13.2'
+ testImplementation 'org.robolectric:robolectric:4.12.2'
androidTestImplementation 'androidx.test.ext:junit:1.1.5'
androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
def camerax_version = "1.3.4"
@@ -109,6 +155,9 @@ dependencies {
implementation 'androidx.activity:activity:1.8.2'
implementation 'com.google.android.play:app-update:2.1.0'
+ // ML Kit OCR (on-device text recognition) - mindpulseDev only
+ mindpulseDevImplementation 'com.google.mlkit:text-recognition:16.0.1'
+
// Firebase temporarily disabled - uncomment after getting new google-services.json
// implementation platform('com.google.firebase:firebase-bom:30.1.0')
// implementation 'com.google.firebase:firebase-analytics'
@@ -182,7 +231,7 @@ def zipalignOutputsTask = tasks.register("zipalignArtifacts") {
}
tasks.configureEach { task ->
- if (task.name.startsWith("assemble") || task.name.startsWith("bundle")) {
+ if ((task.name.startsWith("assemble") || task.name.startsWith("bundle")) && !task.name.contains("Debug")) {
task.finalizedBy(zipalignOutputsTask)
}
}
diff --git a/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java b/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java
index 263fd0a..a217cfd 100644
--- a/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java
+++ b/app/src/androidTest/java/com/screenomics/ExampleInstrumentedTest.java
@@ -22,6 +22,8 @@ public void useAppContext() {
// Context of the app under test.
Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext();
- assertEquals("com.screenomics", appContext.getPackageName());
+ // applicationId is edu.wisc.chm.screenomics (+ ".mindpulse" for the mindpulseDev flavor),
+ // not the "com.screenomics" namespace — assert flavor-agnostically.
+ assertTrue(appContext.getPackageName().startsWith("edu.wisc.chm.screenomics"));
}
}
diff --git a/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java b/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java
new file mode 100644
index 0000000..c9dd9cd
--- /dev/null
+++ b/app/src/androidTest/java/com/screenomics/SecurityFixesTest.java
@@ -0,0 +1,126 @@
+package com.screenomics;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+
+import androidx.preference.PreferenceManager;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.platform.app.InstrumentationRegistry;
+
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import java.io.File;
+import java.io.FileOutputStream;
+
+import okhttp3.OkHttpClient;
+import okhttp3.Request;
+import okhttp3.Response;
+
+/**
+ * On-device runtime verification of the security fixes:
+ * #3 SecureStore (enrollment-token encryption at rest, round-trip, no plaintext, migration)
+ * #5 Logger.sweepStaleTempFiles (orphan temp cleanup)
+ * #1 TLS certificate pinning (real pinned handshake to the SSCC server)
+ */
+@RunWith(AndroidJUnit4.class)
+public class SecurityFixesTest {
+
+ private Context ctx() {
+ return InstrumentationRegistry.getInstrumentation().getTargetContext();
+ }
+
+ @After
+ public void cleanup() {
+ SecureStore.removeSecret(ctx(), "enrollment_token_test");
+ }
+
+ // ---- #3 SecureStore ----
+
+ @Test
+ public void secureStore_roundtrip_and_no_plaintext() {
+ Context c = ctx();
+ String secret = "tok-" + System.nanoTime();
+ SecureStore.putSecret(c, "enrollment_token_test", secret);
+
+ // round-trips correctly
+ assertEquals(secret, SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT"));
+
+ SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(c);
+ // no plaintext copy left behind
+ assertNull("plaintext token must not exist", p.getString("enrollment_token_test", null));
+ // an encrypted blob exists and is NOT the plaintext
+ String enc = p.getString("enrollment_token_test_enc", null);
+ assertNotNull("encrypted blob must exist", enc);
+ assertNotEquals("stored value must be ciphertext, not plaintext", secret, enc);
+
+ // removal clears everything
+ SecureStore.removeSecret(c, "enrollment_token_test");
+ assertEquals("DEFAULT", SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT"));
+ }
+
+ @Test
+ public void secureStore_migrates_legacy_plaintext() {
+ Context c = ctx();
+ SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(c);
+ // simulate an already-enrolled device with a legacy plaintext token
+ p.edit().putString("enrollment_token_test", "legacy-123").remove("enrollment_token_test_enc").apply();
+
+ // read migrates it to encrypted storage and returns the value
+ assertEquals("legacy-123", SecureStore.getSecret(c, "enrollment_token_test", "DEFAULT"));
+ assertNull("legacy plaintext must be removed after migration",
+ p.getString("enrollment_token_test", null));
+ assertNotNull("migrated encrypted blob must exist",
+ p.getString("enrollment_token_test_enc", null));
+ }
+
+ // ---- #5 temp-file sweep ----
+
+ @Test
+ public void sweep_deletes_stale_but_keeps_fresh() throws Exception {
+ Context c = ctx();
+ File dir = c.getExternalFilesDir(null);
+ assertNotNull(dir);
+ File stale = new File(dir, "tmp_test_stale.jpg");
+ File fresh = new File(dir, "tmp_test_fresh.jpg");
+ try (FileOutputStream f = new FileOutputStream(stale)) { f.write(new byte[]{1, 2, 3}); }
+ try (FileOutputStream f = new FileOutputStream(fresh)) { f.write(new byte[]{4}); }
+ assertTrue(stale.setLastModified(System.currentTimeMillis() - 10 * 60 * 1000L)); // 10 min old
+
+ Logger.sweepStaleTempFiles(c);
+
+ assertFalse("stale temp file should be swept", stale.exists());
+ assertTrue("fresh temp file should be kept", fresh.exists());
+ //noinspection ResultOfMethodCallIgnored
+ fresh.delete();
+ }
+
+ // ---- #1 TLS certificate pinning ----
+
+ @Test
+ public void tlsPinning_allows_real_server() {
+ // Runs in the app process => the app's network_security_config (cert pinning) applies.
+ // A successful HTTPS response proves the pinned TLS handshake to the real server works.
+ OkHttpClient client = new OkHttpClient();
+ Request req = new Request.Builder()
+ .url("https://mindpulse.ssc.wisc.edu/api/v1/health").get().build();
+ try (Response resp = client.newCall(req).execute()) {
+ assertTrue("pinned TLS handshake succeeded, HTTP code=" + resp.code(), resp.code() > 0);
+ System.out.println("PINNING TEST: reached server through pinned TLS, HTTP " + resp.code());
+ } catch (javax.net.ssl.SSLPeerUnverifiedException e) {
+ fail("TLS pinning REJECTED the real server (pins are wrong): " + e.getMessage());
+ } catch (java.io.IOException e) {
+ // DNS/timeout/offline — inconclusive for pinning, not a pinning failure.
+ System.out.println("PINNING TEST inconclusive (network issue, not pinning): " + e);
+ }
+ }
+}
diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml
index 0182ffb..e37fd00 100644
--- a/app/src/main/AndroidManifest.xml
+++ b/app/src/main/AndroidManifest.xml
@@ -87,6 +87,7 @@
diff --git a/app/src/main/cpp/CMakeLists.txt b/app/src/main/cpp/CMakeLists.txt
new file mode 100644
index 0000000..a3f7423
--- /dev/null
+++ b/app/src/main/cpp/CMakeLists.txt
@@ -0,0 +1,116 @@
+cmake_minimum_required(VERSION 3.22.1)
+project("screenomics-vlm" C CXX)
+
+# The VLM stack (llama.cpp + mtmd + JNI bridge) is only built for the
+# mindpulseDev flavor, which passes -DVLM_ENABLED=ON. The standard flavor
+# builds only the vlm_noop placeholder so it never needs the llama.cpp
+# checkout or the heavy native toolchain.
+option(VLM_ENABLED "Build the on-device VLM native stack" OFF)
+
+if(NOT VLM_ENABLED)
+ add_library(vlm_noop SHARED noop.cpp)
+ return()
+endif()
+
+if(NOT EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/CMakeLists.txt)
+ message(FATAL_ERROR "VLM_ENABLED=ON but the llama.cpp submodule is missing at "
+ "${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp — run "
+ "'git submodule update --init' before building the mindpulseDev flavor.")
+endif()
+
+# Disable everything we don't need from llama.cpp
+set(LLAMA_BUILD_TESTS OFF CACHE BOOL "" FORCE)
+set(LLAMA_BUILD_EXAMPLES OFF CACHE BOOL "" FORCE)
+set(LLAMA_BUILD_SERVER OFF CACHE BOOL "" FORCE)
+set(LLAMA_BUILD_TOOLS OFF CACHE BOOL "" FORCE)
+set(LLAMA_BUILD_COMMON OFF CACHE BOOL "" FORCE)
+set(LLAMA_CURL OFF CACHE BOOL "" FORCE)
+set(LLAMA_OPENSSL OFF CACHE BOOL "" FORCE)
+set(GGML_OPENMP OFF CACHE BOOL "" FORCE)
+set(GGML_METAL OFF CACHE BOOL "" FORCE)
+set(GGML_CUDA OFF CACHE BOOL "" FORCE)
+# To enable Mali GPU (Vulkan) offload of the ViT: flip this to ON, set use_gpu=true
+# (already toggleable via nativeSetUseGpu), AND provide a HOST C++ compiler so the
+# vulkan-shaders-gen tool can be built (the NDK clang defaults to x86_64-w64-windows-gnu
+# but the machine has no MinGW/MSVC sysroot, so linking a host .exe fails). glslc itself
+# ships in the NDK at ${CMAKE_ANDROID_NDK}/shader-tools/windows-x86_64/glslc.exe.
+set(GGML_VULKAN OFF CACHE BOOL "" FORCE)
+set(BUILD_SHARED_LIBS OFF CACHE BOOL "" FORCE)
+set(LLAMA_TOOLS_INSTALL OFF CACHE BOOL "" FORCE)
+
+# ARM NEON + dotprod optimizations (critical for performance).
+# i8mm removed: it is an ARMv8.6 extension and SIGILLs on armv8.2 cores
+# (Cortex-A55/A75 era); this forced -march bypasses ggml's runtime dispatch.
+set(GGML_CPU_ARM_ARCH "armv8.2-a+dotprod+fp16" CACHE STRING "" FORCE)
+
+# Set version variable that mtmd expects
+if (NOT DEFINED LLAMA_INSTALL_VERSION)
+ set(LLAMA_INSTALL_VERSION "0.0.0")
+endif()
+
+# Build core llama.cpp (ggml + llama libraries)
+add_subdirectory(llama.cpp)
+
+# Build mtmd (multimodal) library manually
+set(MTMD_DIR ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/tools/mtmd)
+add_library(mtmd STATIC
+ ${MTMD_DIR}/mtmd.cpp
+ ${MTMD_DIR}/mtmd-audio.cpp
+ ${MTMD_DIR}/mtmd-image.cpp
+ ${MTMD_DIR}/mtmd-helper.cpp
+ ${MTMD_DIR}/clip.cpp
+ ${MTMD_DIR}/models/cogvlm.cpp
+ ${MTMD_DIR}/models/conformer.cpp
+ ${MTMD_DIR}/models/dotsocr.cpp
+ ${MTMD_DIR}/models/gemma4v.cpp
+ ${MTMD_DIR}/models/glm4v.cpp
+ ${MTMD_DIR}/models/hunyuanocr.cpp
+ ${MTMD_DIR}/models/internvl.cpp
+ ${MTMD_DIR}/models/kimivl.cpp
+ ${MTMD_DIR}/models/kimik25.cpp
+ ${MTMD_DIR}/models/nemotron-v2-vl.cpp
+ ${MTMD_DIR}/models/llama4.cpp
+ ${MTMD_DIR}/models/llava.cpp
+ ${MTMD_DIR}/models/minicpmv.cpp
+ ${MTMD_DIR}/models/paddleocr.cpp
+ ${MTMD_DIR}/models/pixtral.cpp
+ ${MTMD_DIR}/models/qwen2vl.cpp
+ ${MTMD_DIR}/models/qwen3vl.cpp
+ ${MTMD_DIR}/models/step3vl.cpp
+ ${MTMD_DIR}/models/siglip.cpp
+ ${MTMD_DIR}/models/whisper-enc.cpp
+ ${MTMD_DIR}/models/deepseekocr.cpp
+ ${MTMD_DIR}/models/mobilenetv5.cpp
+ ${MTMD_DIR}/models/youtuvl.cpp
+)
+
+target_link_libraries(mtmd PUBLIC ggml llama)
+target_include_directories(mtmd PUBLIC ${MTMD_DIR})
+target_include_directories(mtmd PRIVATE
+ ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp
+ ${CMAKE_CURRENT_SOURCE_DIR}/llama.cpp/vendor
+)
+target_compile_features(mtmd PRIVATE cxx_std_17)
+
+find_package(Threads REQUIRED)
+target_link_libraries(mtmd PRIVATE Threads::Threads)
+
+# JNI bridge library
+add_library(vlm_bridge SHARED vlm_bridge.cpp)
+
+target_link_libraries(vlm_bridge
+ PRIVATE
+ llama
+ ggml
+ mtmd
+ android
+ log
+)
+
+target_include_directories(vlm_bridge PRIVATE
+ llama.cpp/include
+ llama.cpp/ggml/include
+ ${MTMD_DIR}
+)
+
+target_compile_features(vlm_bridge PRIVATE cxx_std_17)
diff --git a/app/src/main/cpp/llama.cpp b/app/src/main/cpp/llama.cpp
new file mode 160000
index 0000000..d6f3030
--- /dev/null
+++ b/app/src/main/cpp/llama.cpp
@@ -0,0 +1 @@
+Subproject commit d6f3030047f85a98b009189e76f441fe818ea44d
diff --git a/app/src/main/cpp/noop.cpp b/app/src/main/cpp/noop.cpp
new file mode 100644
index 0000000..57d45ac
--- /dev/null
+++ b/app/src/main/cpp/noop.cpp
@@ -0,0 +1,5 @@
+// Placeholder translation unit for the standard (production) flavor.
+// The real VLM native stack (vlm_bridge + llama.cpp) is only built when
+// -DVLM_ENABLED=ON is passed by the mindpulseDev flavor. The resulting
+// libvlm_noop.so is excluded from packaging in build.gradle.
+extern "C" int screenomics_vlm_noop(void) { return 0; }
diff --git a/app/src/main/cpp/vlm_bridge.cpp b/app/src/main/cpp/vlm_bridge.cpp
new file mode 100644
index 0000000..4d8b2ab
--- /dev/null
+++ b/app/src/main/cpp/vlm_bridge.cpp
@@ -0,0 +1,547 @@
+#include
+#include
+#include
+#include
+#include
+
+#include "llama.h"
+#include "mtmd.h"
+#include "mtmd-helper.h"
+
+#define LOG_TAG "VLM_BRIDGE"
+#define LOGI(...) __android_log_print(ANDROID_LOG_INFO, LOG_TAG, __VA_ARGS__)
+#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR, LOG_TAG, __VA_ARGS__)
+
+// Global state — single model loaded at a time. All access is serialized by
+// g_mutex: load/unload/inference can be called from different Java threads
+// (worker thread restarts, headless benchmark threads) and must never overlap.
+static std::mutex g_mutex;
+static llama_model * g_model = nullptr;
+static llama_context * g_ctx = nullptr;
+static mtmd_context * g_mtmd = nullptr;
+static llama_sampler * g_sampler = nullptr;
+static bool g_use_gpu = false; // toggled via nativeSetUseGpu(); offloads the ViT (mtmd) to a GPU backend when a GPU backend is compiled in (GGML_VULKAN)
+static bool g_think = false; // toggled via nativeSetThinking(); when false, prefill an empty so the model skips reasoning
+
+// Timing stats
+static double g_last_load_ms = 0.0;
+static double g_last_encode_ms = 0.0; // vision encoder
+static double g_last_decode_ms = 0.0; // text generation
+static double g_last_total_ms = 0.0;
+static int g_last_n_tokens = 0;
+static int g_last_n_embd = 0;
+static long g_peak_mem_bytes = 0;
+
+static double get_time_ms() {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return ts.tv_sec * 1000.0 + ts.tv_nsec / 1e6;
+}
+
+// Free all model state. Caller must hold g_mutex.
+static void unload_locked() {
+ if (g_sampler) { llama_sampler_free(g_sampler); g_sampler = nullptr; }
+ if (g_mtmd) { mtmd_free(g_mtmd); g_mtmd = nullptr; }
+ if (g_ctx) { llama_free(g_ctx); g_ctx = nullptr; }
+ if (g_model) { llama_model_free(g_model); g_model = nullptr; }
+ llama_backend_free();
+}
+
+// NewStringUTF requires *modified* UTF-8 and aborts under CheckJNI on 4-byte
+// sequences (emoji are common in screen-derived model output). Build the
+// String from raw bytes + charset instead.
+static jstring make_jstring_utf8(JNIEnv *env, const std::string &s) {
+ jbyteArray bytes = env->NewByteArray((jsize)s.size());
+ if (!bytes) return nullptr;
+ env->SetByteArrayRegion(bytes, 0, (jsize)s.size(), (const jbyte *)s.data());
+ jclass cls = env->FindClass("java/lang/String");
+ jmethodID ctor = env->GetMethodID(cls, "", "([BLjava/lang/String;)V");
+ jstring charset = env->NewStringUTF("UTF-8");
+ jstring out = (jstring)env->NewObject(cls, ctor, bytes, charset);
+ env->DeleteLocalRef(bytes);
+ env->DeleteLocalRef(charset);
+ env->DeleteLocalRef(cls);
+ return out;
+}
+
+extern "C" {
+
+JNIEXPORT jboolean JNICALL
+Java_com_screenomics_VlmBridge_nativeLoadModel(
+ JNIEnv *env, jclass clazz,
+ jstring model_path_j, jstring mmproj_path_j,
+ jint n_threads, jint n_ctx)
+{
+ std::lock_guard lock(g_mutex);
+
+ const char *model_path = model_path_j ? env->GetStringUTFChars(model_path_j, nullptr) : nullptr;
+ if (!model_path) { LOGE("nativeLoadModel: null model path"); return JNI_FALSE; }
+ const char *mmproj_path = mmproj_path_j ? env->GetStringUTFChars(mmproj_path_j, nullptr) : nullptr;
+
+ LOGI("Loading model: %s", model_path);
+ LOGI("mmproj: %s", mmproj_path ? mmproj_path : "(unified/none)");
+ LOGI("threads=%d ctx=%d", n_threads, n_ctx);
+
+ // A second load without an unload must not leak the previous model
+ // (hundreds of MB of native memory).
+ if (g_model || g_ctx || g_mtmd || g_sampler) {
+ LOGI("Model already loaded; unloading previous instance first");
+ unload_locked();
+ }
+
+ double t0 = get_time_ms();
+
+ // Initialize llama backend
+ llama_backend_init();
+
+ // Load model
+ llama_model_params model_params = llama_model_default_params();
+ model_params.n_gpu_layers = 0; // CPU only on Android
+ g_model = llama_model_load_from_file(model_path, model_params);
+ if (!g_model) {
+ LOGE("Failed to load model from %s", model_path);
+ env->ReleaseStringUTFChars(model_path_j, model_path);
+ if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path);
+ return JNI_FALSE;
+ }
+
+ // Create context
+ llama_context_params ctx_params = llama_context_default_params();
+ ctx_params.n_ctx = n_ctx > 0 ? n_ctx : 2048;
+ ctx_params.n_threads = n_threads > 0 ? n_threads : 4;
+ // n_batch must cover the largest single prompt submission: nativeInferText
+ // decodes the whole prompt in one llama_batch, so a batch smaller than
+ // n_ctx makes prompts over that size fail outright.
+ ctx_params.n_batch = ctx_params.n_ctx;
+ g_ctx = llama_init_from_model(g_model, ctx_params);
+ if (!g_ctx) {
+ LOGE("Failed to create llama context");
+ llama_model_free(g_model);
+ g_model = nullptr;
+ env->ReleaseStringUTFChars(model_path_j, model_path);
+ if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path);
+ return JNI_FALSE;
+ }
+
+ // Initialize multimodal context (for vision)
+ if (mmproj_path) {
+ mtmd_context_params mtmd_params = mtmd_context_params_default();
+ mtmd_params.use_gpu = g_use_gpu; // ViT on GPU when true + a GPU backend is compiled in
+ mtmd_params.n_threads = n_threads > 0 ? n_threads : 4;
+ mtmd_params.warmup = false; // skip warmup to speed up load
+ g_mtmd = mtmd_init_from_file(mmproj_path, g_model, mtmd_params);
+ if (!g_mtmd) {
+ LOGE("Failed to create mtmd context from %s", mmproj_path);
+ // Continue without multimodal — text-only mode
+ } else {
+ LOGI("Multimodal context created, vision=%d", mtmd_support_vision(g_mtmd));
+ }
+ }
+
+ // Create sampler (greedy for benchmark — we just want any output)
+ g_sampler = llama_sampler_chain_init(llama_sampler_chain_default_params());
+ llama_sampler_chain_add(g_sampler, llama_sampler_init_greedy());
+
+ double t1 = get_time_ms();
+ g_last_load_ms = t1 - t0;
+ LOGI("Model loaded in %.1f ms", g_last_load_ms);
+
+ env->ReleaseStringUTFChars(model_path_j, model_path);
+ if (mmproj_path) env->ReleaseStringUTFChars(mmproj_path_j, mmproj_path);
+ return JNI_TRUE;
+}
+
+JNIEXPORT jstring JNICALL
+Java_com_screenomics_VlmBridge_nativeInferImage(
+ JNIEnv *env, jclass clazz,
+ jbyteArray image_data_j, jint width, jint height,
+ jstring prompt_j, jint max_tokens)
+{
+ std::lock_guard lock(g_mutex);
+
+ if (!g_model || !g_ctx) {
+ LOGE("Model not loaded");
+ return env->NewStringUTF("");
+ }
+
+ double t_start = get_time_ms();
+
+ const char *prompt_str = prompt_j ? env->GetStringUTFChars(prompt_j, nullptr) : nullptr;
+ if (!prompt_str) { LOGE("nativeInferImage: null prompt"); return env->NewStringUTF(""); }
+ jbyte *image_bytes = image_data_j ? env->GetByteArrayElements(image_data_j, nullptr) : nullptr;
+ jsize image_len = image_data_j ? env->GetArrayLength(image_data_j) : 0;
+
+ std::string result_text;
+ double t_encode = 0, t_decode = 0;
+
+ // Clear KV cache for fresh inference
+ llama_memory_clear(llama_get_memory(g_ctx), true);
+
+ if (g_mtmd && image_bytes && image_len > 0) {
+ // --- Multimodal path: image + text ---
+ double t0 = get_time_ms();
+
+ // Create bitmap from raw RGB data (width * height * 3 bytes)
+ // If image_data is JPEG/PNG bytes, use helper instead
+ mtmd_bitmap *bitmap = mtmd_helper_bitmap_init_from_buf(
+ g_mtmd, (const unsigned char *)image_bytes, image_len);
+
+ if (!bitmap) {
+ LOGE("Failed to create bitmap from image data (%d bytes)", image_len);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return env->NewStringUTF("");
+ }
+
+ // Build prompt with the Qwen-VL chat template (marker = where the image goes).
+ // Without the chat template the instruct model emits EOS immediately (0 tokens).
+ // When g_think is false, prefill an empty so the (Qwen3) model
+ // skips reasoning and answers directly (saves decode tokens/time). When true, let
+ // it reason first (may help on hard/low-quality frames, at higher token cost).
+ std::string assistant_prefix = g_think
+ ? "<|im_start|>assistant\n"
+ : "<|im_start|>assistant\n\n\n\n\n";
+ std::string full_prompt =
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
+ "<|im_start|>user\n" + std::string(mtmd_default_marker()) + prompt_str + "<|im_end|>\n"
+ + assistant_prefix;
+
+ // Tokenize text + image
+ mtmd_input_chunks *chunks = mtmd_input_chunks_init();
+ mtmd_input_text input_text;
+ input_text.text = full_prompt.c_str();
+ input_text.add_special = true;
+ input_text.parse_special = true;
+
+ const mtmd_bitmap *bitmap_ptr = bitmap;
+ int32_t tok_result = mtmd_tokenize(g_mtmd, chunks, &input_text, &bitmap_ptr, 1);
+ if (tok_result != 0) {
+ LOGE("mtmd_tokenize failed: %d", tok_result);
+ mtmd_bitmap_free(bitmap);
+ mtmd_input_chunks_free(chunks);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return env->NewStringUTF("");
+ }
+
+ double t1 = get_time_ms();
+
+ // Eval all chunks (vision encode + text tokens)
+ llama_pos n_past = 0;
+ int32_t eval_result = mtmd_helper_eval_chunks(
+ g_mtmd, g_ctx, chunks, n_past, 0, 512, true, &n_past);
+
+ double t2 = get_time_ms();
+ t_encode = t2 - t0;
+
+ mtmd_bitmap_free(bitmap);
+ mtmd_input_chunks_free(chunks);
+
+ if (eval_result != 0) {
+ LOGE("mtmd_helper_eval_chunks failed: %d", eval_result);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return env->NewStringUTF("");
+ }
+
+ // Generate text tokens
+ double t_dec_start = get_time_ms();
+ int n_generated = 0;
+ const llama_vocab *vocab = llama_model_get_vocab(g_model);
+
+ for (int i = 0; i < max_tokens; i++) {
+ llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1);
+
+ if (llama_vocab_is_eog(vocab, token)) break;
+
+ char buf[256];
+ int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
+ if (n > 0) {
+ result_text.append(buf, n);
+ }
+ n_generated++;
+
+ // Prepare next decode
+ llama_batch batch = llama_batch_get_one(&token, 1);
+ if (llama_decode(g_ctx, batch) != 0) {
+ LOGE("llama_decode failed at token %d", i);
+ break;
+ }
+ }
+ t_decode = get_time_ms() - t_dec_start;
+ g_last_n_tokens = n_generated;
+
+ } else {
+ // --- Text-only path (fallback if no mmproj) ---
+ double t0 = get_time_ms();
+
+ const llama_vocab *vocab = llama_model_get_vocab(g_model);
+ std::string full_prompt = std::string(prompt_str);
+
+ // Tokenize
+ std::vector tokens(full_prompt.size() + 64);
+ int n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(),
+ tokens.data(), tokens.size(), true, true);
+ if (n_tokens < 0) {
+ tokens.resize(-n_tokens);
+ n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(),
+ tokens.data(), tokens.size(), true, true);
+ }
+ tokens.resize(n_tokens);
+
+ // Eval prompt
+ llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size());
+ if (llama_decode(g_ctx, batch) != 0) {
+ LOGE("llama_decode (prompt) failed");
+ if (image_bytes) env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return env->NewStringUTF("");
+ }
+
+ t_encode = get_time_ms() - t0;
+
+ // Generate
+ double t_dec_start = get_time_ms();
+ int n_generated = 0;
+ for (int i = 0; i < max_tokens; i++) {
+ llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1);
+ if (llama_vocab_is_eog(vocab, token)) break;
+
+ char buf[256];
+ int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
+ if (n > 0) result_text.append(buf, n);
+ n_generated++;
+
+ llama_batch next_batch = llama_batch_get_one(&token, 1);
+ if (llama_decode(g_ctx, next_batch) != 0) break;
+ }
+ t_decode = get_time_ms() - t_dec_start;
+ g_last_n_tokens = n_generated;
+ }
+
+ double t_total = get_time_ms() - t_start;
+
+ g_last_encode_ms = t_encode;
+ g_last_decode_ms = t_decode;
+ g_last_total_ms = t_total;
+
+ LOGI("Inference done: encode=%.0fms decode=%.0fms total=%.0fms tokens=%d (%.1f tok/s)",
+ t_encode, t_decode, t_total, g_last_n_tokens,
+ g_last_n_tokens > 0 ? g_last_n_tokens / (t_decode / 1000.0) : 0);
+
+ if (image_bytes) env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+
+ return make_jstring_utf8(env, result_text);
+}
+
+// Vision-encoder-only path: runs the image through the mmproj (ViT + projector)
+// and returns the projected embedding tokens WITHOUT any LLM decode. This is the
+// "extract embedding on device" cost the caller cares about. If return_embd is
+// false, we skip the (potentially multi-MB) float copy and only record timing —
+// used for the sustained power benchmark loop.
+JNIEXPORT jfloatArray JNICALL
+Java_com_screenomics_VlmBridge_nativeEncodeImageToEmbedding(
+ JNIEnv *env, jclass clazz, jbyteArray image_data_j, jboolean return_embd)
+{
+ std::lock_guard lock(g_mutex);
+
+ if (!g_mtmd || !g_model) {
+ LOGE("nativeEncodeImageToEmbedding: mtmd/model not loaded");
+ return nullptr;
+ }
+ if (!image_data_j) { LOGE("nativeEncodeImageToEmbedding: null image"); return nullptr; }
+
+ jbyte *image_bytes = env->GetByteArrayElements(image_data_j, nullptr);
+ if (!image_bytes) { LOGE("nativeEncodeImageToEmbedding: pin failed"); return nullptr; }
+ jsize image_len = env->GetArrayLength(image_data_j);
+
+ double t0 = get_time_ms();
+
+ mtmd_bitmap *bitmap = mtmd_helper_bitmap_init_from_buf(
+ g_mtmd, (const unsigned char *)image_bytes, image_len);
+ if (!bitmap) {
+ LOGE("nativeEncodeImageToEmbedding: bitmap init failed (%d bytes)", image_len);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ return nullptr;
+ }
+
+ std::string marker = std::string(mtmd_default_marker());
+ mtmd_input_chunks *chunks = mtmd_input_chunks_init();
+ mtmd_input_text input_text;
+ input_text.text = marker.c_str();
+ input_text.add_special = false;
+ input_text.parse_special = true;
+
+ const mtmd_bitmap *bp = bitmap;
+ int32_t tr = mtmd_tokenize(g_mtmd, chunks, &input_text, &bp, 1);
+ if (tr != 0) {
+ LOGE("nativeEncodeImageToEmbedding: mtmd_tokenize failed: %d", tr);
+ mtmd_bitmap_free(bitmap);
+ mtmd_input_chunks_free(chunks);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+ return nullptr;
+ }
+
+ double t_enc0 = get_time_ms(); // after jpeg-decode + preprocess + tokenize
+ jfloatArray result = nullptr;
+ size_t n_chunks = mtmd_input_chunks_size(chunks);
+ for (size_t i = 0; i < n_chunks; i++) {
+ const mtmd_input_chunk *ch = mtmd_input_chunks_get(chunks, i);
+ if (mtmd_input_chunk_get_type(ch) != MTMD_INPUT_CHUNK_TYPE_IMAGE) continue;
+
+ int32_t enc = mtmd_encode_chunk(g_mtmd, ch); // <-- the ViT forward pass
+ if (enc != 0) { LOGE("mtmd_encode_chunk failed: %d", enc); continue; }
+
+ size_t n_tokens = mtmd_input_chunk_get_n_tokens(ch);
+ int n_embd = llama_model_n_embd_inp(g_model);
+ g_last_n_tokens = (int)n_tokens;
+ g_last_n_embd = n_embd;
+
+ if (return_embd) {
+ float *embd = mtmd_get_output_embd(g_mtmd);
+ size_t total = n_tokens * (size_t)n_embd;
+ if (result) env->DeleteLocalRef(result); // multi-chunk: drop the previous array
+ result = env->NewFloatArray((jsize)total);
+ if (!result) {
+ // allocation failed (pending OutOfMemoryError) — clear it and
+ // return null instead of continuing with an exception pending
+ env->ExceptionClear();
+ LOGE("nativeEncodeImageToEmbedding: NewFloatArray(%zu) failed", total);
+ continue;
+ }
+ if (embd) env->SetFloatArrayRegion(result, 0, (jsize)total, embd);
+ }
+ }
+ double t1 = get_time_ms();
+ g_last_encode_ms = t1 - t_enc0; // pure vision-encode (ViT+projector) time
+ g_last_total_ms = t1 - t0; // incl. jpeg decode + preprocess + tokenize
+
+ mtmd_bitmap_free(bitmap);
+ mtmd_input_chunks_free(chunks);
+ env->ReleaseByteArrayElements(image_data_j, image_bytes, JNI_ABORT);
+
+ LOGI("encode-only: n_tokens=%d n_embd=%d encode=%.1fms total(incl.preproc)=%.1fms",
+ g_last_n_tokens, g_last_n_embd, g_last_encode_ms, g_last_total_ms);
+ return result;
+}
+
+JNIEXPORT jdoubleArray JNICALL
+Java_com_screenomics_VlmBridge_nativeGetStats(JNIEnv *env, jclass clazz) {
+ // Returns: [load_ms, encode_ms, decode_ms, total_ms, n_tokens, tok_per_sec]
+ jdoubleArray arr = env->NewDoubleArray(6);
+ double vals[6] = {
+ g_last_load_ms,
+ g_last_encode_ms,
+ g_last_decode_ms,
+ g_last_total_ms,
+ (double)g_last_n_tokens,
+ g_last_n_tokens > 0 ? g_last_n_tokens / (g_last_decode_ms / 1000.0) : 0
+ };
+ env->SetDoubleArrayRegion(arr, 0, 6, vals);
+ return arr;
+}
+
+JNIEXPORT void JNICALL
+Java_com_screenomics_VlmBridge_nativeUnloadModel(JNIEnv *env, jclass clazz) {
+ std::lock_guard lock(g_mutex);
+ LOGI("Unloading model...");
+ unload_locked();
+ LOGI("Model unloaded");
+}
+
+JNIEXPORT jboolean JNICALL
+Java_com_screenomics_VlmBridge_nativeIsLoaded(JNIEnv *env, jclass clazz) {
+ std::lock_guard lock(g_mutex);
+ return (g_model != nullptr && g_ctx != nullptr) ? JNI_TRUE : JNI_FALSE;
+}
+
+JNIEXPORT void JNICALL
+Java_com_screenomics_VlmBridge_nativeSetUseGpu(JNIEnv *env, jclass clazz, jboolean use_gpu) {
+ g_use_gpu = (use_gpu == JNI_TRUE);
+ LOGI("nativeSetUseGpu: %d", (int)g_use_gpu);
+}
+
+JNIEXPORT void JNICALL
+Java_com_screenomics_VlmBridge_nativeSetThinking(JNIEnv *env, jclass clazz, jboolean think) {
+ g_think = (think == JNI_TRUE);
+ LOGI("nativeSetThinking: %d", (int)g_think);
+}
+
+JNIEXPORT jstring JNICALL
+Java_com_screenomics_VlmBridge_nativeInferText(
+ JNIEnv *env, jclass clazz,
+ jstring prompt_j, jint max_tokens)
+{
+ std::lock_guard lock(g_mutex);
+
+ if (!g_model || !g_ctx) {
+ LOGE("Model not loaded");
+ return env->NewStringUTF("");
+ }
+
+ double t_start = get_time_ms();
+ const char *prompt_str = prompt_j ? env->GetStringUTFChars(prompt_j, nullptr) : nullptr;
+ if (!prompt_str) { LOGE("nativeInferText: null prompt"); return env->NewStringUTF(""); }
+
+ // Clear KV cache
+ llama_memory_clear(llama_get_memory(g_ctx), true);
+
+ const llama_vocab *vocab = llama_model_get_vocab(g_model);
+ std::string full_prompt(prompt_str);
+
+ // Tokenize
+ std::vector tokens(full_prompt.size() + 128);
+ int n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(),
+ tokens.data(), tokens.size(), true, true);
+ if (n_tokens < 0) {
+ tokens.resize(-n_tokens);
+ n_tokens = llama_tokenize(vocab, full_prompt.c_str(), full_prompt.size(),
+ tokens.data(), tokens.size(), true, true);
+ }
+ tokens.resize(n_tokens);
+
+ double t_tok = get_time_ms();
+ LOGI("Tokenized %d tokens in %.0fms", n_tokens, t_tok - t_start);
+
+ // Eval prompt
+ llama_batch batch = llama_batch_get_one(tokens.data(), tokens.size());
+ if (llama_decode(g_ctx, batch) != 0) {
+ LOGE("llama_decode (prompt) failed");
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return env->NewStringUTF("");
+ }
+
+ double t_prompt = get_time_ms();
+ g_last_encode_ms = t_prompt - t_start;
+
+ // Generate
+ std::string result_text;
+ int n_generated = 0;
+ for (int i = 0; i < max_tokens; i++) {
+ llama_token token = llama_sampler_sample(g_sampler, g_ctx, -1);
+ if (llama_vocab_is_eog(vocab, token)) break;
+
+ char buf[256];
+ int n = llama_token_to_piece(vocab, token, buf, sizeof(buf), 0, true);
+ if (n > 0) result_text.append(buf, n);
+ n_generated++;
+
+ llama_batch next_batch = llama_batch_get_one(&token, 1);
+ if (llama_decode(g_ctx, next_batch) != 0) break;
+ }
+
+ double t_end = get_time_ms();
+ g_last_decode_ms = t_end - t_prompt;
+ g_last_total_ms = t_end - t_start;
+ g_last_n_tokens = n_generated;
+
+ LOGI("Text inference done: prompt=%.0fms decode=%.0fms total=%.0fms tokens=%d (%.1f tok/s)",
+ g_last_encode_ms, g_last_decode_ms, g_last_total_ms, n_generated,
+ n_generated > 0 ? n_generated / (g_last_decode_ms / 1000.0) : 0);
+
+ env->ReleaseStringUTFChars(prompt_j, prompt_str);
+ return make_jstring_utf8(env, result_text);
+}
+
+} // extern "C"
diff --git a/app/src/main/java/com/screenomics/A11yState.java b/app/src/main/java/com/screenomics/A11yState.java
new file mode 100644
index 0000000..7655c0c
--- /dev/null
+++ b/app/src/main/java/com/screenomics/A11yState.java
@@ -0,0 +1,64 @@
+package com.screenomics;
+
+import android.accessibilityservice.AccessibilityServiceInfo;
+import android.content.ComponentName;
+import android.content.Context;
+import android.content.Intent;
+import android.provider.Settings;
+import android.view.accessibility.AccessibilityManager;
+
+import java.util.List;
+
+/**
+ * API-safe status and bridge for {@link AccessibilityCaptureService}.
+ *
+ * The service class carries a class-level {@code @RequiresApi(R)} (its capture
+ * path uses API 30 APIs), which made every call to its harmless static helpers
+ * a NewApi lint error at API-29-reachable call sites. Everything here uses
+ * API-1-era APIs only and is safe to call on any supported device; the service
+ * writes the status flags, everyone else reads them through this class.
+ */
+public final class A11yState {
+
+ private A11yState() {}
+
+ /** Written by AccessibilityCaptureService on connect/destroy. */
+ static volatile boolean serviceConnected = false;
+ /** Written by AccessibilityCaptureService as capture starts/stops. */
+ static volatile boolean captureActive = false;
+
+ /** VLM benchmark hook (mindpulseDev only) — static because an
+ * AccessibilityService cannot be bound. */
+ static volatile VlmBenchmark vlmBenchmark;
+
+ public static void setVlmBenchmark(VlmBenchmark benchmark) {
+ vlmBenchmark = benchmark;
+ }
+
+ public static boolean isCaptureRunning() {
+ return serviceConnected && captureActive;
+ }
+
+ /** True when the MindPulse accessibility service is enabled in system settings. */
+ public static boolean isServiceEnabled(Context context) {
+ AccessibilityManager accessibilityManager =
+ (AccessibilityManager) context.getSystemService(Context.ACCESSIBILITY_SERVICE);
+ if (accessibilityManager == null) return false;
+ List enabledServices =
+ accessibilityManager.getEnabledAccessibilityServiceList(
+ AccessibilityServiceInfo.FEEDBACK_ALL_MASK
+ );
+ ComponentName componentName = new ComponentName(context, AccessibilityCaptureService.class);
+ String expectedId = componentName.flattenToShortString();
+ for (AccessibilityServiceInfo info : enabledServices) {
+ if (expectedId.equals(info.getId())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static Intent buildAccessibilitySettingsIntent() {
+ return new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS);
+ }
+}
diff --git a/app/src/main/java/com/screenomics/AccessibilityCaptureService.java b/app/src/main/java/com/screenomics/AccessibilityCaptureService.java
index e02f118..4331c90 100644
--- a/app/src/main/java/com/screenomics/AccessibilityCaptureService.java
+++ b/app/src/main/java/com/screenomics/AccessibilityCaptureService.java
@@ -59,8 +59,8 @@ public class AccessibilityCaptureService extends AccessibilityService {
private static final long MIN_FREE_SPACE_BYTES = 200L * 1024L * 1024L;
private static final long LOW_STORAGE_LOG_THROTTLE_MS = 60_000L;
- private static volatile boolean serviceConnected = false;
- private static volatile boolean captureActive = false;
+ // Status flags and the VLM hook live in A11yState so API-29-reachable code
+ // can read them without tripping NewApi lint on this @RequiresApi(R) class.
private final Handler handler = new Handler(Looper.getMainLooper());
private final ExecutorService ioExecutor = Executors.newSingleThreadExecutor();
@@ -128,6 +128,14 @@ public void onSuccess(@NonNull ScreenshotResult screenshotResult) {
Bitmap finalBitmap = bitmap;
String foregroundApp = getForegroundApp();
Log.d(TAG, "Screenshot captured | foreground_app=" + foregroundApp);
+
+ // VLM benchmark: submit a copy with context
+ VlmBenchmark vlm = A11yState.vlmBenchmark;
+ if (vlm != null && vlm.isRunning()) {
+ Bitmap copy = finalBitmap.copy(Bitmap.Config.ARGB_8888, false);
+ if (copy != null) vlm.submitFrame(copy, foregroundApp, 0, 0);
+ }
+
ioExecutor.execute(() -> {
try {
boolean imageSaved = encryptImage(finalBitmap, "image", foregroundApp);
@@ -162,7 +170,7 @@ protected void onServiceConnected() {
super.onServiceConnected();
prefs = PreferenceManager.getDefaultSharedPreferences(this);
prefs.registerOnSharedPreferenceChangeListener(prefChangeListener);
- serviceConnected = true;
+ A11yState.serviceConnected = true;
updateCaptureState();
Logger.i(getApplicationContext(), "AccessibilityCaptureService connected");
Log.i(TAG, "Accessibility capture service connected");
@@ -179,8 +187,8 @@ public void onInterrupt() {
@Override
public boolean onUnbind(Intent intent) {
stopCaptureLoop();
- serviceConnected = false;
- captureActive = false;
+ A11yState.serviceConnected = false;
+ A11yState.captureActive = false;
if (prefs != null) {
prefs.unregisterOnSharedPreferenceChangeListener(prefChangeListener);
}
@@ -191,43 +199,20 @@ public boolean onUnbind(Intent intent) {
public void onDestroy() {
super.onDestroy();
stopCaptureLoop();
- serviceConnected = false;
- captureActive = false;
+ A11yState.serviceConnected = false;
+ A11yState.captureActive = false;
if (prefs != null) {
prefs.unregisterOnSharedPreferenceChangeListener(prefChangeListener);
}
ioExecutor.shutdownNow();
}
- public static boolean isServiceEnabled(Context context) {
- AccessibilityManager accessibilityManager =
- (AccessibilityManager) context.getSystemService(Context.ACCESSIBILITY_SERVICE);
- if (accessibilityManager == null) return false;
- List enabledServices =
- accessibilityManager.getEnabledAccessibilityServiceList(
- AccessibilityServiceInfo.FEEDBACK_ALL_MASK
- );
- ComponentName componentName = new ComponentName(context, AccessibilityCaptureService.class);
- String expectedId = componentName.flattenToShortString();
- for (AccessibilityServiceInfo info : enabledServices) {
- if (expectedId.equals(info.getId())) {
- return true;
- }
- }
- return false;
- }
-
- public static boolean isCaptureRunning() {
- return serviceConnected && captureActive;
- }
-
- public static Intent buildAccessibilitySettingsIntent() {
- return new Intent(Settings.ACTION_ACCESSIBILITY_SETTINGS);
- }
+ // isServiceEnabled / isCaptureRunning / buildAccessibilitySettingsIntent
+ // moved to A11yState (API-safe; callable from any API level without lint noise).
private void updateCaptureState() {
boolean shouldCapture = shouldCapture();
- captureActive = shouldCapture;
+ A11yState.captureActive = shouldCapture;
if (shouldCapture) {
scheduleNextCapture(500L);
UploadScheduler.ensurePeriodicUpload(getApplicationContext());
diff --git a/app/src/main/java/com/screenomics/AutoUploadWorker.java b/app/src/main/java/com/screenomics/AutoUploadWorker.java
index e046920..0756ee2 100644
--- a/app/src/main/java/com/screenomics/AutoUploadWorker.java
+++ b/app/src/main/java/com/screenomics/AutoUploadWorker.java
@@ -22,6 +22,12 @@ public Result doWork() {
// No FGS required; fully Google Play compliant.
HealthChecker.check(getApplicationContext());
+ // Devicestate heartbeat, throttled to 30 min. LocationService normally
+ // covers this every 10 min, but never runs when location permission is
+ // denied -- this keeps the server-side "alive vs broken" signal flowing
+ // for every enrolled participant. Runs on the worker thread (file I/O).
+ DeviceStateCollector.maybeQueueSnapshot(getApplicationContext(), 30 * 60_000L);
+
// Sensor buffer flushing is handled by LocationService on its own 10-min
// cycle (flushCombinedBuffer). Crash recovery of stale JSONL files happens
// in LocationService.onStartCommand() via flushStaleCombinedBuffer().
diff --git a/app/src/main/java/com/screenomics/Batch.java b/app/src/main/java/com/screenomics/Batch.java
index 7a37f2e..82b4b1f 100644
--- a/app/src/main/java/com/screenomics/Batch.java
+++ b/app/src/main/java/com/screenomics/Batch.java
@@ -86,7 +86,7 @@ public String[] sendFiles() {
String baseUrl = prefs.getString("base_url", Constants.BASE_URL);
String pptId = prefs.getString("ppt_id", "");
String studyId = prefs.getString("study_id", "");
- String bearerToken = prefs.getString("enrollment_token", "");
+ String bearerToken = SecureStore.getSecret(context, "enrollment_token", "");
String imagePubPem = prefs.getString("image_public_key", "");
if (studyId.isEmpty() || pptId.isEmpty()) {
diff --git a/app/src/main/java/com/screenomics/BootReceiver.java b/app/src/main/java/com/screenomics/BootReceiver.java
index 610adff..276f71f 100644
--- a/app/src/main/java/com/screenomics/BootReceiver.java
+++ b/app/src/main/java/com/screenomics/BootReceiver.java
@@ -44,6 +44,10 @@ public void onReceive(Context context, Intent intent) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean wasRecording = prefs.getBoolean("recordingState", false);
+ // OMI Glass collection (mindpulseDev only) runs independently of screen
+ // recording. The facade checks the pref and BLE permissions before starting.
+ GlassesFeature.ensureStartedIfEnabled(context);
+
if (!wasRecording) {
Log.i(TAG, "Recording was not active before reboot, nothing to restart");
return;
diff --git a/app/src/main/java/com/screenomics/CaptureService.java b/app/src/main/java/com/screenomics/CaptureService.java
index eca0d7e..da0be42 100644
--- a/app/src/main/java/com/screenomics/CaptureService.java
+++ b/app/src/main/java/com/screenomics/CaptureService.java
@@ -54,6 +54,7 @@
import java.util.TreeMap;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import android.graphics.Bitmap.Config;
public class CaptureService extends Service {
@@ -90,6 +91,13 @@ public class CaptureService extends Service {
private static final String WAKE_LOCK_TAG = "MindPulse:CaptureWakeLock";
private long lastLowStorageLogMs = 0L;
+ // VLM benchmark (dev only)
+ private VlmBenchmark mVlmBenchmark;
+
+ public void setVlmBenchmark(VlmBenchmark benchmark) {
+ mVlmBenchmark = benchmark;
+ }
+
private class ImageAvailableListener implements ImageReader.OnImageAvailableListener {
@Override
public void onImageAvailable(ImageReader reader) {
@@ -225,6 +233,15 @@ private void processCapturedImage(Image image) {
displayWidth + rowPadding / pixelStride,
displayHeight, Bitmap.Config.ARGB_8888);
bitmap.copyPixelsFromBuffer(buffer);
+
+ // VLM benchmark: submit a copy before bitmap is recycled by encryptImage
+ if (mVlmBenchmark != null && mVlmBenchmark.isRunning()) {
+ Bitmap copy = bitmap.copy(Bitmap.Config.ARGB_8888, false);
+ if (copy != null) {
+ mVlmBenchmark.submitFrame(copy);
+ }
+ }
+
encryptImage(bitmap, "image", foregroundApp);
}
@@ -527,7 +544,7 @@ private void stopCapturing() {
if (mBackgroundThread != null) {
// Post pause image before quitSafely - quitSafely processes pending messages
- mBackgroundHandler.post(insertPauseImage);
+ if (mBackgroundHandler != null) mBackgroundHandler.post(insertPauseImage);
mBackgroundThread.quitSafely();
try {
mBackgroundThread.join();
diff --git a/app/src/main/java/com/screenomics/DailyReminderWorker.java b/app/src/main/java/com/screenomics/DailyReminderWorker.java
index 1bba508..328c5b0 100644
--- a/app/src/main/java/com/screenomics/DailyReminderWorker.java
+++ b/app/src/main/java/com/screenomics/DailyReminderWorker.java
@@ -56,7 +56,7 @@ public Result doWork() {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(ctx);
// Guard: skip if user not registered
- String key = prefs.getString("key", "");
+ String key = SecureStore.getSecret(ctx, "key", "");
if (key.isEmpty()) {
Log.i(TAG, "User not registered, skipping daily reminder");
return Result.success();
@@ -134,7 +134,7 @@ private List auditPermissions(Context ctx, SharedPreferences prefs) {
// Accessibility (only if useAccessibilityCapture is true)
boolean useA11y = prefs.getBoolean("useAccessibilityCapture", false);
if (useA11y && Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
- if (!AccessibilityCaptureService.isServiceEnabled(ctx)) {
+ if (!A11yState.isServiceEnabled(ctx)) {
missing.add("Accessibility");
}
}
diff --git a/app/src/main/java/com/screenomics/DevToolsActivity.java b/app/src/main/java/com/screenomics/DevToolsActivity.java
index 8c525d9..c449778 100644
--- a/app/src/main/java/com/screenomics/DevToolsActivity.java
+++ b/app/src/main/java/com/screenomics/DevToolsActivity.java
@@ -1,10 +1,14 @@
package com.screenomics;
import android.app.Activity;
+import android.content.ComponentName;
+import android.content.Context;
import android.content.Intent;
+import android.content.ServiceConnection;
import android.content.SharedPreferences;
import android.os.AsyncTask;
import android.os.Bundle;
+import android.os.IBinder;
import androidx.preference.PreferenceManager;
import android.text.Editable;
import android.text.TextWatcher;
@@ -23,16 +27,39 @@
import java.io.File;
import java.io.IOException;
import java.net.URL;
+import java.util.Locale;
import java.util.concurrent.TimeUnit;
public class DevToolsActivity extends Activity {
String imagesPath;
+ private VlmBenchmark vlmBenchmark;
+ private CaptureService captureService;
+ private boolean serviceBound = false;
+
+ private final ServiceConnection captureConnection = new ServiceConnection() {
+ @Override
+ public void onServiceConnected(ComponentName name, IBinder binder) {
+ captureService = ((CaptureService.LocalBinder) binder).getService();
+ serviceBound = true;
+ }
+ @Override
+ public void onServiceDisconnected(ComponentName name) {
+ captureService = null;
+ serviceBound = false;
+ }
+ };
@Override
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
+ // Defense in depth: even if the isDev pref was set before this gate
+ // existed, the screen must not open in a production release build.
+ if (!MainActivity.isDevToolsAllowed()) {
+ finish();
+ return;
+ }
setContentView(R.layout.dev_tools);
imagesPath = getApplicationContext().getExternalFilesDir(null).getAbsolutePath() + File.separator + "encrypt";
Button resetButton = findViewById(R.id.clearPrefs);
@@ -84,7 +111,7 @@ protected void onCreate(Bundle savedInstanceState) {
accessibilitySettingsButton.setOnClickListener(v -> {
try {
- startActivity(AccessibilityCaptureService.buildAccessibilitySettingsIntent());
+ startActivity(A11yState.buildAccessibilitySettingsIntent());
} catch (Exception e) {
Toast.makeText(this, "Failed to open accessibility settings", Toast.LENGTH_LONG).show();
}
@@ -166,6 +193,136 @@ public void afterTextChanged(Editable edt) {
editor.apply();
finish();
});
+
+ // --- VLM Benchmark (mindpulseDev only; card hidden in standard) ---
+ // Everything below this guard is dev-flavor-only setup.
+ if (!VlmBenchmark.AVAILABLE) {
+ findViewById(R.id.vlmCard).setVisibility(android.view.View.GONE);
+ return;
+ }
+ Switch vlmSwitch = findViewById(R.id.vlmBenchmarkSwitch);
+ EditText vlmModelPath = findViewById(R.id.vlmModelPathInput);
+ EditText vlmMmprojPath = findViewById(R.id.vlmMmprojPathInput);
+ EditText vlmThreadsInput = findViewById(R.id.vlmThreadsInput);
+ TextView vlmStatus = findViewById(R.id.vlmStatusText);
+ TextView vlmMetrics = findViewById(R.id.vlmMetricsText);
+ Button vlmExportLog = findViewById(R.id.vlmExportLogButton);
+
+ // Restore saved model path
+ vlmModelPath.setText(prefs.getString("vlmModelPath", ""));
+ vlmMmprojPath.setText(prefs.getString("vlmMmprojPath", ""));
+
+ // Bind to CaptureService to inject VLM benchmark
+ Intent captureIntent = new Intent(this, CaptureService.class);
+ bindService(captureIntent, captureConnection, Context.BIND_AUTO_CREATE);
+
+ vlmSwitch.setOnCheckedChangeListener((buttonView, isChecked) -> {
+ if (isChecked) {
+ String modelPath = vlmModelPath.getText().toString().trim();
+ if (modelPath.isEmpty()) {
+ Toast.makeText(this, "Set model path first", Toast.LENGTH_SHORT).show();
+ vlmSwitch.setChecked(false);
+ return;
+ }
+ if (!new File(modelPath).exists()) {
+ Toast.makeText(this, "Model file not found: " + modelPath, Toast.LENGTH_LONG).show();
+ vlmSwitch.setChecked(false);
+ return;
+ }
+
+ // Save paths
+ prefs.edit()
+ .putString("vlmModelPath", modelPath)
+ .putString("vlmMmprojPath", vlmMmprojPath.getText().toString().trim())
+ .apply();
+
+ String mmproj = vlmMmprojPath.getText().toString().trim();
+ int threads = 4;
+ try { threads = Integer.parseInt(vlmThreadsInput.getText().toString()); } catch (Exception ignored) {}
+
+ vlmBenchmark = new VlmBenchmark(DevToolsActivity.this);
+ vlmBenchmark.setStatusListener(new VlmBenchmark.StatusListener() {
+ @Override
+ public void onStatusUpdate(String status) {
+ vlmStatus.setText(status);
+ }
+ @Override
+ public void onMetricsUpdate(double lastMs, double avgMs, int total, int skipped, float batteryDrain) {
+ vlmMetrics.setText(String.format(Locale.US,
+ "Last: %.0fms Avg: %.0fms Total: %d Skip: %d Bat: -%.1f%%",
+ lastMs, avgMs, total, skipped, batteryDrain));
+ }
+ });
+
+ vlmBenchmark.start(modelPath, mmproj.isEmpty() ? null : mmproj, threads);
+
+ // Inject into CaptureService and AccessibilityCaptureService
+ if (serviceBound && captureService != null) {
+ captureService.setVlmBenchmark(vlmBenchmark);
+ }
+ A11yState.setVlmBenchmark(vlmBenchmark);
+
+ Toast.makeText(this, "VLM benchmark started", Toast.LENGTH_SHORT).show();
+ } else {
+ if (vlmBenchmark != null) {
+ vlmBenchmark.stop();
+ if (serviceBound && captureService != null) {
+ captureService.setVlmBenchmark(null);
+ }
+ A11yState.setVlmBenchmark(null);
+ vlmBenchmark = null;
+ }
+ vlmStatus.setText("Stopped");
+ Toast.makeText(this, "VLM benchmark stopped", Toast.LENGTH_SHORT).show();
+ }
+ });
+
+ vlmExportLog.setOnClickListener(v -> {
+ if (vlmBenchmark != null && vlmBenchmark.getLogFile() != null) {
+ String path = vlmBenchmark.getLogFile().getAbsolutePath();
+ Toast.makeText(this, "Log: " + path, Toast.LENGTH_LONG).show();
+ Log.i("VLM_DEVTOOLS", "Benchmark CSV: " + path);
+ } else {
+ Toast.makeText(this, "No active benchmark log", Toast.LENGTH_SHORT).show();
+ }
+ });
+
+ // Headless embedding-only benchmark trigger (dev/testing via adb am start).
+ // Example:
+ // adb shell am start -n edu.wisc.chm.screenomics.mindpulse/com.screenomics.DevToolsActivity \
+ // --ez run_embed_bench true \
+ // --es model /sdcard/Android/data/.../files/models/Qwen3.5-0.8B-Q4_K_M.gguf \
+ // --es mmproj /sdcard/Android/data/.../files/models/mmproj-Qwen3.5-0.8B-f16.gguf \
+ // --es image /sdcard/Android/data/.../files/models/testimg.jpg \
+ // --ei threads 6 --ei dur_ms 360000
+ Intent launchIntent = getIntent();
+ if (launchIntent != null && launchIntent.getBooleanExtra("run_embed_bench", false)) {
+ String m = launchIntent.getStringExtra("model");
+ String mm = launchIntent.getStringExtra("mmproj");
+ String img = launchIntent.getStringExtra("image");
+ int th = launchIntent.getIntExtra("threads", 4);
+ long dur = launchIntent.getIntExtra("dur_ms", 360000);
+ boolean useGpu = launchIntent.getBooleanExtra("use_gpu", false);
+ Log.i("VLM_DEVTOOLS", "Embed bench: model=" + m + " mmproj=" + mm + " image=" + img
+ + " threads=" + th + " dur_ms=" + dur + " use_gpu=" + useGpu);
+ Toast.makeText(this, "Embedding benchmark started (see logcat EMBED_BENCH)", Toast.LENGTH_LONG).show();
+ VlmBenchmark.runEmbeddingBenchmark(getApplicationContext(), m, mm, img, th, dur, useGpu);
+ }
+
+ if (launchIntent != null && launchIntent.getBooleanExtra("run_caption_bench", false)) {
+ String m = launchIntent.getStringExtra("model");
+ String mm = launchIntent.getStringExtra("mmproj");
+ String img = launchIntent.getStringExtra("image");
+ int th = launchIntent.getIntExtra("threads", 4);
+ long dur = launchIntent.getIntExtra("dur_ms", 30000);
+ int maxTok = launchIntent.getIntExtra("max_tokens", 40);
+ String prompt = launchIntent.getStringExtra("prompt");
+ if (prompt == null) prompt = "Describe what you see in one short sentence.";
+ boolean think = launchIntent.getBooleanExtra("think", false);
+ Log.i("VLM_DEVTOOLS", "Caption bench: model=" + m + " image=" + img + " maxTok=" + maxTok + " think=" + think);
+ Toast.makeText(this, "Caption benchmark started (see logcat CAP_BENCH)", Toast.LENGTH_LONG).show();
+ VlmBenchmark.runCaptionBenchmark(getApplicationContext(), m, mm, img, th, dur, prompt, maxTok, think);
+ }
}
@@ -178,6 +335,20 @@ protected Void doInBackground(Void... params){
}
}*/
+ @Override
+ protected void onDestroy() {
+ super.onDestroy();
+ if (serviceBound) {
+ unbindService(captureConnection);
+ serviceBound = false;
+ }
+ // The benchmark may keep running in the capture services after this screen
+ // closes, but the listener holds this Activity and its views — detach it.
+ if (vlmBenchmark != null) {
+ vlmBenchmark.setStatusListener(null);
+ }
+ }
+
public Response canReachEndpoint(){
OkHttpClient client = new OkHttpClient();
Request request = new Request.Builder().url(Constants.HEALTHCHECK_ADDRESS).get().build();
diff --git a/app/src/main/java/com/screenomics/DeviceStateCollector.java b/app/src/main/java/com/screenomics/DeviceStateCollector.java
index 0b3c50c..86e592a 100644
--- a/app/src/main/java/com/screenomics/DeviceStateCollector.java
+++ b/app/src/main/java/com/screenomics/DeviceStateCollector.java
@@ -35,6 +35,65 @@ public class DeviceStateCollector {
private static final String TAG = "DeviceStateCollector";
+ private static final String PREF_LAST_HEARTBEAT_MS = "last_devicestate_upload_ms";
+ private static final String PREF_LAST_PIN_WARN_MS = "last_pin_expiry_warn_ms";
+
+ /**
+ * Heartbeat: queue a devicestate snapshot for upload unless one went out within
+ * {@code maxAgeMs}. Called from AutoUploadWorker so the heartbeat reaches the
+ * server even for participants who declined location permission (LocationService,
+ * which normally flushes devicestate every 10 minutes, never runs for them).
+ * Also raises/clears the upload-backlog participant alert from the same snapshot.
+ */
+ public static void maybeQueueSnapshot(Context context, long maxAgeMs) {
+ SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+ if (prefs.getString("image_public_key", "").isEmpty()) {
+ return; // not enrolled yet — nothing to report, avoid warning spam
+ }
+ long now = System.currentTimeMillis();
+ try {
+ JSONObject snapshot = null;
+ if (now - prefs.getLong(PREF_LAST_HEARTBEAT_MS, 0) >= maxAgeMs) {
+ snapshot = collectSnapshot(context);
+ if (Logger.queueTextForUpload(context, snapshot.toString(),
+ "devicestate", "application/json")) {
+ markHeartbeatSent(context);
+ }
+ }
+ if (snapshot != null) {
+ JSONObject storage = snapshot.optJSONObject("storage");
+ if (storage != null) {
+ HealthChecker.notifyUploadBacklog(context,
+ storage.optInt("pending_upload_count", 0),
+ storage.optLong("storage_available_bytes", -1));
+ }
+ }
+ maybeWarnCertPinExpiry(context, prefs, now);
+ } catch (Exception e) {
+ Log.w(TAG, "heartbeat failed", e);
+ }
+ }
+
+ /** Stamp the heartbeat clock; LocationService calls this on its own devicestate
+ * flush so the AutoUploadWorker heartbeat doesn't duplicate it. */
+ public static void markHeartbeatSent(Context context) {
+ PreferenceManager.getDefaultSharedPreferences(context).edit()
+ .putLong(PREF_LAST_HEARTBEAT_MS, System.currentTimeMillis()).apply();
+ }
+
+ /** The pin-set in network_security_config expires 2027-07-08 and then fails
+ * open. Starting a month out, put a daily warning into the app log stream
+ * (uploaded as diagnostics) so staff see it during routine monitoring. */
+ private static void maybeWarnCertPinExpiry(Context context, SharedPreferences prefs, long now) {
+ try {
+ if (java.time.LocalDate.now().isBefore(java.time.LocalDate.of(2027, 6, 8))) return;
+ if (now - prefs.getLong(PREF_LAST_PIN_WARN_MS, 0) < 24L * 3600_000L) return;
+ Logger.e(context, "TLS pin-set in network_security_config expires 2027-07-08 "
+ + "and will FAIL OPEN — ship a release with rotated pins");
+ prefs.edit().putLong(PREF_LAST_PIN_WARN_MS, now).apply();
+ } catch (Exception ignored) {}
+ }
+
public static JSONObject collectSnapshot(Context context) {
JSONObject root = new JSONObject();
try { root.put("battery", collectBattery(context)); } catch (Exception e) { Log.w(TAG, "battery", e); }
@@ -71,6 +130,31 @@ private static JSONObject collectBattery(Context context) throws Exception {
default: plugType = "none"; break;
}
obj.put("battery_plugged_type", plugType);
+ int temp = batteryStatus.getIntExtra(BatteryManager.EXTRA_TEMPERATURE, -1);
+ if (temp >= 0) {
+ obj.put("battery_temp_c", temp / 10.0);
+ }
+ int voltage = batteryStatus.getIntExtra(BatteryManager.EXTRA_VOLTAGE, -1);
+ if (voltage > 0) {
+ obj.put("battery_voltage_mv", voltage);
+ }
+ int health = batteryStatus.getIntExtra(BatteryManager.EXTRA_HEALTH, -1);
+ obj.put("battery_health", health);
+ }
+ BatteryManager bm = (BatteryManager) context.getSystemService(Context.BATTERY_SERVICE);
+ if (bm != null) {
+ int currentNow = bm.getIntProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_NOW);
+ if (currentNow != Integer.MIN_VALUE) {
+ obj.put("battery_current_ua", currentNow);
+ }
+ long energyCounter = bm.getLongProperty(BatteryManager.BATTERY_PROPERTY_ENERGY_COUNTER);
+ if (energyCounter != Long.MIN_VALUE && energyCounter > 0) {
+ obj.put("battery_energy_nwh", energyCounter);
+ }
+ int avgCurrent = bm.getIntProperty(BatteryManager.BATTERY_PROPERTY_CURRENT_AVERAGE);
+ if (avgCurrent != Integer.MIN_VALUE) {
+ obj.put("battery_current_avg_ua", avgCurrent);
+ }
}
PowerManager pm = (PowerManager) context.getSystemService(Context.POWER_SERVICE);
if (pm != null) {
@@ -186,7 +270,7 @@ private static JSONObject collectPermissions(Context context) throws Exception {
}
try {
obj.put("perm_accessibility_enabled",
- AccessibilityCaptureService.isServiceEnabled(context));
+ A11yState.isServiceEnabled(context));
} catch (Exception e) {
Log.w(TAG, "accessibility check failed", e);
}
@@ -199,7 +283,7 @@ private static JSONObject collectServiceState(Context context) throws Exception
obj.put("recording_state", prefs.getBoolean("recordingState", false));
boolean useA11y = prefs.getBoolean("useAccessibilityCapture", false);
obj.put("capture_mode", useA11y ? "accessibility" : "media_projection");
- obj.put("a11y_capture_active", AccessibilityCaptureService.isCaptureRunning());
+ obj.put("a11y_capture_active", A11yState.isCaptureRunning());
obj.put("a11y_last_image_ts", prefs.getLong("a11y_last_image_ts", 0));
obj.put("a11y_last_error", prefs.getString("a11y_last_error", ""));
obj.put("a11y_consecutive_failures", prefs.getInt("a11y_consecutive_failures", 0));
diff --git a/app/src/main/java/com/screenomics/Encryptor.java b/app/src/main/java/com/screenomics/Encryptor.java
index e0202f3..6ba5ccd 100644
--- a/app/src/main/java/com/screenomics/Encryptor.java
+++ b/app/src/main/java/com/screenomics/Encryptor.java
@@ -93,6 +93,20 @@ public static Result encryptFileToEnc(File in, File outEnc, String serverImagePu
return new Result(outEnc, aesKeyEncB64, nonceB64);
}
+ /**
+ * Legacy entry point kept only so the disabled mindpulseDev video-capture code
+ * compiles. It must never silently succeed: callers delete the plaintext source
+ * after this returns, so a no-op here would destroy the recording while logging
+ * success. Throwing keeps the original file intact (callers catch and log).
+ *
+ * @deprecated Port callers to {@link #encryptFileToEnc} before re-enabling video.
+ */
+ @Deprecated
+ public static void encryptFile(byte[] key, String inputPath, String outputPath, byte[] iv) {
+ throw new UnsupportedOperationException(
+ "Legacy encryptFile is not implemented; port to encryptFileToEnc before use");
+ }
+
// ---- helpers ----
private static SecretKey genAesKey(int bits) throws Exception {
diff --git a/app/src/main/java/com/screenomics/HealthChecker.java b/app/src/main/java/com/screenomics/HealthChecker.java
index ee06d20..e8b1640 100644
--- a/app/src/main/java/com/screenomics/HealthChecker.java
+++ b/app/src/main/java/com/screenomics/HealthChecker.java
@@ -24,6 +24,17 @@ public final class HealthChecker {
private static final String ALERT_CHANNEL_ID = "screenomics_alert_id";
static final int NOTIF_ID_A11Y_LOST = 3001;
+ static final int NOTIF_ID_CAPTURE_STALL = 3002;
+ static final int NOTIF_ID_UPLOAD_BACKLOG = 3003;
+
+ /** Consecutive screenshot failures before alerting. Failures only accumulate
+ * during unlocked capture attempts (the loop skips while the keyguard is up),
+ * so overnight non-use can never trip this. ~30 = about a minute of solid
+ * failures at the 1s retry cadence. */
+ private static final int STALL_CONSEC_FAILURES = 30;
+ /** Backlog thresholds for the participant-facing upload alert. */
+ private static final int BACKLOG_ALERT_COUNT = 2000;
+ private static final long LOW_SPACE_ALERT_BYTES = 500L * 1024 * 1024;
private HealthChecker() {}
@@ -44,20 +55,54 @@ public static void check(Context context) {
NotificationManager nm =
(NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
- boolean a11yEnabled = AccessibilityCaptureService.isServiceEnabled(context);
+ boolean a11yEnabled = A11yState.isServiceEnabled(context);
- // Only check: accessibility permission lost
+ // Accessibility permission lost
if (!a11yEnabled) {
showA11yLostNotification(context, nm);
} else {
nm.cancel(NOTIF_ID_A11Y_LOST);
}
+
+ // Capture stalled: permission is fine but screenshots keep failing
+ // (persistent takeScreenshot errors, storage problems, ...)
+ int consecFailures = prefs.getInt("a11y_consecutive_failures", 0);
+ if (a11yEnabled && consecFailures >= STALL_CONSEC_FAILURES) {
+ showCaptureStallNotification(context, nm);
+ } else {
+ nm.cancel(NOTIF_ID_CAPTURE_STALL);
+ }
+ }
+
+ /**
+ * Raise or clear the participant-facing upload-backlog / low-storage alert.
+ * Called from the AutoUploadWorker heartbeat (background thread) with values
+ * from the devicestate snapshot; not from the 1-minute main-thread check.
+ */
+ public static void notifyUploadBacklog(Context context, int pendingCount, long availableBytes) {
+ ensureAlertChannel(context);
+ NotificationManager nm =
+ (NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
+ boolean backlog = pendingCount >= BACKLOG_ALERT_COUNT;
+ boolean lowSpace = availableBytes >= 0 && availableBytes < LOW_SPACE_ALERT_BYTES;
+ if (!backlog && !lowSpace) {
+ nm.cancel(NOTIF_ID_UPLOAD_BACKLOG);
+ return;
+ }
+ String text = lowSpace
+ ? "Phone storage is almost full — connect to Wi-Fi so MindPulse can upload and free space."
+ : pendingCount + " files are waiting to upload — please connect to Wi-Fi.";
+ nm.notify(NOTIF_ID_UPLOAD_BACKLOG, buildOpenAppNotification(context,
+ NOTIF_ID_UPLOAD_BACKLOG, "MindPulse needs to upload", text));
+ Log.w(TAG, "Upload backlog alert: pending=" + pendingCount + " availableBytes=" + availableBytes);
}
public static void dismissAll(Context context) {
NotificationManager nm =
(NotificationManager) context.getSystemService(Context.NOTIFICATION_SERVICE);
nm.cancel(NOTIF_ID_A11Y_LOST);
+ nm.cancel(NOTIF_ID_CAPTURE_STALL);
+ nm.cancel(NOTIF_ID_UPLOAD_BACKLOG);
}
private static void ensureAlertChannel(Context context) {
@@ -92,4 +137,29 @@ private static void showA11yLostNotification(Context context, NotificationManage
Log.w(TAG, "Health check: accessibility service not enabled");
}
+ private static void showCaptureStallNotification(Context context, NotificationManager nm) {
+ nm.notify(NOTIF_ID_CAPTURE_STALL, buildOpenAppNotification(context,
+ NOTIF_ID_CAPTURE_STALL,
+ "MindPulse capture needs attention",
+ "Screen capture keeps failing. Tap to open MindPulse and toggle capture off and on."));
+ Log.w(TAG, "Health check: capture stalled (consecutive screenshot failures)");
+ }
+
+ private static Notification buildOpenAppNotification(
+ Context context, int requestCode, String title, String text) {
+ Intent appIntent = new Intent(context, MainActivity.class);
+ appIntent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK | Intent.FLAG_ACTIVITY_CLEAR_TOP);
+ int flags = PendingIntent.FLAG_IMMUTABLE | PendingIntent.FLAG_UPDATE_CURRENT;
+ PendingIntent pi = PendingIntent.getActivity(context, requestCode, appIntent, flags);
+ return new NotificationCompat.Builder(context, ALERT_CHANNEL_ID)
+ .setSmallIcon(R.drawable.dna)
+ .setContentTitle(title)
+ .setContentText(text)
+ .setStyle(new NotificationCompat.BigTextStyle().bigText(text))
+ .setPriority(NotificationCompat.PRIORITY_HIGH)
+ .setAutoCancel(true)
+ .setContentIntent(pi)
+ .build();
+ }
+
}
diff --git a/app/src/main/java/com/screenomics/LocationService.java b/app/src/main/java/com/screenomics/LocationService.java
index 859d7bc..89ef066 100644
--- a/app/src/main/java/com/screenomics/LocationService.java
+++ b/app/src/main/java/com/screenomics/LocationService.java
@@ -220,6 +220,8 @@ private void flushCombinedBuffer() {
JSONObject deviceState = DeviceStateCollector.collectSnapshot(ctx);
if (Logger.queueTextForUpload(ctx, deviceState.toString(), "devicestate", "application/json")) {
queued++;
+ // Keep the AutoUploadWorker heartbeat from duplicating this flush
+ DeviceStateCollector.markHeartbeatSent(ctx);
}
} catch (Exception e) {
Log.w(TAG, "Failed to flush device state", e);
diff --git a/app/src/main/java/com/screenomics/Logger.java b/app/src/main/java/com/screenomics/Logger.java
index 7afbc23..5f91bdf 100644
--- a/app/src/main/java/com/screenomics/Logger.java
+++ b/app/src/main/java/com/screenomics/Logger.java
@@ -130,6 +130,31 @@ public static boolean queueDiagnosticsForUpload(Context context) {
}
}
+ /**
+ * Defense-in-depth: delete stale plaintext temp files (tmp_*.jpg / tmp_*.log) left in the
+ * files/ root if the process was killed between writing and deleting during encryption.
+ * Normally these are deleted in a finally block within milliseconds; this only catches
+ * crash-orphaned files. Call on collection-service startup.
+ */
+ public static void sweepStaleTempFiles(Context context) {
+ try {
+ File extDir = context.getApplicationContext().getExternalFilesDir(null);
+ if (extDir == null) return;
+ File[] files = extDir.listFiles((dir, name) ->
+ name.startsWith("tmp_") && (name.endsWith(".jpg") || name.endsWith(".log")));
+ if (files == null) return;
+ long cutoff = System.currentTimeMillis() - 5 * 60 * 1000L; // older than 5 minutes
+ for (File f : files) {
+ if (f.lastModified() < cutoff) {
+ //noinspection ResultOfMethodCallIgnored
+ f.delete();
+ }
+ }
+ } catch (Exception ignored) {
+ // best-effort cleanup; never fail the caller
+ }
+ }
+
static boolean queueTextForUpload(
Context context,
String content,
@@ -195,6 +220,94 @@ static boolean queueTextForUpload(
}
}
+ /**
+ * Binary sibling of {@link #queueTextForUpload}: encrypt a JPEG (e.g. an OMI
+ * Glass photo received over BLE) into the same /encrypt upload queue used by
+ * screenshots. Tagged meta type="image" so the Receiver ingests it like a
+ * screenshot, plus source/orientation for provenance. The existing
+ * UploadService/Batch pipeline carries it with no changes.
+ *
+ * @param descriptor filename component (e.g. "glass")
+ * @param source provenance tag written to meta (e.g. "omi_glass")
+ * @param orientation image orientation byte from the device, or -1 if unknown
+ * @return true if an .enc + .meta pair was queued
+ */
+ public static boolean queueImageForUpload(
+ Context context,
+ byte[] jpeg,
+ String descriptor,
+ String source,
+ int orientation
+ ) {
+ SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
+
+ String pubKeyPem = prefs.getString("image_public_key", "");
+ if (pubKeyPem == null || pubKeyPem.trim().isEmpty()) {
+ Log.w(TAG, "No image_public_key available, skipping " + descriptor + " image upload");
+ return false;
+ }
+ if (jpeg == null || jpeg.length == 0) {
+ Log.w(TAG, "Empty jpeg, skipping " + descriptor + " image upload");
+ return false;
+ }
+
+ File extDir = context.getApplicationContext().getExternalFilesDir(null);
+ if (extDir == null) {
+ Log.e(TAG, "getExternalFilesDir returned null, cannot queue " + descriptor + " image");
+ return false;
+ }
+
+ File encryptDir = new File(extDir, "encrypt");
+ if (!encryptDir.exists() && !encryptDir.mkdirs()) {
+ Log.e(TAG, "Failed to create encrypt directory for " + descriptor + " image");
+ return false;
+ }
+
+ String hashFull = prefs.getString("hash", "00000000");
+ String hash = hashFull.substring(0, Math.min(8, hashFull.length()));
+ String baseName = hash + "_" + System.currentTimeMillis() + "_" + descriptor;
+
+ File tempFile = new File(extDir, "tmp_" + UUID.randomUUID() + ".jpg");
+ try {
+ try (java.io.FileOutputStream fos = new java.io.FileOutputStream(tempFile, false)) {
+ fos.write(jpeg);
+ }
+
+ File encFile = new File(encryptDir, baseName + ".enc");
+ Encryptor.Result result = Encryptor.encryptFileToEnc(tempFile, encFile, pubKeyPem);
+
+ JSONObject metaObj = new JSONObject();
+ metaObj.put("aes_key_encrypted_b64", result.aesKeyEncB64);
+ metaObj.put("tag_len_bits", result.tagLenBits);
+ metaObj.put("mime", "image/jpeg");
+ metaObj.put("type", "image");
+ metaObj.put("captured_at", utcIsoMillis().format(new Date()));
+ metaObj.put("epoch_ms", System.currentTimeMillis());
+ if (source != null && !source.isEmpty()) {
+ metaObj.put("source", source);
+ }
+ if (orientation >= 0) {
+ metaObj.put("orientation", orientation);
+ }
+
+ File metaFile = new File(encryptDir, baseName + ".meta");
+ try (FileWriter fw = new FileWriter(metaFile, false)) {
+ fw.write(metaObj.toString());
+ }
+
+ Log.i(TAG, "Queued encrypted " + descriptor + " image: " + encFile.getName());
+ return true;
+ } catch (Exception e) {
+ Log.e(TAG, "Failed to queue " + descriptor + " image for upload", e);
+ return false;
+ } finally {
+ if (tempFile.exists()) {
+ //noinspection ResultOfMethodCallIgnored
+ tempFile.delete();
+ }
+ }
+ }
+
static String captureOwnProcessLogcat() {
String pidArg = "--pid=" + android.os.Process.myPid();
List primaryCmd = Arrays.asList("logcat", "-d", "-t", "800", pidArg, "*:V");
diff --git a/app/src/main/java/com/screenomics/MainActivity.java b/app/src/main/java/com/screenomics/MainActivity.java
index 7603d4e..dadded9 100644
--- a/app/src/main/java/com/screenomics/MainActivity.java
+++ b/app/src/main/java/com/screenomics/MainActivity.java
@@ -81,6 +81,7 @@ public class MainActivity extends AppCompatActivity {
private AlertDialog activeUpdateDialog;
private AppUpdateManager appUpdateManager;
private ActivityResultLauncher locationPermissionRequest;
+ private ActivityResultLauncher blePermissionRequest;
private final InstallStateUpdatedListener installStateUpdatedListener = state -> {
if (state.installStatus() == InstallStatus.DOWNLOADED && appUpdateManager != null) {
@@ -105,7 +106,7 @@ protected void onCreate(Bundle savedInstanceState) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
SharedPreferences.Editor editor = prefs.edit();
- String key = prefs.getString("key", "");
+ String key = SecureStore.getSecret(this, "key", "");
recordingState = prefs.getBoolean("recordingState", false);
continueWithoutWifi = prefs.getBoolean("continueWithoutWifi", false);
if (key.equals("")) {
@@ -131,6 +132,22 @@ protected void onCreate(Bundle savedInstanceState) {
}
});
+ // Register BLE permission launcher for OMI Glass collection (mindpulseDev only)
+ blePermissionRequest = registerForActivityResult(
+ new ActivityResultContracts.RequestMultiplePermissions(), result -> {
+ boolean allGranted = !result.isEmpty();
+ for (Boolean granted : result.values()) {
+ if (!Boolean.TRUE.equals(granted)) allGranted = false;
+ }
+ if (allGranted) {
+ GlassesFeature.start(this);
+ } else {
+ Log.w(TAG, "BLE permission denied; glasses collection unavailable");
+ Toast.makeText(this, "Bluetooth permission is required to connect glasses",
+ Toast.LENGTH_LONG).show();
+ }
+ });
+
maybeEnableAccessibilityModeForAndroid15(prefs);
initInAppUpdate();
@@ -156,7 +173,9 @@ protected void onCreate(Bundle savedInstanceState) {
infoButton.setOnClickListener(v -> {
infoOpenCount++;
- if (infoOpenCount == 5) {
+ // Five-tap dev entry works only in internal builds (any debug build or
+ // the mindpulseDev flavor) — never in the production Play release.
+ if (infoOpenCount == 5 && isDevToolsAllowed()) {
editor.putBoolean("isDev", true);
editor.apply();
devButton.setVisibility(View.VISIBLE);
@@ -306,8 +325,8 @@ protected void onResume() {
}
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(this);
- boolean isDev = prefs.getBoolean("isDev", false);
- if (!isDev) devButton.setVisibility(View.GONE);
+ boolean isDev = prefs.getBoolean("isDev", false) && isDevToolsAllowed();
+ devButton.setVisibility(isDev ? View.VISIBLE : View.GONE);
Intent launchIntent = getIntent();
if (launchIntent != null && launchIntent.getBooleanExtra("start_video_recording", false)) {
@@ -317,6 +336,10 @@ protected void onResume() {
}
}
+ // OMI Glass (mindpulseDev only): restart the service if it should be running
+ // but isn't. No-op when already running, so a live connection is untouched.
+ GlassesFeature.ensureStartedIfEnabled(this);
+
checkForAppUpdate(true);
maybeResumeInProgressUpdate();
}
@@ -406,6 +429,27 @@ public void stopLocationService(){
stopService(intent);
}
+ /** Dev tools are for internal builds only: any debug build, or the mindpulseDev flavor. */
+ static boolean isDevToolsAllowed() {
+ return BuildConfig.DEBUG || BuildConfig.MINDPULSE_ENABLED;
+ }
+
+ // ---- OMI Glass BLE collection (mindpulseDev only; no-op in standard) ----
+
+ /** Start OMI Glass BLE collection, requesting BLE permissions first if needed. */
+ public void startGlassesService() {
+ if (!GlassesFeature.AVAILABLE) return;
+ if (!GlassesFeature.hasBlePermissions(this)) {
+ blePermissionRequest.launch(GlassesFeature.requiredRuntimePermissions());
+ return;
+ }
+ GlassesFeature.start(this);
+ }
+
+ public void stopGlassesService() {
+ GlassesFeature.stop(this);
+ }
+
public void startMediaProjectionRequest() {
// Android 11+ (R): always prefer accessibility capture -- it survives reboots
// and does not require a one-time MediaProjection token.
@@ -471,7 +515,7 @@ public void stopCaptureService() {
public boolean isCaptureServiceRunning() {
if (isUsingAccessibilityCapture()) {
- return AccessibilityCaptureService.isCaptureRunning();
+ return A11yState.isCaptureRunning();
}
return captureServiceBound && captureService != null && captureService.isCapturing();
}
@@ -488,7 +532,7 @@ private void startAccessibilityCaptureRequest() {
return;
}
- if (AccessibilityCaptureService.isServiceEnabled(this)) {
+ if (A11yState.isServiceEnabled(this)) {
Toast.makeText(this, "MindPulse accessibility capture is running!", Toast.LENGTH_SHORT).show();
return;
}
@@ -533,7 +577,7 @@ private void startAccessibilityCaptureRequest() {
.setPositiveButton("Agree and Open Settings", (dialog, which) -> {
try {
- startActivity(AccessibilityCaptureService.buildAccessibilitySettingsIntent());
+ startActivity(A11yState.buildAccessibilitySettingsIntent());
} catch (Exception e) {
Log.e(TAG, "Failed to open accessibility settings", e);
Toast.makeText(this, "Open Settings > Accessibility and enable MindPulse Accessibility Capture", Toast.LENGTH_LONG).show();
diff --git a/app/src/main/java/com/screenomics/PermissionTestActivity.java b/app/src/main/java/com/screenomics/PermissionTestActivity.java
deleted file mode 100644
index cfba051..0000000
--- a/app/src/main/java/com/screenomics/PermissionTestActivity.java
+++ /dev/null
@@ -1,144 +0,0 @@
-package com.screenomics;
-
-import android.Manifest;
-import android.os.Bundle;
-import android.widget.Button;
-import android.widget.TextView;
-import android.widget.Toast;
-import androidx.appcompat.app.AppCompatActivity;
-
-public class PermissionTestActivity extends AppCompatActivity {
-
- private TextView statusTextView;
-
- @Override
- protected void onCreate(Bundle savedInstanceState) {
- super.onCreate(savedInstanceState);
- setContentView(R.layout.activity_permission_test);
-
- statusTextView = findViewById(R.id.permission_status);
-
- Button cameraButton = findViewById(R.id.test_camera_button);
- Button locationButton = findViewById(R.id.test_location_button);
- Button notificationButton = findViewById(R.id.test_notification_button);
- Button allButton = findViewById(R.id.test_all_button);
-
- cameraButton.setOnClickListener(v -> testCameraPermission());
- locationButton.setOnClickListener(v -> testLocationPermission());
- notificationButton.setOnClickListener(v -> testNotificationPermission());
- allButton.setOnClickListener(v -> testAllPermissions());
-
- updateStatus();
- }
-
- @Override
- protected void onResume() {
- super.onResume();
- updateStatus();
- }
-
- private void updateStatus() {
- StringBuilder status = new StringBuilder();
- status.append("Permission Status:\n\n");
-
- status.append("Camera: ")
- .append(PermissionHelper.hasPermission(this, Manifest.permission.CAMERA) ? "✓ Granted" : "✗ Denied")
- .append("\n");
-
- status.append("Location (Fine): ")
- .append(PermissionHelper.hasPermission(this, Manifest.permission.ACCESS_FINE_LOCATION) ? "✓ Granted" : "✗ Denied")
- .append("\n");
-
- status.append("Location (Coarse): ")
- .append(PermissionHelper.hasPermission(this, Manifest.permission.ACCESS_COARSE_LOCATION) ? "✓ Granted" : "✗ Denied")
- .append("\n");
-
- if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.Q) {
- status.append("Background Location: ")
- .append(PermissionHelper.hasPermission(this, Manifest.permission.ACCESS_BACKGROUND_LOCATION) ? "✓ Granted" : "✗ Denied")
- .append("\n");
- }
-
- if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.TIRAMISU) {
- status.append("Notifications: ")
- .append(PermissionHelper.hasPermission(this, Manifest.permission.POST_NOTIFICATIONS) ? "✓ Granted" : "✗ Denied")
- .append("\n");
- }
-
- statusTextView.setText(status.toString());
- }
-
- private void testCameraPermission() {
- PermissionHelper.requestPermissionWithExplanation(
- this,
- Manifest.permission.CAMERA,
- PermissionHelper.PERMISSION_REQUEST_CAMERA,
- createCallback("Camera")
- );
- }
-
- private void testLocationPermission() {
- PermissionHelper.requestLocationPermissions(this, createCallback("Location"));
- }
-
- private void testNotificationPermission() {
- PermissionHelper.requestNotificationPermission(this, createCallback("Notification"));
- }
-
- private void testAllPermissions() {
- String[] allPermissions = {
- Manifest.permission.CAMERA,
- Manifest.permission.ACCESS_FINE_LOCATION,
- Manifest.permission.ACCESS_COARSE_LOCATION
- };
-
- if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.TIRAMISU) {
- allPermissions = new String[]{
- Manifest.permission.CAMERA,
- Manifest.permission.ACCESS_FINE_LOCATION,
- Manifest.permission.ACCESS_COARSE_LOCATION,
- Manifest.permission.POST_NOTIFICATIONS
- };
- }
-
- PermissionHelper.requestPermissionsWithExplanation(
- this,
- allPermissions,
- PermissionHelper.PERMISSION_REQUEST_ALL
- );
- }
-
- private PermissionHelper.PermissionCallback createCallback(String permissionName) {
- return new PermissionHelper.PermissionCallback() {
- @Override
- public void onPermissionGranted() {
- Toast.makeText(PermissionTestActivity.this,
- permissionName + " permission granted!",
- Toast.LENGTH_SHORT).show();
- updateStatus();
- }
-
- @Override
- public void onPermissionDenied() {
- Toast.makeText(PermissionTestActivity.this,
- permissionName + " permission denied",
- Toast.LENGTH_SHORT).show();
- updateStatus();
- }
-
- @Override
- public void onPermissionPermanentlyDenied() {
- Toast.makeText(PermissionTestActivity.this,
- permissionName + " permission permanently denied",
- Toast.LENGTH_LONG).show();
- updateStatus();
- }
- };
- }
-
- @Override
- public void onRequestPermissionsResult(int requestCode, String[] permissions, int[] grantResults) {
- super.onRequestPermissionsResult(requestCode, permissions, grantResults);
- updateStatus();
- }
-}
\ No newline at end of file
diff --git a/app/src/main/java/com/screenomics/RegisterActivity.java b/app/src/main/java/com/screenomics/RegisterActivity.java
index 7b3b735..80643e0 100644
--- a/app/src/main/java/com/screenomics/RegisterActivity.java
+++ b/app/src/main/java/com/screenomics/RegisterActivity.java
@@ -87,6 +87,13 @@ protected void onCreate(Bundle savedInstanceState) {
http = HttpClientProvider.get(this); // includes HttpSignatureInterceptor
+ // Back is intentionally a no-op: registration must complete or the app
+ // has no identity. (Replaces the deprecated onBackPressed() override.)
+ getOnBackPressedDispatcher().addCallback(this,
+ new androidx.activity.OnBackPressedCallback(true) {
+ @Override public void handleOnBackPressed() { }
+ });
+
imagePickerLauncher = registerForActivityResult(
new ActivityResultContracts.GetContent(),
uri -> { if (uri != null) processQRImageFromGallery(uri); }
@@ -273,17 +280,15 @@ private void enrollWithReceiver(String code) {
ed.putString("study_id", studyId);
ed.putString("image_public_key", imagePubPem); // new key name used by Batch
ed.putString("base_url", RECEIVER_BASE);
- ed.putString("key", key);
ed.putString("hash", hash);
- // enrollment_token stored in plain SharedPreferences -- accepted risk.
- // EncryptedSharedPreferences not in project deps; token is also sent as
- // Authorization header on every upload, so SharedPreferences is not the
- // weakest link.
+ // The enrollment key and token are stored encrypted at rest via
+ // SecureStore (AndroidKeyStore AES-256-GCM); never in plaintext prefs.
+ SecureStore.putSecret(getApplicationContext(), "key", key);
if (longToken) {
- ed.putString("enrollment_token", code);
+ SecureStore.putSecret(getApplicationContext(), "enrollment_token", code);
} else {
- ed.remove("enrollment_token");
+ SecureStore.removeSecret(getApplicationContext(), "enrollment_token");
}
// remove any legacy key name
@@ -393,8 +398,6 @@ private String decodeQRCode(Bitmap bitmap) {
}
}
- @SuppressWarnings("deprecation")
- @Override public void onBackPressed() {}
@Override public boolean onKeyDown(int keyCode, KeyEvent event) {
return keyCode == KeyEvent.KEYCODE_BACK || super.onKeyDown(keyCode, event);
}
diff --git a/app/src/main/java/com/screenomics/ScreenLifeFragment.java b/app/src/main/java/com/screenomics/ScreenLifeFragment.java
index 6fe4b54..e4776d1 100644
--- a/app/src/main/java/com/screenomics/ScreenLifeFragment.java
+++ b/app/src/main/java/com/screenomics/ScreenLifeFragment.java
@@ -61,6 +61,10 @@ public class ScreenLifeFragment extends Fragment {
private Timer numImageRefreshTimer;
private UploadService uploadService;
private TextView accessibilityModeStatus;
+ private View glassCard;
+ private View glassStatusDot;
+ private TextView glassStatus;
+ private Button glassConnectButton;
private final ExecutorService statsExecutor = Executors.newSingleThreadExecutor();
private final AtomicBoolean statsRefreshInFlight = new AtomicBoolean(false);
@@ -121,6 +125,13 @@ private void initializeViews(View view) {
// statsSettingsButton removed - each permission row opens its own settings
accessibilityCaptureButton = view.findViewById(R.id.accessibilityCaptureButton);
accessibilityModeStatus = view.findViewById(R.id.accessibilityModeStatus);
+ glassCard = view.findViewById(R.id.glassCard);
+ glassStatusDot = view.findViewById(R.id.glassStatusDot);
+ glassStatus = view.findViewById(R.id.glassStatus);
+ glassConnectButton = view.findViewById(R.id.glassConnectButton);
+ if (!GlassesFeature.AVAILABLE && glassCard != null) {
+ glassCard.setVisibility(View.GONE);
+ }
// Permission status dots
cameraPermissionDot = view.findViewById(R.id.cameraPermissionDot);
@@ -151,7 +162,7 @@ private void setupListeners() {
justStartedCapture = true; // Mark that user just started it
boolean useA11y = Build.VERSION.SDK_INT >= Build.VERSION_CODES.R;
- boolean a11yReady = useA11y && AccessibilityCaptureService.isServiceEnabled(requireContext());
+ boolean a11yReady = useA11y && A11yState.isServiceEnabled(requireContext());
if (!useA11y || a11yReady) {
// Standard capture or accessibility already enabled
@@ -238,6 +249,13 @@ private void setupListeners() {
accessibilityCaptureButton.setOnClickListener(view -> handleAccessibilityCaptureSelection());
+ glassConnectButton.setOnClickListener(v -> {
+ MainActivity act = (MainActivity) requireActivity();
+ if (GlassesFeature.isEnabled(requireContext())) act.stopGlassesService();
+ else act.startGlassesService();
+ v.postDelayed(this::updateGlassUi, 800);
+ });
+
uploadButton.setOnClickListener(v -> {
if (!InternetConnection.checkWiFiConnection(requireContext())) {
AlertDialog alertDialog = new AlertDialog.Builder(requireContext()).create();
@@ -336,11 +354,12 @@ public void onResume() {
startImageRefreshTimer();
updatePermissionStatus();
updateAccessibilityCaptureUi();
+ updateGlassUi();
// If user toggled capture ON but accessibility wasn't enabled yet, check now
if (pendingCaptureStart) {
pendingCaptureStart = false;
- if (AccessibilityCaptureService.isServiceEnabled(requireContext())) {
+ if (A11yState.isServiceEnabled(requireContext())) {
Log.i("ScreenLifeFragment", "Accessibility now enabled -- confirming recordingState");
prefs.edit().putBoolean("recordingState", true).apply();
switchCapture.setChecked(true);
@@ -432,6 +451,7 @@ public void run() {
updatePermissionStatus();
updateAccessibilityCaptureUi();
+ updateGlassUi();
});
} catch (Exception e) {
Log.w(TAG, "Failed to refresh file stats", e);
@@ -531,7 +551,7 @@ public void onServiceDisconnected(ComponentName componentName) { }
private void showUpdateQRCodeDialog() {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(requireContext());
boolean isTester = prefs.getBoolean("isTester", false);
- String currentKey = prefs.getString("key", "");
+ String currentKey = SecureStore.getSecret(requireContext(), "key", "");
String currentHash = prefs.getString("hash", "");
AlertDialog.Builder builder = new AlertDialog.Builder(requireContext());
@@ -622,10 +642,10 @@ private void generateNewTestId() {
}
String hash = hexString.toString();
- // Save to preferences
+ // Save to preferences (the key itself goes to encrypted storage)
+ SecureStore.putSecret(requireContext(), "key", key);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(requireContext());
SharedPreferences.Editor editor = prefs.edit();
- editor.putString("key", key);
editor.putString("hash", hash);
editor.putBoolean("isTester", true);
java.text.SimpleDateFormat testerTsFmt = new java.text.SimpleDateFormat("yyyy-MM-dd HH:mm:ss 'UTC'", java.util.Locale.US);
@@ -682,7 +702,7 @@ private void updatePermissionStatus() {
// Check Accessibility Capture permission (Android 11+)
boolean accessibilityGranted;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
- accessibilityGranted = AccessibilityCaptureService.isServiceEnabled(requireContext());
+ accessibilityGranted = A11yState.isServiceEnabled(requireContext());
} else {
// Not applicable on Android 10, show as granted (N/A)
accessibilityGranted = true;
@@ -778,7 +798,7 @@ private void disableAccessibilityCapture() {
private void openAccessibilitySettings() {
try {
- startActivity(AccessibilityCaptureService.buildAccessibilitySettingsIntent());
+ startActivity(A11yState.buildAccessibilitySettingsIntent());
Toast.makeText(requireContext(),
"Enable MindPulse Accessibility Capture, then return to the app.",
Toast.LENGTH_LONG).show();
@@ -797,7 +817,7 @@ private void updateAccessibilityCaptureUi() {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(requireContext());
boolean accessibilityModeEnabled = prefs.getBoolean(PREF_USE_ACCESSIBILITY_CAPTURE, false);
- boolean accessibilityGranted = AccessibilityCaptureService.isServiceEnabled(requireContext());
+ boolean accessibilityGranted = A11yState.isServiceEnabled(requireContext());
if (!accessibilityModeEnabled) {
accessibilityModeStatus.setText("Standard screen recording is selected.");
@@ -814,6 +834,24 @@ private void updateAccessibilityCaptureUi() {
}
}
+ private void updateGlassUi() {
+ if (!GlassesFeature.AVAILABLE) return;
+ if (getActivity() == null || glassStatus == null) return;
+ boolean enabled = GlassesFeature.isEnabled(requireContext());
+ boolean connected = GlassesFeature.isConnected();
+ updatePermissionDot(glassStatusDot, connected);
+ if (!enabled) {
+ glassStatus.setText("Off");
+ glassConnectButton.setText("Connect OMI Glass");
+ } else {
+ int batteryPct = GlassesFeature.batteryPct();
+ String batt = batteryPct >= 0 ? (batteryPct + "%") : "—";
+ glassStatus.setText((connected ? "Connected" : "Scanning…")
+ + " · photos " + GlassesFeature.photoCount() + " · battery " + batt);
+ glassConnectButton.setText("Disconnect OMI Glass");
+ }
+ }
+
private void openAppSettings() {
try {
Intent intent = new Intent(Settings.ACTION_APPLICATION_DETAILS_SETTINGS);
diff --git a/app/src/main/java/com/screenomics/SecureStore.java b/app/src/main/java/com/screenomics/SecureStore.java
new file mode 100644
index 0000000..4fab599
--- /dev/null
+++ b/app/src/main/java/com/screenomics/SecureStore.java
@@ -0,0 +1,112 @@
+package com.screenomics;
+
+import android.content.Context;
+import android.content.SharedPreferences;
+import android.security.keystore.KeyGenParameterSpec;
+import android.security.keystore.KeyProperties;
+import android.util.Base64;
+import android.util.Log;
+
+import androidx.preference.PreferenceManager;
+
+import java.security.KeyStore;
+
+import javax.crypto.Cipher;
+import javax.crypto.KeyGenerator;
+import javax.crypto.SecretKey;
+import javax.crypto.spec.GCMParameterSpec;
+
+/**
+ * Stores small secrets (e.g. the enrollment bearer token) encrypted at rest using an
+ * AndroidKeyStore-backed AES-256-GCM key. No external dependencies.
+ *
+ * The ciphertext is kept in the app's default SharedPreferences under "<name>_enc";
+ * the key material lives only in the hardware-backed AndroidKeyStore and is never exported.
+ *
+ * Fail-safe by design: if key access or crypto fails for any reason, it transparently
+ * falls back to the previous plaintext SharedPreferences behavior, so authentication can
+ * never break. Reading a legacy plaintext value also migrates it to encrypted storage.
+ */
+public final class SecureStore {
+ private static final String TAG = "SecureStore";
+ private static final String KEYSTORE = "AndroidKeyStore";
+ private static final String KEY_ALIAS = "mindpulse_secure_store_key";
+ private static final String ENC_SUFFIX = "_enc";
+ private static final int GCM_TAG_BITS = 128;
+ private static final int GCM_IV_LEN = 12;
+
+ private SecureStore() {}
+
+ private static SecretKey getOrCreateKey() throws Exception {
+ KeyStore ks = KeyStore.getInstance(KEYSTORE);
+ ks.load(null);
+ KeyStore.Entry entry = ks.getEntry(KEY_ALIAS, null);
+ if (entry instanceof KeyStore.SecretKeyEntry) {
+ return ((KeyStore.SecretKeyEntry) entry).getSecretKey();
+ }
+ KeyGenerator kg = KeyGenerator.getInstance(KeyProperties.KEY_ALGORITHM_AES, KEYSTORE);
+ kg.init(new KeyGenParameterSpec.Builder(KEY_ALIAS,
+ KeyProperties.PURPOSE_ENCRYPT | KeyProperties.PURPOSE_DECRYPT)
+ .setBlockModes(KeyProperties.BLOCK_MODE_GCM)
+ .setEncryptionPaddings(KeyProperties.ENCRYPTION_PADDING_NONE)
+ .setKeySize(256)
+ .build());
+ return kg.generateKey();
+ }
+
+ /** Encrypt and store {@code value} under {@code name}; removes any plaintext copy. */
+ public static void putSecret(Context ctx, String name, String value) {
+ SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(ctx);
+ try {
+ Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
+ c.init(Cipher.ENCRYPT_MODE, getOrCreateKey());
+ byte[] iv = c.getIV();
+ byte[] ct = c.doFinal(value.getBytes("UTF-8"));
+ byte[] blob = new byte[iv.length + ct.length];
+ System.arraycopy(iv, 0, blob, 0, iv.length);
+ System.arraycopy(ct, 0, blob, iv.length, ct.length);
+ p.edit()
+ .putString(name + ENC_SUFFIX, Base64.encodeToString(blob, Base64.NO_WRAP))
+ .remove(name) // never leave a plaintext copy behind
+ .apply();
+ } catch (Exception e) {
+ Log.w(TAG, "putSecret encryption failed; falling back to plaintext for " + name, e);
+ p.edit().putString(name, value).apply();
+ }
+ }
+
+ /** Remove both the encrypted and any legacy plaintext value for {@code name}. */
+ public static void removeSecret(Context ctx, String name) {
+ PreferenceManager.getDefaultSharedPreferences(ctx).edit()
+ .remove(name + ENC_SUFFIX).remove(name).apply();
+ }
+
+ /**
+ * Read the secret. Prefers the encrypted value; if only a legacy plaintext value exists
+ * it is returned and transparently migrated to encrypted storage.
+ */
+ public static String getSecret(Context ctx, String name, String def) {
+ SharedPreferences p = PreferenceManager.getDefaultSharedPreferences(ctx);
+ String enc = p.getString(name + ENC_SUFFIX, null);
+ if (enc != null) {
+ try {
+ byte[] blob = Base64.decode(enc, Base64.NO_WRAP);
+ byte[] iv = new byte[GCM_IV_LEN];
+ System.arraycopy(blob, 0, iv, 0, GCM_IV_LEN);
+ byte[] ct = new byte[blob.length - GCM_IV_LEN];
+ System.arraycopy(blob, GCM_IV_LEN, ct, 0, ct.length);
+ Cipher c = Cipher.getInstance("AES/GCM/NoPadding");
+ c.init(Cipher.DECRYPT_MODE, getOrCreateKey(), new GCMParameterSpec(GCM_TAG_BITS, iv));
+ return new String(c.doFinal(ct), "UTF-8");
+ } catch (Exception e) {
+ Log.w(TAG, "getSecret decryption failed for " + name + "; trying plaintext", e);
+ }
+ }
+ String legacy = p.getString(name, null);
+ if (legacy != null) {
+ putSecret(ctx, name, legacy); // one-time migration plaintext -> encrypted
+ return legacy;
+ }
+ return def;
+ }
+}
diff --git a/app/src/main/res/layout/dev_tools.xml b/app/src/main/res/layout/dev_tools.xml
index 9fb4f4f..063bd3f 100644
--- a/app/src/main/res/layout/dev_tools.xml
+++ b/app/src/main/res/layout/dev_tools.xml
@@ -343,6 +343,156 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+