diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e241e4c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,63 @@ +name: CI + +on: + push: + branches: [ main, ios_support ] + pull_request: + +jobs: + analyze-and-test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@v2 + with: + channel: stable + - name: Install dependencies + run: flutter pub get + - name: Verify formatting + run: dart format --output=none --set-exit-if-changed . + - name: Analyze + run: flutter analyze + - name: Run tests + run: flutter test + - name: Publish dry-run + run: flutter pub publish --dry-run + - name: pana score + run: | + dart pub global activate pana + dart pub global run pana --no-warning --exit-code-threshold 20 . + + build-android: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + - uses: subosito/flutter-action@v2 + with: + channel: stable + - name: Build example APK (compiles Kotlin + native C) + run: | + cd example + flutter pub get + flutter build apk --debug + + build-ios: + runs-on: macos-latest + steps: + - uses: actions/checkout@v4 + - uses: subosito/flutter-action@v2 + with: + channel: stable + - name: Build example iOS (compiles Swift + native C) + run: | + cd example + flutter pub get + flutter build ios --no-codesign --debug + - name: Lint podspec + run: | + cd ios + pod lib lint --allow-warnings || true diff --git a/.github/workflows/native-build.yml b/.github/workflows/native-build.yml deleted file mode 100644 index 83f448c..0000000 --- a/.github/workflows/native-build.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Build Native Binaries - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build-android: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up JDK 17 - uses: actions/setup-java@v3 - with: - java-version: '17' - distribution: 'temurin' - - name: Build Android AAR - run: | - cd android - ./gradlew assembleRelease - - build-ios: - runs-on: macos-latest - steps: - - uses: actions/checkout@v3 - - name: Build iOS Framework - run: | - # In a real project, we'd use xcodebuild - echo "Validating iOS Project..." - cd ios - # Add validation commands here diff --git a/CHANGELOG.md b/CHANGELOG.md index 219d031..3499acb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,54 @@ -## 0.0.1 +# Changelog -* Initial release of `flutter_native_vision_camera`. -* Integrated FFI for high-performance frame processing. -* Support for zero-copy preview textures on Android and iOS. -* Basic camera controls (zoom, focus, flash). -* Vision API for real-time frame access. +## 0.0.4 + +Correctness, honesty, and packaging pass. + +* **Fixed:** removed the deprecated `package` attribute from the Android manifest (broke builds under AGP 8.x; + the namespace is set in `build.gradle`). +* **Fixed:** Android video recording no longer crashes β€” `RECORD_AUDIO` is now declared and audio is gated behind a + runtime permission check, falling back to video-only when the mic is unavailable. +* **Fixed:** `requestMicrophonePermission` now actually awaits the permission dialog result. +* **Fixed:** frame processor no longer routes frames through a no-op relay isolate. The Dart callback is now + delivered directly via `NativeCallable.listener`; documentation corrected to state it runs on the main isolate + event loop (not a background isolate). +* **Fixed:** `CameraPreview` now rebuilds when the controller updates (e.g. after async init or device switch). +* **Fixed:** Android tap-to-focus now targets the correct metering point. +* **Improved:** Android frames now expose all YUV planes with correct row strides (previously only the Y plane). +* **Improved:** iOS reports the real frame pixel format and retains preview buffers safely. +* **Packaging:** real podspec metadata, version lockstep, honest README/feature matrix, `topics`/`issue_tracker`, + CI (format/analyze/test/dry-run/pana + per-platform native builds), dropped the unused + `plugin_platform_interface` dependency. +* **iOS:** implemented video recording (AVAssetWriter, keeps the frame stream live), `setFocusDistance`, + barcode symbology filtering, teardown-on-reinit, `CVPixelBuffer` retain/lock balance, honest pixel format, + main-thread results. Device-verified on iOS 18 (preview, photo, video, scanning). +* **Orientation (structural):** preview rotation is now read from the camera framework + (Android `SurfaceRequest.TransformationInfo`, iOS sensor-relative) and applied once in `CameraPreview` β€” + fixing the recurring 90Β° tilt. Added `controller.previewRotation` / `displayPreviewSize`; scanner overlay + and camera page no longer compensate for rotation themselves. iOS Vision boxes now use the buffer orientation. +* **Mirroring:** a single `mirror` flag on `initialize` drives both the front-camera preview and the saved + photo/video; the preview no longer double-mirrors on Android. `CameraPreview` gains `ResizeMode.contain`. +* **Android:** dynamic capture orientation, correct tap-to-focus metering, real photo orientation. +* **API:** exposed per-plane strides (`Frame.planeBytesPerRow`/`planePixelStride`) so chroma planes can be + walked correctly; added `CameraDevices.getCameraFormat(...)` to pick a format by resolution/fps. +* **Docs:** comprehensive README (compile-correct quickstart with permissions, photo/video/scan/lifecycle + snippets, a correct native C/C++ plugin guide, requirements, expanded platform matrix, Limitations & Roadmap); + rewrote the example README; corrected the "background isolate" claim in the API docs to match reality + (main isolate); documented `takeSnapshot` (iOS-only), `setExposure` units, and `initialize`'s + `pixelFormat`/`mirror`. +* **Example:** the Native Vision Camera page now reads the raw frame buffer over FFI and shows a live average + brightness, demonstrating the headline feature; fixed the C++ sample's row-stride indexing. -### 0.0.2 +## 0.0.3 + +* Migrated the Android embedding to AndroidX for better compatibility. + +## 0.0.2 * Fixed pubspec metadata (homepage, repository). -### 0.0.3 +## 0.0.1 -* Replaced Android2 with AndroidX for better compatibility. \ No newline at end of file +* Initial release of `flutter_native_vision_camera`. +* FFI pipeline for frame access, zero-copy preview textures, basic camera controls (zoom, focus, flash), + and real-time barcode scanning. diff --git a/README.md b/README.md index 4996f00..274d46d 100644 --- a/README.md +++ b/README.md @@ -1,148 +1,292 @@ # Flutter Native Vision Camera -A high-performance, FFI-powered camera plugin for Flutter that puts efficiency and hardware control first. - -Built for developers who need more than just a preview: **Real-time AI, low-latency filters, and professional-grade video control.** +A high-performance, FFI-powered camera plugin for Flutter, built for **real-time +on-device vision**: direct, low-overhead access to camera frames for your own +ML/CV code, integrated barcode/QR scanning, and full hardware control. [![pub package](https://img.shields.io/pub/v/flutter_native_vision_camera.svg)](https://pub.dev/packages/flutter_native_vision_camera) -## Why use this instead of `camera`? +> **Status:** Active development (`0.0.x`) β€” the API may change before `1.0.0`. +> **Device-verified** on Android (Pixel 8) and iOS 18: preview, photo, video +> recording, barcode/QR scanning, zoom/torch/focus, orientation, and mirroring. -The official `camera` package is great for simple use cases, but it often falls short when building high-performance vision applications (AI, QR scanning, custom filters). This package was built to solve those limitations. +## Why use this instead of `camera`? -### πŸš€ Key Advantages +The official [`camera`](https://pub.dev/packages/camera) package is excellent for +general capture. This package targets the niche it doesn't cover: a **real-time +frame pipeline** where you read raw camera buffers for your own vision code, +with integrated scanning, all sharing one GPU-texture preview. | Feature | `camera` (standard) | `flutter_native_vision_camera` | |---------|---------------------|--------------------------------| -| **Rendering** | Platform Channels (YUV -> Bitmap -> Skia) | **Zero-Copy GPU Textures** (OES -> Flutter Texture) | -| **Orientation** | Matches UI Orientation (Buggy for video) | **Hardware-Level Sensing** (Correct even if UI is locked) | -| **Frame Processing** | Async via Isolate (High Latency) | **Synchronous Background FFI** (Near Zero Latency) | -| **QR/Barcodes** | Third-party plugin required | **Optimized MLKit Integration** built-in | -| **Zoom/Focus** | Basic level support | Unified request management (Works during recording) | +| **Preview** | Platform texture | **GPU texture** (Android zero-copy `SurfaceProducer`; iOS `CVPixelBuffer`) | +| **Frame access** | `startImageStream` over the platform channel (serialized) | **Direct FFI pointer access** to plane buffers β€” no channel serialization | +| **Native frame hook** | β€” | **Synchronous C/C++ plugin** on the camera thread (zero-latency) | +| **Barcodes / QR** | Separate plugin | **Integrated** (Android MLKit, iOS Vision) | ## Features -- **Zero-Copy Rendering**: Frames go directly from the camera sensor to the GPU and then to Flutter's `Texture` widget. No expensive copying or CPU-side bitmap conversion. -- **Physical Orientation Engine**: High-fidelity rotation sensing using native `OrientationEventListener`. Recorded videos and previews always have the correct orientation, even on orientation-locked apps. -- **Integrated MLKit**: High-speed, hardware-accelerated barcode and QR code scanning. -- **Unified Request Loop**: Change zoom, torch, exposure, or recording state without dropping frame buffers. -- **FFI Background Pipeline**: Process camera frames in C++/Rust/Dart FFI with zero memory copying. +- **GPU-texture preview** with automatic orientation. +- **FFI frame access** β€” read raw Y/U/V (Android) or BGRA (iOS) planes directly. +- **Synchronous native C/C++ plugins** invoked on the camera thread. +- **Integrated barcode/QR scanning** (Android MLKit, iOS Vision). +- **Hardware controls** β€” zoom, torch, exposure, tap-to-focus, video recording. -## Frame Processors +## Requirements -Frame processors allow you to run code for every frame the camera captures. Unlike the standard `camera` plugin, `flutter_native_vision_camera` runs these **synchronously on a background thread**. +- **Flutter** β‰₯ 3.22, **Dart** β‰₯ 3.8 +- **iOS** 13.0+, **Android** `minSdk` 21+ +- A **physical device** β€” simulators/emulators have no real camera. -### How it works -1. **Zero Latency**: Frames are delivered to the processor as soon as they are available from the sensor. -2. **Background Execution**: Your code runs on a separate background Isolate or in Native C++, so the main UI thread stays at a silky smooth 60/120 FPS. -3. **Memory Safety**: The `Frame` object is valid only for the duration of the callback. If you need to keep data, copy it out of the frame's buffer. +## Getting Started -### Dart Frame Processor +### Install -Use for ML tasks (using `google_mlkit_*` or `tflite_flutter`) or image manipulation in Dart. +```bash +flutter pub add flutter_native_vision_camera +``` -```dart -await controller.setFrameProcessor((frame) { - // 'frame' contains pointers to YUV/RGB buffers - final bytes = frame.getPlane(0).bytes; // Direct access to Y-plane - - // Do your analysis here... -}); +### Permissions + +**iOS** β€” add to `ios/Runner/Info.plist`: + +```xml +NSCameraUsageDescription +This app needs camera access to capture photos and video. +NSMicrophoneUsageDescription +This app needs microphone access to record video with audio. ``` -### πŸš€ High-Performance FFI Plugins +**Android** β€” `CAMERA` and `RECORD_AUDIO` are declared by the plugin; request +them at runtime (shown below). -One of the unique features of `flutter_native_vision_camera` is the ability to write **Synchronous C++ Plugins**. +### Basic usage β€” preview -Instead of sending expensive image data back and forth over Method Channels or into separate Dart Isolates, you can hook directly into the native frame loop. +```dart +import 'package:flutter/material.dart'; +import 'package:flutter_native_vision_camera/flutter_native_vision_camera.dart'; -```cpp -// Your high-performance C++ code -void onFrame(FrameHandle frame) { - void* y_plane = Frame_getPlanePointer(frame, 0); - // Do heavy AI math here on the background thread +// Once, at startup (e.g. in main()): +initializeVisionCamera(); + +class CameraScreen extends StatefulWidget { + const CameraScreen({super.key}); + @override + State createState() => _CameraScreenState(); } -``` -## Getting Started +class _CameraScreenState extends State { + final controller = CameraController(); + + @override + void initState() { + super.initState(); + _start(); + } + + Future _start() async { + // 0. Permission (otherwise you get a black preview). + if (await CameraPermissions.requestCameraPermission() != + PermissionStatus.granted) { + return; + } + // 1. Pick a device. + final devices = await CameraDevices.getAvailableCameraDevices(); + final back = + CameraDevices.getCameraDevice(devices, CameraPosition.back) ?? + devices.first; + // 2. Initialize + start. + await controller.initialize(back, enablePhoto: true); + await controller.setActive(true); + if (mounted) setState(() {}); + } + + @override + void dispose() { + controller.dispose(); // Release the hardware. + super.dispose(); + } + + @override + Widget build(BuildContext context) => + CameraPreview(controller: controller, resizeMode: ResizeMode.cover); +} +``` -### Installation +### Take a photo -Add to your `pubspec.yaml`: +`takePhoto()` requires `enablePhoto: true` at init. -```yaml -dependencies: - flutter_native_vision_camera: ^1.0.0 +```dart +final photo = await controller.takePhoto(); +Image.file(File(photo.path)); // photo.width / photo.height / photo.path ``` -### Basic Usage +### Record video + +Initialize with `enableVideo: true`, then: ```dart -final controller = CameraController(); +final dir = await getTemporaryDirectory(); +await controller.startRecording('${dir.path}/clip.mp4'); +// ... seamlessly zoom / toggle torch while recording ... +await controller.stopRecording(); +``` -// 1. Initialize with a device -await controller.initialize( - devices.first, - enableVideo: true, - codeScanner: CodeScannerConfiguration(), -); +> The `mirror` flag on `initialize` controls the front-camera selfie mirror for +> **both** the preview and the saved photo/video (default `true`; set `false` to +> save what the camera actually sees). -// 2. Start streaming -await controller.setActive(true); +### Scan barcodes / QR codes -// 3. Listen for codes +```dart +await controller.initialize( + device, + codeScanner: CodeScannerConfiguration( + codeTypes: [CodeType.qr, CodeType.ean13, CodeType.code128], + ), +); controller.onCodeScanned.listen((codes) { - print('Detected: ${codes.first.value}'); + for (final code in codes) debugPrint('${code.type}: ${code.value}'); }); - -// 4. Render in your widget tree -ValueListenableBuilder( - valueListenable: controller, - builder: (context, state, _) { - if (state == CameraState.uninitialized) return CircularProgressIndicator(); - return Texture(textureId: controller.textureId!); - }, -) ``` -### Lifecycle Management +### Lifecycle management -Keeping the camera hardware active is resource-intensive. You **must** dispose of the controller when it's no longer needed to release the hardware and stop background threads. +Release the hardware on `dispose()`, and pause/resume with the app lifecycle: ```dart -@override -void dispose() { - // Shuts down the camera, sessions, and background isolates - controller.dispose(); - super.dispose(); +class _S extends State with WidgetsBindingObserver { + @override + void initState() { super.initState(); WidgetsBinding.instance.addObserver(this); } + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + controller.dispose(); + super.dispose(); + } + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (!controller.isInitialized) return; + if (state == AppLifecycleState.inactive || + state == AppLifecycleState.paused) { + controller.setActive(false); + } else if (state == AppLifecycleState.resumed) { + controller.setActive(true); + } + } } ``` -### Video Recording & Zooming +## Frame Processors -Unlike many other plugins, you can smoothly zoom or toggle the torch *while* recording video without causing any frame drops or freezes. +Run code for every camera frame β€” the package's headline feature. + +### Threading model β€” read this + +- **Dart frame callback:** delivered **asynchronously on the main isolate's + event loop** (via `dart:ffi` `NativeCallable.listener`). It does **not** run + on a background isolate and does **not** block the camera thread. Keep the work + light, or copy data out and hand it to your own isolate. (A true off-isolate + worklet model is on the roadmap.) +- **Native C/C++ hook:** runs **synchronously on the camera thread** with zero + added latency β€” use this for the heaviest work. ```dart -await controller.startRecording('/path/to/video.mp4'); +await controller.setFrameProcessor((frame) { + // FFI hot path: read pixels here for ML/CV. + final yPlane = frame.getPlaneData(0); // Uint8List view + final stride = frame.planeBytesPerRow(0); // bytes per row (>= width) + final avgLuma = frame.computeLuminance(0, 0, frame.width, frame.height); +}); +``` -// This call seamlessly updates the ongoing recording request -await controller.setZoom(2.5); +> `getPlaneData(i)` returns a **direct view** of native memory. Use +> `planeBytesPerRow(i)` / `planePixelStride(i)` to walk it correctly (rows are +> padded). `computeLuminance` is **YUV/Android-only** β€” on iOS (BGRA) it returns +> `0.0`; read `getPlaneData(0)` instead. -await controller.stopRecording(); +### Keeping a frame past the callback + +The `Frame` and its buffers are valid **only during the callback**. To use the +data later (e.g. on another isolate), either copy it out synchronously: + +```dart +final bytes = Uint8List.fromList(frame.getPlaneData(0)); // owns a copy ``` +…or extend the native lifetime by balancing the ref-count exactly once: + +```dart +frame.incrementRefCount(); // keep the buffer alive +// ... use frame asynchronously ... +frame.decrementRefCount(); // release it (mandatory) +``` + +### High-performance native C/C++ plugin + +For the heaviest work, hook the synchronous frame loop in C++ (`src/VisionCamera.hpp`): + +```cpp +#include "VisionCamera.hpp" + +class BrightnessPlugin : public vision::Plugin { +public: + const char* name() const override { return "brightness"; } + void onFrame(const vision::Frame& frame) override { + const uint8_t* y = frame.data(); // Y / first plane + int stride = frame.bytesPerRow(); + // ...heavy SIMD/AI math on the camera thread... + } +}; + +// Register once (e.g. from an init function you call via FFI at startup): +extern "C" __attribute__((visibility("default"))) void registerMyPlugins() { + vision::Registry::instance().addPlugin(std::make_shared()); +} +``` + +Put your `.cpp` alongside the plugin sources; it links via CMake on Android and +the podspec on iOS. See `src/VisionCamera_NativePluginExample.cpp` for a full +working example. + ## Platform Support -| Platform | Support | Notes | -|----------|---------|-------| -| **Android** | βœ… Full | CameraX, MLKit, FFI, Recording | -| **iOS** | ⚠️ Partial | AVFoundation, Vision API, Preview only. Recording & FFI coming soon. | +| Capability | Android | iOS | +|------------|:-------:|:---:| +| Preview (GPU texture) | βœ… | βœ… | +| Photo capture | βœ… | βœ… | +| Video recording (+ audio) | βœ… CameraX | βœ… AVAssetWriter | +| Barcode / QR scanning | βœ… MLKit | βœ… Vision | +| Frame processor (Dart, main isolate) | βœ… YUV planes | βœ… BGRA | +| Native C/C++ plugin (camera thread) | βœ… | βœ… | +| Zoom / torch / exposure / tap-focus | βœ… | βœ… | +| Manual focus distance | ⬜ | βœ… | +| `takeSnapshot` | ⬜ | βœ… | +| `regionOfInterest` (scanning) | ⬜ | βœ… | +| Minimum OS | `minSdk` 21 | iOS 13.0 | + +Legend: βœ… supported Β· ⬜ not implemented yet. Symbology coverage differs slightly +between MLKit (Android) and Vision (iOS). + +## Limitations & Roadmap + +- **No web / desktop** β€” Android + iOS only. +- **Frame processor runs on the main isolate** today; a true off-isolate worklet + model is planned. +- **Recording options** (`RecordVideoOptions`, codec/HDR) and **pause/resume** + are not yet wired on all platforms; `startRecording` takes a path string. +- **Mirror** is set at `initialize` time (no runtime toggle yet). +- **Controls** (`setZoom`/`setTorch`/etc.) are no-ops until `setActive(true)`. +- No RAW capture / multi-camera; `takeSnapshot` is iOS-only. +- Planned API polish: typed `CameraException`, `TorchMode` enum, `switchCamera`, + richer error reporting. ## Credits & Attribution -This package is a Flutter port and expansion of the concepts introduced by [react-native-vision-camera](https://github.com/mrousavy/react-native-vision-camera), originally created by [Marc Rousavy](https://github.com/mrousavy). - -We aim to bring the same high-performance, low-level camera control to the Flutter ecosystem, while leveraging Flutter's unique strengths like synchronous FFI and specialized `Texture` rendering. +Inspired by [react-native-vision-camera](https://github.com/mrousavy/react-native-vision-camera) +by [Marc Rousavy](https://github.com/mrousavy) β€” bringing the same high-performance, +low-level camera philosophy to Flutter via synchronous FFI and `Texture` rendering. ## License diff --git a/android/build.gradle b/android/build.gradle index b69b5f6..6275bc0 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -1,7 +1,7 @@ // The Android Gradle Plugin builds the native code with the Android NDK. group = "dev.jentejan.flutter_native_vision_camera" -version = "1.0" +version = "0.0.4" buildscript { repositories { diff --git a/android/src/main/AndroidManifest.xml b/android/src/main/AndroidManifest.xml index fd0a1f2..79538e8 100644 --- a/android/src/main/AndroidManifest.xml +++ b/android/src/main/AndroidManifest.xml @@ -1,4 +1,7 @@ - + + + + + diff --git a/android/src/main/kotlin/dev/jentejan/flutter_native_vision_camera/FlutterNativeVisionCameraPlugin.kt b/android/src/main/kotlin/dev/jentejan/flutter_native_vision_camera/FlutterNativeVisionCameraPlugin.kt index 70b45f2..dd9fffc 100644 --- a/android/src/main/kotlin/dev/jentejan/flutter_native_vision_camera/FlutterNativeVisionCameraPlugin.kt +++ b/android/src/main/kotlin/dev/jentejan/flutter_native_vision_camera/FlutterNativeVisionCameraPlugin.kt @@ -57,9 +57,16 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi private lateinit var lifecycleRegistry: LifecycleRegistry override val lifecycle: Lifecycle get() = lifecycleRegistry - // Native dispatcher - private external fun nativeDispatchFrame(buffer: java.nio.ByteBuffer, width: Int, height: Int, format: Int, orientation: Int, timestamp: Double, id: Long, address: Long) - private external fun nativeSetFrameProcessorCallback(callback: Long) + // Native dispatcher. Passes every image plane (with its row/pixel stride and + // byte size) so the Dart/C++ side can read true multi-plane YUV. + private external fun nativeDispatchFrame( + p0: java.nio.ByteBuffer, p1: java.nio.ByteBuffer?, p2: java.nio.ByteBuffer?, + rs0: Int, rs1: Int, rs2: Int, + ps0: Int, ps1: Int, ps2: Int, + sz0: Int, sz1: Int, sz2: Int, + numPlanes: Int, + width: Int, height: Int, format: Int, orientation: Int, timestamp: Double, id: Long + ) private lateinit var channel: MethodChannel private lateinit var textureRegistry: TextureRegistry @@ -67,6 +74,7 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi private var activity: Activity? = null private var binding: ActivityPluginBinding? = null private var pendingPermissionResult: MethodChannel.Result? = null + private var pendingMicPermissionResult: MethodChannel.Result? = null // CameraX components private var cameraProvider: ProcessCameraProvider? = null @@ -92,6 +100,7 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi private var barcodeScanner: BarcodeScanner? = null @Volatile private var isCodeScannerEnabled = false private var currentFormat: Map? = null + private var mirrorCaptures: Boolean = false private var recordingStartTime: Long = 0 private var pendingVideoResult: MethodChannel.Result? = null @@ -112,6 +121,7 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi companion object { private const val CHANNEL_NAME = "dev.jentejan.flutter_native_vision_camera/camera" private const val CAMERA_PERMISSION_REQUEST = 1001 + private const val MIC_PERMISSION_REQUEST = 1002 private val frameIdCounter = AtomicLong(0) @@ -135,6 +145,13 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi return currentCount } + @JvmStatic + @androidx.annotation.Keep + fun retainFrame(id: Long): Int { + val managed = activeFrames[id] ?: return 0 + return managed.refCount.incrementAndGet() + } + fun clearFrames() { activeFrames.forEach { (id, managed) -> try { @@ -178,13 +195,21 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi orientationEventListener = object : OrientationEventListener(context) { override fun onOrientationChanged(orientation: Int) { if (orientation == ORIENTATION_UNKNOWN) return - physicalOrientation = when { + val newRotation = when { orientation < 45 || orientation > 315 -> Surface.ROTATION_0 orientation in 45..134 -> Surface.ROTATION_270 orientation in 135..224 -> Surface.ROTATION_180 orientation in 225..314 -> Surface.ROTATION_90 else -> Surface.ROTATION_0 } + if (newRotation != physicalOrientation) { + physicalOrientation = newRotation + // Keep capture outputs correctly oriented from the physical + // sensor angle, even when the UI is orientation-locked. + imageCapture?.targetRotation = newRotation + videoCapture?.targetRotation = newRotation + imageAnalysis?.targetRotation = newRotation + } } } orientationEventListener?.enable() @@ -229,6 +254,7 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi val enablePhoto = call.argument("enablePhoto") ?: false val enableVideo = call.argument("enableVideo") ?: false val codeScanner = call.argument>("codeScanner") + mirrorCaptures = call.argument("mirror") ?: true initializeCamera(deviceId, format, enablePhoto, enableVideo, codeScanner, result) } "setActive" -> { @@ -469,10 +495,10 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi CameraSelector.DEFAULT_BACK_CAMERA } else { CameraSelector.Builder().addCameraFilter { cameras -> - cameras.filter { + cameras.filter { val info = Camera2CameraInfo.from(it) if (info.cameraId == deviceId) { - isFrontCamera = it.cameraInfo.lensFacing == CameraSelector.LENS_FACING_FRONT + isFrontCamera = it.lensFacing == CameraSelector.LENS_FACING_FRONT true } else false } @@ -507,9 +533,27 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi .build() previewUseCase.setSurfaceProvider(cameraExecutor) { request -> + // CameraX is the authority on how much the preview buffer must be + // rotated to display upright β€” it accounts for sensor orientation, + // target rotation AND use-case negotiation (which can pre-rotate the + // buffer). Report it to Dart so a single source of truth drives the + // preview rotation. This also re-fires on device rotation. + request.setTransformationInfoListener(cameraExecutor) { info -> + val degrees = info.rotationDegrees + // CameraX mirrors the front-camera preview itself; report that so + // the Dart side doesn't double-mirror it. + val mirrored = isFrontCamera + mainHandler.post { + channel.invokeMethod( + "onPreviewConfigurationChanged", + mapOf("rotationDegrees" to degrees, "mirrored" to mirrored) + ) + } + } + val res = request.resolution Log.d("CameraPlugin", "CameraX requesting preview surface: ${res.width}x${res.height}") - + // Important: Set size before providing surface producer.setSize(res.width, res.height) val surface = producer.surface @@ -552,6 +596,10 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi .build() videoCapture = VideoCapture.Builder(recorder) .setTargetRotation(rotation) + .setMirrorMode( + if (mirrorCaptures) MirrorMode.MIRROR_MODE_ON_FRONT_ONLY + else MirrorMode.MIRROR_MODE_OFF + ) .build() } @@ -642,18 +690,31 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi } } - // Dispatch to Native/C++ (Synchronous) + // Dispatch to Native/C++ (synchronous). Pass every plane with its + // row/pixel stride and byte size so Dart can read true multi-plane YUV. managed.refCount.incrementAndGet() - val yBuffer = image.planes[0].buffer + val planes = image.planes + val b0 = planes[0].buffer + val b1 = if (planes.size > 1) planes[1].buffer else null + val b2 = if (planes.size > 2) planes[2].buffer else null nativeDispatchFrame( - yBuffer, - image.width, - image.height, + b0, b1, b2, + planes[0].rowStride, + if (planes.size > 1) planes[1].rowStride else 0, + if (planes.size > 2) planes[2].rowStride else 0, + planes[0].pixelStride, + if (planes.size > 1) planes[1].pixelStride else 0, + if (planes.size > 2) planes[2].pixelStride else 0, + b0.remaining(), + b1?.remaining() ?: 0, + b2?.remaining() ?: 0, + planes.size, + image.width, + image.height, 0x23, // YUV_420_888 - image.imageInfo.rotationDegrees, - image.imageInfo.timestamp.toDouble() / 1e9, - id, - 0L + image.imageInfo.rotationDegrees, + image.imageInfo.timestamp.toDouble() / 1e9, + id ) } finally { releaseFrame(id) @@ -681,7 +742,14 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi File(context.cacheDir, "photo_${System.currentTimeMillis()}.jpg") } - val outputOptions = ImageCapture.OutputFileOptions.Builder(file).build() + val metadata = ImageCapture.Metadata().apply { + // Mirror the saved image only when explicitly requested (selfie + // mirror); otherwise save what the camera actually sees. + isReversedHorizontal = mirrorCaptures && isFrontCamera + } + val outputOptions = ImageCapture.OutputFileOptions.Builder(file) + .setMetadata(metadata) + .build() capture.takePicture(outputOptions, cameraExecutor, object : ImageCapture.OnImageSavedCallback { override fun onImageSaved(outputFileResults: ImageCapture.OutputFileResults) { @@ -690,8 +758,8 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi "path" to file.absolutePath, "width" to (currentFormat?.get("photoWidth") ?: 1920), "height" to (currentFormat?.get("photoHeight") ?: 1080), - "orientation" to "portrait", - "isMirrored" to isFrontCamera + "orientation" to rotationToOrientationString(physicalOrientation), + "isMirrored" to (mirrorCaptures && isFrontCamera) )) } } @@ -723,10 +791,21 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi val file = File(outputFilePath) val outOptions = FileOutputOptions.Builder(file).build() - activeRecording = capture.output - .prepareRecording(context, outOptions) - .withAudioEnabled() - .start(cameraExecutor) { event -> + // Only enable audio when RECORD_AUDIO has actually been granted β€” + // calling withAudioEnabled() without the permission throws. + val hasAudio = ContextCompat.checkSelfPermission( + context, Manifest.permission.RECORD_AUDIO + ) == PackageManager.PERMISSION_GRANTED + + try { + var pending = capture.output.prepareRecording(context, outOptions) + if (hasAudio) { + pending = pending.withAudioEnabled() + } else { + Log.w("CameraPlugin", "RECORD_AUDIO not granted β€” recording video without audio") + } + + activeRecording = pending.start(cameraExecutor) { event -> when (event) { is VideoRecordEvent.Start -> { recordingStartTime = System.currentTimeMillis() @@ -735,7 +814,20 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi is VideoRecordEvent.Finalize -> { if (event.hasError()) { Log.e("CameraPlugin", "Video recording error: ${event.error}") - // Handle error if needed + // Drop the dangling recording so a later stopRecording() + // doesn't hang, and surface the failure to whichever call + // is still pending plus the controller's error stream. + activeRecording = null + mainHandler.post { + safeResult.error("RECORDING_ERROR", "Recording failed (code ${event.error})", null) + pendingVideoResult?.error("RECORDING_ERROR", "Recording failed (code ${event.error})", null) + pendingVideoResult = null + channel.invokeMethod("onError", mapOf( + "code" to "RECORDING_ERROR", + "message" to "Recording failed (code ${event.error})" + )) + } + return@start } val duration = (event.recordingStats.recordedDurationNanos / 1e9) val metadata = mapOf( @@ -751,6 +843,10 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi } } } + } catch (e: Exception) { + Log.e("CameraPlugin", "Failed to start recording: ${e.message}") + safeResult.error("RECORDING_ERROR", "Failed to start recording: ${e.message}", null) + } } private fun stopRecording(result: MethodChannel.Result) { @@ -798,11 +894,10 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi } private fun focus(x: Double, y: Double, result: MethodChannel.Result) { - val videoWidth = currentFormat?.get("videoWidth") as? Int ?: 1920 - val videoHeight = currentFormat?.get("videoHeight") as? Int ?: 1080 - - val factory = SurfaceOrientedMeteringPointFactory(videoWidth.toFloat(), videoHeight.toFloat()) - val point = factory.createPoint(x.toFloat(), y.toFloat()) + // x,y arrive already normalized to 0..1 in sensor space (the widget maps + // through BoxFit and front-mirror), so use a unit-sized factory. + val factory = SurfaceOrientedMeteringPointFactory(1f, 1f) + val point = factory.createPoint(x.toFloat().coerceIn(0f, 1f), y.toFloat().coerceIn(0f, 1f)) val action = FocusMeteringAction.Builder(point, FocusMeteringAction.FLAG_AF or FocusMeteringAction.FLAG_AE) .setAutoCancelDuration(5, TimeUnit.SECONDS) .build() @@ -817,6 +912,14 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi result.notImplemented() } + private fun rotationToOrientationString(rotation: Int): String = when (rotation) { + Surface.ROTATION_0 -> "portrait" + Surface.ROTATION_90 -> "landscape-right" + Surface.ROTATION_180 -> "portrait-upside-down" + Surface.ROTATION_270 -> "landscape-left" + else -> "portrait" + } + // ─── Code Scanner Utilities ──────────────────────────────────────── private fun getMlKitRotation(): Int { @@ -925,6 +1028,16 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi pendingPermissionResult = null return true } + if (requestCode == MIC_PERMISSION_REQUEST) { + val pendingResult = pendingMicPermissionResult ?: return false + if (grantResults.isNotEmpty() && grantResults[0] == PackageManager.PERMISSION_GRANTED) { + pendingResult.success("granted") + } else { + pendingResult.success("denied") + } + pendingMicPermissionResult = null + return true + } return false } @@ -934,12 +1047,20 @@ class FlutterNativeVisionCameraPlugin : FlutterPlugin, MethodCallHandler, Activi } private fun requestMicrophonePermission(result: MethodChannel.Result) { + if (ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED) { + result.success("granted") + return + } val act = activity ?: run { result.error("NO_ACTIVITY", "Activity not available", null) return } - ActivityCompat.requestPermissions(act, arrayOf(Manifest.permission.RECORD_AUDIO), CAMERA_PERMISSION_REQUEST + 1) - result.success("granted") + if (pendingMicPermissionResult != null) { + result.error("PERMISSION_REQUEST_IN_PROGRESS", "A microphone permission request is already in progress", null) + return + } + pendingMicPermissionResult = result + ActivityCompat.requestPermissions(act, arrayOf(Manifest.permission.RECORD_AUDIO), MIC_PERMISSION_REQUEST) } // ─── Lifecycle & Cleanup ────────────────────────────────────────── diff --git a/example/README.md b/example/README.md index c6536af..0633e23 100644 --- a/example/README.md +++ b/example/README.md @@ -1,16 +1,33 @@ -# flutter_native_vision_camera_example +# flutter_native_vision_camera β€” example -Demonstrates how to use the flutter_native_vision_camera plugin. +A showcase of [`flutter_native_vision_camera`](../) with three pages. -## Getting Started +## Run it -This project is a starting point for a Flutter application. +```bash +cd example +flutter run # use a PHYSICAL device β€” simulators/emulators have no camera +``` -A few resources to get you started if this is your first Flutter project: +Grant the camera (and microphone, for video) permission prompts on first launch. -- [Lab: Write your first Flutter app](https://docs.flutter.dev/get-started/codelab) -- [Cookbook: Useful Flutter samples](https://docs.flutter.dev/cookbook) +## What's inside -For help getting started with Flutter development, view the -[online documentation](https://docs.flutter.dev/), which offers tutorials, -samples, guidance on mobile development, and a full API reference. +- **Native Vision Camera** β€” the FFI preview with a live **frame processor**: + an FPS meter *and* an average-brightness readout computed from the raw pixel + buffer (`Frame.computeLuminance` / `getPlaneData`), plus photo, video, + zoom, torch, tap-to-focus, and front/back switching. This is the page that + demonstrates the package's headline feature β€” reading frames for your own + ML/CV. The hot path is in `lib/native_camera_page.dart` (`setFrameProcessor`). +- **Barcode / QR Scanner** β€” real-time scanning (MLKit on Android, Vision on + iOS) with a live bounding-box overlay. See `lib/code_scanner_page.dart`. +- **Standard Camera** β€” the official `camera` package, side-by-side, so you can + compare delivery FPS and latency. See `lib/standard_camera_page.dart`. + +## Read this first + +- `lib/native_camera_page.dart` β†’ `setFrameProcessor(...)` is where per-frame + pixel data is read on the FFI hot path. +- `../src/VisionCamera_NativePluginExample.cpp` is a native C++ frame plugin + (registered via `initializeNativeExamplePlugin()` in `lib/main.dart`) β€” the + template for shipping your own zero-latency C/C++ vision code. diff --git a/example/android/gradle.properties b/example/android/gradle.properties index f018a61..475a628 100644 --- a/example/android/gradle.properties +++ b/example/android/gradle.properties @@ -1,3 +1,7 @@ org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true android.enableJetifier=true +# This builtInKotlin flag was added automatically by Flutter migrator +android.builtInKotlin=false +# This newDsl flag was added automatically by Flutter migrator +android.newDsl=false diff --git a/example/ios/Flutter/AppFrameworkInfo.plist b/example/ios/Flutter/AppFrameworkInfo.plist index 7c56964..391a902 100644 --- a/example/ios/Flutter/AppFrameworkInfo.plist +++ b/example/ios/Flutter/AppFrameworkInfo.plist @@ -20,7 +20,5 @@ ???? CFBundleVersion 1.0 - MinimumOSVersion - 12.0 diff --git a/example/ios/Flutter/Debug.xcconfig b/example/ios/Flutter/Debug.xcconfig index 592ceee..ec97fc6 100644 --- a/example/ios/Flutter/Debug.xcconfig +++ b/example/ios/Flutter/Debug.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig" #include "Generated.xcconfig" diff --git a/example/ios/Flutter/Release.xcconfig b/example/ios/Flutter/Release.xcconfig index 592ceee..c4855bf 100644 --- a/example/ios/Flutter/Release.xcconfig +++ b/example/ios/Flutter/Release.xcconfig @@ -1 +1,2 @@ +#include? "Pods/Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig" #include "Generated.xcconfig" diff --git a/example/ios/Podfile b/example/ios/Podfile new file mode 100644 index 0000000..620e46e --- /dev/null +++ b/example/ios/Podfile @@ -0,0 +1,43 @@ +# Uncomment this line to define a global platform for your project +# platform :ios, '13.0' + +# CocoaPods analytics sends network stats synchronously affecting flutter build latency. +ENV['COCOAPODS_DISABLE_STATS'] = 'true' + +project 'Runner', { + 'Debug' => :debug, + 'Profile' => :release, + 'Release' => :release, +} + +def flutter_root + generated_xcode_build_settings_path = File.expand_path(File.join('..', 'Flutter', 'Generated.xcconfig'), __FILE__) + unless File.exist?(generated_xcode_build_settings_path) + raise "#{generated_xcode_build_settings_path} must exist. If you're running pod install manually, make sure flutter pub get is executed first" + end + + File.foreach(generated_xcode_build_settings_path) do |line| + matches = line.match(/FLUTTER_ROOT\=(.*)/) + return matches[1].strip if matches + end + raise "FLUTTER_ROOT not found in #{generated_xcode_build_settings_path}. Try deleting Generated.xcconfig, then run flutter pub get" +end + +require File.expand_path(File.join('packages', 'flutter_tools', 'bin', 'podhelper'), flutter_root) + +flutter_ios_podfile_setup + +target 'Runner' do + use_frameworks! + + flutter_install_all_ios_pods File.dirname(File.realpath(__FILE__)) + target 'RunnerTests' do + inherit! :search_paths + end +end + +post_install do |installer| + installer.pods_project.targets.each do |target| + flutter_additional_ios_build_settings(target) + end +end diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock new file mode 100644 index 0000000..6524034 --- /dev/null +++ b/example/ios/Podfile.lock @@ -0,0 +1,28 @@ +PODS: + - Flutter (1.0.0) + - flutter_native_vision_camera (0.0.4): + - Flutter + - permission_handler_apple (9.3.0): + - Flutter + +DEPENDENCIES: + - Flutter (from `Flutter`) + - flutter_native_vision_camera (from `.symlinks/plugins/flutter_native_vision_camera/ios`) + - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`) + +EXTERNAL SOURCES: + Flutter: + :path: Flutter + flutter_native_vision_camera: + :path: ".symlinks/plugins/flutter_native_vision_camera/ios" + permission_handler_apple: + :path: ".symlinks/plugins/permission_handler_apple/ios" + +SPEC CHECKSUMS: + Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 + flutter_native_vision_camera: f9cae7b27180d93bf6fafad999e37e5086ae477a + permission_handler_apple: 4ed2196e43d0651e8ff7ca3483a069d469701f2d + +PODFILE CHECKSUM: 3c63482e143d1b91d2d2560aee9fb04ecc74ac7e + +COCOAPODS: 1.16.2 diff --git a/example/ios/Runner.xcodeproj/project.pbxproj b/example/ios/Runner.xcodeproj/project.pbxproj index 003fb5e..4da7cfc 100644 --- a/example/ios/Runner.xcodeproj/project.pbxproj +++ b/example/ios/Runner.xcodeproj/project.pbxproj @@ -8,12 +8,15 @@ /* Begin PBXBuildFile section */ 1498D2341E8E89220040F4C2 /* GeneratedPluginRegistrant.m in Sources */ = {isa = PBXBuildFile; fileRef = 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */; }; + 25E05AC18BAA1742AE008D0B /* Pods_RunnerTests.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 9AA1E4A555F330674FA0257F /* Pods_RunnerTests.framework */; }; 331C808B294A63AB00263BE5 /* RunnerTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 331C807B294A618700263BE5 /* RunnerTests.swift */; }; 3B3967161E833CAA004F5970 /* AppFrameworkInfo.plist in Resources */ = {isa = PBXBuildFile; fileRef = 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */; }; 74858FAF1ED2DC5600515810 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 74858FAE1ED2DC5600515810 /* AppDelegate.swift */; }; 97C146FC1CF9000F007C117D /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FA1CF9000F007C117D /* Main.storyboard */; }; 97C146FE1CF9000F007C117D /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FD1CF9000F007C117D /* Assets.xcassets */; }; 97C147011CF9000F007C117D /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 97C146FF1CF9000F007C117D /* LaunchScreen.storyboard */; }; + B070447B347C8198B44400FA /* Pods_Runner.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = C80BCFC421C8CE89A7CAD129 /* Pods_Runner.framework */; }; + 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */ = {isa = PBXBuildFile; productRef = 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ @@ -42,12 +45,16 @@ /* Begin PBXFileReference section */ 1498D2321E8E86230040F4C2 /* GeneratedPluginRegistrant.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = GeneratedPluginRegistrant.h; sourceTree = ""; }; 1498D2331E8E89220040F4C2 /* GeneratedPluginRegistrant.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = GeneratedPluginRegistrant.m; sourceTree = ""; }; + 1CDAD4F3BA37BEA5F590F902 /* Pods-RunnerTests.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.debug.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.debug.xcconfig"; sourceTree = ""; }; 331C807B294A618700263BE5 /* RunnerTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RunnerTests.swift; sourceTree = ""; }; 331C8081294A63A400263BE5 /* RunnerTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = RunnerTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; name = AppFrameworkInfo.plist; path = Flutter/AppFrameworkInfo.plist; sourceTree = ""; }; + 5B723E7A019946C908861B8D /* Pods-RunnerTests.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.release.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.release.xcconfig"; sourceTree = ""; }; + 62488AA5DBE2D68084E27215 /* Pods-Runner.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.profile.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.profile.xcconfig"; sourceTree = ""; }; 74858FAD1ED2DC5600515810 /* Runner-Bridging-Header.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = "Runner-Bridging-Header.h"; sourceTree = ""; }; 74858FAE1ED2DC5600515810 /* AppDelegate.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 7AFA3C8E1D35360C0083082E /* Release.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; name = Release.xcconfig; path = Flutter/Release.xcconfig; sourceTree = ""; }; + 8D7386B18D8A338CEF6A8AA4 /* Pods-Runner.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.release.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.release.xcconfig"; sourceTree = ""; }; 9740EEB21CF90195004384FC /* Debug.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Debug.xcconfig; path = Flutter/Debug.xcconfig; sourceTree = ""; }; 9740EEB31CF90195004384FC /* Generated.xcconfig */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.xcconfig; name = Generated.xcconfig; path = Flutter/Generated.xcconfig; sourceTree = ""; }; 97C146EE1CF9000F007C117D /* Runner.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Runner.app; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -55,6 +62,13 @@ 97C146FD1CF9000F007C117D /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 97C147001CF9000F007C117D /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 97C147021CF9000F007C117D /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 9AA1E4A555F330674FA0257F /* Pods_RunnerTests.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_RunnerTests.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + C80BCFC421C8CE89A7CAD129 /* Pods_Runner.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Runner.framework; sourceTree = BUILT_PRODUCTS_DIR; }; + DD93BF017D387770CE323B3C /* Pods-RunnerTests.profile.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-RunnerTests.profile.xcconfig"; path = "Target Support Files/Pods-RunnerTests/Pods-RunnerTests.profile.xcconfig"; sourceTree = ""; }; + F8E8BA6A34C34B33A521968F /* Pods-Runner.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Runner.debug.xcconfig"; path = "Target Support Files/Pods-Runner/Pods-Runner.debug.xcconfig"; sourceTree = ""; }; + 78E0A7A72DC9AD7400C4905E /* FlutterGeneratedPluginSwiftPackage */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = FlutterGeneratedPluginSwiftPackage; path = Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; sourceTree = ""; }; + 784666492D4C4C64000A1A5F /* FlutterFramework */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = FlutterFramework; path = Flutter/ephemeral/Packages/.packages/FlutterFramework; sourceTree = ""; }; + 78DABEA22ED26510000E7860 /* flutter_native_vision_camera */ = {isa = PBXFileReference; lastKnownFileType = wrapper; name = flutter_native_vision_camera; path = ../../ios/flutter_native_vision_camera; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -62,12 +76,36 @@ isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( + 78A318202AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage in Frameworks */, + B070447B347C8198B44400FA /* Pods_Runner.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; + A20A03935EBAB1868797F310 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 25E05AC18BAA1742AE008D0B /* Pods_RunnerTests.framework in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 05EE07D64D92BD77DE4FFA53 /* Pods */ = { + isa = PBXGroup; + children = ( + F8E8BA6A34C34B33A521968F /* Pods-Runner.debug.xcconfig */, + 8D7386B18D8A338CEF6A8AA4 /* Pods-Runner.release.xcconfig */, + 62488AA5DBE2D68084E27215 /* Pods-Runner.profile.xcconfig */, + 1CDAD4F3BA37BEA5F590F902 /* Pods-RunnerTests.debug.xcconfig */, + 5B723E7A019946C908861B8D /* Pods-RunnerTests.release.xcconfig */, + DD93BF017D387770CE323B3C /* Pods-RunnerTests.profile.xcconfig */, + ); + name = Pods; + path = Pods; + sourceTree = ""; + }; 331C8082294A63A400263BE5 /* RunnerTests */ = { isa = PBXGroup; children = ( @@ -76,9 +114,21 @@ path = RunnerTests; sourceTree = ""; }; + 5F202EE044A26F442EA2FF52 /* Frameworks */ = { + isa = PBXGroup; + children = ( + C80BCFC421C8CE89A7CAD129 /* Pods_Runner.framework */, + 9AA1E4A555F330674FA0257F /* Pods_RunnerTests.framework */, + ); + name = Frameworks; + sourceTree = ""; + }; 9740EEB11CF90186004384FC /* Flutter */ = { isa = PBXGroup; children = ( + 78DABEA22ED26510000E7860 /* flutter_native_vision_camera */, + 784666492D4C4C64000A1A5F /* FlutterFramework */, + 78E0A7A72DC9AD7400C4905E /* FlutterGeneratedPluginSwiftPackage */, 3B3967151E833CAA004F5970 /* AppFrameworkInfo.plist */, 9740EEB21CF90195004384FC /* Debug.xcconfig */, 7AFA3C8E1D35360C0083082E /* Release.xcconfig */, @@ -94,6 +144,8 @@ 97C146F01CF9000F007C117D /* Runner */, 97C146EF1CF9000F007C117D /* Products */, 331C8082294A63A400263BE5 /* RunnerTests */, + 05EE07D64D92BD77DE4FFA53 /* Pods */, + 5F202EE044A26F442EA2FF52 /* Frameworks */, ); sourceTree = ""; }; @@ -128,8 +180,10 @@ isa = PBXNativeTarget; buildConfigurationList = 331C8087294A63A400263BE5 /* Build configuration list for PBXNativeTarget "RunnerTests" */; buildPhases = ( + 52CEFB409F4817C72A5DA21E /* [CP] Check Pods Manifest.lock */, 331C807D294A63A400263BE5 /* Sources */, 331C807F294A63A400263BE5 /* Resources */, + A20A03935EBAB1868797F310 /* Frameworks */, ); buildRules = ( ); @@ -142,15 +196,21 @@ productType = "com.apple.product-type.bundle.unit-test"; }; 97C146ED1CF9000F007C117D /* Runner */ = { + packageProductDependencies = ( + 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */, + ); isa = PBXNativeTarget; buildConfigurationList = 97C147051CF9000F007C117D /* Build configuration list for PBXNativeTarget "Runner" */; buildPhases = ( + 432D3CAC0163436231B3D0CE /* [CP] Check Pods Manifest.lock */, 9740EEB61CF901F6004384FC /* Run Script */, 97C146EA1CF9000F007C117D /* Sources */, 97C146EB1CF9000F007C117D /* Frameworks */, 97C146EC1CF9000F007C117D /* Resources */, 9705A1C41CF9048500538489 /* Embed Frameworks */, 3B06AD1E1E4923F5004D2608 /* Thin Binary */, + 93745E2F4E9D10842BAE316E /* [CP] Embed Pods Frameworks */, + 9803E967E5AE86604E1AD115 /* [CP] Copy Pods Resources */, ); buildRules = ( ); @@ -165,6 +225,9 @@ /* Begin PBXProject section */ 97C146E61CF9000F007C117D /* Project object */ = { + packageReferences = ( + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */, + ); isa = PBXProject; attributes = { BuildIndependentTargetsInParallel = YES; @@ -238,6 +301,67 @@ shellPath = /bin/sh; shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" embed_and_thin"; }; + 432D3CAC0163436231B3D0CE /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-Runner-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 52CEFB409F4817C72A5DA21E /* [CP] Check Pods Manifest.lock */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + ); + inputPaths = ( + "${PODS_PODFILE_DIR_PATH}/Podfile.lock", + "${PODS_ROOT}/Manifest.lock", + ); + name = "[CP] Check Pods Manifest.lock"; + outputFileListPaths = ( + ); + outputPaths = ( + "$(DERIVED_FILE_DIR)/Pods-RunnerTests-checkManifestLockResult.txt", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "diff \"${PODS_PODFILE_DIR_PATH}/Podfile.lock\" \"${PODS_ROOT}/Manifest.lock\" > /dev/null\nif [ $? != 0 ] ; then\n # print error to STDERR\n echo \"error: The sandbox is not in sync with the Podfile.lock. Run 'pod install' or update your CocoaPods installation.\" >&2\n exit 1\nfi\n# This output is used by Xcode 'outputs' to avoid re-running this script phase.\necho \"SUCCESS\" > \"${SCRIPT_OUTPUT_FILE_0}\"\n"; + showEnvVarsInLog = 0; + }; + 93745E2F4E9D10842BAE316E /* [CP] Embed Pods Frameworks */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Embed Pods Frameworks"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-frameworks.sh\"\n"; + showEnvVarsInLog = 0; + }; 9740EEB61CF901F6004384FC /* Run Script */ = { isa = PBXShellScriptBuildPhase; alwaysOutOfDate = 1; @@ -253,6 +377,23 @@ shellPath = /bin/sh; shellScript = "/bin/sh \"$FLUTTER_ROOT/packages/flutter_tools/bin/xcode_backend.sh\" build"; }; + 9803E967E5AE86604E1AD115 /* [CP] Copy Pods Resources */ = { + isa = PBXShellScriptBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + inputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources-${CONFIGURATION}-input-files.xcfilelist", + ); + name = "[CP] Copy Pods Resources"; + outputFileListPaths = ( + "${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources-${CONFIGURATION}-output-files.xcfilelist", + ); + runOnlyForDeploymentPostprocessing = 0; + shellPath = /bin/sh; + shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-Runner/Pods-Runner-resources.sh\"\n"; + showEnvVarsInLog = 0; + }; /* End PBXShellScriptBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -346,7 +487,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; @@ -378,6 +519,7 @@ }; 331C8088294A63A400263BE5 /* Debug */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 1CDAD4F3BA37BEA5F590F902 /* Pods-RunnerTests.debug.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; @@ -395,6 +537,7 @@ }; 331C8089294A63A400263BE5 /* Release */ = { isa = XCBuildConfiguration; + baseConfigurationReference = 5B723E7A019946C908861B8D /* Pods-RunnerTests.release.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; @@ -410,6 +553,7 @@ }; 331C808A294A63A400263BE5 /* Profile */ = { isa = XCBuildConfiguration; + baseConfigurationReference = DD93BF017D387770CE323B3C /* Pods-RunnerTests.profile.xcconfig */; buildSettings = { BUNDLE_LOADER = "$(TEST_HOST)"; CODE_SIGN_STYLE = Automatic; @@ -472,7 +616,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = YES; ONLY_ACTIVE_ARCH = YES; SDKROOT = iphoneos; @@ -523,7 +667,7 @@ GCC_WARN_UNINITIALIZED_AUTOS = YES_AGGRESSIVE; GCC_WARN_UNUSED_FUNCTION = YES; GCC_WARN_UNUSED_VARIABLE = YES; - IPHONEOS_DEPLOYMENT_TARGET = 12.0; + IPHONEOS_DEPLOYMENT_TARGET = 13.0; MTL_ENABLE_DEBUG_INFO = NO; SDKROOT = iphoneos; SUPPORTED_PLATFORMS = iphoneos; @@ -611,6 +755,18 @@ defaultConfigurationName = Release; }; /* End XCConfigurationList section */ +/* Begin XCLocalSwiftPackageReference section */ + 781AD8BC2B33823900A9FFBB /* XCLocalSwiftPackageReference "Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage" */ = { + isa = XCLocalSwiftPackageReference; + relativePath = Flutter/ephemeral/Packages/FlutterGeneratedPluginSwiftPackage; + }; +/* End XCLocalSwiftPackageReference section */ +/* Begin XCSwiftPackageProductDependency section */ + 78A3181F2AECB46A00862997 /* FlutterGeneratedPluginSwiftPackage */ = { + isa = XCSwiftPackageProductDependency; + productName = FlutterGeneratedPluginSwiftPackage; + }; +/* End XCSwiftPackageProductDependency section */ }; rootObject = 97C146E61CF9000F007C117D /* Project object */; } diff --git a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme index e3773d4..c3fedb2 100644 --- a/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme +++ b/example/ios/Runner.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme @@ -5,6 +5,24 @@ + + + + + + + + + + + + diff --git a/example/ios/Runner/AppDelegate.swift b/example/ios/Runner/AppDelegate.swift index 6266644..c30b367 100644 --- a/example/ios/Runner/AppDelegate.swift +++ b/example/ios/Runner/AppDelegate.swift @@ -2,12 +2,15 @@ import Flutter import UIKit @main -@objc class AppDelegate: FlutterAppDelegate { +@objc class AppDelegate: FlutterAppDelegate, FlutterImplicitEngineDelegate { override func application( _ application: UIApplication, didFinishLaunchingWithOptions launchOptions: [UIApplication.LaunchOptionsKey: Any]? ) -> Bool { - GeneratedPluginRegistrant.register(with: self) return super.application(application, didFinishLaunchingWithOptions: launchOptions) } + + func didInitializeImplicitFlutterEngine(_ engineBridge: FlutterImplicitEngineBridge) { + GeneratedPluginRegistrant.register(with: engineBridge.pluginRegistry) + } } diff --git a/example/ios/Runner/Info.plist b/example/ios/Runner/Info.plist index afb656c..c86d091 100644 --- a/example/ios/Runner/Info.plist +++ b/example/ios/Runner/Info.plist @@ -2,6 +2,8 @@ + CADisableMinimumFrameDurationOnPhone + CFBundleDevelopmentRegion $(DEVELOPMENT_LANGUAGE) CFBundleDisplayName @@ -24,6 +26,29 @@ $(FLUTTER_BUILD_NUMBER) LSRequiresIPhoneOS + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneClassName + UIWindowScene + UISceneConfigurationName + flutter + UISceneDelegateClassName + FlutterSceneDelegate + UISceneStoryboardFile + Main + + + + + UIApplicationSupportsIndirectInputEvents + UILaunchStoryboardName LaunchScreen UIMainStoryboardFile @@ -41,9 +66,11 @@ UIInterfaceOrientationLandscapeLeft UIInterfaceOrientationLandscapeRight - CADisableMinimumFrameDurationOnPhone - - UIApplicationSupportsIndirectInputEvents - + NSCameraUsageDescription + This app needs camera access to capture photos and videos. + NSMicrophoneUsageDescription + This app needs microphone access for video recording. + NSPhotoLibraryUsageDescription + This app needs access to your photo library to save captured media. diff --git a/example/lib/code_scanner_page.dart b/example/lib/code_scanner_page.dart index 146d7d6..8b0f7ac 100644 --- a/example/lib/code_scanner_page.dart +++ b/example/lib/code_scanner_page.dart @@ -129,30 +129,19 @@ class _CodeScannerPageState extends State // High-Tech Viewfinder const Center(child: ViewfinderGuide()), - // Overlay Boxes + // Overlay Boxes β€” sized against the controller's upright display + // size (single source of truth), so boxes stay aligned with the + // rotation CameraPreview already applies. No manual W/H swap. if (_isInitialized) Positioned.fill( child: Builder( builder: (context) { - final orientation = - _controller.device?.sensorOrientation ?? - Orientation.portrait; - final bool isSwapped = - orientation == Orientation.landscapeLeft || - orientation == Orientation.landscapeRight; - - final previewWidth = isSwapped - ? (_controller.previewHeight?.toDouble() ?? 1080) - : (_controller.previewWidth?.toDouble() ?? 1920); - final previewHeight = isSwapped - ? (_controller.previewWidth?.toDouble() ?? 1920) - : (_controller.previewHeight?.toDouble() ?? 1080); - + final display = _controller.displayPreviewSize; return CustomPaint( painter: ScannerOverlayPainter( _scannedCodes, - previewWidth, - previewHeight, + display?.width ?? 1080, + display?.height ?? 1920, ), ); }, @@ -323,7 +312,7 @@ class ScannerOverlayPainter extends CustomPainter { ..strokeWidth = 3.0; final fillPaint = Paint() - ..color = Colors.greenAccent.withOpacity(0.2) + ..color = Colors.greenAccent.withValues(alpha: 0.2) ..style = PaintingStyle.fill; // Calculate BoxFit.cover mapping diff --git a/example/lib/main.dart b/example/lib/main.dart index c7db1a5..c59aa13 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -60,7 +60,7 @@ class _HomePageState extends State { _buildFeatureCard( context, title: 'Native Vision Camera', - subtitle: 'High-performance zero-copy preview', + subtitle: 'FFI frame processor + live pixel read', icon: Icons.camera_enhance, color: Colors.blue, onTap: () { @@ -72,8 +72,8 @@ class _HomePageState extends State { const SizedBox(height: 16), _buildFeatureCard( context, - title: 'MLKit Code Scanner', - subtitle: 'Real-time barcode & QR detection', + title: 'Barcode / QR Scanner', + subtitle: 'MLKit on Android Β· Vision on iOS', icon: Icons.qr_code_scanner, color: Colors.green, onTap: () { @@ -129,7 +129,7 @@ class _HomePageState extends State { Container( padding: const EdgeInsets.all(12), decoration: BoxDecoration( - color: color.withOpacity(0.1), + color: color.withValues(alpha: 0.1), borderRadius: BorderRadius.circular(12), ), child: Icon(icon, color: color, size: 32), diff --git a/example/lib/native_camera_page.dart b/example/lib/native_camera_page.dart index 11085ab..ba6b415 100644 --- a/example/lib/native_camera_page.dart +++ b/example/lib/native_camera_page.dart @@ -19,13 +19,16 @@ class _NativeCameraPageState extends State CameraDevice? _currentDevice; CameraDeviceFormat? _currentFormat; bool _isInitialized = false; + String? _error; double _zoom = 1.0; String? _lastMediaPath; bool _isVideo = false; - // FPS Meter + // FPS Meter + a live average-brightness readout computed from the raw frame + // buffer over FFI (demonstrates the package's headline feature). double _fps = 0; + double _brightness = 0; int _frameCount = 0; DateTime? _lastFpsUpdate; @@ -56,18 +59,37 @@ class _NativeCameraPageState extends State } Future _initialize() async { - final camStatus = await CameraPermissions.requestCameraPermission(); - if (camStatus != PermissionStatus.granted) return; + try { + final camStatus = await CameraPermissions.requestCameraPermission(); + if (camStatus != PermissionStatus.granted) { + if (mounted) { + setState(() => _error = "Camera permission denied."); + } + return; + } - final devices = await CameraDevices.getAvailableCameraDevices(); - if (devices.isEmpty) return; + final devices = await CameraDevices.getAvailableCameraDevices(); + if (devices.isEmpty) { + if (mounted) { + setState( + () => _error = + "No camera devices found. (If on simulator, check settings)", + ); + } + return; + } - _devices = devices; - _currentDevice = - CameraDevices.getCameraDevice(devices, CameraPosition.back) ?? - devices.first; + _devices = devices; + _currentDevice = + CameraDevices.getCameraDevice(devices, CameraPosition.back) ?? + devices.first; - await _startCamera(); + await _startCamera(); + } catch (e) { + if (mounted) { + setState(() => _error = "Initialization failed: $e"); + } + } } Future _startCamera() async { @@ -96,21 +118,29 @@ class _NativeCameraPageState extends State format: _currentFormat, enablePhoto: true, enableVideo: true, + // One flag drives BOTH the preview and the saved image: true = selfie + // mirror, false = save what the camera actually sees. + mirror: true, ); - // Start FPS counter via frame processor + // Frame processor: FPS counter + average-brightness read from the raw + // pixel buffer. This is the FFI hot path β€” read pixels here for ML/CV. await _controller.setFrameProcessor((frame) { _frameCount++; final now = DateTime.now(); _lastFpsUpdate ??= now; if (now.difference(_lastFpsUpdate!).inMilliseconds >= 1000) { + // computeLuminance reads the Y plane directly over FFI (YUV/Android; + // on iOS BGRA it returns 0.0 β€” read frame.getPlaneData(0) instead). + final luma = frame.computeLuminance(0, 0, frame.width, frame.height); if (mounted) { setState(() { _fps = _frameCount * 1000 / now.difference(_lastFpsUpdate!).inMilliseconds; + _brightness = luma; _frameCount = 0; _lastFpsUpdate = now; }); @@ -185,6 +215,40 @@ class _NativeCameraPageState extends State @override Widget build(BuildContext context) { + if (_error != null) { + return Scaffold( + backgroundColor: Colors.black, + body: Center( + child: Padding( + padding: const EdgeInsets.all(24.0), + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + const Icon( + Icons.error_outline, + color: Colors.redAccent, + size: 64, + ), + const SizedBox(height: 16), + Text( + _error!, + textAlign: TextAlign.center, + style: const TextStyle(color: Colors.white, fontSize: 16), + ), + const SizedBox(height: 32), + ElevatedButton( + onPressed: () { + setState(() => _error = null); + _initialize(); + }, + child: const Text("Retry"), + ), + ], + ), + ), + ), + ); + } return ValueListenableBuilder( valueListenable: _controller, builder: (context, state, _) { @@ -193,18 +257,15 @@ class _NativeCameraPageState extends State body: Stack( children: [ if (_isInitialized) + // CameraPreview self-orients via controller.previewRotation β€” + // no AspectRatio/RotatedBox compensation needed here. Positioned.fill( - child: Center( - child: AspectRatio( - aspectRatio: - (_currentFormat?.videoHeight ?? 1) / - (_currentFormat?.videoWidth ?? 1), - child: CameraPreview( - controller: _controller, - resizeMode: ResizeMode.cover, - onTapToFocus: true, - ), - ), + child: CameraPreview( + controller: _controller, + // Fit the whole frame inside the view (letterboxed) so the + // full image is visible instead of cropped to fill. + resizeMode: ResizeMode.contain, + onTapToFocus: true, ), ) else @@ -231,14 +292,30 @@ class _NativeCameraPageState extends State color: Colors.black54, borderRadius: BorderRadius.circular(4), ), - child: Text( - "FPS: ${_fps.toStringAsFixed(1)}", - style: const TextStyle( - color: Colors.greenAccent, - fontWeight: FontWeight.bold, - fontSize: 12, - fontFamily: "monospace", - ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Text( + "FPS: ${_fps.toStringAsFixed(1)}", + style: const TextStyle( + color: Colors.greenAccent, + fontWeight: FontWeight.bold, + fontSize: 12, + fontFamily: "monospace", + ), + ), + // Proof of the FFI frame read (avg luminance from the Y plane). + Text( + "LUMA: ${_brightness.toStringAsFixed(0)}", + style: const TextStyle( + color: Colors.amberAccent, + fontWeight: FontWeight.bold, + fontSize: 12, + fontFamily: "monospace", + ), + ), + ], ), ), ); diff --git a/example/lib/standard_camera_page.dart b/example/lib/standard_camera_page.dart index 7d87530..5e98703 100644 --- a/example/lib/standard_camera_page.dart +++ b/example/lib/standard_camera_page.dart @@ -164,10 +164,11 @@ class _StandardCameraPageState extends State { try { await _controller!.setFocusPoint(offset); await _controller!.setExposurePoint(offset); - if (mounted) + if (mounted) { _showSnackbar( 'Focused at ${offset.dx.toStringAsFixed(2)}, ${offset.dy.toStringAsFixed(2)}', ); + } } on official.CameraException catch (e) { debugPrint('Focus failed: $e'); } diff --git a/example/pubspec.lock b/example/pubspec.lock index 4332f2c..4dceae1 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -61,10 +61,10 @@ packages: dependency: transitive description: name: characters - sha256: f71061c654a3380576a52b451dd5532377954cf9dbd272a78fc8479606670803 + sha256: faf38497bda5ead2a8c7615f4f7939df04333478bf32e4173fcb06d428b5716b url: "https://pub.dev" source: hosted - version: "1.4.0" + version: "1.4.1" clock: dependency: transitive description: @@ -140,7 +140,7 @@ packages: path: ".." relative: true source: path - version: "0.0.1" + version: "0.0.4" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -171,26 +171,26 @@ packages: dependency: transitive description: name: leak_tracker - sha256: "6bb818ecbdffe216e81182c2f0714a2e62b593f4a4f13098713ff1685dfb6ab0" + sha256: "33e2e26bdd85a0112ec15400c8cbffea70d0f9c3407491f672a2fad47915e2de" url: "https://pub.dev" source: hosted - version: "10.0.9" + version: "11.0.2" leak_tracker_flutter_testing: dependency: transitive description: name: leak_tracker_flutter_testing - sha256: f8b613e7e6a13ec79cfdc0e97638fddb3ab848452eff057653abd3edba760573 + sha256: "1dbc140bb5a23c75ea9c4811222756104fbcd1a27173f0c34ca01e16bea473c1" url: "https://pub.dev" source: hosted - version: "3.0.9" + version: "3.0.10" leak_tracker_testing: dependency: transitive description: name: leak_tracker_testing - sha256: "6ba465d5d76e67ddf503e1161d1f4a6bc42306f9d66ca1e8f079a47290fb06d3" + sha256: "8d5a2d49f4a66b49744b23b018848400d23e54caf9463f4eb20df3eb8acb2eb1" url: "https://pub.dev" source: hosted - version: "3.0.1" + version: "3.0.2" lints: dependency: transitive description: @@ -203,26 +203,26 @@ packages: dependency: transitive description: name: matcher - sha256: dc58c723c3c24bf8d3e2d3ad3f2f9d7bd9cf43ec6feaa64181775e60190153f2 + sha256: dc0b7dc7651697ea4ff3e69ef44b0407ea32c487a39fff6a4004fa585e901861 url: "https://pub.dev" source: hosted - version: "0.12.17" + version: "0.12.19" material_color_utilities: dependency: transitive description: name: material_color_utilities - sha256: f7142bb1154231d7ea5f96bc7bde4bda2a0945d2806bb11670e30b850d56bdec + sha256: "9c337007e82b1889149c82ed242ed1cb24a66044e30979c44912381e9be4c48b" url: "https://pub.dev" source: hosted - version: "0.11.1" + version: "0.13.0" meta: dependency: transitive description: name: meta - sha256: e3641ec5d63ebf0d9b41bd43201a66e3fc79a65db5f61fc181f04cd27aab950c + sha256: "1741988757a65eb6b36abe716829688cf01910bbf91c34354ff7ec1c3de2b349" url: "https://pub.dev" source: hosted - version: "1.16.0" + version: "1.18.0" path: dependency: transitive description: @@ -400,18 +400,18 @@ packages: dependency: transitive description: name: test_api - sha256: fb31f383e2ee25fbbfe06b40fe21e1e458d14080e3c67e7ba0acfde4df4e0bbd + sha256: "949a932224383300f01be9221c39180316445ecb8e7547f70a41a35bf421fb9e" url: "https://pub.dev" source: hosted - version: "0.7.4" + version: "0.7.11" vector_math: dependency: transitive description: name: vector_math - sha256: "80b3257d1492ce4d091729e3a67a60407d227c27241d6927be0130c98e741803" + sha256: d530bd74fea330e6e364cda7a85019c434070188383e1cd8d9777ee586914c5b url: "https://pub.dev" source: hosted - version: "2.1.4" + version: "2.2.0" video_player: dependency: "direct main" description: @@ -477,5 +477,5 @@ packages: source: hosted version: "1.1.0" sdks: - dart: ">=3.8.1 <4.0.0" + dart: ">=3.10.0-0 <4.0.0" flutter: ">=3.32.0" diff --git a/ios/.vscode/tasks.json b/ios/.vscode/tasks.json new file mode 100644 index 0000000..88c2622 --- /dev/null +++ b/ios/.vscode/tasks.json @@ -0,0 +1,10 @@ +{ + "version": "2.0.0", + "tasks": [ + { + "label": "Remove duplicate header", + "type": "shell", + "command": "rm /Volumes/External/FlutterProjects/flutter_native_vision_camera/ios/Classes/FlutterNativeVisionCameraPlugin.h" + } + ] +} \ No newline at end of file diff --git a/ios/Classes/FlutterNativeVisionCameraPlugin.h b/ios/Classes/FlutterNativeVisionCameraPlugin.h index 2b9e727..035303b 100644 --- a/ios/Classes/FlutterNativeVisionCameraPlugin.h +++ b/ios/Classes/FlutterNativeVisionCameraPlugin.h @@ -1,10 +1,15 @@ #ifndef FlutterNativeVisionCameraPlugin_h #define FlutterNativeVisionCameraPlugin_h +// Forward everything to the Swift-implemented functionality by using a module import. +// This is required because Flutter's GeneratedPluginRegistrant.m still expects to +// find the header, but defining the interface here would collide with the +// automatic Swift-to-ObjC bridging header. #import -#include "../../src/flutter_native_vision_camera.h" +#import "VisionCamera_FFI.h" -@interface FlutterNativeVisionCameraPlugin : NSObject +// Objective-C interface that the Flutter registrant expects. +@interface FlutterNativeVisionCameraPlugin : NSObject @end #endif /* FlutterNativeVisionCameraPlugin_h */ diff --git a/ios/Classes/FlutterNativeVisionCameraPlugin.m b/ios/Classes/FlutterNativeVisionCameraPlugin.m new file mode 100644 index 0000000..2988245 --- /dev/null +++ b/ios/Classes/FlutterNativeVisionCameraPlugin.m @@ -0,0 +1,20 @@ +#import "FlutterNativeVisionCameraPlugin.h" + +// Check for the existence of the modular Swift header. +// This is generated by Xcode when building the framework. +#if __has_include() +#import +#else +#import "flutter_native_vision_camera-Swift.h" +#endif + +@implementation FlutterNativeVisionCameraPlugin + ++ (void)registerWithRegistrar:(NSObject*)registrar { + // Relay the registration to the Swift implementation. + // The Swift class was renamed to SwiftFlutterNativeVisionCameraPlugin + // to avoid name collision in the module map. + [SwiftFlutterNativeVisionCameraPlugin registerWithRegistrar:registrar]; +} + +@end diff --git a/ios/Classes/FlutterNativeVisionCameraPlugin.swift b/ios/Classes/FlutterNativeVisionCameraPlugin.swift index 68d48d2..49b46d3 100644 --- a/ios/Classes/FlutterNativeVisionCameraPlugin.swift +++ b/ios/Classes/FlutterNativeVisionCameraPlugin.swift @@ -6,7 +6,7 @@ import Vision /// /// Uses AVFoundation for camera access, FlutterTextureRegistry for /// zero-copy GPU preview, and FlutterMethodChannel for control commands. -public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { +public class SwiftFlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { private var channel: FlutterMethodChannel! private var textureRegistry: FlutterTextureRegistry! @@ -15,15 +15,25 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { private var captureDevice: AVCaptureDevice? private var videoOutput: AVCaptureVideoDataOutput? private var photoOutput: AVCapturePhotoOutput? + private var audioOutput: AVCaptureAudioDataOutput? + private var audioInput: AVCaptureDeviceInput? private var isFrameProcessorEnabled = false private var textureId: Int64? private var pixelBufferRenderer: PixelBufferRenderer? - private var pendingPhotoResult: FlutterResult? - + fileprivate var pendingPhotoResult: FlutterResult? + + + // Recording state (AVAssetWriter reuses the live frame stream so frames keep + // flowing to the preview/processor while recording). + private var recorder: VideoRecorder? + private var videoPath: String? + private var enableVideo = false + private var mirrorCaptures = false + private var lastZoom: Float = 1.0 private var lastAFTriggerZoom: Float = 1.0 private var isManualFocusActive = false - + private var codeScannerRequest: VNDetectBarcodesRequest? private var isCodeScannerEnabled = false @@ -34,7 +44,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { name: "dev.jentejan.flutter_native_vision_camera/camera", binaryMessenger: registrar.messenger() ) - let instance = FlutterNativeVisionCameraPlugin() + let instance = SwiftFlutterNativeVisionCameraPlugin() instance.channel = channel instance.textureRegistry = registrar.textures() registrar.addMethodCallDelegate(instance, channel: channel) @@ -51,7 +61,9 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { } let deviceId = args["deviceId"] as? String ?? "" let codeScanner = args["codeScanner"] as? [String: Any] - initializeCamera(deviceId: deviceId, codeScanner: codeScanner, result: result) + let enableVideo = args["enableVideo"] as? Bool ?? false + self.mirrorCaptures = args["mirror"] as? Bool ?? true + initializeCamera(deviceId: deviceId, enableVideo: enableVideo, codeScanner: codeScanner, result: result) case "setActive": guard let args = call.arguments as? [String: Any], let isActive = args["isActive"] as? Bool else { @@ -88,22 +100,29 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { return } focus(at: CGPoint(x: x, y: y), result: result) + case "setFocusDistance": + guard let args = call.arguments as? [String: Any], + let distance = args["distance"] as? Double else { + result(FlutterError(code: "INVALID_ARGS", message: "Expected distance", details: nil)) + return + } + setFocusDistance(Float(distance), result: result) case "takePhoto": guard let args = call.arguments as? [String: Any] else { result(FlutterError(code: "INVALID_ARGS", message: "Expected arguments", details: nil)) return } - takePhoto(args: args, result: result) + takePhoto(options: args, result: result) case "startRecording": - result(FlutterError(code: "NOT_IMPLEMENTED", message: "Recording not implemented yet", details: nil)) + let args = call.arguments as? [String: Any] ?? [:] + startRecording(options: args, result: result) case "stopRecording": - result(FlutterError(code: "NOT_IMPLEMENTED", message: "Recording not implemented yet", details: nil)) - case "pauseRecording": - result(FlutterError(code: "NOT_IMPLEMENTED", message: "Recording not implemented yet", details: nil)) - case "resumeRecording": - result(FlutterError(code: "NOT_IMPLEMENTED", message: "Recording not implemented yet", details: nil)) + stopRecording(result: result) case "cancelRecording": - result(FlutterError(code: "NOT_IMPLEMENTED", message: "Recording not implemented yet", details: nil)) + cancelRecording(result: result) + case "pauseRecording", "resumeRecording": + // AVAssetWriter does not expose a hardware pause; Android supports it. + result(FlutterError(code: "NOT_SUPPORTED", message: "pause/resume recording is not supported on iOS", details: nil)) case "setFrameProcessor": guard let args = call.arguments as? [String: Any], let enabled = args["enabled"] as? Bool else { @@ -138,12 +157,25 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { // MARK: - Device Discovery private func getAvailableCameraDevices(result: @escaping FlutterResult) { + var deviceTypes: [AVCaptureDevice.DeviceType] = [ + .builtInWideAngleCamera, + .builtInTelephotoCamera, + .builtInUltraWideCamera, + ] + + if #available(iOS 13.0, *) { + deviceTypes.append(.builtInDualCamera) + deviceTypes.append(.builtInTripleCamera) + deviceTypes.append(.builtInDualWideCamera) + } + + // External cameras (USB-C / Continuity) are supported from iOS 17. + if #available(iOS 17.0, *) { + deviceTypes.append(.external) + } + let discoverySession = AVCaptureDevice.DiscoverySession( - deviceTypes: [ - .builtInWideAngleCamera, - .builtInTelephotoCamera, - .builtInUltraWideCamera, - ], + deviceTypes: deviceTypes, mediaType: .video, position: .unspecified ) @@ -178,7 +210,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { "autoFocusSystem": format.autoFocusSystem == .phaseDetection ? "phase-detection" : "contrast-detection", "videoStabilizationModes": ["off"], // Simplified - "pixelFormats": ["yuv"], + "pixelFormats": ["rgb"], ] } @@ -196,6 +228,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { "maxExposure": device.maxExposureTargetBias, "supportsLowLightBoost": device.isLowLightBoostSupported, "supportsFocus": device.isFocusPointOfInterestSupported, + "minFocusDistance": 0.0, "hardwareLevel": "full", "sensorOrientation": "portrait", "physicalDevices": ["wide-angle-camera"], @@ -208,9 +241,15 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { // MARK: - Camera Initialization - private func initializeCamera(deviceId: String, codeScanner: [String: Any]?, result: @escaping FlutterResult) { + private func initializeCamera(deviceId: String, enableVideo: Bool, codeScanner: [String: Any]?, result: @escaping FlutterResult) { sessionQueue.async { [weak self] in guard let self = self else { return } + + // Tear down any prior session/texture so re-init and device switching + // don't leak the previous camera. + self.teardownSession() + + self.enableVideo = enableVideo self.updateCodeScanner(config: codeScanner) let device: AVCaptureDevice? @@ -221,13 +260,14 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { } guard let device = device else { - result(FlutterError(code: "DEVICE_NOT_FOUND", message: "Device \(deviceId) not found", details: nil)) + DispatchQueue.main.async { + result(FlutterError(code: "DEVICE_NOT_FOUND", message: "Device \(deviceId) not found", details: nil)) + } return } - + self.captureDevice = device - - // Enable HDR if supported + do { try device.lockForConfiguration() if device.activeFormat.isVideoHDRSupported { @@ -240,7 +280,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { session.sessionPreset = .high do { - let input = try AVCaptureDeviceInput(device: captureDevice) + let input = try AVCaptureDeviceInput(device: device) if session.canAddInput(input) { session.addInput(input) } @@ -251,90 +291,254 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { return } - // Set up video output for preview texture + // Video output for preview texture + frame processing + recording source. let videoOutput = AVCaptureVideoDataOutput() videoOutput.videoSettings = [ kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA ] + videoOutput.alwaysDiscardsLateVideoFrames = true let renderer = PixelBufferRenderer() videoOutput.setSampleBufferDelegate(renderer, queue: self.sessionQueue) - if session.canAddOutput(videoOutput) { session.addOutput(videoOutput) } - // Set up photo output + // Photo output. let photoOutput = AVCapturePhotoOutput() if session.canAddOutput(photoOutput) { session.addOutput(photoOutput) } self.photoOutput = photoOutput + // Audio input + output for recording (only when a mic is permitted). + if enableVideo, + AVCaptureDevice.authorizationStatus(for: .audio) == .authorized, + let audioDevice = AVCaptureDevice.default(for: .audio), + let audioIn = try? AVCaptureDeviceInput(device: audioDevice) { + if session.canAddInput(audioIn) { + session.addInput(audioIn) + self.audioInput = audioIn + let audioOut = AVCaptureAudioDataOutput() + audioOut.setSampleBufferDelegate(self, queue: self.sessionQueue) + if session.canAddOutput(audioOut) { + session.addOutput(audioOut) + self.audioOutput = audioOut + } + } + } + self.videoOutput = videoOutput self.pixelBufferRenderer = renderer self.captureSession = session - // Register texture with Flutter + let format = device.activeFormat + let dims = CMVideoFormatDescriptionGetDimensions(format.formatDescription) + DispatchQueue.main.async { let textureId = self.textureRegistry.register(renderer) self.textureId = textureId - renderer.plugin = self // Link back for code scanning + renderer.plugin = self renderer.textureRegistry = self.textureRegistry renderer.textureId = textureId - - session.startRunning() - - result(["textureId": textureId]) + self.setupRotationCoordinator(for: device) + + self.sessionQueue.async { + session.startRunning() + DispatchQueue.main.async { + result([ + "textureId": textureId, + "previewWidth": Int(dims.width), + "previewHeight": Int(dims.height) + ]) + } + } } } } // MARK: - Photo Capture - private func takePhoto(options: [String: Any], result: @escaping FlutterResult) { - guard let photoOutput = photoOutput else { - result(FlutterError(code: "NOT_INITIALIZED", message: "Photo output not initialized", details: nil)) - return + private func takePhoto(options: [String: Any], result: @escaping FlutterResult) { + // pendingPhotoResult is owned by the session queue; all access (set here, + // read in the delegate, clear in teardown) happens on it. + sessionQueue.async { [weak self] in + guard let self = self else { return } + guard let photoOutput = self.photoOutput else { + DispatchQueue.main.async { + result(FlutterError(code: "NOT_INITIALIZED", message: "Photo output not initialized", details: nil)) + } + return + } + if self.pendingPhotoResult != nil { + DispatchQueue.main.async { + result(FlutterError(code: "CAPTURE_IN_PROGRESS", message: "A photo capture is already in progress", details: nil)) + } + return + } + + let settings = AVCapturePhotoSettings() + if let flash = options["flash"] as? String { + switch flash { + case "on": settings.flashMode = .on + case "auto": settings.flashMode = .auto + default: settings.flashMode = .off + } + } + if let enableHdr = options["enableHdr"] as? Swift.Bool, enableHdr { + if #available(iOS 13.0, *) { + settings.photoQualityPrioritization = .quality + } + } + + // Mirror the saved photo only when explicitly requested (selfie + // mirror); otherwise capture what the camera actually sees. + if let conn = photoOutput.connection(with: .video), conn.isVideoMirroringSupported { + conn.automaticallyAdjustsVideoMirroring = false + conn.isVideoMirrored = self.mirrorCaptures && (self.captureDevice?.position == .front) + } + + self.pendingPhotoResult = result + photoOutput.capturePhoto(with: settings, delegate: self) } + } + + // MARK: - Video Recording + + private func startRecording(options: [String: Any], result: @escaping FlutterResult) { + sessionQueue.async { [weak self] in + guard let self = self else { return } + guard self.enableVideo else { + DispatchQueue.main.async { + result(FlutterError(code: "NOT_INITIALIZED", message: "Camera was not initialized with enableVideo: true", details: nil)) + } + return + } + guard self.recorder == nil else { + DispatchQueue.main.async { + result(FlutterError(code: "ALREADY_RECORDING", message: "A recording is already in progress", details: nil)) + } + return + } + + let path = (options["path"] as? String) + ?? FileManager.default.temporaryDirectory + .appendingPathComponent("video_\(Int(Date().timeIntervalSince1970)).mp4").path + self.videoPath = path + + // Torch during recording, matching Android. + if let flash = options["flash"] as? String, flash == "on" { + try? self.captureDevice?.lockForConfiguration() + if self.captureDevice?.hasTorch == true { self.captureDevice?.torchMode = .on } + self.captureDevice?.unlockForConfiguration() + } + + // Use the live buffer dimensions when available, else the active format. + var width = 1920 + var height = 1080 + if let buf = self.pixelBufferRenderer?.getCurrentBuffer() { + width = CVPixelBufferGetWidth(buf) + height = CVPixelBufferGetHeight(buf) + } else if let dev = self.captureDevice { + let dims = CMVideoFormatDescriptionGetDimensions(dev.activeFormat.formatDescription) + width = Int(dims.width); height = Int(dims.height) + } + + let mirror = self.mirrorCaptures && (self.captureDevice?.position == .front) + let transform = Self.portraitTransform(mirror: mirror) - let settings = AVCapturePhotoSettings() - - // Flash - if let flash = options["flash"] as? String { - switch flash { - case "on": settings.flashMode = .on - case "auto": settings.flashMode = .auto - default: settings.flashMode = .off + do { + self.recorder = try VideoRecorder( + url: URL(fileURLWithPath: path), + width: width, + height: height, + audio: self.audioOutput != nil, + transform: transform + ) + DispatchQueue.main.async { result(nil) } + } catch { + self.recorder = nil + DispatchQueue.main.async { + result(FlutterError(code: "RECORDING_ERROR", message: "Failed to start recording: \(error.localizedDescription)", details: nil)) + } } } - - // HDR - if let enableHdr = options["enableHdr"] as? Swift.Bool, enableHdr { - if photoOutput.isAutoPhotoHDRSupported { - settings.isAutoPhotoHDREnabled = true + } + + private func stopRecording(result: @escaping FlutterResult) { + sessionQueue.async { [weak self] in + guard let self = self, let recorder = self.recorder else { + DispatchQueue.main.async { + result(FlutterError(code: "NOT_RECORDING", message: "No recording in progress", details: nil)) + } + return + } + let path = self.videoPath ?? recorder.url.path + // The recording transform rotates 90Β°, so the played file is + // portrait β€” report the oriented (display) dimensions. + let width = recorder.height + let height = recorder.width + recorder.finish { [weak self] duration in + // Mutate plugin state on the session queue (where the frame-append + // path reads `recorder`), not on the asset-writer's queue. + self?.sessionQueue.async { + self?.recorder = nil + self?.turnTorchOff() + } + DispatchQueue.main.async { + result([ + "path": path, + "duration": duration, + "width": width, + "height": height + ]) + } } } + } - // Location - if let locationMap = options["location"] as? [String: Any], - let lat = locationMap["latitude"] as? Double, - let lon = locationMap["longitude"] as? Double { - // In a real app, we'd use CoreLocation and set metadata - // For now, we'll assume we can pass it to the delegate + private func cancelRecording(result: @escaping FlutterResult) { + sessionQueue.async { [weak self] in + guard let self = self else { return } + let path = self.videoPath + self.recorder?.cancel() + self.recorder = nil + self.turnTorchOff() + if let path = path { try? FileManager.default.removeItem(atPath: path) } + DispatchQueue.main.async { result(nil) } } + } - self.pendingPhotoResult = result - photoOutput.capturePhoto(with: settings, delegate: self) + private func turnTorchOff() { + guard let device = captureDevice, device.hasTorch else { return } + try? device.lockForConfiguration() + device.torchMode = .off + device.unlockForConfiguration() + } + + /// Display transform for a portrait-oriented recording from a landscape sensor buffer. + private static func portraitTransform(mirror: Bool) -> CGAffineTransform { + var t = CGAffineTransform(rotationAngle: .pi / 2) + if mirror { t = t.scaledBy(x: 1, y: -1) } + return t + } + + // Called from the renderer (on sessionQueue) for every video sample buffer. + fileprivate func appendRecordingVideo(_ sampleBuffer: CMSampleBuffer) { + recorder?.appendVideo(sampleBuffer) } // MARK: - Camera Controls private func setActive(_ active: Bool, result: @escaping FlutterResult) { sessionQueue.async { [weak self] in + guard let session = self?.captureSession else { + DispatchQueue.main.async { result(nil) } + return + } if active { - self?.captureSession?.startRunning() + if !session.isRunning { session.startRunning() } } else { - self?.captureSession?.stopRunning() + if session.isRunning { session.stopRunning() } } DispatchQueue.main.async { result(nil) } } @@ -347,18 +551,19 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { } do { try device.lockForConfiguration() - let zoom = CGFloat(max(1.0, min(factor, Float(device.maxAvailableVideoZoomFactor)))) + let minZoom = Float(device.minAvailableVideoZoomFactor) + let maxZoom = Float(device.maxAvailableVideoZoomFactor) + let zoom = CGFloat(max(minZoom, min(factor, maxZoom))) device.videoZoomFactor = zoom device.unlockForConfiguration() - + lastZoom = factor - - // Pro-Tip: Re-trigger focus if zoom change is significant (> 0.1x) + if abs(lastZoom - lastAFTriggerZoom) > 0.1 && !isManualFocusActive { lastAFTriggerZoom = lastZoom triggerAutoFocus() } - + result(nil) } catch { result(FlutterError(code: "CAMERA_ERROR", message: error.localizedDescription, details: nil)) @@ -417,27 +622,25 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { result(FlutterError(code: "CAMERA_ERROR", message: "No device", details: nil)) return } - + do { try device.lockForConfiguration() - - // Map point to focusPointOfInterest (0,0 - 1,1) - // Note: point (x,y) from Dart is 0..1 relative to the preview widget. - // On iOS, focusPointOfInterest is in normalized coordinates (0,0) top-left to (1,1) bottom-right. + + // Dart sends a point normalized 0..1 in the preview's sensor space. + let clamped = CGPoint(x: min(max(point.x, 0), 1), y: min(max(point.y, 0), 1)) if device.isFocusPointOfInterestSupported { - device.focusPointOfInterest = point + device.focusPointOfInterest = clamped device.focusMode = .autoFocus } - + if device.isExposurePointOfInterestSupported { - device.exposurePointOfInterest = point + device.exposurePointOfInterest = clamped device.exposureMode = .continuousAutoExposure } - + isManualFocusActive = true device.unlockForConfiguration() - - // Revert to continuous focus after 5 seconds + DispatchQueue.main.asyncAfter(deadline: .now() + 5.0) { [weak self] in guard let self = self, let device = self.captureDevice else { return } do { @@ -449,7 +652,28 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { device.unlockForConfiguration() } catch {} } - + + result(nil) + } catch { + result(FlutterError(code: "CAMERA_ERROR", message: error.localizedDescription, details: nil)) + } + } + + private func setFocusDistance(_ distance: Float, result: @escaping FlutterResult) { + guard let device = captureDevice else { + result(FlutterError(code: "CAMERA_ERROR", message: "No device", details: nil)) + return + } + guard device.isLockingFocusWithCustomLensPositionSupported else { + result(FlutterError(code: "NOT_SUPPORTED", message: "Manual focus distance not supported on this device", details: nil)) + return + } + do { + try device.lockForConfiguration() + let lens = min(max(distance, 0.0), 1.0) + isManualFocusActive = true + device.setFocusModeLocked(lensPosition: lens, completionHandler: nil) + device.unlockForConfiguration() result(nil) } catch { result(FlutterError(code: "CAMERA_ERROR", message: error.localizedDescription, details: nil)) @@ -468,35 +692,85 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { isCodeScannerEnabled = true let request = VNDetectBarcodesRequest { [weak self] request, error in guard error == nil, let results = request.results as? [VNBarcodeObservation], !results.isEmpty else { return } - + let codes = results.map { observation -> [String: Any] in + // Vision's boundingBox is normalized with a bottom-left origin; + // flip Y to a top-left origin to match Android. + let box = observation.boundingBox return [ - "type": observation.symbology.rawValue, + "type": Self.symbologyToString(observation.symbology), "value": observation.payloadStringValue ?? "", "frame": [ - "x": observation.boundingBox.origin.x, - "y": observation.boundingBox.origin.y, - "width": observation.boundingBox.size.width, - "height": observation.boundingBox.size.height + "x": box.origin.x, + "y": 1.0 - box.origin.y - box.size.height, + "width": box.size.width, + "height": box.size.height ] ] } - + DispatchQueue.main.async { - self?.channel.invokeMethod("onCodeScanned", codes) + self?.channel.invokeMethod("onCodeScanned", arguments: codes) } } - - // TODO: Filter symbologies based on config["types"] + + if let types = config["codeTypes"] as? [String], !types.isEmpty { + request.symbologies = types.compactMap { Self.stringToSymbology($0) } + } self.codeScannerRequest = request } func scanBarcodes(in pixelBuffer: CVPixelBuffer) { guard isCodeScannerEnabled, let request = codeScannerRequest else { return } - let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer, options: [:]) + // Tell Vision the buffer's orientation so the returned bounding boxes are + // normalized in the UPRIGHT (displayed) frame β€” matching the rotated + // preview and the Android coordinate convention. Without this, boxes are + // normalized against the raw landscape buffer and appear stretched/ + // misplaced once the preview is rotated upright. + let orientation: CGImagePropertyOrientation = + (captureDevice?.position == .front) ? .leftMirrored : .right + let handler = VNImageRequestHandler( + cvPixelBuffer: pixelBuffer, + orientation: orientation, + options: [:] + ) try? handler.perform([request]) } + private static func symbologyToString(_ s: VNBarcodeSymbology) -> String { + switch s { + case .qr: return "qr" + case .ean13: return "ean-13" + case .ean8: return "ean-8" + case .code128: return "code-128" + case .code39: return "code-39" + case .code93: return "code-93" + case .dataMatrix: return "data-matrix" + case .upce: return "upc-e" + case .pdf417: return "pdf-417" + case .aztec: return "aztec" + case .itf14, .i2of5: return "itf" + default: return "unknown" + } + } + + private static func stringToSymbology(_ s: String) -> VNBarcodeSymbology? { + switch s { + case "qr": return .qr + case "ean-13": return .ean13 + case "ean-8": return .ean8 + case "code-128": return .code128 + case "code-39": return .code39 + case "code-93": return .code93 + case "data-matrix": return .dataMatrix + case "upc-e": return .upce + case "pdf-417": return .pdf417 + case "aztec": return .aztec + case "itf": return .itf14 + default: return nil + } + } + // MARK: - Snapshot private func takeSnapshot(result: @escaping FlutterResult) { @@ -511,7 +785,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to create image", details: nil)) return } - + let uiImage = UIImage(cgImage: cgImage) guard let data = uiImage.jpegData(compressionQuality: 0.8) else { result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to encode JPEG", details: nil)) @@ -520,7 +794,7 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { let tempDir = FileManager.default.temporaryDirectory let fileURL = tempDir.appendingPathComponent("snapshot_\(Int(Date().timeIntervalSince1970)).jpg") - + do { try data.write(to: fileURL) result([ @@ -573,61 +847,235 @@ public class FlutterNativeVisionCameraPlugin: NSObject, FlutterPlugin { } } + // MARK: - Preview Rotation + + /// Reports the preview rotation to Dart (single source of truth, mirrors the + /// Android `onPreviewConfigurationChanged` path). + /// + /// `AVCaptureVideoDataOutput` always delivers the buffer in the sensor's + /// (landscape) orientation, so for a portrait UI a fixed 90Β° (back) / 270Β° + /// (front) rotation displays it upright β€” deterministic and matching Android. + /// We deliberately do NOT use `RotationCoordinator`'s horizon-level angle: + /// that follows the device gyro, which is wrong for an orientation-locked UI. + private func setupRotationCoordinator(for device: AVCaptureDevice) { + // Both the back and front sensor buffers need a 90Β° rotation to display + // upright in a portrait UI. The front camera's horizontal selfie-mirror + // is applied separately by CameraPreview, so it does not change this. + reportPreviewRotation(90) + } + + private func reportPreviewRotation(_ angle: CGFloat) { + let degrees = Int(angle.rounded()) + DispatchQueue.main.async { + self.channel.invokeMethod( + "onPreviewConfigurationChanged", + // AVCaptureVideoDataOutput delivers an un-mirrored buffer, so the + // Dart side applies the front-camera selfie mirror itself. + arguments: ["rotationDegrees": degrees, "mirrored": false] + ) + } + } + // MARK: - Cleanup + /// Stops and releases the session, outputs, and texture. Runs on sessionQueue. + private func teardownSession() { + recorder?.cancel() + recorder = nil + captureSession?.stopRunning() + captureSession = nil + captureDevice = nil + videoOutput = nil + photoOutput = nil + audioOutput = nil + audioInput = nil + if let textureId = self.textureId { + DispatchQueue.main.async { + self.textureRegistry.unregisterTexture(textureId) + } + } + pixelBufferRenderer = nil + textureId = nil + // Fail any in-flight photo capture so the Dart future resolves instead + // of hanging forever on dispose / device-switch. + if let pending = pendingPhotoResult { + DispatchQueue.main.async { + pending(FlutterError(code: "CANCELLED", message: "Camera disposed during photo capture", details: nil)) + } + } + pendingPhotoResult = nil + } + private func disposeCamera() { sessionQueue.async { [weak self] in - self?.captureSession?.stopRunning() - self?.captureSession = nil - self?.captureDevice = nil - self?.videoOutput = nil - self?.photoOutput = nil - if let textureId = self?.textureId { + self?.teardownSession() + } + } +} + +// MARK: - Audio sample delegate (recording) + +extension SwiftFlutterNativeVisionCameraPlugin: AVCaptureAudioDataOutputSampleBufferDelegate { + public func captureOutput( + _ output: AVCaptureOutput, + didOutput sampleBuffer: CMSampleBuffer, + from connection: AVCaptureConnection + ) { + // Only the audio output is delegated to the plugin; the video output is + // delegated to PixelBufferRenderer. + recorder?.appendAudio(sampleBuffer) + } +} + +// MARK: - AVCapturePhotoCaptureDelegate + +extension SwiftFlutterNativeVisionCameraPlugin: AVCapturePhotoCaptureDelegate { + public func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) { + // Read/clear pendingPhotoResult on its owning queue (the session queue). + sessionQueue.async { [weak self] in + guard let self = self, let result = self.pendingPhotoResult else { return } + self.pendingPhotoResult = nil + + if let error = error { + DispatchQueue.main.async { + result(FlutterError(code: "CAPTURE_ERROR", message: error.localizedDescription, details: nil)) + } + return + } + + guard let data = photo.fileDataRepresentation() else { + DispatchQueue.main.async { + result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to get photo data", details: nil)) + } + return + } + + let tempDir = FileManager.default.temporaryDirectory + let fileName = "photo_\(Int(Date().timeIntervalSince1970)).jpg" + let fileURL = tempDir.appendingPathComponent(fileName) + let isMirrored = self.mirrorCaptures && (self.captureDevice?.position == .front) + + do { + try data.write(to: fileURL) + let dims = photo.resolvedSettings.photoDimensions DispatchQueue.main.async { - self?.textureRegistry.unregisterTexture(textureId) + result([ + "path": fileURL.path, + "width": Int(dims.width), + "height": Int(dims.height), + "isRawPhoto": false, + "orientation": "portrait", + "isMirrored": isMirrored + ]) + } + } catch { + DispatchQueue.main.async { + result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to save photo: \(error.localizedDescription)", details: nil)) } } - self?.pixelBufferRenderer = nil - self?.textureId = nil - self?.pendingPhotoResult = nil } } } -// MARK: - AVCapturePhotoCaptureDelegate +// MARK: - VideoRecorder (AVAssetWriter) + +/// Records the live BGRA frame stream (and optional audio) to an .mp4 via +/// AVAssetWriter, so the preview/frame-processor stream keeps running. +final class VideoRecorder { + let url: URL + let width: Int + let height: Int + + private let assetWriter: AVAssetWriter + private let videoInput: AVAssetWriterInput + private let audioInput: AVAssetWriterInput? + private var started = false + private var finished = false + private var lastTimestamp: CMTime = .zero + private var startTimestamp: CMTime = .zero + + init(url: URL, width: Int, height: Int, audio: Bool, transform: CGAffineTransform) throws { + self.url = url + self.width = width + self.height = height + + try? FileManager.default.removeItem(at: url) + assetWriter = try AVAssetWriter(outputURL: url, fileType: .mp4) + + let videoSettings: [String: Any] = [ + AVVideoCodecKey: AVVideoCodecType.h264, + AVVideoWidthKey: width, + AVVideoHeightKey: height + ] + videoInput = AVAssetWriterInput(mediaType: .video, outputSettings: videoSettings) + videoInput.expectsMediaDataInRealTime = true + videoInput.transform = transform + if assetWriter.canAdd(videoInput) { assetWriter.add(videoInput) } + + if audio { + let audioSettings: [String: Any] = [ + AVFormatIDKey: kAudioFormatMPEG4AAC, + AVNumberOfChannelsKey: 1, + AVSampleRateKey: 44100.0 + ] + let input = AVAssetWriterInput(mediaType: .audio, outputSettings: audioSettings) + input.expectsMediaDataInRealTime = true + if assetWriter.canAdd(input) { + assetWriter.add(input) + audioInput = input + } else { + audioInput = nil + } + } else { + audioInput = nil + } + } -extension FlutterNativeVisionCameraPlugin: AVCapturePhotoCaptureDelegate { - public func photoOutput(_ output: AVCapturePhotoOutput, didFinishProcessingPhoto photo: AVCapturePhoto, error: Error?) { - guard let result = pendingPhotoResult else { return } - pendingPhotoResult = nil + func appendVideo(_ sampleBuffer: CMSampleBuffer) { + guard !finished, CMSampleBufferDataIsReady(sampleBuffer) else { return } + let ts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) + if !started { + // If startWriting fails, leave `started` false so a later frame can + // retry rather than appending into a non-writing writer. + guard assetWriter.startWriting() else { return } + started = true + startTimestamp = ts + assetWriter.startSession(atSourceTime: ts) + } + lastTimestamp = ts + if assetWriter.status == .writing, videoInput.isReadyForMoreMediaData { + videoInput.append(sampleBuffer) + } + } - if let error = error { - result(FlutterError(code: "CAPTURE_ERROR", message: error.localizedDescription, details: nil)) - return + func appendAudio(_ sampleBuffer: CMSampleBuffer) { + guard started, !finished, let audioInput = audioInput, + CMSampleBufferDataIsReady(sampleBuffer) else { return } + if assetWriter.status == .writing, audioInput.isReadyForMoreMediaData { + audioInput.append(sampleBuffer) } + } - guard let data = photo.fileDataRepresentation() else { - result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to get photo data", details: nil)) + func finish(completion: @escaping (Double) -> Void) { + guard started, !finished, assetWriter.status == .writing else { + finished = true + completion(0) return } + finished = true + let duration = CMTimeGetSeconds(CMTimeSubtract(lastTimestamp, startTimestamp)) + videoInput.markAsFinished() + audioInput?.markAsFinished() + assetWriter.finishWriting { + completion(max(0, duration)) + } + } - let tempDir = FileManager.default.temporaryDirectory - let fileName = "photo_\(Int(Date().timeIntervalSince1970)).jpg" - let fileURL = tempDir.appendingPathComponent(fileName) - - do { - try data.write(to: fileURL) - let dims = CMVideoFormatDescriptionGetDimensions(photo.formatDescription) - result([ - "path": fileURL.path, - "width": Int(dims.width), - "height": Int(dims.height), - "isRawPhoto": false, - "orientation": "portrait", - "isMirrored": false - ]) - } catch { - result(FlutterError(code: "CAPTURE_ERROR", message: "Failed to save photo: \(error.localizedDescription)", details: nil)) + func cancel() { + guard !finished else { return } + finished = true + if assetWriter.status == .writing { + assetWriter.cancelWriting() } } } @@ -636,22 +1084,28 @@ extension FlutterNativeVisionCameraPlugin: AVCapturePhotoCaptureDelegate { /// Bridges AVCaptureVideoDataOutput to Flutter's texture registry. /// -/// Each frame's CVPixelBuffer is held and provided to Flutter when +/// Each frame's CVPixelBuffer is retained and provided to Flutter when /// it requests the texture β€” enabling zero-copy GPU rendering. class PixelBufferRenderer: NSObject, FlutterTexture, AVCaptureVideoDataOutputSampleBufferDelegate { var textureRegistry: FlutterTextureRegistry? var textureId: Int64 = 0 var isFrameProcessorEnabled = false - weak var plugin: FlutterNativeVisionCameraPlugin? + weak var plugin: SwiftFlutterNativeVisionCameraPlugin? + private var latestPixelBuffer: CVPixelBuffer? + private let bufferLock = NSLock() func copyPixelBuffer() -> Unmanaged? { + bufferLock.lock() + defer { bufferLock.unlock() } guard let buffer = latestPixelBuffer else { return nil } return Unmanaged.passRetained(buffer) } func getCurrentBuffer() -> CVPixelBuffer? { + bufferLock.lock() + defer { bufferLock.unlock() } return latestPixelBuffer } @@ -661,26 +1115,34 @@ class PixelBufferRenderer: NSObject, FlutterTexture, AVCaptureVideoDataOutputSam from connection: AVCaptureConnection ) { guard let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) else { return } + + // Retain the buffer we hand to the texture registry; release the prior one. + bufferLock.lock() latestPixelBuffer = pixelBuffer - + bufferLock.unlock() + + // Feed the recorder (no-op when not recording). Runs on the session queue. + plugin?.appendRecordingVideo(sampleBuffer) + if isFrameProcessorEnabled { let width = Int32(CVPixelBufferGetWidth(pixelBuffer)) let height = Int32(CVPixelBufferGetHeight(pixelBuffer)) let timestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer).seconds - + let metadata = FrameMetadata( width: width, height: height, - pixelFormat: 0, // YUV placeholder - orientation: 0, // Portrait placeholder + pixelFormat: 1, // BGRA (maps to PixelFormat.rgb on the Dart side) + orientation: 0, timestamp: timestamp ) - - // Retain the buffer so it stays alive during asynchronous FFI processing - CFRetain(pixelBuffer) - VisionCamera_dispatchFrame(pixelBuffer, metadata) + + // Retain the buffer so it stays alive during asynchronous FFI processing. + // VisionCamera_dispatchFrame locks it; Frame_decrementRefCount unlocks + releases. + let handle = Unmanaged.passRetained(pixelBuffer).toOpaque() + VisionCamera_dispatchFrame(handle, metadata) } - + plugin?.scanBarcodes(in: pixelBuffer) DispatchQueue.main.async { [weak self] in diff --git a/ios/Classes/VisionCamera_FFI.h b/ios/Classes/VisionCamera_FFI.h new file mode 100644 index 0000000..4d902a6 --- /dev/null +++ b/ios/Classes/VisionCamera_FFI.h @@ -0,0 +1,29 @@ +#ifndef VisionCamera_FFI_h +#define VisionCamera_FFI_h + +#import + +#ifdef __cplusplus +extern "C" +{ +#endif + + // Redefining types to avoid relative include issues in frameworks + typedef void *FrameHandle; + + typedef struct + { + int32_t width; + int32_t height; + int32_t pixelFormat; // 0 for YUV_420_888, 1 for BGRA + int32_t orientation; // 0, 90, 180, 270 + double timestamp; // Presentation timestamp in seconds + } FrameMetadata; + + void VisionCamera_dispatchFrame(FrameHandle handle, FrameMetadata metadata); + +#ifdef __cplusplus +} +#endif + +#endif /* VisionCamera_FFI_h */ diff --git a/ios/Classes/VisionCamera_NativePluginExample.cpp b/ios/Classes/VisionCamera_NativePluginExample.cpp new file mode 100644 index 0000000..4b53c5b --- /dev/null +++ b/ios/Classes/VisionCamera_NativePluginExample.cpp @@ -0,0 +1,2 @@ +// Relative import to be able to reuse the C++ sources. +#include "../../src/VisionCamera_NativePluginExample.cpp" diff --git a/ios/Resources/PrivacyInfo.xcprivacy b/ios/Resources/PrivacyInfo.xcprivacy new file mode 100644 index 0000000..64e785c --- /dev/null +++ b/ios/Resources/PrivacyInfo.xcprivacy @@ -0,0 +1,23 @@ + + + + + NSPrivacyTracking + + NSPrivacyTrackingDomains + + NSPrivacyCollectedDataTypes + + NSPrivacyAccessedAPITypes + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + + C617.1 + + + + + diff --git a/ios/flutter_native_vision_camera.podspec b/ios/flutter_native_vision_camera.podspec index 91e9615..0a452e5 100644 --- a/ios/flutter_native_vision_camera.podspec +++ b/ios/flutter_native_vision_camera.podspec @@ -4,25 +4,35 @@ # Pod::Spec.new do |s| s.name = 'flutter_native_vision_camera' - s.version = '0.0.1' - s.summary = 'A new Flutter FFI plugin project.' + s.version = '0.0.4' + s.summary = 'High-performance Flutter FFI camera plugin with zero-copy preview and real-time frame access.' s.description = <<-DESC -A new Flutter FFI plugin project. +A high-performance camera plugin for Flutter built on AVFoundation (iOS) and CameraX (Android), +providing zero-copy preview textures, integrated barcode/QR scanning, and low-latency native +frame access via FFI for real-time on-device vision. DESC - s.homepage = 'http://example.com' + s.homepage = 'https://github.com/JenteJan/flutter_native_vision_camera' s.license = { :file => '../LICENSE' } - s.author = { 'Your Company' => 'email@example.com' } + s.author = { 'Jente Jan de Waart' => 'jentedewaart@gmail.com' } # This will ensure the source files in Classes/ are included in the native # builds of apps using this FFI plugin. Podspec does not support relative # paths, so Classes contains a forwarder C file that relatively imports # `../src/*` so that the C sources can be shared among all target platforms. - s.source = { :path => '.' } - s.source_files = 'Classes/**/*' + s.source = { :http => 'https://github.com/JenteJan/flutter_native_vision_camera' } + s.source_files = 'Classes/**/*.{h,c,cpp,m,swift}' + s.public_header_files = 'Classes/**/*.h' + s.resource_bundles = { 'flutter_native_vision_camera_privacy' => ['Resources/PrivacyInfo.xcprivacy'] } s.dependency 'Flutter' - s.platform = :ios, '12.0' + s.platform = :ios, '13.0' + + s.frameworks = 'AVFoundation', 'CoreMedia', 'CoreVideo', 'CoreImage', 'Vision', 'UIKit' # Flutter.framework does not contain a i386 slice. - s.pod_target_xcconfig = { 'DEFINES_MODULE' => 'YES', 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386' } + s.pod_target_xcconfig = { + 'DEFINES_MODULE' => 'YES', + 'EXCLUDED_ARCHS[sdk=iphonesimulator*]' => 'i386', + 'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/../src"' + } s.swift_version = '5.0' end diff --git a/lib/flutter_native_vision_camera.dart b/lib/flutter_native_vision_camera.dart index 2ef33d4..e2ea25c 100644 --- a/lib/flutter_native_vision_camera.dart +++ b/lib/flutter_native_vision_camera.dart @@ -43,10 +43,12 @@ void initializeVisionCamera() { initializeFrameBindings(_dylib); } -/// Initializes the C++ Native Plugin showcase. +/// Initializes the bundled **demo** C++ frame plugin (the `BrightnessPlugin` +/// showcase in `src/VisionCamera_NativePluginExample.cpp`). /// -/// This demonstrates how other developers can register C++ plugins -/// that hook into the camera pipeline with zero latency. +/// This is a reference/demo only β€” you do **not** need to call it in your app. +/// It exists to show how to register a native C/C++ plugin that hooks the +/// camera pipeline with zero latency; ship your own plugin the same way. void initializeNativeExamplePlugin() { final init = _dylib.lookupFunction( 'VisionCamera_initExamplePlugin', diff --git a/lib/src/camera_controller.dart b/lib/src/camera_controller.dart index 8a5db27..0ceaadd 100644 --- a/lib/src/camera_controller.dart +++ b/lib/src/camera_controller.dart @@ -34,7 +34,8 @@ enum CameraState { /// * **Zero-Copy Preview**: Uses `TextureRegistry` for direct GPU rendering. /// * **Physical Orientation**: Correctly handles hardware sensor orientation. /// * **Integrated ML**: Built-in high-speed Barcode/QR scanning via MLKit. -/// * **FFI Frame Processing**: Synchronous background frame analysis. +/// * **FFI Frame Processing**: Low-overhead frame access via `dart:ffi` +/// (callback on the main isolate; native C/C++ plugins on the camera thread). /// * **Unified API**: Easy-to-use reactive state via [ValueNotifier]. class CameraController extends ValueNotifier { int? _textureId; @@ -46,6 +47,9 @@ class CameraController extends ValueNotifier { FrameProcessorPipeline? _frameProcessorPipeline; int? _previewWidth; int? _previewHeight; + int? _previewRotationDegrees; + bool _previewMirrored = false; + bool _mirror = true; // Configuration State double _zoom = 1.0; @@ -79,12 +83,57 @@ class CameraController extends ValueNotifier { /// The currently active camera device. CameraDevice? get device => _device; - /// The width of the preview texture. + /// The width of the preview texture, in raw sensor space (un-rotated). int? get previewWidth => _previewWidth; - /// The height of the preview texture. + /// The height of the preview texture, in raw sensor space (un-rotated). int? get previewHeight => _previewHeight; + /// The clockwise quarter-turns needed to rotate the raw preview texture so + /// it displays upright. + /// + /// **This is the single source of truth for preview rotation.** The value is + /// reported by the native layer, which is the only place that knows how much + /// the preview buffer was already rotated (the camera stack may pre-rotate it + /// depending on the bound use-cases, device orientation and sensor mount). + /// [CameraPreview] applies this for you; custom previews/overlays must use + /// this value (or [displayPreviewSize]) instead of swapping/rotating + /// dimensions themselves. Falls back to the device's + /// [CameraDevice.sensorOrientation] before the native value arrives. + int get previewRotation { + final degrees = + _previewRotationDegrees ?? _device?.sensorOrientation.degrees ?? 0; + return (degrees ~/ 90) % 4; + } + + /// The raw preview buffer size, in sensor space (before [previewRotation]). + Size? get rawPreviewSize => (_previewWidth != null && _previewHeight != null) + ? Size(_previewWidth!.toDouble(), _previewHeight!.toDouble()) + : null; + + /// The preview size in display (upright) space, accounting for + /// [previewRotation]. Lay out overlays against this so they align with the + /// rotated preview. + Size? get displayPreviewSize { + final raw = rawPreviewSize; + if (raw == null) return null; + return previewRotation.isOdd ? Size(raw.height, raw.width) : raw; + } + + /// Whether the native preview texture is already horizontally mirrored + /// relative to the true scene. + /// + /// Reported by the native layer because the camera stacks differ: Android's + /// CameraX mirrors the front-camera preview itself, while iOS delivers an + /// un-mirrored buffer. [CameraPreview] uses this so the front preview looks + /// like a mirror on both platforms without double-mirroring. + bool get previewMirrored => _previewMirrored; + + /// Whether the front camera is mirrored (the "selfie" look) for **both** the + /// preview and the captured photo/video. Set via [initialize]'s `mirror` + /// argument. Has no effect on back cameras. + bool get mirror => _mirror; + /// Fires when a runtime error occurs in the native layer. Stream get onError => _onErrorController.stream; @@ -99,8 +148,14 @@ class CameraController extends ValueNotifier { /// Initializes the camera with the specified [device]. /// /// [format] controls resolution and FPS. - /// [enablePhoto] and [enableVideo] prepare the underlying pipeline. + /// [pixelFormat] hints the desired frame-processor format (use + /// [PixelFormat.rgb] if your processor expects RGB/BGRA). + /// [enablePhoto] and [enableVideo] prepare the underlying pipeline β€” they are + /// required before [takePhoto] / [startRecording] respectively. /// [codeScanner] enables the high-speed barcode scanning features. + /// [mirror] mirrors the **front** camera (the selfie look) for both the + /// preview and the captured photo/video; defaults to `true`. Ignored for back + /// cameras. See [mirror]. Future initialize( CameraDevice device, { CameraDeviceFormat? format, @@ -108,11 +163,27 @@ class CameraController extends ValueNotifier { bool enablePhoto = false, bool enableVideo = false, CodeScannerConfiguration? codeScanner, + bool mirror = true, }) async { if (value == CameraState.disposed) return; try { + // Tear down any frame processor from a previous session before re-init / + // device-switch, so frames already queued from the old session don't read + // buffers the native side is about to recycle (use-after-free). + _frameProcessorPipeline?.stop(); + _frameProcessorPipeline = null; + _device = device; + _previewRotationDegrees = null; + _previewMirrored = false; + _mirror = mirror; + // Reset per-session state so re-init / device-switch starts clean. + _isActive = false; + _isRecording = false; + _textureId = null; + _previewWidth = null; + _previewHeight = null; _activeHandler = this; _channel.setMethodCallHandler(_handleMethodCall); @@ -127,6 +198,7 @@ class CameraController extends ValueNotifier { 'enablePhoto': enablePhoto, 'enableVideo': enableVideo, 'codeScanner': codeScanner?.toMap(), + 'mirror': mirror, }); if (result != null) { @@ -189,7 +261,11 @@ class CameraController extends ValueNotifier { } } - /// Changes the manual exposure compensation. + /// Changes the exposure compensation, in the device's exposure units + /// (an EV-bias index on Android; an exposure-target bias on iOS). Clamp to + /// [CameraDevice.minExposure]..[CameraDevice.maxExposure]. + /// + /// No-op unless the camera is active (call [setActive] first). Future setExposure(double exposure) async { if (!_isActive) return; try { @@ -251,8 +327,12 @@ class CameraController extends ValueNotifier { /// Sets the frame processor for this camera session. /// - /// The [callback] will be executed on a background isolate for every frame. - /// Set to `null` to disable frame processing. + /// The [callback] is invoked for every frame, delivered **asynchronously on + /// the main isolate's event loop** (via `dart:ffi` `NativeCallable.listener`). + /// It does **not** run on a background isolate and does **not** block the + /// camera thread β€” keep the work light, or copy data out and hand it to your + /// own isolate. For the heaviest work, register a native C/C++ plugin, which + /// runs synchronously on the camera thread. Pass `null` to disable. Future setFrameProcessor(FrameProcessorCallback? callback) async { _frameProcessorPipeline?.stop(); _frameProcessorPipeline = null; @@ -286,6 +366,9 @@ class CameraController extends ValueNotifier { /// Takes a snapshot of the current preview. /// /// Snapshots are usually faster than high-resolution photos. + /// + /// **iOS only** β€” not implemented on Android; use [takePhoto] there. On + /// Android this throws a [PlatformException] with code `NOT_IMPLEMENTED`. Future takeSnapshot([TakeSnapshotOptions? options]) async { final result = await _channel.invokeMapMethod( 'takeSnapshot', @@ -340,6 +423,12 @@ class CameraController extends ValueNotifier { case 'onInitialized': _isInitialized = true; break; + case 'onPreviewConfigurationChanged': + final args = Map.from(call.arguments as Map); + _previewRotationDegrees = (args['rotationDegrees'] as num).toInt(); + _previewMirrored = (args['mirrored'] as bool?) ?? false; + notifyListeners(); + break; case 'onStarted': _isActive = true; break; diff --git a/lib/src/camera_devices.dart b/lib/src/camera_devices.dart index 5af8ed0..0fde18d 100644 --- a/lib/src/camera_devices.dart +++ b/lib/src/camera_devices.dart @@ -56,4 +56,44 @@ class CameraDevices { final multiCam = matching.where((d) => d.isMultiCam); return multiCam.isNotEmpty ? multiCam.first : matching.first; } + + /// Picks the [CameraDeviceFormat] from [device] that best matches the desired + /// video resolution ([targetWidth] x [targetHeight]) and supports [targetFps]. + /// + /// Pass it to `CameraController.initialize(device, format: ...)`. With no + /// targets it returns the highest-resolution format. Returns `null` if the + /// device exposes no formats. + static CameraDeviceFormat? getCameraFormat( + CameraDevice device, { + int? targetWidth, + int? targetHeight, + int? targetFps, + }) { + if (device.formats.isEmpty) return null; + + var candidates = device.formats; + if (targetFps != null) { + final supported = candidates + .where((f) => f.minFps <= targetFps && targetFps <= f.maxFps) + .toList(); + if (supported.isNotEmpty) candidates = supported; + } + + if (targetWidth != null && targetHeight != null) { + final targetArea = targetWidth * targetHeight; + return ([...candidates]..sort((a, b) { + final da = (a.videoWidth * a.videoHeight - targetArea).abs(); + final db = (b.videoWidth * b.videoHeight - targetArea).abs(); + return da.compareTo(db); + })) + .first; + } + + return ([...candidates]..sort( + (a, b) => (b.videoWidth * b.videoHeight).compareTo( + a.videoWidth * a.videoHeight, + ), + )) + .first; + } } diff --git a/lib/src/camera_preview.dart b/lib/src/camera_preview.dart index 2f56da0..f93c056 100644 --- a/lib/src/camera_preview.dart +++ b/lib/src/camera_preview.dart @@ -5,9 +5,15 @@ import 'types/types.dart'; /// The camera preview widget. /// -/// Displays the live camera feed using Flutter's [Texture] widget, -/// which renders directly from the GPU surface provided by the native -/// camera session β€” zero-copy preview as per project rules. +/// Displays the live camera feed using Flutter's [Texture] widget. +/// +/// ## Orientation +/// The camera sensor is physically mounted at an angle, so the texture arrives +/// rotated relative to the screen. **This widget is the single place that +/// rotation is corrected** β€” it applies [CameraController.previewRotation] so +/// the preview is always upright. Do not wrap it in `RotatedBox`/`AspectRatio` +/// hacks; if you draw an overlay on top, size it against +/// [CameraController.displayPreviewSize] so it stays aligned. /// /// ## Usage /// ```dart @@ -42,62 +48,115 @@ class CameraPreview extends StatelessWidget { this.isMirrored, }); - bool get _shouldMirror { + /// Whether the preview should be displayed mirror-like (the selfie look). + /// + /// By default this follows the controller's `mirror` setting for front + /// cameras (so one variable drives both the preview and the captured image); + /// pass [isMirrored] to override the preview independently. + bool get _wantsMirrorLike { if (isMirrored != null) return isMirrored!; - return controller.device?.position == CameraPosition.front; + return controller.device?.position == CameraPosition.front && + controller.mirror; } @override Widget build(BuildContext context) { - final textureId = controller.textureId; - if (textureId == null || !controller.isInitialized) { - return const ColoredBox(color: Colors.black); - } - - // The dimensions from the native stream - // Since we set targetRotation in CameraX, these dimensions already - // reflect the correct orientation for the current display. - final nativeWidth = controller.previewWidth?.toDouble() ?? 1920.0; - final nativeHeight = controller.previewHeight?.toDouble() ?? 1080.0; - - Widget previewWidget = SizedBox( - width: nativeWidth, - height: nativeHeight, - child: Texture(textureId: textureId), - ); - - final fit = resizeMode == ResizeMode.cover ? BoxFit.cover : BoxFit.contain; - - Widget outputWidget = SizedBox.expand( - child: FittedBox( - fit: fit, - clipBehavior: Clip.hardEdge, - child: _shouldMirror - ? Transform.scale( - scaleX: -1, // Mirror horizontally - child: previewWidget, - ) - : previewWidget, - ), + // Rebuild whenever the controller changes (e.g. after async init or + // device switch) so the preview appears instead of staying black. + return ListenableBuilder( + listenable: controller, + builder: (context, _) { + final textureId = controller.textureId; + if (textureId == null || !controller.isInitialized) { + return const ColoredBox(color: Colors.black); + } + + // Raw (sensor-space) texture dimensions and the rotation needed to + // make them upright. Rotation is applied here and ONLY here. + final rawWidth = controller.previewWidth?.toDouble() ?? 1920.0; + final rawHeight = controller.previewHeight?.toDouble() ?? 1080.0; + final turns = controller.previewRotation; + final fit = resizeMode == ResizeMode.cover + ? BoxFit.cover + : BoxFit.contain; + + Widget content = SizedBox( + width: rawWidth, + height: rawHeight, + child: Texture(textureId: textureId), + ); + // 1. Rotate the raw texture upright. + content = RotatedBox(quarterTurns: turns, child: content); + // 2. Flip horizontally only when needed to reach the desired mirror-like + // look β€” the native preview may already be mirrored (e.g. Android front + // camera), so a blind flip would double-mirror it. + final flip = _wantsMirrorLike != controller.previewMirrored; + if (flip) { + content = Transform.scale(scaleX: -1, child: content); + } + + final output = SizedBox.expand( + child: FittedBox( + fit: fit, + clipBehavior: Clip.hardEdge, + child: content, + ), + ); + + if (!onTapToFocus) return output; + + // Upright (display-space) preview size, used to map taps through the fit. + final displaySize = + controller.displayPreviewSize ?? Size(rawWidth, rawHeight); + + return LayoutBuilder( + builder: (context, constraints) { + final widgetSize = constraints.biggest; + return GestureDetector( + onTapUp: (details) { + if (!widgetSize.isFinite || + widgetSize.isEmpty || + !controller.isActive) { + return; + } + final fitted = applyBoxFit(fit, displaySize, widgetSize); + final dest = fitted.destination; + if (dest.width <= 0 || dest.height <= 0) return; + + final dx0 = (widgetSize.width - dest.width) / 2; + final dy0 = (widgetSize.height - dest.height) / 2; + final ndx = ((details.localPosition.dx - dx0) / dest.width) + .clamp(0.0, 1.0); + final ndy = ((details.localPosition.dy - dy0) / dest.height) + .clamp(0.0, 1.0); + // Invert the *actual* rendered transform (rotation + the flip we + // applied), not the desired mirror-like state, so the focus + // point matches what the user sees. + controller.focus(_displayToSensor(ndx, ndy, turns, flip)); + }, + child: output, + ); + }, + ); + }, ); + } - // Tap-to-focus gesture - if (onTapToFocus) { - outputWidget = GestureDetector( - onTapUp: (details) { - final box = context.findRenderObject() as RenderBox; - final size = box.size; - final normalizedPoint = Point( - x: details.localPosition.dx / size.width, - y: details.localPosition.dy / size.height, - ); - controller.focus(normalizedPoint); - }, - child: outputWidget, - ); + /// Maps a normalized point in upright **display** space back to **sensor** + /// space, inverting [previewRotation] and the front-camera mirror so the + /// native focus call targets the location the user actually tapped. + static Point _displayToSensor(double dx, double dy, int turns, bool mirror) { + if (mirror) dx = 1.0 - dx; + switch (turns % 4) { + case 1: + return Point(x: dy, y: 1.0 - dx); + case 2: + return Point(x: 1.0 - dx, y: 1.0 - dy); + case 3: + return Point(x: 1.0 - dy, y: dx); + default: + return Point(x: dx, y: dy); } - - return outputWidget; } } diff --git a/lib/src/frame.dart b/lib/src/frame.dart index 24d8801..2f6b39d 100644 --- a/lib/src/frame.dart +++ b/lib/src/frame.dart @@ -25,7 +25,9 @@ final class FrameMetadataNative extends Struct { /// A single frame from the camera. /// /// This class is backed by a native memory pointer (zero-copy). -/// It allows high-performance access to the raw image data on a background Isolate. +/// It allows low-overhead access to the raw image data. The frame-processor +/// callback is delivered on the main isolate's event loop; copy data out (or +/// balance [incrementRefCount]/[decrementRefCount]) before using it elsewhere. /// /// Maps to `Frame` from react-native-vision-camera. class Frame { @@ -66,6 +68,18 @@ class Frame { /// The number of planes in the frame (e.g., 3 for YUV, 1 for RGB). int get planesCount => _getPlanesCount(_pointer); + /// The byte stride (bytes per row) of the given [planeIndex]. This may exceed + /// `width * pixelStride` because of hardware row padding, so always use it + /// (not [width]) when indexing into [getPlaneData]. + int planeBytesPerRow(int planeIndex) => + _getPlaneBytesPerRow(_pointer, planeIndex); + + /// The pixel stride (bytes between consecutive samples) of [planeIndex]. + /// For Android `YUV_420_888` chroma planes this is often `2` (interleaved + /// CbCr); tightly-packed planes are `1`; iOS BGRA is `4`. + int planePixelStride(int planeIndex) => + _getPlanePixelStride(_pointer, planeIndex); + /// Returns a [Uint8List] view of the frame's data for the given [planeIndex]. /// /// This is a **direct view** of the native memory. Modifying it will @@ -99,11 +113,14 @@ class Frame { /// (startX, startY, endX, endY) are pixel coordinates. double computeLuminance(int startX, int startY, int endX, int endY) { if (pixelFormat != PixelFormat.yuv) return 0.0; - // Use the native function directly on the Y plane pointer + // Use the native function directly on the Y plane pointer. The Y plane's + // row stride (which may exceed [width] due to hardware padding) is passed + // so indexing stays correct. return _computeLuminance( _getPlanePointer(_pointer, 0).cast(), width, height, + bytesPerRow, startX, startY, endX, @@ -123,6 +140,11 @@ typedef _GetBytesPerRowFunc = Int32 Function(Pointer); typedef _GetBytesPerRow = int Function(Pointer); late _GetBytesPerRow _getBytesPerRow; +typedef _GetPlaneStrideFunc = Int32 Function(Pointer, Int32); +typedef _GetPlaneStride = int Function(Pointer, int); +late _GetPlaneStride _getPlaneBytesPerRow; +late _GetPlaneStride _getPlanePixelStride; + typedef _GetPlanesCountFunc = Int32 Function(Pointer); typedef _GetPlanesCount = int Function(Pointer); late _GetPlanesCount _getPlanesCount; @@ -166,6 +188,16 @@ void initializeFrameBindings(DynamicLibrary dylib) { _getBytesPerRow = dylib .lookup>('Frame_getBytesPerRow') .asFunction(); + _getPlaneBytesPerRow = dylib + .lookup>( + 'Frame_getPlaneBytesPerRow', + ) + .asFunction(); + _getPlanePixelStride = dylib + .lookup>( + 'Frame_getPlanePixelStride', + ) + .asFunction(); _getPlanesCount = dylib .lookup>('Frame_getPlanesCount') .asFunction(); @@ -219,11 +251,12 @@ typedef _ComputeLuminanceFunc = Pointer yPlane, Int32 width, Int32 height, + Int32 rowStride, Int32 startX, Int32 startY, Int32 endX, Int32 endY, ); typedef _ComputeLuminance = - double Function(Pointer, int, int, int, int, int, int); + double Function(Pointer, int, int, int, int, int, int, int); late _ComputeLuminance _computeLuminance; diff --git a/lib/src/frame_processor.dart b/lib/src/frame_processor.dart index 1923dc7..aaa8575 100644 --- a/lib/src/frame_processor.dart +++ b/lib/src/frame_processor.dart @@ -1,6 +1,7 @@ import 'dart:async'; import 'dart:ffi'; -import 'dart:isolate'; + +import 'package:flutter/foundation.dart'; import 'frame.dart'; import 'types/orientation.dart'; @@ -8,136 +9,113 @@ import 'types/pixel_format.dart'; /// The signature of a frame processor callback. /// -/// This callback is executed on a background Isolate for every frame. +/// This callback is invoked for every camera frame. +/// +/// ## Threading +/// The callback is delivered **asynchronously on the main isolate's event +/// loop** via `dart:ffi` [NativeCallable.listener]. It does **not** run on a +/// separate background isolate and it does **not** block the camera thread. +/// Keep the work light, or copy the data out and hand it to your own isolate. +/// For the heaviest work, register a synchronous native C/C++ plugin instead, +/// which runs on the camera thread with zero added latency. typedef FrameProcessorCallback = void Function(Frame frame); -/// Manages the background Isolate and communication for frame processors. +/// Manages the native frame-processor callback for a camera session. +/// +/// Each pipeline owns its own [NativeCallable]; only one pipeline should be +/// active per native frame source at a time (the C layer holds a single +/// callback slot). class FrameProcessorPipeline { - final FrameProcessorCallback callback; - final ReceivePort _receivePort = ReceivePort(); - Isolate? _isolate; - SendPort? _mainToIsolateSendPort; - + /// Creates a pipeline that forwards native frames to [callback]. FrameProcessorPipeline(this.callback); - static late NativeCallable _nativeCallable; - static SendPort? _currentIsolateSendPort; - - /// Initializes the pipeline and starts the background Isolate. - Future start() async { - _nativeCallable = NativeCallable.listener(_staticFrameCallback); - setNativeFrameProcessorCallback(_nativeCallable.nativeFunction); - - _isolate = await Isolate.spawn(_isolateEntry, _receivePort.sendPort); + /// The user-provided frame processor. + final FrameProcessorCallback callback; - // Wait for the isolate to send its SendPort - final completer = Completer(); - _receivePort.listen((message) { - if (message is SendPort) { - completer.complete(message); - } else { - _handleMessage(message); - } - }); + NativeCallable? _nativeCallable; + bool _stopped = false; - _mainToIsolateSendPort = await completer.future; - _currentIsolateSendPort = _mainToIsolateSendPort; + /// Registers the native callback so frames begin flowing to [callback]. + Future start() async { + _stopped = false; + final callable = NativeCallable.listener( + _onNativeFrame, + ); + _nativeCallable = callable; + setNativeFrameProcessorCallback(callable.nativeFunction); } - void _handleMessage(dynamic message) { - if (message is Map) { - // Map native format codes to PixelFormat enum - final int nativeFormat = message['pixelFormat'] as int; - PixelFormat pixelFormat; - switch (nativeFormat) { - case 35: // android.graphics.ImageFormat.YUV_420_888 - case 842094169: // android.graphics.ImageFormat.YV12 - pixelFormat = PixelFormat.yuv; - break; - case 1: // android.graphics.ImageFormat.RGB_565 (approx) - case 22: // android.graphics.ImageFormat.RGBA_8888 - pixelFormat = PixelFormat.rgb; - break; - default: - if (nativeFormat >= 0 && nativeFormat < PixelFormat.values.length) { - pixelFormat = PixelFormat.values[nativeFormat]; - } else { - pixelFormat = PixelFormat.unknown; - } - } - - // Map native orientation degrees to Orientation enum - final int nativeOrientation = message['orientation'] as int; - Orientation orientation; - switch (nativeOrientation) { - case 0: - orientation = Orientation.portrait; - break; - case 90: - orientation = Orientation.landscapeLeft; - break; - case 180: - orientation = Orientation.portraitUpsideDown; - break; - case 270: - orientation = Orientation.landscapeRight; - break; - default: - if (nativeOrientation >= 0 && - nativeOrientation < Orientation.values.length) { - orientation = Orientation.values[nativeOrientation]; - } else { - orientation = Orientation.portrait; - } - } - - final frame = Frame( - Pointer.fromAddress(message['pointer'] as int), - width: message['width'] as int, - height: message['height'] as int, - pixelFormat: pixelFormat, - orientation: orientation, - timestamp: message['timestamp'] as double, - ); - - try { - callback(frame); - } finally { - frame.decrementRefCount(); + /// Invoked asynchronously on the main isolate for each dispatched frame. + /// + /// The native side has taken a reference on our behalf; we are responsible + /// for releasing it via [Frame.decrementRefCount] exactly once β€” even if the + /// pipeline was stopped between dispatch and delivery, or the user callback + /// throws. + void _onNativeFrame(Pointer handle, FrameMetadataNative metadata) { + final frame = Frame( + handle, + width: metadata.width, + height: metadata.height, + pixelFormat: _mapPixelFormat(metadata.pixelFormat), + orientation: _mapOrientation(metadata.orientation), + timestamp: metadata.timestamp, + ); + try { + if (!_stopped) callback(frame); + } catch (e, stack) { + // A throwing frame processor must not tear down the listener or leak + // the frame; log and continue. + if (kDebugMode) { + debugPrint('FrameProcessor callback threw: $e\n$stack'); } + } finally { + frame.decrementRefCount(); } } - static void _staticFrameCallback( - Pointer handle, - FrameMetadataNative metadata, - ) { - _currentIsolateSendPort?.send({ - 'pointer': handle.address, - 'width': metadata.width, - 'height': metadata.height, - 'pixelFormat': metadata.pixelFormat, - 'orientation': metadata.orientation, - 'timestamp': metadata.timestamp, - }); + /// Unregisters the native callback and releases the [NativeCallable]. + void stop() { + _stopped = true; + // Detach the native side first so no new frames are dispatched into a + // callable we are about to close. + setNativeFrameProcessorCallback(nullptr); + _nativeCallable?.close(); + _nativeCallable = null; } - static void _isolateEntry(SendPort mainSendPort) { - final receivePort = ReceivePort(); - mainSendPort.send(receivePort.sendPort); - - receivePort.listen((message) { - mainSendPort.send(message); - }); + static PixelFormat _mapPixelFormat(int nativeFormat) { + switch (nativeFormat) { + case 35: // android.graphics.ImageFormat.YUV_420_888 + case 842094169: // android.graphics.ImageFormat.YV12 + return PixelFormat.yuv; + case 1: // BGRA (iOS) / RGB family + case 22: // android.graphics.ImageFormat.RGBA_8888 + return PixelFormat.rgb; + default: + if (nativeFormat >= 0 && nativeFormat < PixelFormat.values.length) { + return PixelFormat.values[nativeFormat]; + } + return PixelFormat.unknown; + } } - /// Stops the pipeline and kills the Isolate. - void stop() { - _nativeCallable.close(); - setNativeFrameProcessorCallback(nullptr); - _isolate?.kill(); - _receivePort.close(); - _currentIsolateSendPort = null; + static Orientation _mapOrientation(int nativeOrientation) { + switch (nativeOrientation) { + case 0: + return Orientation.portrait; + case 90: + return Orientation.landscapeLeft; + case 180: + return Orientation.portraitUpsideDown; + case 270: + return Orientation.landscapeRight; + default: + if (nativeOrientation >= 0 && + nativeOrientation < Orientation.values.length) { + return Orientation.values[nativeOrientation]; + } + return Orientation.portrait; + } } } @@ -145,11 +123,14 @@ class FrameProcessorPipeline { /// /// Maps to `runAtTargetFps` from react-native-vision-camera. class FrameProcessorThrottler { + /// Creates a throttler that admits at most [targetFps] frames per second. + FrameProcessorThrottler({required this.targetFps}); + + /// The maximum number of frames to process per second. final int targetFps; int _lastProcessedTimestamp = 0; - FrameProcessorThrottler({required this.targetFps}); - + /// Returns `true` if a frame at [timestampMs] should be processed. bool shouldProcess(int timestampMs) { final interval = 1000 ~/ targetFps; if (timestampMs - _lastProcessedTimestamp >= interval) { diff --git a/pubspec.yaml b/pubspec.yaml index f09d0d6..50e1699 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,25 +1,32 @@ name: flutter_native_vision_camera -description: "High-performance Flutter FFI camera plugin with zero-copy preview and real-time frame processing." -version: 0.0.3 +description: "High-performance Flutter FFI camera plugin with zero-copy preview textures, integrated MLKit/Vision barcode scanning, and low-latency native frame access for real-time on-device vision." +version: 0.0.4 homepage: https://github.com/JenteJan/flutter_native_vision_camera repository: https://github.com/JenteJan/flutter_native_vision_camera +issue_tracker: https://github.com/JenteJan/flutter_native_vision_camera/issues + +topics: + - camera + - ffi + - barcode + - mlkit + - vision environment: sdk: ^3.8.1 - flutter: '>=3.3.0' + flutter: '>=3.22.0' dependencies: flutter: sdk: flutter ffi: ^2.1.3 - plugin_platform_interface: ^2.0.2 dev_dependencies: ffigen: ^13.0.0 flutter_test: sdk: flutter flutter_lints: ^5.0.0 - + flutter: plugin: platforms: @@ -29,4 +36,4 @@ flutter: ffiPlugin: true ios: pluginClass: FlutterNativeVisionCameraPlugin - ffiPlugin: true \ No newline at end of file + ffiPlugin: true diff --git a/src/VisionCamera_NativePluginExample.cpp b/src/VisionCamera_NativePluginExample.cpp index b29bb70..da03349 100644 --- a/src/VisionCamera_NativePluginExample.cpp +++ b/src/VisionCamera_NativePluginExample.cpp @@ -28,18 +28,19 @@ class BrightnessPlugin : public vision::Plugin { // Perform a super-fast brightness calculation on a 100x100 center crop int width = frame.width(); int height = frame.height(); + int stride = frame.bytesPerRow(); // row stride (>= width due to padding) int centerX = width / 2; int centerY = height / 2; - + long long sum = 0; int count = 0; - - // Sampling loop + + // Sampling loop β€” index by the row stride, NOT width, or padded buffers skew. for (int y = centerY - 50; y < centerY + 50; ++y) { if (y < 0 || y >= height) continue; for (int x = centerX - 50; x < centerX + 50; ++x) { if (x < 0 || x >= width) continue; - sum += yData[y * width + x]; + sum += yData[y * stride + x]; count++; } } diff --git a/src/flutter_native_vision_camera.c b/src/flutter_native_vision_camera.c index 7f9e492..e03fb7b 100644 --- a/src/flutter_native_vision_camera.c +++ b/src/flutter_native_vision_camera.c @@ -4,14 +4,20 @@ static JavaVM* g_javaVM = NULL; static jclass g_pluginClass = NULL; static jmethodID g_releaseFrameMethod = NULL; +static jmethodID g_retainFrameMethod = NULL; #define FRAME_MAGIC 0xFEEDFACE +// Android frame handle: holds direct pointers + strides for every plane so the +// Dart/C++ side can read true multi-plane YUV (not just the Y plane). typedef struct { uint32_t magic; - uint32_t padding; + uint32_t numPlanes; uint64_t id; - void* address; + void* planes[3]; + int32_t rowStrides[3]; + int32_t pixelStrides[3]; + int32_t planeSizes[3]; } NativeFrame; JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { @@ -22,18 +28,40 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { } jclass localClass = (*env)->FindClass(env, "dev/jentejan/flutter_native_vision_camera/FlutterNativeVisionCameraPlugin"); if (!localClass) return JNI_ERR; - + g_pluginClass = (*env)->NewGlobalRef(env, localClass); - // Updated signature: returns Int (I) g_releaseFrameMethod = (*env)->GetStaticMethodID(env, g_pluginClass, "releaseFrame", "(J)I"); - - if (!g_releaseFrameMethod) { - __android_log_print(ANDROID_LOG_ERROR, "VisionCamera", "Failed to find releaseFrame(J)I"); + g_retainFrameMethod = (*env)->GetStaticMethodID(env, g_pluginClass, "retainFrame", "(J)I"); + + if (!g_releaseFrameMethod || !g_retainFrameMethod) { + __android_log_print(ANDROID_LOG_ERROR, "VisionCamera", "Failed to find releaseFrame/retainFrame methods"); return JNI_ERR; } - + return JNI_VERSION_1_6; } + +// Invokes a static int(long) method on the plugin class, attaching the current +// thread to the JVM if necessary. +static int call_frame_jni(jmethodID method, uint64_t id) { + if (g_javaVM == NULL || method == NULL) { + __android_log_print(ANDROID_LOG_ERROR, "VisionCamera", "JNI not initialized for frame %lld", (long long)id); + return 0; + } + JNIEnv* env; + int status = (*g_javaVM)->GetEnv(g_javaVM, (void**)&env, JNI_VERSION_1_6); + int attached = 0; + if (status == JNI_EDETACHED) { + status = (*g_javaVM)->AttachCurrentThread(g_javaVM, (void**)&env, NULL); + attached = 1; + } + int result = 0; + if (status == JNI_OK && env != NULL) { + result = (*env)->CallStaticIntMethod(env, g_pluginClass, method, (jlong)id); + if (attached) (*g_javaVM)->DetachCurrentThread(g_javaVM); + } + return result; +} #else #include #endif @@ -41,6 +69,7 @@ JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM* vm, void* reserved) { #include #include #include +#include #include "flutter_native_vision_camera.h" // ─── Plugin System ─────────────────────────────────────────────────── @@ -71,154 +100,200 @@ FFI_PLUGIN_EXPORT void VisionCamera_unregisterPlugin(const char* name) { // ─── Frame Accessors ───────────────────────────────────────────────── FFI_PLUGIN_EXPORT int32_t Frame_getBytesPerRow(FrameHandle handle) { + if (handle == NULL) return 0; #ifdef ANDROID - // In YUV_420_888, planes might have different strides. - // This is a simplified return for the Y plane. - return 0; // Better to get this from metadata or a separate helper + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return 0; + return frame->rowStrides[0]; #else - if (handle == NULL) return 0; return (int32_t)CVPixelBufferGetBytesPerRow((CVPixelBufferRef)handle); #endif } -FFI_PLUGIN_EXPORT int32_t Frame_getPlanesCount(FrameHandle handle) { +FFI_PLUGIN_EXPORT int32_t Frame_getPlaneBytesPerRow(FrameHandle handle, int32_t planeIndex) { + if (handle == NULL) return 0; #ifdef ANDROID - return 3; // Y, U, V + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return 0; + if (planeIndex < 0 || planeIndex >= (int32_t)frame->numPlanes) return 0; + return frame->rowStrides[planeIndex]; #else + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; + if (CVPixelBufferIsPlanar(pixelBuffer)) { + return (int32_t)CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, planeIndex); + } + return (int32_t)CVPixelBufferGetBytesPerRow(pixelBuffer); +#endif +} + +FFI_PLUGIN_EXPORT int32_t Frame_getPlanePixelStride(FrameHandle handle, int32_t planeIndex) { if (handle == NULL) return 0; - return (int32_t)CVPixelBufferGetPlaneCount((CVPixelBufferRef)handle); +#ifdef ANDROID + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return 0; + if (planeIndex < 0 || planeIndex >= (int32_t)frame->numPlanes) return 0; + return frame->pixelStrides[planeIndex]; +#else + // BGRA is interleaved (4 bytes/pixel); planar buffers are tightly packed. + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; + return CVPixelBufferIsPlanar(pixelBuffer) ? 1 : 4; #endif } -FFI_PLUGIN_EXPORT void* Frame_getPlanePointer(FrameHandle handle, int32_t planeIndex) { +FFI_PLUGIN_EXPORT int32_t Frame_getPlanesCount(FrameHandle handle) { + if (handle == NULL) return 0; #ifdef ANDROID + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return 0; + return (int32_t)frame->numPlanes; +#else + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; + if (CVPixelBufferIsPlanar(pixelBuffer)) { + return (int32_t)CVPixelBufferGetPlaneCount(pixelBuffer); + } + return 1; +#endif +} + +FFI_PLUGIN_EXPORT void* Frame_getPlanePointer(FrameHandle handle, int32_t planeIndex) { if (handle == NULL) return NULL; +#ifdef ANDROID NativeFrame* frame = (NativeFrame*)handle; if (frame->magic != FRAME_MAGIC) return NULL; - return frame->address; + if (planeIndex < 0 || planeIndex >= (int32_t)frame->numPlanes) return NULL; + return frame->planes[planeIndex]; #else - if (handle == NULL) return NULL; + // The buffer is locked once for the lifetime of the frame in + // VisionCamera_dispatchFrame and unlocked in Frame_decrementRefCount. CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; - CVPixelBufferLockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); if (CVPixelBufferIsPlanar(pixelBuffer)) { return CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, planeIndex); - } else { - return CVPixelBufferGetBaseAddress(pixelBuffer); } + return CVPixelBufferGetBaseAddress(pixelBuffer); #endif } FFI_PLUGIN_EXPORT int32_t Frame_getPlaneSize(FrameHandle handle, FrameMetadata metadata, int32_t planeIndex) { if (handle == NULL) return 0; #ifdef ANDROID - if (planeIndex == 0) return metadata.width * metadata.height; - return (metadata.width / 2) * (metadata.height / 2); + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return 0; + if (planeIndex < 0 || planeIndex >= (int32_t)frame->numPlanes) return 0; + return frame->planeSizes[planeIndex]; #else CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; if (CVPixelBufferIsPlanar(pixelBuffer)) { return (int32_t)(CVPixelBufferGetHeightOfPlane(pixelBuffer, planeIndex) * CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, planeIndex)); - } else { - return (int32_t)(metadata.height * CVPixelBufferGetBytesPerRow(pixelBuffer)); } + return (int32_t)(metadata.height * CVPixelBufferGetBytesPerRow(pixelBuffer)); #endif } FFI_PLUGIN_EXPORT void Frame_incrementRefCount(FrameHandle handle) { -#ifndef ANDROID - if (handle != NULL) { - CFRetain((CVPixelBufferRef)handle); - } + if (handle == NULL) return; +#ifdef ANDROID + NativeFrame* frame = (NativeFrame*)handle; + if (frame->magic != FRAME_MAGIC) return; + call_frame_jni(g_retainFrameMethod, frame->id); +#else + // Each reference holds one lock; every decrement unlocks exactly once, so a + // retain must take a matching lock or the buffer would be over-unlocked. + CVPixelBufferLockBaseAddress((CVPixelBufferRef)handle, kCVPixelBufferLock_ReadOnly); + CFRetain((CVPixelBufferRef)handle); #endif } FFI_PLUGIN_EXPORT void Frame_decrementRefCount(FrameHandle handle) { -#ifdef ANDROID if (handle == NULL) return; +#ifdef ANDROID NativeFrame* frame = (NativeFrame*)handle; - - // Safety check: only process and free if magic matches + + // Safety check: only process and free if magic matches. if (frame->magic != FRAME_MAGIC) { __android_log_print(ANDROID_LOG_WARN, "VisionCamera", "Attempted to release invalid handle %p!", handle); return; } - if (g_javaVM != NULL && g_releaseFrameMethod != NULL) { - JNIEnv* env; - int status = (*g_javaVM)->GetEnv(g_javaVM, (void**)&env, JNI_VERSION_1_6); - int attached = 0; - if (status == JNI_EDETACHED) { - status = (*g_javaVM)->AttachCurrentThread(g_javaVM, (void**)&env, NULL); - attached = 1; - } - - if (status == JNI_OK && env != NULL) { - // Call Kotlin releaseFrame - (*env)->CallStaticIntMethod(env, g_pluginClass, g_releaseFrameMethod, (jlong)frame->id); - - if (attached) (*g_javaVM)->DetachCurrentThread(g_javaVM); - } else { - __android_log_print(ANDROID_LOG_ERROR, "VisionCamera", "Failed to get JNIEnv to release frame %lld", (long long)frame->id); - } - } else { - __android_log_print(ANDROID_LOG_ERROR, "VisionCamera", "JNI not initialized, leaking frame %lld", (long long)frame->id); + int remaining = call_frame_jni(g_releaseFrameMethod, frame->id); + + // Only free the struct once the underlying image is fully released (refcount + // reached zero). A Dart-side incrementRefCount()/retain bumps the count, so + // the struct (and its plane pointers) must stay valid until the final + // decrement β€” freeing on the first decrement would be a use-after-free. + if (remaining <= 0) { + // Mark as invalid BEFORE freeing to catch double-frees. + frame->magic = 0; + free(frame); } - - // Mark as invalid BEFORE freeing to catch double-frees - frame->magic = 0; - free(frame); #else - if (handle != NULL) { - CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; - CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); - CFRelease(pixelBuffer); - } + CVPixelBufferRef pixelBuffer = (CVPixelBufferRef)handle; + CVPixelBufferUnlockBaseAddress(pixelBuffer, kCVPixelBufferLock_ReadOnly); + CFRelease(pixelBuffer); #endif } -static FrameProcessorCallback g_frameProcessorCallback = NULL; +static _Atomic(FrameProcessorCallback) g_frameProcessorCallback = NULL; +static pthread_mutex_t g_cbMutex = PTHREAD_MUTEX_INITIALIZER; FFI_PLUGIN_EXPORT void VisionCamera_setFrameProcessorCallback(FrameProcessorCallback callback) { - g_frameProcessorCallback = callback; + // Serialize with dispatch so detaching the callback (and the Dart-side + // NativeCallable.close()) can't race a call that is mid-flight. + pthread_mutex_lock(&g_cbMutex); + atomic_store(&g_frameProcessorCallback, callback); + pthread_mutex_unlock(&g_cbMutex); } FFI_PLUGIN_EXPORT void VisionCamera_dispatchFrame(FrameHandle handle, FrameMetadata metadata) { - // 1. Notify C/C++ Plugins first (Zero latency, synchronous) +#ifndef ANDROID + // iOS: lock the pixel buffer once for the whole frame lifetime. It is + // unlocked exactly once in Frame_decrementRefCount. + CVPixelBufferLockBaseAddress((CVPixelBufferRef)handle, kCVPixelBufferLock_ReadOnly); +#endif + + // 1. Notify C/C++ Plugins first (zero latency, synchronous). for (int i = 0; i < g_pluginCount; i++) { g_plugins[i].onFrame(handle, metadata); } - // 2. Notify Dart (FFI Isolate dispatch) - if (g_frameProcessorCallback != NULL) { - // Dart will be responsible for calling Frame_decrementRefCount - g_frameProcessorCallback(handle, metadata); - } else { - // No Dart listener, release the reference taken by the native side + // 2. Notify Dart (FFI dispatch). Dart is then responsible for calling + // Frame_decrementRefCount exactly once. Hold the mutex across the + // load+call so teardown can't close the callable mid-dispatch. + pthread_mutex_lock(&g_cbMutex); + FrameProcessorCallback cb = atomic_load(&g_frameProcessorCallback); + if (cb != NULL) { + cb(handle, metadata); + } + pthread_mutex_unlock(&g_cbMutex); + + if (cb == NULL) { + // No Dart listener; release the reference taken by the native side. Frame_decrementRefCount(handle); } } // ─── Image Processing Helpers ──────────────────────────────────────── -FFI_PLUGIN_EXPORT double VisionCamera_computeLuminance(const uint8_t* yPlane, int32_t width, int32_t height, int32_t startX, int32_t startY, int32_t endX, int32_t endY) { +FFI_PLUGIN_EXPORT double VisionCamera_computeLuminance(const uint8_t* yPlane, int32_t width, int32_t height, int32_t rowStride, int32_t startX, int32_t startY, int32_t endX, int32_t endY) { if (yPlane == NULL) return 0.0; - + if (rowStride <= 0) rowStride = width; + if (startX < 0) startX = 0; if (startY < 0) startY = 0; if (endX >= width) endX = width - 1; if (endY >= height) endY = height - 1; - + if (startX > endX || startY > endY) return 0.0; - + uint64_t sum = 0; int32_t count = 0; - + for (int y = startY; y <= endY; y++) { for (int x = startX; x <= endX; x++) { - sum += yPlane[y * width + x]; + sum += yPlane[y * rowStride + x]; count++; } } - + if (count == 0) return 0.0; return (double)sum / (double)count; } @@ -226,18 +301,28 @@ FFI_PLUGIN_EXPORT double VisionCamera_computeLuminance(const uint8_t* yPlane, in #ifdef ANDROID JNIEXPORT void JNICALL Java_dev_jentejan_flutter_1native_1vision_1camera_FlutterNativeVisionCameraPlugin_nativeDispatchFrame( - JNIEnv* env, jobject thiz, jobject buffer, jint width, jint height, jint format, jint orientation, jdouble timestamp, jlong id, jlong address) { - + JNIEnv* env, jobject thiz, + jobject b0, jobject b1, jobject b2, + jint rs0, jint rs1, jint rs2, + jint ps0, jint ps1, jint ps2, + jint sz0, jint sz1, jint sz2, + jint numPlanes, + jint width, jint height, jint format, jint orientation, jdouble timestamp, jlong id) { + NativeFrame* frame = (NativeFrame*)malloc(sizeof(NativeFrame)); + if (frame == NULL) return; + frame->magic = FRAME_MAGIC; frame->id = (uint64_t)id; - - // If address wasn't passed or is 0, try to get it from the direct buffer - if (address == 0 && buffer != NULL) { - frame->address = (*env)->GetDirectBufferAddress(env, buffer); - } else { - frame->address = (void*)address; - } + frame->numPlanes = (uint32_t)numPlanes; + + frame->planes[0] = (b0 != NULL) ? (*env)->GetDirectBufferAddress(env, b0) : NULL; + frame->planes[1] = (b1 != NULL) ? (*env)->GetDirectBufferAddress(env, b1) : NULL; + frame->planes[2] = (b2 != NULL) ? (*env)->GetDirectBufferAddress(env, b2) : NULL; + + frame->rowStrides[0] = rs0; frame->rowStrides[1] = rs1; frame->rowStrides[2] = rs2; + frame->pixelStrides[0] = ps0; frame->pixelStrides[1] = ps1; frame->pixelStrides[2] = ps2; + frame->planeSizes[0] = sz0; frame->planeSizes[1] = sz1; frame->planeSizes[2] = sz2; FrameMetadata metadata = { .width = width, @@ -249,12 +334,6 @@ Java_dev_jentejan_flutter_1native_1vision_1camera_FlutterNativeVisionCameraPlugi VisionCamera_dispatchFrame((FrameHandle)frame, metadata); } - -JNIEXPORT void JNICALL -Java_dev_jentejan_flutter_1native_1vision_1camera_FlutterNativeVisionCameraPlugin_nativeSetFrameProcessorCallback( - JNIEnv* env, jobject thiz, jlong callback) { - VisionCamera_setFrameProcessorCallback((FrameProcessorCallback)callback); -} #endif FFI_PLUGIN_EXPORT int sum(int a, int b) { return a + b; } diff --git a/src/flutter_native_vision_camera.h b/src/flutter_native_vision_camera.h index 940bc2d..2f73e04 100644 --- a/src/flutter_native_vision_camera.h +++ b/src/flutter_native_vision_camera.h @@ -1,3 +1,6 @@ +#ifndef FLUTTER_NATIVE_VISION_CAMERA_H +#define FLUTTER_NATIVE_VISION_CAMERA_H + #include #include #include @@ -12,60 +15,67 @@ #if _WIN32 #define FFI_PLUGIN_EXPORT __declspec(dllexport) #else -#define FFI_PLUGIN_EXPORT +#define FFI_PLUGIN_EXPORT __attribute__((visibility("default"))) #endif #ifdef __cplusplus -extern "C" { +extern "C" +{ #endif -// Opaque handle for a native frame. -// Android: NativeFrame struct (id + address) -// iOS: CVPixelBufferRef -typedef void* FrameHandle; + // Opaque handle for a native frame. + // Android: NativeFrame struct (id + per-plane pointers/strides) + // iOS: CVPixelBufferRef + typedef void *FrameHandle; -/** - * Metadata for a single camera frame. - */ -typedef struct { - int32_t width; - int32_t height; - int32_t pixelFormat; // 0 for YUV_420_888, 1 for BGRA - int32_t orientation; // 0, 90, 180, 270 - double timestamp; // Presentation timestamp in seconds -} FrameMetadata; + /** + * Metadata for a single camera frame. + */ + typedef struct + { + int32_t width; + int32_t height; + int32_t pixelFormat; // 0 for YUV_420_888, 1 for BGRA + int32_t orientation; // 0, 90, 180, 270 + double timestamp; // Presentation timestamp in seconds + } FrameMetadata; -typedef void (*FrameProcessorCallback)(FrameHandle handle, FrameMetadata metadata); + typedef void (*FrameProcessorCallback)(FrameHandle handle, FrameMetadata metadata); -/** - * A native C/C++ plugin for Vision Camera. - */ -typedef struct VisionCameraPlugin { - const char* name; - // Called when a new frame is available. - // Return 1 to consume/process, 0 to ignore. - void (*onFrame)(FrameHandle handle, FrameMetadata metadata); - // Called when the plugin is removed. - void (*onDestroy)(); -} VisionCameraPlugin; + /** + * A native C/C++ plugin for Vision Camera. + */ + typedef struct VisionCameraPlugin + { + const char *name; + // Called when a new frame is available. + // Return 1 to consume/process, 0 to ignore. + void (*onFrame)(FrameHandle handle, FrameMetadata metadata); + // Called when the plugin is removed. + void (*onDestroy)(); + } VisionCameraPlugin; -FFI_PLUGIN_EXPORT int32_t Frame_getBytesPerRow(FrameHandle handle); -FFI_PLUGIN_EXPORT int32_t Frame_getPlanesCount(FrameHandle handle); -FFI_PLUGIN_EXPORT void* Frame_getPlanePointer(FrameHandle handle, int32_t planeIndex); -FFI_PLUGIN_EXPORT int32_t Frame_getPlaneSize(FrameHandle handle, FrameMetadata metadata, int32_t planeIndex); -FFI_PLUGIN_EXPORT void Frame_incrementRefCount(FrameHandle handle); -FFI_PLUGIN_EXPORT void Frame_decrementRefCount(FrameHandle handle); + FFI_PLUGIN_EXPORT int32_t Frame_getBytesPerRow(FrameHandle handle); + FFI_PLUGIN_EXPORT int32_t Frame_getPlaneBytesPerRow(FrameHandle handle, int32_t planeIndex); + FFI_PLUGIN_EXPORT int32_t Frame_getPlanePixelStride(FrameHandle handle, int32_t planeIndex); + FFI_PLUGIN_EXPORT int32_t Frame_getPlanesCount(FrameHandle handle); + FFI_PLUGIN_EXPORT void *Frame_getPlanePointer(FrameHandle handle, int32_t planeIndex); + FFI_PLUGIN_EXPORT int32_t Frame_getPlaneSize(FrameHandle handle, FrameMetadata metadata, int32_t planeIndex); + FFI_PLUGIN_EXPORT void Frame_incrementRefCount(FrameHandle handle); + FFI_PLUGIN_EXPORT void Frame_decrementRefCount(FrameHandle handle); -// Registration for C-level plugins (Zero-latency processing) -FFI_PLUGIN_EXPORT void VisionCamera_registerPlugin(VisionCameraPlugin plugin); -FFI_PLUGIN_EXPORT void VisionCamera_unregisterPlugin(const char* name); + // Registration for C-level plugins (Zero-latency processing) + FFI_PLUGIN_EXPORT void VisionCamera_registerPlugin(VisionCameraPlugin plugin); + FFI_PLUGIN_EXPORT void VisionCamera_unregisterPlugin(const char *name); -// Bridge to Dart (High-level processing) -FFI_PLUGIN_EXPORT void VisionCamera_setFrameProcessorCallback(FrameProcessorCallback callback); -FFI_PLUGIN_EXPORT void VisionCamera_dispatchFrame(FrameHandle handle, FrameMetadata metadata); + // Bridge to Dart (High-level processing) + FFI_PLUGIN_EXPORT void VisionCamera_setFrameProcessorCallback(FrameProcessorCallback callback); + FFI_PLUGIN_EXPORT void VisionCamera_dispatchFrame(FrameHandle handle, FrameMetadata metadata); -FFI_PLUGIN_EXPORT double VisionCamera_computeLuminance(const uint8_t* yPlane, int32_t width, int32_t height, int32_t startX, int32_t startY, int32_t endX, int32_t endY); + FFI_PLUGIN_EXPORT double VisionCamera_computeLuminance(const uint8_t *yPlane, int32_t width, int32_t height, int32_t rowStride, int32_t startX, int32_t startY, int32_t endX, int32_t endY); #ifdef __cplusplus } #endif + +#endif // FLUTTER_NATIVE_VISION_CAMERA_H diff --git a/test/camera_controller_test.dart b/test/camera_controller_test.dart index 4bb2d76..93989ee 100644 --- a/test/camera_controller_test.dart +++ b/test/camera_controller_test.dart @@ -121,6 +121,95 @@ void main() { expect(controller.isActive, true); }); + CameraDevice makeDevice({ + Orientation sensorOrientation = Orientation.portrait, + CameraPosition position = CameraPosition.back, + }) { + return CameraDevice( + id: 'cam', + name: 'Camera', + position: position, + hasFlash: true, + hasTorch: true, + minFocusDistance: 0.0, + isMultiCam: false, + minZoom: 1.0, + maxZoom: 10.0, + neutralZoom: 1.0, + minExposure: -2.0, + maxExposure: 2.0, + supportsLowLightBoost: false, + supportsRawCapture: false, + supportsFocus: true, + hardwareLevel: HardwareLevel.full, + sensorOrientation: sensorOrientation, + physicalDevices: const [PhysicalCameraDeviceType.wideAngleCamera], + formats: const [], + ); + } + + // Simulates a native -> Dart method call (e.g. onPreviewConfigurationChanged). + Future sendNative(String method, dynamic arguments) { + return TestDefaultBinaryMessengerBinding.instance.defaultBinaryMessenger + .handlePlatformMessage( + channel.name, + const StandardMethodCodec().encodeMethodCall( + MethodCall(method, arguments), + ), + (_) {}, + ); + } + + test( + 'previewRotation falls back to sensorOrientation before native reports', + () async { + await controller.initialize( + makeDevice(sensorOrientation: Orientation.landscapeLeft), + ); + expect(controller.previewRotation, 1); // 90 / 90 + // Landscape preview (1280x720) is swapped to portrait in display space. + expect(controller.displayPreviewSize, const Size(720, 1280)); + }, + ); + + test( + 'previewRotation uses the native rotationDegrees once reported', + () async { + await controller.initialize( + makeDevice(sensorOrientation: Orientation.landscapeLeft), + ); + await sendNative('onPreviewConfigurationChanged', { + 'rotationDegrees': 0, + 'mirrored': false, + }); + expect(controller.previewRotation, 0); + expect( + controller.displayPreviewSize, + const Size(1280, 720), + ); // not swapped + }, + ); + + test('previewMirrored reflects the native report', () async { + await controller.initialize(makeDevice(position: CameraPosition.front)); + expect(controller.previewMirrored, false); + await sendNative('onPreviewConfigurationChanged', { + 'rotationDegrees': 90, + 'mirrored': true, + }); + expect(controller.previewMirrored, true); + }); + + test('mirror reflects the init argument (defaults to true)', () async { + await controller.initialize(makeDevice()); + expect(controller.mirror, true); + + final c2 = CameraController(); + await c2.initialize(makeDevice(), mirror: false); + expect(c2.mirror, false); + c2.dispose(); + }); + test('dispose clean up correctly', () { controller.dispose(); expect(controller.value, CameraState.disposed); diff --git a/test/models_test.dart b/test/models_test.dart index c9aa53c..258068f 100644 --- a/test/models_test.dart +++ b/test/models_test.dart @@ -33,5 +33,26 @@ void main() { Orientation.portraitUpsideDown, ); }); + + test('Orientation.degrees (drives preview rotation)', () { + expect(Orientation.portrait.degrees, 0); + expect(Orientation.landscapeLeft.degrees, 90); + expect(Orientation.portraitUpsideDown.degrees, 180); + expect(Orientation.landscapeRight.degrees, 270); + }); + }); + + group('FrameProcessorThrottler', () { + test('admits at most targetFps frames per second', () { + final throttler = FrameProcessorThrottler( + targetFps: 10, + ); // 100ms interval + // Real camera timestamps are large, so the first frame is admitted. + expect(throttler.shouldProcess(1000), true); + expect(throttler.shouldProcess(1050), false); + expect(throttler.shouldProcess(1100), true); + expect(throttler.shouldProcess(1150), false); + expect(throttler.shouldProcess(1200), true); + }); }); }