diff --git a/.pubignore b/.pubignore index 2ca9ac5..628c0ec 100644 --- a/.pubignore +++ b/.pubignore @@ -5,6 +5,10 @@ # Demo media doc/ +# The example's bundled ML model (git-tracked so the example runs, but ~5 MB — +# kept out of the published package). +**/*.tflite + # Build & tooling artifacts (mirrors .gitignore) build/ **/build/ diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c81a4..6ae2fd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Changelog +## 0.0.6 + +* **New:** `CameraController.previewRectFromFrame(rect, sourceRotationDegrees:)` maps a normalized + detection box from a frame's coordinate space into preview-display space, accounting for the live preview + rotation and the front-camera mirror — so ML overlays line up with the preview **at any device + orientation** (the preview rotation and frame orientation are tracked independently, so boxes drawn + without it drift as the phone turns). +* **Example:** added a real-time **Object Detector** page — the FFI frame pipeline drives an on-device + EfficientDet-Lite0 detector (90 COCO classes) in a background isolate, with labelled boxes and a + cat-certainty voting trigger. Demonstrates YUV→RGB preprocessing, isolate inference, and + `previewRectFromFrame` for correctly-oriented overlays. + ## 0.0.5 * **Docs:** added demo GIFs to the README (live preview + orientation, barcode/QR scanning with overlay, diff --git a/README.md b/README.md index 6690c32..8586f3b 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,24 @@ frame.incrementRefCount(); // keep the buffer alive frame.decrementRefCount(); // release it (mandatory) ``` +### Drawing ML detection boxes on the preview + +Detection boxes are in the frame's coordinate space; the preview is rotated (and +maybe mirrored) for display, and the two are tracked **independently** — so a box +drawn naively drifts as the device turns. Map it with `previewRectFromFrame`: + +```dart +final previewRect = controller.previewRectFromFrame( + detection.boundingBox, // normalized 0..1 from your model + sourceRotationDegrees: frame.orientation.degrees, // rotation you fed the model +); +// Scale previewRect onto the preview widget's rect (use displayPreviewSize for BoxFit). +``` + +It accounts for the live `previewRotation` and the front-camera mirror, so boxes +stay aligned at any orientation. See the example's **Object Detector** page for a +full EfficientDet-Lite0 overlay. + ### High-performance native C/C++ plugin For the heaviest work, hook the synchronous frame loop in C++ (`src/VisionCamera.hpp`): diff --git a/android/build.gradle b/android/build.gradle index a80820a..32b1113 100644 --- a/android/build.gradle +++ b/android/build.gradle @@ -1,7 +1,7 @@ // The Android Gradle Plugin builds the native code with the Android NDK. group = "dev.jentejan.flutter_native_vision_camera" -version = "0.0.5" +version = "0.0.6" buildscript { repositories { diff --git a/example/android/gradle.properties b/example/android/gradle.properties index 475a628..c696804 100644 --- a/example/android/gradle.properties +++ b/example/android/gradle.properties @@ -1,6 +1,10 @@ org.gradle.jvmargs=-Xmx8G -XX:MaxMetaspaceSize=4G -XX:ReservedCodeCacheSize=512m -XX:+HeapDumpOnOutOfMemoryError android.useAndroidX=true android.enableJetifier=true +# Some plugins (e.g. tflite_flutter 0.11.x) leave Java at 1.8 while Kotlin uses +# the toolchain's 21. On Android everything dexes to the same level, so downgrade +# the JVM-target consistency check from a hard error to a warning. +kotlin.jvm.target.validation.mode=warning # This builtInKotlin flag was added automatically by Flutter migrator android.builtInKotlin=false # This newDsl flag was added automatically by Flutter migrator diff --git a/example/assets/coco_labels.txt b/example/assets/coco_labels.txt new file mode 100644 index 0000000..695772d --- /dev/null +++ b/example/assets/coco_labels.txt @@ -0,0 +1,90 @@ +person +bicycle +car +motorcycle +airplane +bus +train +truck +boat +traffic light +fire hydrant +??? +stop sign +parking meter +bench +bird +cat +dog +horse +sheep +cow +elephant +bear +zebra +giraffe +??? +backpack +umbrella +??? +??? +handbag +tie +suitcase +frisbee +skis +snowboard +sports ball +kite +baseball bat +baseball glove +skateboard +surfboard +tennis racket +bottle +??? +wine glass +cup +fork +knife +spoon +bowl +banana +apple +sandwich +orange +broccoli +carrot +hot dog +pizza +donut +cake +chair +couch +potted plant +bed +??? +dining table +??? +??? +toilet +??? +tv +laptop +mouse +remote +keyboard +cell phone +microwave +oven +toaster +sink +refrigerator +??? +book +clock +vase +scissors +teddy bear +hair drier +toothbrush diff --git a/example/assets/efficientdet.tflite b/example/assets/efficientdet.tflite new file mode 100644 index 0000000..57bea4a Binary files /dev/null and b/example/assets/efficientdet.tflite differ diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock index 6524034..f661db1 100644 --- a/example/ios/Podfile.lock +++ b/example/ios/Podfile.lock @@ -1,14 +1,42 @@ PODS: - Flutter (1.0.0) - - flutter_native_vision_camera (0.0.4): + - flutter_native_vision_camera (0.0.5): - Flutter - permission_handler_apple (9.3.0): - Flutter + - TensorFlowLiteC (2.12.0): + - TensorFlowLiteC/Core (= 2.12.0) + - TensorFlowLiteC/Core (2.12.0) + - TensorFlowLiteC/CoreML (2.12.0): + - TensorFlowLiteC/Core + - TensorFlowLiteC/Metal (2.12.0): + - TensorFlowLiteC/Core + - TensorFlowLiteSwift (2.12.0): + - TensorFlowLiteSwift/Core (= 2.12.0) + - TensorFlowLiteSwift/Core (2.12.0): + - TensorFlowLiteC (= 2.12.0) + - TensorFlowLiteSwift/CoreML (2.12.0): + - TensorFlowLiteC/CoreML (= 2.12.0) + - TensorFlowLiteSwift/Core (= 2.12.0) + - TensorFlowLiteSwift/Metal (2.12.0): + - TensorFlowLiteC/Metal (= 2.12.0) + - TensorFlowLiteSwift/Core (= 2.12.0) + - tflite_flutter (0.0.1): + - Flutter + - TensorFlowLiteSwift (= 2.12.0) + - TensorFlowLiteSwift/CoreML (= 2.12.0) + - TensorFlowLiteSwift/Metal (= 2.12.0) DEPENDENCIES: - Flutter (from `Flutter`) - flutter_native_vision_camera (from `.symlinks/plugins/flutter_native_vision_camera/ios`) - permission_handler_apple (from `.symlinks/plugins/permission_handler_apple/ios`) + - tflite_flutter (from `.symlinks/plugins/tflite_flutter/ios`) + +SPEC REPOS: + trunk: + - TensorFlowLiteC + - TensorFlowLiteSwift EXTERNAL SOURCES: Flutter: @@ -17,11 +45,16 @@ EXTERNAL SOURCES: :path: ".symlinks/plugins/flutter_native_vision_camera/ios" permission_handler_apple: :path: ".symlinks/plugins/permission_handler_apple/ios" + tflite_flutter: + :path: ".symlinks/plugins/tflite_flutter/ios" SPEC CHECKSUMS: Flutter: cabc95a1d2626b1b06e7179b784ebcf0c0cde467 - flutter_native_vision_camera: f9cae7b27180d93bf6fafad999e37e5086ae477a + flutter_native_vision_camera: 241fc00d5a123d3e560daa047824120dd6af5c03 permission_handler_apple: 4ed2196e43d0651e8ff7ca3483a069d469701f2d + TensorFlowLiteC: 20785a69299185a379ba9852b6625f00afd7984a + TensorFlowLiteSwift: 3a4928286e9e35bdd3e17970f48e53c80d25e793 + tflite_flutter: 64b192e11352fe36943ab6656e1d49207f1a5595 PODFILE CHECKSUM: 3c63482e143d1b91d2d2560aee9fb04ecc74ac7e diff --git a/example/lib/main.dart b/example/lib/main.dart index c59aa13..34c5c99 100644 --- a/example/lib/main.dart +++ b/example/lib/main.dart @@ -3,6 +3,7 @@ import 'package:flutter_native_vision_camera/flutter_native_vision_camera.dart'; import 'native_camera_page.dart'; import 'standard_camera_page.dart'; import 'code_scanner_page.dart'; +import 'object_detector_page.dart'; import 'package:flutter/services.dart'; @@ -83,6 +84,19 @@ class _HomePageState extends State { }, ), const SizedBox(height: 16), + _buildFeatureCard( + context, + title: 'Object Detector', + subtitle: 'Frame processor → EfficientDet (90 classes) + boxes', + icon: Icons.center_focus_strong, + color: Colors.deepPurple, + onTap: () { + Navigator.of(context).push( + MaterialPageRoute(builder: (_) => const ObjectDetectorPage()), + ); + }, + ), + const SizedBox(height: 16), _buildFeatureCard( context, title: 'Standard Camera', diff --git a/example/lib/object_detector_page.dart b/example/lib/object_detector_page.dart new file mode 100644 index 0000000..aabce73 --- /dev/null +++ b/example/lib/object_detector_page.dart @@ -0,0 +1,539 @@ +import 'dart:ui' as ui; + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart' hide Orientation; +import 'package:flutter/services.dart' show rootBundle; +import 'package:flutter_native_vision_camera/flutter_native_vision_camera.dart'; +import 'package:tflite_flutter/tflite_flutter.dart'; + +/// A real-time multi-class object detector built on the package's FFI frame +/// pipeline. +/// +/// Pipeline: [CameraController.setFrameProcessor] delivers each frame → we read +/// the raw YUV/BGRA buffers over FFI and resize+rotate them into the model's +/// input tensor → an **EfficientDet-Lite0** detector (90 COCO classes) runs in a +/// background isolate → we draw a labelled box for every object. +/// +/// On top of the general detection it keeps a **single-class certainty trigger** +/// for "cat": temporal voting turns the jittery per-frame score into a stable +/// "certain" verdict — the technique you'd use for a reliable real-time alert. +/// +/// The frame pipeline is generic, so swapping EfficientDet for your own +/// `.tflite` is a one-line change — that's the whole point of the FFI access. +class ObjectDetectorPage extends StatefulWidget { + const ObjectDetectorPage({super.key}); + + @override + State createState() => _ObjectDetectorPageState(); +} + +class _ObjectDetectorPageState extends State + with WidgetsBindingObserver { + final CameraController _controller = CameraController(); + final FrameProcessorThrottler _throttler = FrameProcessorThrottler( + targetFps: 5, + ); + + Interpreter? _interpreter; + IsolateInterpreter? _isolate; + List _labels = []; + int _inputSize = 320; + List> _outShapes = []; + + static const double _displayGate = 0.40; // min score to draw a box + // Temporal voting for the "cat" trigger. + final List _catWindow = []; + static const int _windowSize = 8; + static const int _needHits = 4; + + bool _isInitialized = false; + bool _busy = false; + int _lastFrameDeg = -1; // diagnostics: log rotation only when it changes + List<_Obj> _objects = []; + double _catScore = 0; + bool _catCertain = false; + String? _error; + + @override + void initState() { + super.initState(); + WidgetsBinding.instance.addObserver(this); + _init(); + } + + @override + void dispose() { + WidgetsBinding.instance.removeObserver(this); + _controller.dispose(); + _isolate?.close(); + _interpreter?.close(); + super.dispose(); + } + + @override + void didChangeAppLifecycleState(AppLifecycleState state) { + if (!_isInitialized) return; + if (state == AppLifecycleState.inactive || + state == AppLifecycleState.paused) { + _controller.setActive(false); + } else if (state == AppLifecycleState.resumed) { + _controller.setActive(true); + } + } + + Future _init() async { + try { + if (await CameraPermissions.requestCameraPermission() != + PermissionStatus.granted) { + setState(() => _error = 'Camera permission denied.'); + return; + } + await _loadModel(); + + final devices = await CameraDevices.getAvailableCameraDevices(); + if (devices.isEmpty) { + setState(() => _error = 'No camera found.'); + return; + } + final device = + CameraDevices.getCameraDevice(devices, CameraPosition.back) ?? + devices.first; + + await _controller.initialize(device, pixelFormat: PixelFormat.yuv); + await _controller.setFrameProcessor(_onFrame); + await _controller.setActive(true); + setState(() => _isInitialized = true); + } catch (e) { + setState(() => _error = 'Init failed: $e'); + } + } + + Future _loadModel() async { + final interpreter = await Interpreter.fromAsset( + 'assets/efficientdet.tflite', + ); + _interpreter = interpreter; + _isolate = await IsolateInterpreter.create(address: interpreter.address); + + _inputSize = interpreter.getInputTensor(0).shape[1]; + _outShapes = interpreter + .getOutputTensors() + .map((t) => t.shape) + .toList(growable: false); + + final raw = await rootBundle.loadString('assets/coco_labels.txt'); + _labels = raw + .split('\n') + .map((e) => e.trim()) + .where((e) => e.isNotEmpty) + .toList(); + + debugPrint( + 'ObjectDetector: input=${interpreter.getInputTensor(0).shape}, ' + 'outputs=$_outShapes, labels=${_labels.length}', + ); + } + + void _onFrame(Frame frame) { + if (_isolate == null) return; + if (!_throttler.shouldProcess((frame.timestamp * 1000).toInt())) return; + if (_busy) return; + _busy = true; + + // DIAGNOSTIC: how do the frame's orientation and the preview's rotation + // relate as the phone turns? (Logged only when the frame orientation flips.) + final fdeg = _degreesOf(frame.orientation); + if (fdeg != _lastFrameDeg) { + _lastFrameDeg = fdeg; + debugPrint( + 'ObjRot: frame.orientation=$fdeg° previewQuarterTurns=' + '${_controller.previewRotation} mirrored=${_controller.previewMirrored} ' + 'displaySize=${_controller.displayPreviewSize}', + ); + } + + // Read+resize+rotate the frame into the model input NOW (the FFI buffer is + // only valid inside this callback). + final Uint8List input; + try { + input = _frameToRgb(frame, _inputSize, fdeg); + } catch (_) { + _busy = false; + return; + } + + _detect(input, fdeg).then(_apply).whenComplete(() => _busy = false); + } + + Future<_Detection> _detect(Uint8List rgb, int frameDeg) async { + try { + final s = _inputSize; + final inputs = [rgb.reshape([1, s, s, 3])]; + final outputs = { + for (var i = 0; i < _outShapes.length; i++) i: _alloc(_outShapes[i]), + }; + await _isolate!.runForMultipleInputs(inputs, outputs); + + // EfficientDet's 4 outputs come back in a converter-dependent order, so + // classify them by shape (and value range) instead of assuming. + List>? boxes; // [N][4] ymin,xmin,ymax,xmax + final twoDim = >[]; // candidate scores/classes + for (var i = 0; i < _outShapes.length; i++) { + final shape = _outShapes[i]; + if (shape.length == 3 && shape.last == 4) { + boxes = _flatten2(outputs[i]); + } else if (shape.length == 2) { + twoDim.add(_flatten1(outputs[i])); + } + } + if (boxes == null || twoDim.length < 2) return const _Detection(0, []); + + // Of the two [1,N] tensors, classes holds indices (0..89) → larger max; + // scores are probabilities in [0,1]. + twoDim.sort((a, b) => _max(b).compareTo(_max(a))); + final classes = twoDim[0]; + final scores = twoDim[1]; + + var catScore = 0.0; + final objs = <_Obj>[]; + for (var j = 0; j < scores.length; j++) { + if (scores[j] < _displayGate) continue; + final ci = classes[j].round(); + if (ci < 0 || ci >= _labels.length) continue; + final label = _labels[ci]; + if (label == '???') continue; // COCO index gaps + final b = boxes[j]; // ymin, xmin, ymax, xmax (in the upright frame) + final isCat = label == 'cat'; + // Map the box into preview-display space with the package helper so it + // tracks the preview at any device orientation. + final box = _controller.previewRectFromFrame( + Rect.fromLTRB(b[1], b[0], b[3], b[2]), + sourceRotationDegrees: frameDeg, + ); + objs.add(_Obj(box, label, scores[j], isCat)); + if (isCat && scores[j] > catScore) catScore = scores[j]; + } + return _Detection(catScore, objs); + } catch (e) { + debugPrint('ObjectDetector inference error: $e'); + return const _Detection(0, []); + } + } + + void _apply(_Detection d) { + if (!mounted) return; + _catWindow.add(d.catScore); + if (_catWindow.length > _windowSize) _catWindow.removeAt(0); + final hits = _catWindow.where((s) => s >= _displayGate).length; + setState(() { + _objects = d.objects; + _catScore = d.catScore; + _catCertain = _catWindow.length >= _windowSize && hits >= _needHits; + }); + } + + /// Samples the YUV_420_888 frame into a packed RGB `uint8` buffer of + /// [size]x[size], applying [rotationDegrees] so the model sees an upright + /// image. Stride-correct (honours row/pixel strides) — the reason the package + /// exposes [Frame.planeBytesPerRow] / [Frame.planePixelStride]. + Uint8List _frameToRgb(Frame frame, int size, int rotationDegrees) { + final fw = frame.width, fh = frame.height; + final y = frame.getPlaneData(0); + final u = frame.getPlaneData(1); + final v = frame.getPlaneData(2); + final yRow = frame.planeBytesPerRow(0); + final uRow = frame.planeBytesPerRow(1); + final vRow = frame.planeBytesPerRow(2); + final uPix = frame.planePixelStride(1); + final vPix = frame.planePixelStride(2); + + final out = Uint8List(size * size * 3); + var o = 0; + for (var oy = 0; oy < size; oy++) { + final tv = (oy + 0.5) / size; // normalized in upright image + for (var ox = 0; ox < size; ox++) { + final tu = (ox + 0.5) / size; + // Map upright (tu,tv) back to the sensor-oriented frame. + final double nu, nv; + switch (rotationDegrees) { + case 90: + nu = tv; + nv = 1 - tu; + break; + case 180: + nu = 1 - tu; + nv = 1 - tv; + break; + case 270: + nu = 1 - tv; + nv = tu; + break; + default: + nu = tu; + nv = tv; + } + var fx = (nu * fw).toInt(); + var fy = (nv * fh).toInt(); + if (fx < 0) fx = 0; + if (fx >= fw) fx = fw - 1; + if (fy < 0) fy = 0; + if (fy >= fh) fy = fh - 1; + + final yy = y[fy * yRow + fx]; + final cx = fx >> 1, cy = fy >> 1; + final uu = u[cy * uRow + cx * uPix] - 128; + final vv = v[cy * vRow + cx * vPix] - 128; + + out[o++] = _clip(yy + ((1436 * vv) >> 10)); // R + out[o++] = _clip(yy - ((352 * uu + 731 * vv) >> 10)); // G + out[o++] = _clip(yy + ((1814 * uu) >> 10)); // B + } + } + return out; + } + + static int _clip(int v) => v < 0 ? 0 : (v > 255 ? 255 : v); + + static int _degreesOf(Orientation o) { + switch (o) { + case Orientation.landscapeLeft: + return 90; + case Orientation.portraitUpsideDown: + return 180; + case Orientation.landscapeRight: + return 270; + case Orientation.portrait: + return 0; + } + } + + // Output-buffer helpers. + static Object _alloc(List shape) { + if (shape.length == 1) return List.filled(shape[0], 0); + return List.generate(shape[0], (_) => _alloc(shape.sublist(1))); + } + + static List _flatten1(Object? o) => + ((o as List)[0] as List).cast().map((e) => e.toDouble()).toList(); + + static List> _flatten2(Object? o) => ((o as List)[0] as List) + .map((r) => (r as List).cast().map((e) => e.toDouble()).toList()) + .toList(); + + static double _max(List l) { + var m = double.negativeInfinity; + for (final v in l) { + if (v > m) m = v; + } + return m; + } + + @override + Widget build(BuildContext context) { + return Scaffold( + backgroundColor: Colors.black, + appBar: AppBar( + title: const Text('Object Detector'), + backgroundColor: Colors.black, + foregroundColor: Colors.white, + ), + body: _error != null + ? Center( + child: Padding( + padding: const EdgeInsets.all(24), + child: Text( + _error!, + textAlign: TextAlign.center, + style: const TextStyle(color: Colors.white), + ), + ), + ) + : !_isInitialized + ? const Center( + child: Column( + mainAxisSize: MainAxisSize.min, + children: [ + CircularProgressIndicator(), + SizedBox(height: 16), + Text( + 'Loading detector…', + style: TextStyle(color: Colors.white70), + ), + ], + ), + ) + : Stack( + children: [ + Positioned.fill( + child: CameraPreview( + controller: _controller, + resizeMode: ResizeMode.contain, + ), + ), + Positioned.fill( + child: CustomPaint( + painter: _BoxPainter( + _objects, + _controller.displayPreviewSize, + _catCertain ? Colors.green : Colors.amber, + ), + ), + ), + Positioned(left: 16, right: 16, bottom: 32, child: _panel()), + ], + ), + ); + } + + Widget _panel() { + return Container( + padding: const EdgeInsets.all(16), + decoration: BoxDecoration( + color: (_catCertain ? Colors.green : Colors.black).withValues( + alpha: 0.72, + ), + borderRadius: BorderRadius.circular(16), + ), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + mainAxisSize: MainAxisSize.min, + children: [ + Row( + children: [ + Expanded( + child: Text( + _catCertain ? '🐱 CAT — certain' : '👀 watching for a cat…', + style: const TextStyle( + color: Colors.white, + fontSize: 18, + fontWeight: FontWeight.bold, + ), + ), + ), + Text( + '${_objects.length} object${_objects.length == 1 ? '' : 's'}', + style: const TextStyle( + color: Colors.white, + fontWeight: FontWeight.bold, + ), + ), + ], + ), + const SizedBox(height: 10), + Row( + children: [ + const Text( + 'cat', + style: TextStyle(color: Colors.white70, fontSize: 12), + ), + const SizedBox(width: 8), + Expanded( + child: ClipRRect( + borderRadius: BorderRadius.circular(4), + child: LinearProgressIndicator( + value: _catScore, + minHeight: 8, + backgroundColor: Colors.white24, + valueColor: AlwaysStoppedAnimation( + _catScore >= _displayGate + ? Colors.greenAccent + : Colors.amberAccent, + ), + ), + ), + ), + const SizedBox(width: 8), + Text( + '${(_catScore * 100).round()}%', + style: const TextStyle( + color: Colors.white, + fontFamily: 'monospace', + ), + ), + ], + ), + const SizedBox(height: 6), + const Text( + 'EfficientDet-Lite0 · 90 COCO classes · cat-certainty by voting', + style: TextStyle(color: Colors.white70, fontSize: 11), + ), + ], + ), + ); + } +} + +/// One frame's detections. +class _Detection { + final double catScore; // best cat score this frame (drives the cat trigger) + final List<_Obj> objects; + const _Detection(this.catScore, this.objects); +} + +class _Obj { + final Rect box; // normalized 0..1 in the upright image + final String label; + final double score; + final bool isCat; + const _Obj(this.box, this.label, this.score, this.isCat); +} + +/// Draws each detection, mapping a normalized box in the upright image onto the +/// `contain`-fitted preview rect. Cats use [catColor]; other objects are cyan. +class _BoxPainter extends CustomPainter { + final List<_Obj> objects; + final Size? previewSize; // upright preview dimensions + final Color catColor; + + _BoxPainter(this.objects, this.previewSize, this.catColor); + + @override + void paint(Canvas canvas, Size size) { + final ps = previewSize; + Rect img; + if (ps == null || ps.width == 0 || ps.height == 0) { + img = Offset.zero & size; + } else { + final scale = (size.width / ps.width) < (size.height / ps.height) + ? size.width / ps.width + : size.height / ps.height; + final w = ps.width * scale, h = ps.height * scale; + img = Rect.fromLTWH((size.width - w) / 2, (size.height - h) / 2, w, h); + } + + for (final obj in objects) { + final color = obj.isCat ? catColor : Colors.cyanAccent; + // obj.box is already in preview-display space (see previewRectFromFrame). + final r = Rect.fromLTRB( + img.left + obj.box.left * img.width, + img.top + obj.box.top * img.height, + img.left + obj.box.right * img.width, + img.top + obj.box.bottom * img.height, + ); + canvas.drawRRect( + RRect.fromRectAndRadius(r, const Radius.circular(8)), + Paint() + ..style = PaintingStyle.stroke + ..strokeWidth = obj.isCat ? 4 : 3 + ..color = color, + ); + final tp = TextPainter( + text: TextSpan( + text: ' ${obj.label} ${(obj.score * 100).round()}% ', + style: TextStyle( + color: Colors.black, + fontSize: 13, + fontWeight: FontWeight.bold, + background: Paint()..color = color, + ), + ), + textDirection: ui.TextDirection.ltr, + )..layout(); + tp.paint(canvas, Offset(r.left, (r.top - 18).clamp(0.0, size.height))); + } + } + + @override + bool shouldRepaint(_BoxPainter old) => + old.objects != objects || old.catColor != catColor; +} diff --git a/example/pubspec.lock b/example/pubspec.lock index 4dceae1..5cefceb 100644 --- a/example/pubspec.lock +++ b/example/pubspec.lock @@ -140,7 +140,7 @@ packages: path: ".." relative: true source: path - version: "0.0.4" + version: "0.0.5" flutter_plugin_android_lifecycle: dependency: transitive description: @@ -343,6 +343,14 @@ packages: url: "https://pub.dev" source: hosted version: "2.1.8" + quiver: + dependency: transitive + description: + name: quiver + sha256: ea0b925899e64ecdfbf9c7becb60d5b50e706ade44a85b2363be2a22d88117d2 + url: "https://pub.dev" + source: hosted + version: "3.2.2" sky_engine: dependency: transitive description: flutter @@ -404,6 +412,14 @@ packages: url: "https://pub.dev" source: hosted version: "0.7.11" + tflite_flutter: + dependency: "direct main" + description: + name: tflite_flutter + sha256: "48e6fde2ad97162bb66a16a142f4c4698add9e8cd397ce9d1cc7451b55537ac1" + url: "https://pub.dev" + source: hosted + version: "0.11.0" vector_math: dependency: transitive description: diff --git a/example/pubspec.yaml b/example/pubspec.yaml index 3c07eca..b5b5fd8 100644 --- a/example/pubspec.yaml +++ b/example/pubspec.yaml @@ -46,6 +46,9 @@ dependencies: camera: ^0.11.2+1 permission_handler: ^12.0.1 video_player: ^2.9.2 + # On-device object detection (EfficientDet-Lite0, COCO "cat" class) for the + # cat-detector demo. Example-only dependency — the plugin itself doesn't use it. + tflite_flutter: ^0.11.0 dev_dependencies: flutter_test: @@ -69,6 +72,10 @@ flutter: # the material Icons class. uses-material-design: true + assets: + - assets/efficientdet.tflite + - assets/coco_labels.txt + # To add assets to your application, add an assets section, like this: # assets: # - images/a_dot_burr.jpeg diff --git a/ios/flutter_native_vision_camera.podspec b/ios/flutter_native_vision_camera.podspec index 261ea9d..8e1ad81 100644 --- a/ios/flutter_native_vision_camera.podspec +++ b/ios/flutter_native_vision_camera.podspec @@ -4,7 +4,7 @@ # Pod::Spec.new do |s| s.name = 'flutter_native_vision_camera' - s.version = '0.0.5' + s.version = '0.0.6' s.summary = 'High-performance Flutter FFI camera plugin with zero-copy preview and real-time frame access.' s.description = <<-DESC A high-performance camera plugin for Flutter built on AVFoundation (iOS) and CameraX (Android), diff --git a/lib/src/camera_controller.dart b/lib/src/camera_controller.dart index 0ceaadd..b95490e 100644 --- a/lib/src/camera_controller.dart +++ b/lib/src/camera_controller.dart @@ -134,6 +134,71 @@ class CameraController extends ValueNotifier { /// argument. Has no effect on back cameras. bool get mirror => _mirror; + /// Maps a [normalizedRect] (each side in 0..1) from a camera frame's own + /// coordinate space into normalized preview-display coordinates (0..1) — the + /// missing piece for drawing ML detection boxes on top of [CameraPreview]. + /// + /// Pass the rotation you applied to the frame before running your model in + /// [sourceRotationDegrees] — typically `frame.orientation.degrees` (the same + /// value you hand to MLKit/Vision). Pass `0` if you ran the model on the raw, + /// unrotated frame. The result accounts for the live [previewRotation] and the + /// front-camera mirror, so the box lines up with what the preview shows **at + /// any device orientation**. + /// + /// Scale the returned rect onto the preview widget's rect (use + /// [displayPreviewSize] to honour the `BoxFit`). Without this, boxes drift as + /// the phone turns, because the preview rotation and the frame orientation are + /// tracked independently. + /// + /// ```dart + /// final previewRect = controller.previewRectFromFrame( + /// detection.boundingBox, // normalized 0..1 from your model + /// sourceRotationDegrees: frame.orientation.degrees, + /// ); + /// ``` + Rect previewRectFromFrame( + Rect normalizedRect, { + required int sourceRotationDegrees, + }) { + final delta = + (((previewRotation * 90 - sourceRotationDegrees) % 360) + 360) % 360; + var r = _rotateUnitRect(normalizedRect, delta ~/ 90); + // The delivered frame is the un-mirrored scene; the preview is mirrored only + // for a front camera with [mirror] on. + if (_device?.position == CameraPosition.front && _mirror) { + r = Rect.fromLTRB(1 - r.right, r.top, 1 - r.left, r.bottom); + } + return r; + } + + /// Rotates a unit-square rect by [k] quarter-turns clockwise, returning the + /// axis-aligned bounds of the rotated corners. + static Rect _rotateUnitRect(Rect b, int k) { + if (k % 4 == 0) return b; + var minX = 1.0, minY = 1.0, maxX = 0.0, maxY = 0.0; + for (final c in [b.topLeft, b.topRight, b.bottomRight, b.bottomLeft]) { + final double px, py; + switch (k % 4) { + case 1: + px = 1 - c.dy; + py = c.dx; + break; + case 2: + px = 1 - c.dx; + py = 1 - c.dy; + break; + default: + px = c.dy; + py = 1 - c.dx; + } + if (px < minX) minX = px; + if (py < minY) minY = py; + if (px > maxX) maxX = px; + if (py > maxY) maxY = py; + } + return Rect.fromLTRB(minX, minY, maxX, maxY); + } + /// Fires when a runtime error occurs in the native layer. Stream get onError => _onErrorController.stream; diff --git a/pubspec.yaml b/pubspec.yaml index 10ada7d..3a50c0e 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -1,6 +1,6 @@ name: flutter_native_vision_camera description: "High-performance Flutter FFI camera plugin with zero-copy preview textures, integrated MLKit/Vision barcode scanning, and low-latency native frame access for real-time on-device vision." -version: 0.0.5 +version: 0.0.6 homepage: https://github.com/JenteJan/flutter_native_vision_camera repository: https://github.com/JenteJan/flutter_native_vision_camera issue_tracker: https://github.com/JenteJan/flutter_native_vision_camera/issues diff --git a/test/camera_controller_test.dart b/test/camera_controller_test.dart index 93989ee..84bb6ed 100644 --- a/test/camera_controller_test.dart +++ b/test/camera_controller_test.dart @@ -190,6 +190,34 @@ void main() { }, ); + test('previewRectFromFrame maps boxes into preview space', () async { + await controller.initialize( + makeDevice(sensorOrientation: Orientation.landscapeLeft), + mirror: false, + ); + await sendNative('onPreviewConfigurationChanged', { + 'rotationDegrees': 90, // preview is one quarter-turn + 'mirrored': false, + }); + const box = Rect.fromLTRB(0.1, 0.2, 0.4, 0.6); + + // Frame rotated to match the preview (90°) → identity (the portrait case). + expect( + controller.previewRectFromFrame(box, sourceRotationDegrees: 90), + box, + ); + + // 90° difference → one quarter-turn clockwise: (x,y) -> (1-y, x). + final rotated = controller.previewRectFromFrame( + box, + sourceRotationDegrees: 0, + ); + expect(rotated.left, closeTo(0.4, 1e-9)); + expect(rotated.top, closeTo(0.1, 1e-9)); + expect(rotated.right, closeTo(0.8, 1e-9)); + expect(rotated.bottom, closeTo(0.4, 1e-9)); + }); + test('previewMirrored reflects the native report', () async { await controller.initialize(makeDevice(position: CameraPosition.front)); expect(controller.previewMirrored, false);