From ade2b541d8ca4a785bbd1effdfd33eb2a306f150 Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 10 Oct 2026 22:00:28 -0400 Subject: [PATCH 1/2] Add causal image-motion alpha, forecast replay and measured benchmarks --- .github/workflows/ci.yml | 35 +- README.md | 30 +- benchmarks/motion-alpha-20261010/README.md | 137 +++++ .../motion-alpha-20261010/protocol.json | 1 + benchmarks/motion-alpha-20261010/result.json | 1 + docs/motion-observations.md | 214 ++++++++ docs/motion-prediction-plan.md | 173 +++++++ docs/observation-v0.2-plan.md | 275 ++++++++++ docs/roadmap.md | 24 +- pyproject.toml | 1 + src/dvidia_training/motion.py | 249 ++++++++++ src/dvidia_training/motion_benchmark.py | 448 +++++++++++++++++ src/dvidia_training/motion_cli.py | 75 +++ src/dvidia_training/motion_studio.html | 105 ++++ src/dvidia_training/motion_studio.py | 281 +++++++++++ src/dvidia_training/motion_video.py | 469 ++++++++++++++++++ tests/browser/motion_replay.cjs | 42 ++ tests/test_motion.py | 227 +++++++++ tests/test_motion_benchmark.py | 260 ++++++++++ tests/test_motion_studio.py | 422 ++++++++++++++++ tests/test_motion_video.py | 274 ++++++++++ 21 files changed, 3740 insertions(+), 3 deletions(-) create mode 100644 benchmarks/motion-alpha-20261010/README.md create mode 100644 benchmarks/motion-alpha-20261010/protocol.json create mode 100644 benchmarks/motion-alpha-20261010/result.json create mode 100644 docs/motion-observations.md create mode 100644 docs/motion-prediction-plan.md create mode 100644 docs/observation-v0.2-plan.md create mode 100644 src/dvidia_training/motion.py create mode 100644 src/dvidia_training/motion_benchmark.py create mode 100644 src/dvidia_training/motion_cli.py create mode 100644 src/dvidia_training/motion_studio.html create mode 100644 src/dvidia_training/motion_studio.py create mode 100644 src/dvidia_training/motion_video.py create mode 100644 tests/browser/motion_replay.cjs create mode 100644 tests/test_motion.py create mode 100644 tests/test_motion_benchmark.py create mode 100644 tests/test_motion_studio.py create mode 100644 tests/test_motion_video.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e151b5..f30ba53 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -65,9 +65,42 @@ jobs: dvidia-motor-memory --help dvidia-observe --help dvidia-observation-studio --help + dvidia-motion --help dvidia-training-doctor --json - name: Test footage, visual training and loopback studio - run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_footage_benchmark tests.test_training_studio tests.test_doctor tests.test_coverage tests.test_episode_tape tests.test_reaction tests.test_reaction_metrics tests.test_reaction_convergence tests.test_reaction_benchmark tests.test_motor_memory tests.test_motor_metrics tests.test_motor_benchmark tests.test_observations tests.test_observation_models -v + run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_footage_benchmark tests.test_training_studio tests.test_doctor tests.test_coverage tests.test_episode_tape tests.test_reaction tests.test_reaction_metrics tests.test_reaction_convergence tests.test_reaction_benchmark tests.test_motor_memory tests.test_motor_metrics tests.test_motor_benchmark tests.test_observations tests.test_observation_models tests.test_motion tests.test_motion_benchmark -v + + motion: + name: Causal image motion / Linux / Python 3.12 + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + - name: Install CPU optical-flow runtime + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + python -m pip install ".[observations]" + python -m pip check + dvidia-motion --help + - name: Verify source clocks, causal rows, scoring and local replay + run: python -m unittest tests.test_motion tests.test_motion_video tests.test_motion_benchmark tests.test_motion_studio -v + - name: Run and inspect authored tracking and prediction fixtures + run: | + dvidia-motion benchmark --output runs/image-motion-ci + dvidia-motion inspect runs/image-motion-ci + - name: Preserve every attempted synthetic motion case + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: authored-image-motion + path: runs/image-motion-ci + if-no-files-found: ignore observations: name: Observation detectors / Linux / Python 3.12 diff --git a/README.md b/README.md index 3472d63..f9109ad 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,31 @@ six public excerpts. This establishes the integration, not accuracy: labels can be wrong, outcomes remain unknown, and there is no 3D reconstruction, automatic action understanding, or human-video-to-robot control bridge. +## Estimate and forecast image motion + +The [first image-motion alpha](docs/motion-observations.md) is available in the +candidate source checkout. Manually seed one first-frame region; causal CPU +feature tracking estimates its image-anchor speed and forecasts a path at +100, 250 and 500 ms. Lost tracks stay lost. Actual integer video timestamps, +tracking support, units, source/implementation hashes and failed attempts are +preserved. This feature trains no model and requires no detector weights. + +```sh +.venv/bin/python -m pip install '.[observations]' +.venv/bin/dvidia-motion benchmark --output runs/image-motion-001 +.venv/bin/dvidia-motion inspect runs/image-motion-001 +.venv/bin/dvidia-motion studio runs/image-motion-001 --port 8286 +``` + +Use the [candidate source installation instructions](docs/motion-observations.md#install-from-source), +not the frozen `observation-lab-v0.1.0` tag, for `dvidia-motion`. The authored +benchmark and read-only local replay test image tracking and prediction, not +real detector accuracy. Speeds are analysis-image pixels/second; physical speed, +moving-camera compensation, live robot latency and interaction prediction remain +unqualified. This is one component toward [the full Observation Lab v0.2](docs/observation-v0.2-plan.md). +See the [measured authored benchmark](benchmarks/motion-alpha-20261010/README.md), +including conditional errors, failed tracking and forecast tails. + ## Try the pipeline Generate an explicitly synthetic example, then train and verify its output: @@ -132,4 +157,7 @@ Each command uses a fresh output directory where applicable. The model and compl The demo presents saved candidate and open-jaw control playback from one exact simulation scene. Training runs locally; the dataset and model cards document reproducible inputs, CPU measurements and the limits of this research alpha. [Maintainer publishing tools](publishing/huggingface/README.md) stage only the already-public synthetic release and use a credential from macOS Keychain without printing or caching it. -MIT licensed. Research and measured limitations: [research.dvidia.org](https://research.dvidia.org). Contact: [hello@dvidia.org](mailto:hello@dvidia.org). +Original DVIDIA code is MIT licensed; observation detector adaptations retain +Apache 2.0 notices and upstream models retain their own licenses. Input footage +has separate rights. Research and measured limitations: [research.dvidia.org](https://research.dvidia.org). +Contact: [hello@dvidia.org](mailto:hello@dvidia.org). diff --git a/benchmarks/motion-alpha-20261010/README.md b/benchmarks/motion-alpha-20261010/README.md new file mode 100644 index 0000000..9da0278 --- /dev/null +++ b/benchmarks/motion-alpha-20261010/README.md @@ -0,0 +1,137 @@ +# DVIDIA image-motion alpha — 10 October 2026 + +**Six authored fixtures processed successfully; real-world accuracy and live +robot latency remain unmeasured.** This evaluates sparse tracking of a manually +seeded image region and passive constant-velocity forecasting. It evaluates +neither an object detector nor a trained interaction or robot-control model. + +The [frozen protocol](protocol.json) and [aggregate results](result.json) +preserve parameters, input/reference hashes, software, attempted counts, +conditional errors, abstentions and timing scope. They contain no input footage, +model weights, private data or credentials. The generated fixtures and these +DVIDIA artifacts are MIT licensed. The existing project notices continue to +apply to upstream dependencies. + +## Frozen inputs and causal separation + +Each case is a four-second, 640 × 360, 30 fps video: 120 source frames, 720 frames +and 24 seconds across the batch. One 80 × 64 pixel textured rectangle translates +over a uniform background. The seed is manually authored inside that rectangle +on the first frame. Its center is the image anchor; it is not a measured physical +centroid. The low-texture negative uses an untextured rectangle. The occlusion +case removes the rectangle for frames 45–64 while its authored trajectory +continues. + +The deterministic generator freezes trajectories, encoded source hashes and +actual frame presentation timestamps before tracking. Reference timestamps come +from integer video PTS multiplied by its rational time base. The tracker receives +only the manifest, initial seed and processing cadence. Future reference +positions are opened by the evaluator after processing and checked against their +frozen hashes. No future reference, simulator state or video frame enters an +earlier motion estimate or forecast. + +The tracker reports the initial seed center plus accumulated robust feature +translation. Its current estimate is the observed anchor in this unsmoothed +baseline. Velocity uses two contiguous actual observations and their actual +time difference. A lost track stays lost, with no replacement, interpolation or +automatic reacquisition. Missing evidence becomes an abstention. + +## Tracking results + +| Authored case | Tracked / visible reference frames | Visible track coverage | Anchor error p95, px | Velocity error p95, px/s | +| --- | ---: | ---: | ---: | ---: | +| Stationary | 120 / 120 | 100% | 0 | 0 | +| Constant velocity | 120 / 120 | 100% | 0.00691 | 0.0574 | +| Stop/start | 120 / 120 | 100% | 0.00698 | 0.0621 | +| Direction change | 120 / 120 | 100% | 0.00714 | 0.0627 | +| Brief occlusion | 45 / 100 | 45% | 0.00722 | 0.0582 | +| Low texture | 0 / 120 | 0% | Unavailable | Unavailable | + +All 720 source frames have attempted observation rows. All four textured cases +without occlusion retained their anchors. The occlusion case lost the track +when the object disappeared and remained lost after reappearance: 75 lost rows, +including 55 visible rows. The low-texture case abstained on all 120 frames. +Neither negative case is a perfect prediction result; the accuracy sample +counts become zero where no valid anchor/forecast exists. Velocity scores +exclude the two-frame neighborhood of authored speed discontinuities. + +## Forecast results and failure tails + +Errors are image-plane endpoint distances in pixels. Hold-current and +constant-velocity forecasts below share the same evaluable current anchors, +reference targets and sample counts. Scores are conditional on producing a +forecast; their answer coverage is reported separately. + +| Authored case | 100 ms CV mean / hold mean, px | 500 ms CV mean / hold mean, px | 500 ms CV p95 / worst, px | Forecast samples at 100 / 500 ms | +| --- | ---: | ---: | ---: | ---: | +| Stationary | 0 / 0 | 0 / 0 | 0 / 0 | 116 / 104 | +| Constant velocity | 0.00675 / 9.005 | 0.0252 / 45.005 | 0.0337 / 0.0914 | 116 / 104 | +| Stop/start | 0.317 / 5.903 | 6.937 / 27.699 | 38.531 / 45.007 | 116 / 104 | +| Direction change | 0.317 / 8.897 | 6.945 / 41.769 | 59.109 / 89.975 | 116 / 104 | +| Brief occlusion | 0.00664 / 9.006 | 0.0252 / 45.006 | 0.0342 / 0.0914 | 44 / 44 | +| Low texture | Unavailable | Unavailable | Unavailable | 0 / 0 | + +Every case has 117 available future reference frames at 100 ms and 105 at +500 ms. Terminal forecasts are censored: three and fifteen rows respectively. +The first tracked row has no velocity and therefore no forecast. The four +continuous textured cases answer on 116/117 and 104/105 available targets. +Occlusion answers on 44/117 and 44/105; low texture answers on zero. The remaining +rows are explicit abstentions, not zero-error samples. All 250 ms metrics and +denominators are also retained in the result file. + +Current anchors match the same acquisition PTS within a 2 μs numeric +serialization tolerance. Off-grid observations cannot borrow nearby reference +truth. Future targets use the nearest actual source frame within half a frame +(1/60 second), with the earlier frame chosen on an exact tie. The 250 ms horizon +lands between 30 Hz frames; its reference is therefore approximately 16.67 ms +from the nominal target. The constant-velocity case consequently has about +1.49 px mean error there even though its 100/500 ms error is much smaller. +These are nearest-frame endpoint scores, with offsets disclosed, rather than +exact-time 250 ms accuracy. No backward interpolation produces new references. + +The worst direction-change forecast misses by approximately 90 px at 500 ms. +Its low whole-case mean does not make it reliable at the turn. Passive +extrapolation cannot know an unannounced future reversal or contact action. +Interaction-conditioned models and evaluation slices remain subsequent work. +The propagated uncertainty radii are heuristic and uncalibrated; their empirical +coverage and size are recorded without probability claims. + +## Processing and reproducibility + +Six of six attempted cases completed with no pipeline failures. Processing took +**1.454 seconds**, plus **1.514 seconds** for fixture generation. The offline +batch-time/source-duration ratio is **0.0606**. Processing includes intake, +probing, streaming decode, causal tracking/forecast, writes and scoring; it +excludes fixture generation and provisioning. The ratio is withheld for any +failed batch so early failure cannot appear to improve throughput. + +This was one run on Darwin 25.5.0, ARM64, ten logical CPUs, Python 3.12.9, +OpenCV 5.0.0 and NumPy 2.5.3. One OpenCV thread was requested, while OpenCV +reported ten. Actual peak thread concurrency, enforcement of the requested +limit and a minimum hardware requirement are unmeasured. +The process-lifetime resident high-water mark was **83,591,168 bytes +(79.72 MiB)**, including fixture generation; it is not a per-stage limit. +No GPU or paid inference API is used in this baseline. A fast batch result does +not establish live capture-to-action latency, worst-case deadlines, long-running +stability or robot-control benefit. + +From an installed checkout containing the motion alpha, with the observations +extra and FFmpeg/ffprobe installed: + +```sh +dvidia-motion benchmark --output runs/my-motion-benchmark +dvidia-motion inspect runs/my-motion-benchmark +dvidia-motion studio runs/my-motion-benchmark --port 8286 +``` + +Use a new output directory each time. Complete local replay bundles retain +generated videos, actual-clock frame metadata, causal rows, seed evidence and +all attempted receipts. This public aggregate is not itself a replay bundle. +Generator and implementation hashes bind the protocol; encoder/platform +differences may change encoded-file hashes, so compare declared software and +parameters before interpreting reproduction differences. + +These six cases share one authoring recipe and texture. They are engineering +checks rather than six independent real-world task attempts or a held-out +generalization cohort. Next evidence should include unseen permitted footage, +camera movement, blur, deformation and separate live stream measurements. diff --git a/benchmarks/motion-alpha-20261010/protocol.json b/benchmarks/motion-alpha-20261010/protocol.json new file mode 100644 index 0000000..8e7a671 --- /dev/null +++ b/benchmarks/motion-alpha-20261010/protocol.json @@ -0,0 +1 @@ +{"baseline_comparison":"hold current on the identical evaluable anchored rows with a constant-velocity forecast","cases":["stationary","constant_velocity","stop_start","direction_change","brief_occlusion","low_texture"],"claim_scope":"authored synthetic image tracking and passive forecasting; no detection, real-world generalization or robot control","created_at":"2026-10-11T01:52:48.199306+00:00","fixture_recipe":{"background_rgb":[230,230,230],"encoder":"libx264 fast crf18 yuv420p single-thread; no metadata","fps":30,"frames":120,"height":360,"object_size_pixels":[80,64],"seed_inset_pixels":4,"texture_rng":"numpy.default_rng(4702)","trajectory_code_sha256":"d371dcabd905948670e9d9e49bdb4c2c7abb3f8a761d4daf33b68ca73e72919d","width":640},"fixtures":[{"frame_count":120,"id":"stationary","raw_rgb_bytes":82944000,"raw_rgb_sha256":"f4eea8057b6a26f818e06938a1c668bccb506455baedaa92c7a66c4d909df07f","reference_path":"stationary.reference.json","reference_sha256":"e53dbab795cd350e7d9aa6c185a75c4568083cf2a95f020fb62a4f233cb727a3","seed_box_xyxy_normalized":[0.19375,0.4222222222222222,0.30625,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":10152,"video_path":"stationary.mp4","video_sha256":"661290c74c0e3a7bede5a1290dcd7ead9f72a3ad3b834e38c36fe2d7d26dc777"},{"frame_count":120,"id":"constant_velocity","raw_rgb_bytes":82944000,"raw_rgb_sha256":"18331c7f1e2aa2d24cb11bed156e4b195ce0c3d62e5e31372af28e4bacec5758","reference_path":"constant_velocity.reference.json","reference_sha256":"f6f2ef67f55c901a1d1e00b6cf53884df1231e25cccdc386e3e0370e89c7ec86","seed_box_xyxy_normalized":[0.1,0.4222222222222222,0.2125,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":26211,"video_path":"constant_velocity.mp4","video_sha256":"6534e4b76997dce97ce5db45727aae5b2317ca9d4a2ca4cd0b64117de7777f24"},{"frame_count":120,"id":"stop_start","raw_rgb_bytes":82944000,"raw_rgb_sha256":"f2848c6e07955ece5034c4fb8dcf3ab03d1bd7b95b87fa08b92774c6c7889ad2","reference_path":"stop_start.reference.json","reference_sha256":"3dcf4bd249a915e2bd1f0c36d65e7b818c0eadfe4881aaa1fa3e6d761a5407c1","seed_box_xyxy_normalized":[0.1,0.4222222222222222,0.2125,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":23690,"video_path":"stop_start.mp4","video_sha256":"7ff0245ab68b43158888f00938352dceda59f23da4fff69c41e0ceaccb2fff5c"},{"frame_count":120,"id":"direction_change","raw_rgb_bytes":82944000,"raw_rgb_sha256":"746ae09cc463a5a65ed2b91c477daf968254a8b48896634ce6aa26f75d633db7","reference_path":"direction_change.reference.json","reference_sha256":"e3914ebbc1f494bd8babbb0b03913230c27a10839a46a5e1f2bb6f566611e9ce","seed_box_xyxy_normalized":[0.1,0.4222222222222222,0.2125,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":33361,"video_path":"direction_change.mp4","video_sha256":"db277f4b419a4ac7d07ecbf7be2c8716c4782ca9541795ac6f3ce663c2e01fd7"},{"frame_count":120,"id":"brief_occlusion","raw_rgb_bytes":82944000,"raw_rgb_sha256":"3acfa2a20af83918a939c649b58e6cb1e6b45728c617e06e399b7502fa663283","reference_path":"brief_occlusion.reference.json","reference_sha256":"6f328d0f73683ab5076a98f0d3a82949dbd5f19b5959317b520ac42563ff5727","seed_box_xyxy_normalized":[0.1,0.4222222222222222,0.2125,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":31840,"video_path":"brief_occlusion.mp4","video_sha256":"6ed24e1feb03789cd4af64a6c1d9c95a91c10bc7c85081b24c93b875ff5f8815"},{"frame_count":120,"id":"low_texture","raw_rgb_bytes":82944000,"raw_rgb_sha256":"fe4c0c3422e76dcea4d89f833d09933c5088a50cfba6b5cb745aaec32a57768a","reference_path":"low_texture.reference.json","reference_sha256":"e53dbab795cd350e7d9aa6c185a75c4568083cf2a95f020fb62a4f233cb727a3","seed_box_xyxy_normalized":[0.19375,0.4222222222222222,0.30625,0.5777777777777777],"source_duration_seconds":4.0,"video_bytes":5543,"video_path":"low_texture.mp4","video_sha256":"4eeb0a63d2cfc7d4f09b6079f2f8c9ab90af61a37c1d90f6decaedd34a455fd9"}],"hardware":{"logical_cpu_count":10,"machine":"arm64","os_release":"25.5.0","processor":"arm","python":"3.12.9","system":"Darwin"},"horizons_seconds":[0.1,0.25,0.5],"kind":"dvidia.motion-benchmark-protocol.v1","reference_access":"scorer only; never a tracker argument","sample_hz":30,"software":{"ffmpeg":"ffmpeg version 9.0.1 Copyright (c) 2000-2026 the FFmpeg developers","ffprobe":"ffprobe version 9.0.1 Copyright (c) 2007-2026 the FFmpeg developers"},"sources_manifest_sha256":"329c65421a25cf73e70f55603ed78ad6ec55aae707dfc3d990c8868a4f807ae3"} \ No newline at end of file diff --git a/benchmarks/motion-alpha-20261010/result.json b/benchmarks/motion-alpha-20261010/result.json new file mode 100644 index 0000000..2300856 --- /dev/null +++ b/benchmarks/motion-alpha-20261010/result.json @@ -0,0 +1 @@ +{"attempted":6,"cases":[{"frames_sha256":"d30213bd00667013e7ef084d288a61ef9cff3897e54c9153a876e37dc794699c","id":"stationary","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":10152,"original_video_sha256":"661290c74c0e3a7bede5a1290dcd7ead9f72a3ad3b834e38c36fe2d7d26dc777","reference_sha256":"e53dbab795cd350e7d9aa6c185a75c4568083cf2a95f020fb62a4f233cb727a3","rows_sha256":"555da6c47514e1a2ee5f796bfc02f276141f6560c68aded6051edef7372fc582","run_sha256":"ca5d674536e94a51b022d4a3a8035265b6b681c62b23aa5c9cec3c9d3971247b","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":116,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9914529914529915,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":117,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"hold_current_paired":{"count":116,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"horizon_seconds":0.1,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":116,"evaluated":116,"radius_pixels":{"count":116,"max":7.000000000000021,"mean":7.0,"p50":7.000000000000002,"p95":7.000000000000021}},"velocity_warmup_abstentions":1},"0.25":{"attempted_rows":120,"constant_velocity":{"count":111,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9910714285714286,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":112,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"hold_current_paired":{"count":111,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"horizon_seconds":0.25,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":111,"evaluated":111,"radius_pixels":{"count":111,"max":16.000000000000053,"mean":16.0,"p50":16.000000000000004,"p95":16.000000000000053}},"velocity_warmup_abstentions":1},"0.5":{"attempted_rows":120,"constant_velocity":{"count":104,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9904761904761905,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":105,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"hold_current_paired":{"count":104,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"horizon_seconds":0.5,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":104,"evaluated":104,"radius_pixels":{"count":104,"max":31.000000000000107,"mean":31.0,"p50":31.000000000000007,"p95":31.000000000000107}},"velocity_warmup_abstentions":1}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":120,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":0,"lost_visible_rows":0,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":120,"track_coverage_visible":1.0,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":120,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":119,"max":0.0,"mean":0.0,"p50":0.0,"p95":0.0},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"3ea3621b09c64d5e3524ba9f7c9c65e4b35b05505f938006b295e3bad2cf2392","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.04228057689033449,"dependency_setup_seconds":0.10640266607515514,"elapsed_seconds":0.30841808393597603,"intake_seconds":0.03896083286963403,"manifest_validation_seconds":0.0013036250602453947,"seed_evidence_write_seconds":0.0028465830255299807,"tracking_estimation_seconds":0.11458237562328577},"tracking_protocol_sha256":"d03f2f69c1132fdd17a636d786ffc02ad6e44c9946c757a6a9df2f71b132585f"},{"frames_sha256":"97a5cfcc2fce6fbcdc1b0867927cc9e9ae64aca53d8b8409aae5235a04d26f3f","id":"constant_velocity","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":26211,"original_video_sha256":"6534e4b76997dce97ce5db45727aae5b2317ca9d4a2ca4cd0b64117de7777f24","reference_sha256":"f6f2ef67f55c901a1d1e00b6cf53884df1231e25cccdc386e3e0370e89c7ec86","rows_sha256":"505fe676a40a7f621c2a731dcdc77a482e2e167231e110c4f6763d2a05f858a9","run_sha256":"de60e3a2100fe16f1334220cda516ef76d4a458f8979a1d75fcf1d80de702d29","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":116,"max":0.022845276351064172,"mean":0.006754073806171163,"p50":0.008091396403095774,"p95":0.01161441681654032},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9914529914529915,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":117,"max":9.007263238200208,"mean":9.00483222987847,"p50":9.004882851607267,"p95":9.006774955029895},"hold_current_paired":{"count":116,"max":9.007263238200208,"mean":9.004873887032597,"p50":9.004898223024806,"p95":9.006778769246319},"horizon_seconds":0.1,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":116,"evaluated":116,"radius_pixels":{"count":116,"max":7.291032183874096,"mean":7.149188011015067,"p50":7.150040800255353,"p95":7.27633951055272}},"velocity_warmup_abstentions":1},"0.25":{"attempted_rows":120,"constant_velocity":{"count":111,"max":1.5105596732368236,"mean":1.494887434022298,"p50":1.487145314065471,"p95":1.5093790284626327},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9910714285714286,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":112,"max":21.007263207007306,"mean":21.004860118269782,"p50":21.00492864643218,"p95":21.00679399738877},"hold_current_paired":{"count":111,"max":21.007263207007306,"mean":21.004903903119057,"p50":21.004943849451948,"p95":21.006797811879878},"horizon_seconds":0.25,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":111,"evaluated":111,"radius_pixels":{"count":111,"max":16.635860925088302,"mean":16.32668915224473,"p50":16.329267039000552,"p95":16.603530828228656}},"velocity_warmup_abstentions":1},"0.5":{"attempted_rows":120,"constant_velocity":{"count":104,"max":0.09138110540425669,"mean":0.025224172370874594,"p50":0.02461743908987751,"p95":0.03366410430616778},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9904761904761905,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":105,"max":45.00726319452209,"mean":45.004930646075245,"p50":45.004943881865664,"p95":45.006820688099836},"hold_current_paired":{"count":104,"max":45.00726319452209,"mean":45.00497805613366,"p50":45.00496675341145,"p95":45.00682450270088},"horizon_seconds":0.5,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":104,"evaluated":104,"radius_pixels":{"count":104,"max":32.15239492918857,"mean":31.59480168672492,"p50":31.600303403608173,"p95":32.09567609257275}},"velocity_warmup_abstentions":1}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":120,"max":0.007342712577631533,"mean":0.0051325319155877585,"p50":0.005164744659611852,"p95":0.006908432662795484},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":0,"lost_visible_rows":0,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":120,"track_coverage_visible":1.0,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":120,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":119,"max":0.17133957263298127,"mean":0.04822677166089786,"p50":0.04689788259838519,"p95":0.05742003998326581},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"879d42420d388f4c11a642302add026291e5f1cbf86c5be292e5a3d5af5785b2","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.040842545218765736,"dependency_setup_seconds":0.00015354203060269356,"elapsed_seconds":0.20844908291473985,"intake_seconds":0.039412250043824315,"manifest_validation_seconds":0.0008144169114530087,"seed_evidence_write_seconds":0.0010588751174509525,"tracking_estimation_seconds":0.12311465828679502},"tracking_protocol_sha256":"e40acfd2dc41ad0a1134bc1257458c2b94be948156384b9b317aa4b0cd12386f"},{"frames_sha256":"12b75ff33b655c6cb7271fcfb0eb586747f42b333e171fd2ce60a94852736d77","id":"stop_start","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":23690,"original_video_sha256":"7ff0245ab68b43158888f00938352dceda59f23da4fff69c41e0ceaccb2fff5c","reference_sha256":"3dcf4bd249a915e2bd1f0c36d65e7b818c0eadfe4881aaa1fa3e6d761a5407c1","rows_sha256":"f27f24acbfb8f86b6ef6cfcdff507e0af5fdc869713b98bd0d0d7f80febf4dc0","run_sha256":"892950bee55c990cd625cc7b7481a6e9412a79339e65a97c2832ea4c7fe76828","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":116,"max":9.006881718054936,"mean":0.3169244562818782,"p50":0.006887192670339969,"p95":0.7646702470682728},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9914529914529915,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":117,"max":9.007263238200208,"mean":5.929202724428996,"p50":9.004959238322945,"p95":9.006900028345887},"hold_current_paired":{"count":116,"max":9.007263238200208,"mean":5.902730334122349,"p50":9.004959238322945,"p95":9.006904605918626},"horizon_seconds":0.1,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":114,"evaluated":116,"radius_pixels":{"count":116,"max":7.20471975907866,"mean":7.104313812765775,"p50":7.106337435100483,"p95":7.189031856045821}},"velocity_warmup_abstentions":1},"0.25":{"attempted_rows":120,"constant_velocity":{"count":111,"max":22.476638940128556,"mean":2.471415784830678,"p50":1.4841551058837237,"p95":14.256630183104338},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9910714285714286,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":112,"max":21.007263207007306,"mean":13.506132439312077,"p50":21.00491336892957,"p95":21.00687333331391},"hold_current_paired":{"count":111,"max":21.007263207007306,"mean":13.438620118945519,"p50":21.004928637838827,"p95":21.006874095345616},"horizon_seconds":0.25,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":106,"evaluated":111,"radius_pixels":{"count":111,"max":16.43703161289762,"mean":16.228476440535605,"p50":16.24305699451529,"p95":16.399816439008426}},"velocity_warmup_abstentions":1},"0.5":{"attempted_rows":120,"constant_velocity":{"count":104,"max":45.00688171470525,"mean":6.936910460346599,"p50":0.026501380329358802,"p95":38.5308435381523},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9904761904761905,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":105,"max":45.00726319452209,"mean":27.86330659065632,"p50":42.00671387497222,"p95":45.00687866464219},"hold_current_paired":{"count":104,"max":45.00726319452209,"mean":27.698530692489555,"p50":40.50679779490319,"p95":45.006879427157955},"horizon_seconds":0.5,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":94,"evaluated":104,"radius_pixels":{"count":104,"max":31.756860862514966,"mean":31.41803702572802,"p50":31.47092292687337,"p95":31.69545790919534}},"velocity_warmup_abstentions":1}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":120,"max":0.007342712577631533,"mean":0.006169472011604436,"p50":0.006651212916078117,"p95":0.0069781413284217934},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":0,"lost_visible_rows":0,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":120,"track_coverage_visible":1.0,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":120,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":109,"max":0.17133957263298127,"mean":0.034101178565717816,"p50":0.0426021592306349,"p95":0.06205490952458018},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"38115751fec09b0d51afbe8731e5c94bbb3ecd45f7e4dcb8ce545873b16788d3","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.043357622576877475,"dependency_setup_seconds":0.0002630839589983225,"elapsed_seconds":0.20326795801520348,"intake_seconds":0.0388634589035064,"manifest_validation_seconds":0.0010594581253826618,"seed_evidence_write_seconds":0.0009600420016795397,"tracking_estimation_seconds":0.1162147531285882},"tracking_protocol_sha256":"01fbfee19604c03baa83e52d84993e262edbc270c68266656a946695d132a7c2"},{"frames_sha256":"93450d101d76c5270cf57b0888b80c7b8cb7c306ad8c9e88c5916483e2210dcf","id":"direction_change","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":33361,"original_video_sha256":"db277f4b419a4ac7d07ecbf7be2c8716c4782ca9541795ac6f3ce663c2e01fd7","reference_sha256":"e3914ebbc1f494bd8babbb0b03913230c27a10839a46a5e1f2bb6f566611e9ce","rows_sha256":"76efe7cd24eb7f6e7217def68d99bfdd3919bc9b72b64391d861d4c6acf52f10","run_sha256":"e50ba3ccab934111ec59bd071344c12bf493653eaafc775c556bca8002e3ba8f","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":116,"max":17.990768432675424,"mean":0.3170607356037151,"p50":0.008616647441587993,"p95":0.013273719554140232},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9914529914529915,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":117,"max":9.007263238200208,"mean":8.89743656358346,"p50":8.996109091609096,"p95":9.006774955029895},"hold_current_paired":{"count":116,"max":9.007263238200208,"mean":8.896552396028145,"p50":8.99606331319769,"p95":9.006778769246319},"horizon_seconds":0.1,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":114,"evaluated":116,"radius_pixels":{"count":116,"max":7.304632353324073,"mean":7.152253324150148,"p50":7.150040800255353,"p95":7.28828101134134}},"velocity_warmup_abstentions":1},"0.25":{"attempted_rows":120,"constant_velocity":{"count":111,"max":43.484725975303085,"mean":3.012701736561463,"p50":1.5032274597122262,"p95":10.495310515070196},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9910714285714286,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":112,"max":21.007263207007306,"mean":20.357233649816134,"p50":20.99571244252764,"p95":21.00679399738877},"hold_current_paired":{"count":111,"max":21.007263207007306,"mean":20.35144296197664,"p50":20.995407395267012,"p95":21.006797811879878},"horizon_seconds":0.25,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":106,"evaluated":111,"radius_pixels":{"count":111,"max":16.663572383453022,"mean":16.332671105863422,"p50":16.329267039000552,"p95":16.628256744759565}},"velocity_warmup_abstentions":1},"0.5":{"attempted_rows":120,"constant_velocity":{"count":104,"max":89.97465522776868,"mean":6.944765087569379,"p50":0.027241236301779874,"p95":59.10893143607173},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.9904761904761905,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":105,"max":45.00726319452209,"mean":41.80009259175852,"p50":44.99530035432862,"p95":45.006820688099836},"hold_current_paired":{"count":104,"max":45.00726319452209,"mean":41.76932425129466,"p50":44.99528508317886,"p95":45.00682450270088},"horizon_seconds":0.5,"lost_abstentions":0,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":94,"evaluated":104,"radius_pixels":{"count":104,"max":32.19860172272123,"mean":31.6037398421089,"p50":31.600303403608173,"p95":32.13254787873087}},"velocity_warmup_abstentions":1}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":120,"max":0.007342712577631533,"mean":0.00575393820965248,"p50":0.005725857389611774,"p95":0.007136618807570478},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":0,"lost_visible_rows":0,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":120,"track_coverage_visible":1.0,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":120,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":114,"max":0.17133957263298127,"mean":0.04981999452982742,"p50":0.04845919259937384,"p95":0.06269123156549102},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"f76e098d07141d83d00113fa376113d548f99cb22edd1d1dc8bd48c9d06ff646","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.13951924978755414,"dependency_setup_seconds":0.00013074977323412895,"elapsed_seconds":0.3695645839907229,"intake_seconds":0.03942295792512596,"manifest_validation_seconds":0.0008724171202629805,"seed_evidence_write_seconds":0.0011576670221984386,"tracking_estimation_seconds":0.18556857877410948},"tracking_protocol_sha256":"49528cdefb082f771112c81fde54e3a72ce7346d7f7e600b94d78e502b7bee3a"},{"frames_sha256":"1cd8cfde54ad95c7dac0c3be4dee53fc1aecfc38fe4bce613a5d55ce0a4be8fb","id":"brief_occlusion","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":31840,"original_video_sha256":"6ed24e1feb03789cd4af64a6c1d9c95a91c10bc7c85081b24c93b875ff5f8815","reference_sha256":"6f328d0f73683ab5076a98f0d3a82949dbd5f19b5959317b520ac42563ff5727","rows_sha256":"0a33d8df3c9017abad3ba6928b2e59e0cffb31a8da952e38dde0470cfbd3b545","run_sha256":"84bdaecff897e85155439a20d8d1bc1fe1fe28dc43e59066ae32f4a37301944a","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":44,"max":0.022845276351064172,"mean":0.006638027378661549,"p50":0.00986014789574376,"p95":0.012161009534050023},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.37606837606837606,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":45,"max":9.007263238200208,"mean":9.005665628452459,"p50":9.005638164558214,"p95":9.007113732847435},"hold_current_paired":{"count":44,"max":9.007263238200208,"mean":9.005794392735469,"p50":9.005680124780136,"p95":9.007118692866978},"horizon_seconds":0.1,"lost_abstentions":72,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":44,"evaluated":44,"radius_pixels":{"count":44,"max":7.112589085817852,"mean":7.060095422198605,"p50":7.060895049595651,"p95":7.10743520674284}},"velocity_warmup_abstentions":1},"0.25":{"attempted_rows":120,"constant_velocity":{"count":44,"max":1.5097901680731785,"mean":1.4932497692281623,"p50":1.4850669326991457,"p95":1.5084528255225105},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.39285714285714285,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":45,"max":21.007263207007306,"mean":21.005665605560125,"p50":21.005638140564873,"p95":21.00711368407168},"hold_current_paired":{"count":44,"max":21.007263207007306,"mean":21.005794369322857,"p50":21.005680101614047,"p95":21.00711864356964},"horizon_seconds":0.25,"lost_abstentions":67,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":44,"evaluated":44,"radius_pixels":{"count":44,"max":16.257124425305975,"mean":16.13712380636397,"p50":16.13896081838557,"p95":16.24535053195909}},"velocity_warmup_abstentions":1},"0.5":{"attempted_rows":120,"constant_velocity":{"count":44,"max":0.09138110540425669,"mean":0.025153919162584748,"p50":0.024892702859294465,"p95":0.03418248676642159},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.41904761904761906,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":45,"max":45.00726319452209,"mean":45.00566559639809,"p50":45.00563813096273,"p95":45.00711366454904},"hold_current_paired":{"count":44,"max":45.00726319452209,"mean":45.00579435995259,"p50":45.005680092342935,"p95":45.007118623838224},"horizon_seconds":0.5,"lost_abstentions":60,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":44,"evaluated":44,"radius_pixels":{"count":44,"max":31.498016657786177,"mean":31.26550444663957,"p50":31.269070433035438,"p95":31.47520940731951}},"velocity_warmup_abstentions":1}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":45,"max":0.007342712577631533,"mean":0.005723447530421149,"p50":0.005711319087766043,"p95":0.007220899950055743},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":75,"lost_visible_rows":55,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":100,"track_coverage_visible":0.45,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":45,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":44,"max":0.17133957263298127,"mean":0.04822155926494726,"p50":0.044489092251521346,"p95":0.058217328320159356},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"babf6fb6d280c8f97316880c7379a9fd691a56b294abdcc6fc4b0defdf704fd2","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.07179962727241218,"dependency_setup_seconds":0.00010775006376206875,"elapsed_seconds":0.18860533297993243,"intake_seconds":0.04391525010578334,"manifest_validation_seconds":0.0009034581016749144,"seed_evidence_write_seconds":0.0017752498388290405,"tracking_estimation_seconds":0.0676984612364322},"tracking_protocol_sha256":"0bc3e296b1bbf29455ced468f4bcbce7f39324a80ef7079b89baad1236a579c9"},{"frames_sha256":"96338ff73d29fc5e31f73a5ad2a25c8008831e7c0fd98b150e0beba2b7076b58","id":"low_texture","opencv_threads_reported":10,"opencv_threads_requested":1,"original_video_bytes":5543,"original_video_sha256":"4eeb0a63d2cfc7d4f09b6079f2f8c9ab90af61a37c1d90f6decaedd34a455fd9","reference_sha256":"e53dbab795cd350e7d9aa6c185a75c4568083cf2a95f020fb62a4f233cb727a3","rows_sha256":"4fe8bc67faabea350d74e0befecf0cf018f13187ac104004407e02c89afadf89","run_sha256":"619244450d2ab81ca23b60a1b925a84888480a9b9f046ee7ada8e0cefbb3b643","scores":{"claim_scope":"authored fixed-camera image-anchor tracking and passive forecasting only","forecasts":{"0.1":{"attempted_rows":120,"constant_velocity":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.0,"future_reference_available":117,"future_reference_gap":0,"hold_current_all_tracked":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"hold_current_paired":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"horizon_seconds":0.1,"lost_abstentions":117,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":117,"max":4.440892098500626e-16,"mean":-6.333964691771726e-17,"p50":0.0,"p95":0.0},"terminal_censored":3,"uncertainty":{"calibrated":false,"covered":0,"evaluated":0,"radius_pixels":{"count":0,"max":null,"mean":null,"p50":null,"p95":null}},"velocity_warmup_abstentions":0},"0.25":{"attempted_rows":120,"constant_velocity":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.0,"future_reference_available":112,"future_reference_gap":0,"hold_current_all_tracked":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"hold_current_paired":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"horizon_seconds":0.25,"lost_abstentions":112,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":0.01666666666666705,"reference_time_offset_seconds":{"count":112,"max":-0.016666666666666163,"mean":-0.01666666666666667,"p50":-0.016666666666666607,"p95":-0.016666666666666607},"terminal_censored":8,"uncertainty":{"calibrated":false,"covered":0,"evaluated":0,"radius_pixels":{"count":0,"max":null,"mean":null,"p50":null,"p95":null}},"velocity_warmup_abstentions":0},"0.5":{"attempted_rows":120,"constant_velocity":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"current_reference_unmatched":0,"endpoint_error_units":"pixels","forecast_answer_coverage":0.0,"future_reference_available":105,"future_reference_gap":0,"hold_current_all_tracked":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"hold_current_paired":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"horizon_seconds":0.5,"lost_abstentions":105,"missing_forecast_abstentions":0,"reference_max_absolute_time_offset_seconds":4.440892098500626e-16,"reference_time_offset_seconds":{"count":105,"max":4.440892098500626e-16,"mean":1.0573552615477682e-18,"p50":0.0,"p95":8.881784197001221e-17},"terminal_censored":15,"uncertainty":{"calibrated":false,"covered":0,"evaluated":0,"radius_pixels":{"count":0,"max":null,"mean":null,"p50":null,"p95":null}},"velocity_warmup_abstentions":0}},"kind":"dvidia.motion-scores.v1","reference_matching":{"current_reference_rule":"same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched","current_time_tolerance_seconds":2e-06,"fps":30,"maximum_time_tolerance_seconds":0.016666666666666666,"observations":"unsampled references remain gaps; no extra tracking is run for scoring","rule":"nearest actual reference PTS; earlier frame on a half-frame tie","terminal_rule":"targets after the last reference frame are censored, never rounded backward"},"tracking":{"anchor_error_pixels":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"attempted_observation_rows":120,"automatic_reacquisition_supported":false,"lost_rows":120,"lost_visible_rows":120,"matched_reference_frames":120,"reacquisitions_after_reported_loss":0,"reference_frames":120,"reference_visible_frames":120,"track_coverage_visible":0.0,"tracked_during_reference_occlusion_rows":0,"tracked_visible_reference_frames":0,"unmatched_observation_rows":0,"unsampled_reference_frames":0,"velocity_error_pixels_per_second":{"count":0,"max":null,"mean":null,"p50":null,"p95":null},"velocity_scope":"visible frames with available estimate; authored discontinuity neighborhoods excluded"},"units":{"position":"pixels","velocity":"pixels/second"}},"scores_sha256":"392105671c9dfc83c79e4f55938724a434095369a1d2138c46183c143dfa5f5a","software":{"numpy":"2.5.3","opencv":"5.0.0"},"status":"completed","timings":{"decode_wait_seconds":0.07321779313497245,"dependency_setup_seconds":0.0003685841802507639,"elapsed_seconds":0.1414784169755876,"intake_seconds":0.0618451670743525,"manifest_validation_seconds":0.0009581670165061951,"seed_evidence_write_seconds":0.0009149161633104086,"tracking_estimation_seconds":0.0025478308089077473},"tracking_protocol_sha256":"a4693264946f4a7f825ae400ace1a9162bb64132befda33e66287a203cf4ac02"}],"completed":6,"curated_from":{"source_protocol_sha256":"595368b65754d655d618ad2a7617fd13c35c9aa7030a7bece0c5258fd89f80f5","source_report_sha256":"f0a3499a87cc775bc103e9ba90c08d40e23ef1d9fdccef80afcb55685050c36d","source_run_label":"motion-benchmark-20261010-v3","transformation":"Preserved aggregate scores, timings and evidence hashes; omitted media and per-frame rows. This is not a replay bundle."},"failed":0,"hardware":{"logical_cpu_count":10,"machine":"arm64","os_release":"25.5.0","processor":"arm","python":"3.12.9","system":"Darwin"},"kind":"dvidia.motion-benchmark-summary.v1","license":"MIT","limitations":["Six authored fixtures are engineering integration evidence, not held-out real footage.","Image anchors and pixel/s have no calibrated physical scale.","Seeded sparse tracking is not object detection or semantic task understanding.","Offline source-duration ratio does not establish live latency or deadline guarantees.","No automatic reacquisition or robot commands; losses abstain.","Uncertainty radii are heuristic and uncalibrated."],"memory":{"bytes":83591168,"scope":"process-lifetime resident high-water mark including fixture generation; not a stage cap"},"robot_skill_qualified":false,"timing":{"attempted_source_duration_seconds":24.0,"batch_elapsed_seconds":1.4538089579436928,"batch_processing_time_source_duration_ratio":0.060575373247653864,"completed_source_duration_seconds":24.0,"fixture_generation_seconds":1.5141763330902904,"live_sensor_to_action_latency_measured":false,"ratio_availability":"all cases must complete; failed batches cannot appear faster through early failure","scope":"batch intake, probing, streaming decode, causal tracking/forecast, writes and scoring; fixture generation/provisioning excluded"},"trained_model":false} \ No newline at end of file diff --git a/docs/motion-observations.md b/docs/motion-observations.md new file mode 100644 index 0000000..582d1f6 --- /dev/null +++ b/docs/motion-observations.md @@ -0,0 +1,214 @@ +# DVIDIA image-motion alpha + +Available in the [candidate source checkout](https://github.com/Dvidia-Inference/dvidia-training/tree/feat/motion-observation-v0.2), +10 October 2026. This is the first +motion component toward [Observation Lab v0.2](observation-v0.2-plan.md); the +full v0.2 remains planned. The published `observation-lab-v0.1.0` tag and +Hugging Face review demo do not include this component. + +The tool follows **one human-seeded image region** in permitted local footage, +estimates image velocity and forecasts an image anchor at 100, 250 and 500 ms. +It runs on a CPU and downloads no detector models or weights. It does not train +a model, infer semantic actions or drive a robot. + +## Install from source + +Install Python 3.12+ and FFmpeg, including `ffprobe`, first. The candidate branch +contains the alpha; cloning the older release tag will not install `dvidia-motion`. +Record the exact checked-out commit when reproducing a result. + +```sh +git clone --branch feat/motion-observation-v0.2 https://github.com/Dvidia-Inference/dvidia-training.git +cd dvidia-training +python3.12 -m venv .venv +.venv/bin/python -m pip install '.[observations]' +.venv/bin/dvidia-motion --help +``` + +The observations extra supplies OpenCV; no detector provisioning command is +needed for this motion baseline. Software installation is separate from local +analysis. Once installed, the analysis, benchmark and replay require no network, +account, API key, CUDA or GPU. On Windows, replace `.venv/bin/` with +`.venv\Scripts\`; see [installation instructions](installation.md). + +## First run: authored fixtures + +Each output directory must be new; existing evidence is never overwritten. + +```sh +.venv/bin/dvidia-motion benchmark --output runs/image-motion-001 +.venv/bin/dvidia-motion inspect runs/image-motion-001 +.venv/bin/dvidia-motion studio runs/image-motion-001 --port 8286 +``` + +Open [http://127.0.0.1:8286](http://127.0.0.1:8286) to play or scrub the original +generated clips and inspect observations, speed and forecast paths. This is a +read-only local replay. The forecast uses evidence up to the displayed source +time, not frames after it. A source recording clock is not a live wall clock. + +The benchmark generates six neutral, authored cases: stationary, constant +velocity, stop/start, direction change, brief occlusion and low texture. The +manual seed is frozen before analysis. Future reference trajectories are passed +only to the scorer, never to the tracker or motion estimator. These fixtures +test tracking and prediction behavior; they cannot establish object detector +accuracy, real-scene generalization or physical robot competence. + +The [first measured batch](../benchmarks/motion-alpha-20261010/README.md) retains +coverage, paired baseline errors, failure tails and hardware scope. All six +authored cases completed; occlusion and low texture still lost their tracks. +At 500 ms, the direction-change case reached approximately 90 pixels worst +forecast error. This is a limitation to improve, not a qualified robot forecast. + +H.264 benchmark clips need a browser that supports that codec. The replay +reports an unsupported video format instead of substituting a different source. + +## Analyze your permitted footage + +Use a stationary camera and a textured, clearly visible target for the first +trial. Keep original frame times and the complete motion/outcome. The input +uses the existing strict [`dvidia.observation-source.v1` manifest](observations.md#supply-permitted-videos), +including original SHA-256, session/recording groups, credit, license and an +explicit `local_analysis` permission declaration. The declaration records the +permission you obtained; it does not acquire consent or grant publication rights. + +Choose one listed episode and manually specify its first-frame region: + +```sh +.venv/bin/dvidia-motion analyze /path/to/source.json \ + --episode attempt-001 --box 0.2,0.2,0.5,0.5 --label 'Cup image anchor' \ + --sample-hz 30 --output runs/cup-motion-001 +.venv/bin/dvidia-motion inspect runs/cup-motion-001 +.venv/bin/dvidia-motion studio runs/cup-motion-001 --port 8286 +``` + +`--box` is normalized `x1,y1,x2,y2`, from 0 to 1, in the decoded source raster. +It is a human-authored region, not a model detection or proof of physical object +identity. Automatic rotation is disabled; a phone recording's rotation metadata +is not applied to this coordinate system. The first decoded image is retained +as `seed.png` for inspection. Box drawing/correction and automatic semantic +object selection remain subsequent interface work. + +## What the baseline computes + +```mermaid +flowchart LR + V[Permitted original video] --> T[Actual integer PTS and rational time base] + T --> F[Selected decoded frame] + S[Human-authored first-frame region] --> K[Initial sparse features] + F --> K + K --> A[Robust causal feature translation] + A --> E[Two-observation image velocity] + E --> P[Constant-velocity forecasts and heuristic envelope] + P --> R[Receipt and read-only replay] + G[Authored future references] --> B[Separate benchmark scorer] + P --> B +``` + +The tracker initializes sparse corner features only inside the manual region. +Forward/backward optical-flow checks, minimum support and a robust displacement +consensus reject some ambiguous evidence. The initial region center is translated +by the feature displacement. This **image anchor is not an estimated physical +centroid**. Initial/background feature selection, deformation, rotation and +occlusion can produce drift or loss. + +Velocity is the finite difference between two actual, contiguous image anchors. +It is expressed in **analysis-image pixels/second**. Images retain their aspect +ratio and are scaled only when needed to fit a maximum side of 640 pixels; both +original and analysis dimensions and their scale are recorded. Pixel speed +depends on camera geometry and image scaling. It cannot be read as metres/second. + +Forecasts extrapolate the current velocity; they do not know that a robot will +push, grasp or release an object. A stop or turn can invalidate the hypothesis. +Coordinates remain unclipped, with out-of-frame forecasts explicitly marked. +The supplied anchor-error radius and its forecast propagation are uncalibrated +heuristics. They are not calibrated confidence regions or safety margins. + +Missing evidence is not zero motion. Insufficient or ambiguous support, an anchor +leaving the frame or a tracking gap greater than 250 ms makes the track terminally +lost. No features are replenished and no new object is attached to the old track. +There is no interpolation or forecast through the loss. The velocity estimator +requires observations no more than 150 ms apart; low source or sampling cadence +can therefore prevent all forecasts even when the region is still visible. +Default sampling is 30 Hz; useful cadence is determined by actual source times. + +## Timestamps and preserved evidence + +FFmpeg streams bounded raw frames. For each selected input frame, the decoder's +integer presentation timestamp is multiplied by its declared rational time base. +The formatted `pts_time` string, nominal frame rate and OpenCV frame-position +values are not substituted for that clock. Selected timestamps begin at zero +and strictly increase. Nonzero container/video starts fail instead of silently +shifting annotations relative to the original media. + +Each analysis directory contains: + +| File | Evidence | +| --- | --- | +| `original.` | Byte-preserved copy of the selected video | +| `seed.png` | First decoded analysis image | +| `protocol.json` | Seed/source hashes, recipe, implementation hashes, geometry, clock, software and hardware declarations | +| `rows.json` | Strict causal observation, velocity and forecast rows | +| `frames.json` | Parallel frame index, integer PTS, rational time base and tracking-support metadata | +| `run.json` | Source/protocol/media/row hashes, all errors, coverage and timing | + +Once intake begins, a processing failure retains a failed attempt receipt and +any partial rows. Invalid arguments/manifests are rejected before output creation. +`inspect` checks recorded hashes, media bytes and causal row consistency. These +checks establish internal consistency, not authenticated capture history or +independent prediction accuracy. Use trusted local bundles. + +## Evaluation and resource scope + +The scorer compares constant-velocity predictions with hold-last-position on the +same eligible anchors. It reports current image-anchor and velocity errors, +endpoint error by horizon, denominators, tracking and answer coverage, abstentions, +missing future references and empirical envelope coverage/size. Cases near +velocity discontinuities and occlusions remain explicit. A good conditional +error on a few surviving tracks is not good coverage. + +The new command checks one selected source per invocation. The source manifest +may list up to 50 local clips, bounded to 128 MiB each and 512 MiB total. Selected +videos must be at most 60 seconds and 4096 pixels per source side. Analysis is +limited to 1800 frames, a 640-pixel side and requested sampling between 1 and +30 Hz. A lower frame cap can leave later motion unobserved; the receipt records +the cap and final observed source time. + +The decoder has bounded timestamp/diagnostic queues and a 120-second watchdog. +A 150-second processing budget is checked between processed frames; native +tracking calls are not preempted. FFmpeg/OpenCV thread settings are requests; +the receipt records OpenCV's reported setting. Per-analysis memory and actual +thread concurrency are unmeasured; the authored benchmark separately records +the process-lifetime resident high-water mark, including fixture generation. +Universal minimum hardware is unmeasured. Allow disk for the source +copy, seed image, receipts and installed software. + +Recorded decode/tracking wall times and source-duration ratios describe an +offline batch on the named machine. They do not establish camera-to-command +latency, deadline compliance or live 30 Hz sensing. A fast average replay can +still miss a control deadline. + +## Next evidence gates + +1. Freeze a development recipe and evaluate independent, permitted real clips + with separately reviewed anchor references, including blur, stop/turn, + occlusion, deformation, camera motion and failures. +2. Add measured, bounded streaming replay, then an observation-only live camera + adapter with capture/delivery/update clocks, dropped frames and stale behavior. +3. Validate geometry/depth before reporting physical speed. Moving cameras need + independently evaluated camera-motion compensation. +4. Synchronize robot commands, joint/gripper state and available contact sensors + before action-conditioned interaction prediction or a robot consumer. + +The wider roadmap still includes independent annotation scoring, box/event +correction, reviewed dataset exports and skill-specific simulation benchmarks. +Those capabilities are not completed by this image-motion alpha. See the +[motion/interaction plan](motion-prediction-plan.md) and [execution map](roadmap.md). + +## Licensing + +Original DVIDIA motion, fixture, scoring and replay code is covered by the +repository's [MIT license](../LICENSE). The existing observation detector +adaptations/model notices retain their separate Apache 2.0 terms in +[`third_party/observation-models`](../third_party/observation-models). No model +weights are bundled with this motion component. A source video's own license +and contributor permissions remain independent of the code license. diff --git a/docs/motion-prediction-plan.md b/docs/motion-prediction-plan.md new file mode 100644 index 0000000..cd78bd8 --- /dev/null +++ b/docs/motion-prediction-plan.md @@ -0,0 +1,173 @@ +# Motion-aware observation and robot interaction + +Proposed 10 October 2026 as an extension of the +[Observation Lab v0.2 plan](observation-v0.2-plan.md). The first +[image-motion alpha](motion-observations.md) is now available in the candidate +source checkout. The full v0.2, calibrated motion and live interaction remain +planned; none are features of the published v0.1 Space. + +## Objective and current foundation + +Estimate what an object is doing now, predict its short-horizon motion, and +eventually predict its response to a robot action. Update the estimate as new +evidence arrives. Keep observation, estimation, prediction and command separate. + +The published observation lab samples two frames/second and associates same-class +boxes by image overlap. The new, separate motion alpha uses a manually seeded +first-frame region and causal sparse feature translation at actual selected +source timestamps. It supplies image-anchor velocity and constant-velocity +forecasts, authored reference scoring and local replay. It has no trained motion +model, calibrated physical speed, automatic semantic object selection or live +camera adapter. The existing reaction monitor consumes metric positions, velocities +and sensor age, currently from synthetic sensed state. Motor memory predicts +one-axis actuator response from delayed observations and acknowledged commands. +Those experiments provide reusable contracts; they do not establish visual +prediction or whole-arm hardware control. + +## Incremental deliverables + +| Stage | Build | Output and limitation | Gate | +| --- | --- | --- | --- | +| A, first alpha available; v0.2 extension planned | Actual-PTS manual-region feature tracking, two-observation finite-difference velocity and constant-velocity forecasts implemented; detector refresh, forward filter and real-scene evaluation remain planned | Image anchor, direction, analysis-image pixel/s and future path; no metric speed claim | Authored fixture scorer compares hold-last-position; independently reviewed real motion/occlusion clips still required | +| B, calibrated extension | Validated table-plane mapping, then calibrated stereo/RGB-D and camera-to-robot transforms | Planar m/s only for tracked points known to lie on the calibrated plane; full 3D position/velocity requires sufficient calibrated evidence | Compare against independent metric references; reject unsupported geometry and account for camera movement | +| C, interaction evidence | Synchronize video, encoders, gripper aperture, command acknowledgements and available contact/force/tactile signals | Evidence for free motion, approach, attachment, sliding, release or slip; conflicting/missing evidence remains unknown | Clock, freshness, sensor availability and interaction-state accuracy checks | +| D, learned interaction prediction | A compact dynamics model conditioned on state/history and declared robot action | Competing object trajectories and outcomes, with measured uncertainty, at a declared horizon | Improve on simple action-conditioned dynamics in unseen objects/loads/contact cases | +| E, live use | Bounded live adapter and asynchronous state feed; begin in shadow mode with no robot command authority | Fresh state estimates and causal predictions delivered to a robot-specific consumer | End-to-end deadlines, fault behavior and matched closed-loop comparisons; physical qualification remains separate | + +For v0.2, ship A alongside independent reference scoring and review improvements. +Add wall-clock-paced replay to test streaming behavior. The first live camera +adapter should run in shadow mode. Recorded replay is not proof of live sensing; +shadow predictions are not proof of successful robot control. Stages B–D are +separate extensions and should not delay a measured image-motion release. + +## What movement and speed mean + +Use actual acquisition/presentation times, not assumed frame spacing. Track +object-supported features where possible; changing detector rectangles can move +their centers without the object moving. Retain track quality, identity changes, +visibility and uncertainty. Use a causal filter: future frames and backward +smoothing cannot enter a prediction made earlier. + +A normal RGB stream first supports image velocity in pixel/s. Camera motion +also produces image motion. Fixed-camera trials simplify the first experiment; +moving/wrist-camera trials need independently evaluated motion compensation. +Depth/calibration supplies the scale and coordinate transforms needed for m/s. +A plane mapping is invalid for a lifted cup or arbitrary 3D motion; do not apply +it to a box centroid and call that calibrated world speed. + +Store position, velocity and prediction units explicitly. Angular velocity and +acceleration require additional evidence and validation; box aspect-ratio +changes alone are not reliable rigid-object rotation. Numerical differentiation +can amplify measurement noise. Missing depth or ambiguous identity must not +become a zero speed or fabricated coordinate. + +## Predict interaction, including the robot's action + +Passive extrapolation asks where the object will go if its current motion +continues. Interaction prediction asks what happens under an action, such as a +gripper approach, push, lift or release. The latter needs robot action/state +history and an interaction model. Requested movement and measured movement +remain distinct; include acknowledged command application where available. + +Use simple hypotheses first: stationary/free-moving object, object following a +held tool transform, sliding and uncertain attachment. A visual overlap is not +confirmation of contact or a successful grasp. Force/tactile and relative-motion +evidence can help distinguish regimes, but each sensor's accuracy and freshness +must be evaluated. Friction, mass, compliance and occlusion can make several +futures plausible. Increase uncertainty or abstain when evidence cannot select +a regime; do not silently keep a free-motion extrapolation across contact. + +Train a later dynamics model from aligned interaction evidence. Simulation can +provide labels and scoring truth, but deployed inputs must use the declared +sensor observations. Privileged simulator object/contact state must not masquerade +as camera output. Use renderer/sensor adapters for visual-to-control experiments. + +## Real-time architecture + +```mermaid +flowchart LR + C[Timestamped camera / depth] --> P[Asynchronous tracking and prediction] + P --> S[Latest state, age and uncertainty] + E[Encoders / gripper / available force sensors] --> R[Robot servo and reaction loop] + S --> R + R --> A[Robot-specific bounded commands] + A --> E + A --> P +``` + +The observation worker and robot servo/reaction loop run asynchronously. A slow +detector or language model must not block actuator feedback. Prefer a bounded +latest-frame queue; record skipped frames and use the real time gaps. Keep +monotonic delivery/decision times and an explicit mapping to acquisition clocks, +including offset/drift uncertainty. Never refresh a reading's acquisition time +merely because a consumer reads it again. + +Predict to an explicit target time, including expected command-application delay +where the robot adapter can measure it. Freshness, uncertainty and stopping/load +behavior belong to the robot/task envelope. On stale or missing inputs, invoke +the declared robot-specific bounded response rather than advancing with a stale +target. A hold does not automatically recover a slipping load or justify resuming. + +An initial development target is a 30 Hz image-state stream on one named CPU +machine, using lower-rate detector refreshes where helpful. It is unmeasured, +not a deadline guarantee or a universal requirement. Choose the final camera +and robot cadence from measured task speeds, precision, delay and stopping +behavior. The robot's servo cadence follows its supported control interface. +At 0.5 m/s, 100 ms of unaccounted delay corresponds to 5 cm of travel; throughput +alone is not the relevant control result. + +## Experiments and measurements + +Start with stationary-camera cup/table motion, stop/start, changes in direction, +occlusion, lift/release and entry into the scene. Add controlled interaction +examples when suitable robot evidence exists. Include no-motion negatives, +camera motion, blur, variable timestamps, dropped frames and calibration drift. +Keep complete attempts and linked recording/object/session groups together. + +Freeze a candidate and protocol before opening fresh confirmation data. At each +prediction time, permit only evidence delivered by that time. Score predictions +against future references without feeding those references into the estimator. +Declared future commands may condition a forecast, but future measurements or +unknown future command acknowledgements may not. + +| Measurement | Required evidence | +| --- | --- | +| Current motion | Image position/velocity error separately from calibrated metric position/velocity error; identity switches, track coverage and reacquisition | +| Prediction | Endpoint and average displacement error at proposed 100, 250 and 500 ms horizons; sample counts and error by motion/contact/occlusion condition | +| Uncertainty | Empirical interval/region coverage, region size and abstention; an uncertainty estimate is not calibrated until evaluated | +| Latency | Capture, delivery, inference, state update, decision, request and acknowledged application times; p50/p95/worst, sensor age, jitter and deadline misses | +| Stream health | Queue depth, dropped/late frames, blackout/stale duration, CPU/memory and every failed attempt | +| Control benefit | Current-state-only versus predictive consumer under matched conditions: task success, clearance, overshoot, actual braking path/time, contact and load retention | + +Measure long-running stream behavior as well as short clips. Warm offline +processing-time/source-duration ratios remain useful batch metrics; they do not +establish live latency or worst-case execution time. Prediction errors under +contact need their own slices rather than being diluted by easy stationary data. +Physically measured stopping and contact behavior are separate from simulated +scores. Start control integration with the existing bounded fixture, then one +declared robot/gripper configuration; whole-arm geometry requires its own checks. + +## Interface and release claims + +Show observed boxes, estimated current state and predicted future path with +different visual treatments. Display horizon, acquisition age, units, track +visibility and uncertainty. If calibration is absent, show "metric speed +unavailable". Simulated/synthetic demonstrations remain visibly labeled and +cannot be used as real detector or hardware accuracy evidence. + +A motion-aware software alpha can ship once schemas, causal replay, baselines, +review and scoring work. Live-performance claims wait for named-machine stream +measurements. Robot-control claims wait for the compatible consumer and separate +closed-loop results. Keep the public v0.1 tag and its evidence unchanged. + +## Primary references + +- [OpenCV optical flow](https://docs.opencv.org/4.13.0/d4/dee/tutorial_optical_flow.html) + describes image motion and sparse tracking, including apparent camera motion. + It is a candidate baseline, not proof of our achievable throughput. +- [RealSense metric measurement example](https://github.com/realsenseai/librealsense/blob/master/examples/measure/readme.md) + explains depth/intrinsic-based conversion from pixels to 3D points. Our + calibration and metric error still require independent validation. +- [MoveIt Servo](https://moveit.picknik.ai/main/doc/examples/realtime_servo/realtime_servo_tutorial.html) + documents servo interfaces, encoder feedback and control-loop jitter concerns. + We have not implemented or qualified a MoveIt/hardware adapter in this lab. diff --git a/docs/observation-v0.2-plan.md b/docs/observation-v0.2-plan.md new file mode 100644 index 0000000..74b8ccd --- /dev/null +++ b/docs/observation-v0.2-plan.md @@ -0,0 +1,275 @@ +# DVIDIA Observation Lab v0.2 plan + +Proposed 10 October 2026. This is an implementation and evaluation plan, not a +release announcement or measured result. It concerns our Observation Lab; +the motor-memory framework has its own v0.2 and a separate scope. + +## Product objective + +Turn permitted Skillspace footage into portable, reviewed evidence and causal +motion estimates that we can measure and use in learning experiments. The next +release includes image direction/speed and short-horizon forecasts alongside +annotation quality, uncertainty, correction effort and CPU cost. Calibrated +metric motion, action-conditioned interaction prediction and live robot +consumers follow explicit gates in the [motion prediction plan](motion-prediction-plan.md). + +The longer product path remains: + +```mermaid +flowchart LR + A[Skillspace recordings] --> B[Measured observations and human review] + B --> C[Versioned dataset] + C --> D[Visual task model] + C --> E[Aligned robot supervision] + E --> F[Candidate control policy] + F --> G[Task-specific simulation benchmark] + G --> H[Compatible installable capsule] + H --> I[Separate physical qualification] +``` + +Visual descriptions and robot commands have different contracts. A corrected +object box or a description of moving a cup does not supply joint targets, +gripper force, calibrated depth or a deployable movement policy. + +## Baseline and gaps + +The public baseline is `observation-lab-v0.1.0`, source commit `3cc4a3b`. +Local commit `9fdf2ad` adds grouped review and autosave. Keep both identifiable +when comparing interface behavior. Existing immutable inputs, timestamped +evidence, revisions, conflict handling and local export are the foundation. + +Current detection samples two frames/second at a maximum image side of 640. +Same-class box overlap associates detections; palm/object overlap is a 2D +proximity proposal. Six excerpts established integration, with accuracy, +correction effort and peak resources unmeasured. Current reviews edit labels +and intervals, but cannot correct object boxes. The visual trainer does not +consume observation labels, and its movement lane does not consume episode tapes. + +## Implementation order + +Each row is a separately reviewable change. Evaluation infrastructure can ship +before footage arrives; annotation quality claims require independent evidence. + +| Order | Deliverable | Acceptance evidence | Owner role | +| --- | --- | --- | --- | +| 1 | Reference format, annotation guide and scorer | Synthetic exact-match, miss, duplicate, timing and unknown fixtures; unsupported references fail explicitly; missing references return not assessed | Evaluation | +| 2 | Box and task-event correction | Draw/resize/add boxes at real timestamps; split/merge events; keyboard decisions; raw evidence remains immutable; revisions survive reload and conflicts | Product + data | +| 3 | Budgeted batch jobs | Preflight, progress, cancellation, restart and per-clip failure receipts; worker timeout and measured memory/thread use | Runtime | +| 4 | Small independent cohort and review study | Frozen grouped development/confirmation partitions; blinded references; manual versus assisted comparison | Data + evaluation | +| 5 | Temporal proposal experiments | Matched baseline, denser sampling and association ablations; optional captions evaluated separately; complete quality/cost receipts | Learning | +| 6 | Reviewed dataset export and coverage connection | Round-trip validation, stable groups, intended-use permissions and accepted label lineage; unsupported training consumers are explicit | Data + training | +| 7, parallel to 3–5 | Motion estimates and causal image forecasts | Hold-last-position versus constant-velocity baseline; image velocity/prediction error, timestamp/identity checks and occlusion coverage | Perception + evaluation | +| 8 | Streaming replay and shadow live adapter | Bounded newest-frame processing, measured sensor age/latency/jitter, missed deadlines and explicit stale behavior; no robot authority in shadow mode | Runtime + robotics | + +The first implementation should be the reference schema/scorer, followed by box +correction. A stronger model cannot be selected credibly before these exist. + +## First evidence cohort + +Use one bounded task: move a cup onto a clearly marked surface. The cohort is +planned, not collected. Begin with 30–50 complete attempts across several +contributors, sessions, physical cup instances and camera arrangements. Include +successful placement, wrong target, incomplete attempt, occlusion, drops and +recovery. Add short no-cup/no-palm negatives. Record the initial state, complete +attempt and visible outcome; retain longer originals when a 5–6 second excerpt +omits relevant context. + +Keep linked originals, sessions, contributor groups, physical instances and +derivatives together. Build split groups from their connected relationships, +including exact duplicates and reviewed near-duplicate candidates. Freeze the +split before fitting recipes. Reserve unseen physical instances and at least +one camera arrangement for confirmation where feasible. If the actual data +cannot support those partitions, report that limitation rather than inventing +independent groups. + +Two annotators label confirmation recordings independently, blinded to model +proposals, then adjudicate disagreements. Define visible box extent, occlusion, +identity continuity, event vocabulary, boundary uncertainty and observable +outcome before annotation. References are a separate evidence layer, not ordinary +proposal approvals. Ambiguous or off-camera outcomes remain unknown. + +Use this cohort to diagnose the process. Its size is a workload estimate, not +proof that training or rare-error reliability has enough data. Once confirmation +results are inspected, those groups become regression evidence; later +qualification requires fresh confirmation groups. + +Shoe pairing/alignment can be the next independent task. Shoe tying requires +deformable simulation work and a different evidence contract. Neither inherits +cup-placement results. + +## Measurements and initial targets + +The following targets are proposed and unmeasured. Freeze definitions and +thresholds before the confirmation run. Publish denominators, source-group +counts, uncertainty intervals, failures and per-condition results alongside +aggregate values. Small samples may leave a conclusion inconclusive. + +| Area | Measurement | Initial development target | +| --- | --- | --- | +| Detection | Same-class one-to-one matches at IoU >= 0.50; precision/recall/F1, separate cup/palm and occluded/small/negative slices | Visible cup precision >= 0.90 and recall >= 0.85; palm results reported separately | +| Tracking | Identity switches, fragmentation and visible-time coverage, separating detection and association failures | Improve on frozen overlap baseline without manufacturing continuity through gaps | +| Observable events | One-to-one same-vocabulary interval matches at temporal IoU >= 0.50; missed/extra events; signed boundary errors | Matched boundary median absolute error <= 0.5 s and p95 <= 1.0 s; report misses alongside errors | +| Outcomes | Confusion matrix with unknown; answer coverage, selective accuracy and false completion | No stronger completion claim until failures and uncertainty are adequately represented | +| Review | Active correction time, elapsed time, additions/rejections and final agreement with independent references | Median active review time >= 25% below manual; final quality within 3 percentage points of manual, subject to uncertainty | +| CPU processing | Cold start and warm job stages; p50/p95 wall-time/source-duration ratio, CPU time, child-process peak RAM, scratch storage and all failed jobs | On the declared CPU reference machine: warm job ratio p50 <= 1 and p95 <= 2, peak process-tree RAM <= 2 GiB | +| Portability | Strict import/export, hashes, revision/split lineage, offline operation and fresh installation | Every supported fixture round-trips; incompatible versions fail with a reason | + +Detection quality at sampled timestamps and coverage of short events are +different measurements. Predeclare a common source-frame grid for recipe +comparisons; match actual source-frame identities and presentation times. +Unsampled frames remain absent evidence/coverage gaps. Nearest-frame boxes +cannot become detections at those timestamps, and extra inference solely for +evaluation must not go unreported. Report native sampled-frame detection quality +separately from common-grid coverage. A predicted position or interpolation, if +introduced, must be marked separately from a detected box. + +Compute the runtime target from each job's wall-time/source-duration ratio, +then report its percentiles. Keep failure/timeout counts and durations alongside +successful-job ratios, preserving all attempted jobs in the run receipt. + +The review study should use at least four reviewers with counterbalanced manual +and assisted assignments on different matched clips. A reviewer should not see +the same recording in both conditions. Predeclare the quality scoring rule and +retain disagreements. Fewer interface cards alone do not establish time savings. + +Rare-error claims need a larger independent failure cohort. For illustration, +zero false completions in 20 independent failure attempts still gives an upper +one-sided 95% binomial bound of about 14%. Correlated attempts weaken the premise. +Always show counts; abstaining on everything must not appear to be mastery. + +## Performance and runtime design + +Keep the small CPU detector recipe as the baseline and default installation. +Measure it on macOS ARM64 and a named Linux x86-64 server. A proposed Linux test +profile is four logical CPUs and 8 GiB host RAM; this is a validation profile, +not a measured minimum specification. Windows installation/offline smoke tests +are separate from runtime qualification. Optional GPU recipes get their own +VRAM, throughput, startup and quality results. + +Use a bounded worker process so inference can be cancelled or timed out without +losing completed evidence. Enforce and test platform-specific resource limits +where available, and label monitoring-only limits honestly elsewhere. Record +requested threads separately from observed threads; the current OpenCV thread +request is not an established concurrency cap. Cache only artifacts keyed by +source, model and recipe hashes. Resume verifies each artifact and records which +work was reused; it must not mix recipes or convert a failed attempt into a +success silently. + +Compare 2, 4 and 8 fps recipes and motion-triggered dense windows within explicit +frame/byte budgets. Increasing sampling requires changing the current 120-frame +cap deliberately. Test camera motion and occlusion as failure cases for adaptive +sampling. Measure missed short events and human effort as well as runtime. +Compare a motion-aware association candidate separately from sampling changes. + +Optional temporal descriptions should link each proposal to its actual frames +and window, distinguish proposal scores from calibrated accuracy, and abstain +when evidence is insufficient. Compare detector-only, temporal-only and combined +recipes at matched final annotation quality. Keep only improvements that justify +their processing and correction cost. Do not require a large caption model for +the basic local tool. + +## Evidence schema and user experience + +Use distinct records for source media, raw model proposals, corrected boxes, +reviewed observable events, visible outcomes, independent references, robot +commands and measured robot signals. Version strict schemas and provide an +explicit v1 import/migration path without rewriting frozen v1 artifacts. +Retain raw scores and an unassessed/calibrated status separately from review +priority. Prioritization heuristics are not probabilities of correctness. + +The review screen should answer: what happened, what evidence supports it, what +is uncertain, and what needs correction. Add frame-level box editing, event +split/merge, uncertainty reasons and raw/corrected comparison. Preserve autosave, +explicit completion, conflict protection and export of unsaved drafts. Keep +unsupported gaps visible. Validate actual phone/Safari behavior separately from +headless layout checks. + +The import flow starts from a local folder or permitted Skillspace export. It +preflights rights declarations, hashes, grouping and media compatibility before +processing. A later remote-link adapter must verify the real Skillspace access +contract; a folder of images is not a video or robot demonstration. Nonzero +timestamps can use an explicit derivative with a recorded source-to-derivative +mapping; the original remains unchanged. + +Contribution missions request a missing condition, for example: "Record a +complete placement with an occluded approach, including the outcome." Show +received, checked, reviewed, accepted or correction-needed status and explain +why a contribution is useful. Recognition should reward accepted independent +coverage, useful failures and recoveries rather than uploaded seconds or repeated +near-duplicates. Public credit is opt-in. Any monetary incentive needs its own +budget and anti-duplication rules before launch. + +Keep collection coverage, usable visual evidence, robot supervision, trained +candidate, simulation qualification and hardware qualification as separate +statuses. There is no overall "skill learned" percentage derived from clip count. + +## Portable exports and the robotics bridge + +Export reviewed annotations, accepted labels, source/derivative hashes, +review and recipe IDs, frozen split assignments, intended-use permissions and +failure receipts. Media inclusion must follow its actual permission; local +analysis permission alone does not permit training or public redistribution. +Validate reimport and trace every label back to its original evidence. + +Reuse the existing footage intake and coverage tooling. The current footage +schema contains a shoe-pair grouping field; a task-general adapter needs an +explicit schema change, not invented shoe-pair IDs for cups. Accepting an +export into the current pixel-based trainer does not make that trainer consume +task annotations. Label-consuming learners require a separately tested consumer. + +The next research stage joins permitted video to robot episode tapes on one +timeline. Declare clock offset, drift, synchronization uncertainty, missingness +and signal freshness. Current tapes contain simulation time; correspondence +cannot be guessed. Requested, issued and applied commands remain separate from +measured joint positions, velocities, forces and gripper aperture. + +For the first learned task model, select deployable sensor inputs and compare +against simple baselines. Keep privileged object state and authored phase labels +out of deployment inputs. Robot-specific control learning additionally needs +aligned action supervision or an independently validated human-to-robot bridge. +Every candidate then needs a versioned simulator task, observable success/failure +rules, fresh confirmation scenarios and complete attempted-trial receipts. + +An eventual capsule should declare robot adapter, degrees of freedom, coordinate +frames, units, joint/gripper limits, sensors, calibration, operating envelope, +policy hash and artifact-bound simulation results. Installation checks +compatibility; it does not itself qualify hardware. + +## Release gates and subsequent versions + +- **v0.2-alpha software gate:** reference import/scoring, box/event review, + bounded runs, round-trip exports, causal image-motion/forecast baselines and + offline installation work. Accuracy is explicitly pending where no independent + references exist; forecasts do not supply calibrated metric speed. +- **v0.2 evidence gate:** publish the grouped cohort protocol, blinded comparison, + review study, named hardware measurements and observed results. Report unmet + targets or insufficient evidence; do not rename them as passed. +- **v0.2 live gate:** measured streaming replay and then a shadow camera adapter + on named hardware; replay, live perception and robot-control results are + separate claims. +- **v0.3 research objective:** calibrated motion, synchronized video/robot + inspector and compact evaluated task-state/interaction prediction on one + supported task. +- **Later control release:** action-supervised policy, task-specific simulation + benchmark, compatible capsule and separate physical qualification. + +Each release updates GitHub installation docs, the research report and the +Hugging Face demo with the same artifact/recipe identity. Public synthetic +fixtures remain labeled workflow tests. A public accuracy dataset requires +appropriate redistribution permission; private evaluations publish aggregates +and redacted receipts with their scope stated. + +## Implementation references + +- Our [current observations](observations.md), [review contract](observation-ui.md), + [pilot evidence](../benchmarks/observation-20261010/README.md), + [collection planner](skillspace-coverage.md), [episode tapes](episode-tape.md) + and [product roadmap](roadmap.md) define the existing contracts. +- [LeRobotDataset v3 documentation](https://huggingface.co/docs/lerobot/lerobot-dataset-v3) + describes multimodal robot time-series and video storage. Evaluate an adapter + when we have aligned robot evidence; do not invent action/state columns from + human footage. +- [SAM 2's official repository](https://github.com/facebookresearch/sam2) + provides promptable video segmentation. It is an optional comparison candidate, + not a committed dependency or a performance claim for our CPU recipe. Review + the exact code/weight licenses and measure its benefit before adoption. diff --git a/docs/roadmap.md b/docs/roadmap.md index cd9406b..4244a08 100644 --- a/docs/roadmap.md +++ b/docs/roadmap.md @@ -1,6 +1,6 @@ # Execution map: footage to installable skills -Working map, 9 October 2026. This is the team's sequence of builds and evidence +Working map, updated 10 October 2026. This is the team's sequence of builds and evidence gates. A completed software component does not establish robot competence. ## The product loop @@ -100,6 +100,28 @@ outcome model on the existing placement environment. The first real Skillspace will then test the collection and review loop. See [collection design](skillspace-coverage.md), [episode evidence](episode-tape.md) and [existing benchmark protocol](benchmarks.md). +## Observation Lab next version + +[Observation Lab v0.2 plan](observation-v0.2-plan.md) prioritizes independent +reference scoring, box and event correction, bounded offline batches, reviewed +dataset export and [causal motion prediction](motion-prediction-plan.md). Image +speed/forecasts, calibrated metric motion and real-time robot interaction have +separate evidence gates. It separates the software release from measured annotation +quality and keeps visual labels distinct from robot action supervision. Grouped +review and autosave are already implemented; the next evidence gate is the +planned complete-task cohort with blinded references and measured review effort. + +The [first image-motion alpha](motion-observations.md) is now runnable from the +candidate source checkout: one manually seeded region, causal sparse-feature +translation, image-anchor velocity, 100/250/500 ms constant-velocity forecasts, +authored benchmark scoring and read-only replay. Its original source clocks, +hashes, tracking losses and conditional score denominators are retained. +The published Observation Lab v0.1 tag/Space remain unchanged and the full v0.2 +has not shipped. Real-scene anchor references, correction/export integration, +bounded live observation, calibration and action-conditioned interaction still +need their respective evidence gates. Motion speed is analysis-image pixels/s; +no physical speed or robot control follows from these forecasts. + ## Reaction and stopping evidence [Reaction framework v0.1](reaction-framework.md) supplies the first reusable diff --git a/pyproject.toml b/pyproject.toml index ecfcc9d..da66afb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -38,6 +38,7 @@ dvidia-reaction = "dvidia_training.reaction_benchmark:main" dvidia-motor-memory = "dvidia_training.motor_benchmark:main" dvidia-observe = "dvidia_training.observation_pipeline:main" dvidia-observation-studio = "dvidia_training.observation_studio:main" +dvidia-motion = "dvidia_training.motion_cli:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/src/dvidia_training/motion.py b/src/dvidia_training/motion.py new file mode 100644 index 0000000..2c6cff4 --- /dev/null +++ b/src/dvidia_training/motion.py @@ -0,0 +1,249 @@ +"""Causal image-anchor motion, without calibration or robot command authority. + +Only two actual, contiguous observations enter each velocity estimate. Missing +observations never become interpolated measurements. Forecasts are un-clipped +constant-velocity hypotheses, not object dynamics or calibrated probabilities. +""" +from __future__ import annotations + +import math + + +KIND = 'dvidia.image-motion.v1' +ALGORITHM = 'two-observation-finite-difference' +UNCERTAINTY_SEMANTICS = 'empirical-envelope-not-calibrated-probability' +UNITS = {'position': 'image_pixels', 'velocity': 'image_pixels_per_second', + 'time': 'seconds'} + + +def _number(value, name, *, minimum=None, positive=False): + if type(value) not in (int, float): + raise ValueError(f'{name} must be a finite number, not a boolean.') + try: + result = float(value) + except (OverflowError, ValueError): + raise ValueError(f'{name} must be finite.') from None + if (not math.isfinite(result) or (minimum is not None and result < minimum) + or (positive and result <= 0)): + raise ValueError(f'{name} must be finite and within its supported range.') + return result + + +def _dimensions(width, height): + if type(width) is not int or type(height) is not int or width <= 0 or height <= 0: + raise ValueError('Image width and height must be positive integers.') + # The coordinates, not only integer dimensions, must fit finite float space. + _number(width, 'Image width', positive=True) + _number(height, 'Image height', positive=True) + + +def _anchor(value, width, height): + if type(value) is not list or len(value) != 2: + raise ValueError('An observed anchor requires a two-number pixel list.') + center = [_number(v, 'Anchor coordinate', minimum=0) for v in value] + if center[0] >= width or center[1] >= height: + raise ValueError('An observed anchor must be inside the image.') + return center + + +def _support(value): + if type(value) is not int or value < 0: + raise ValueError('Support count must be a nonnegative integer.') + return value + + +class MotionEstimator: + """Causal, unsmoothed finite-difference baseline for one image feature anchor. + + ``update`` takes actual acquisition times in one declared recording clock. + They must strictly increase, including missing frames. A tracked center must + be inside the image. ``None`` marks missing evidence; explicit ``lost`` also + requires ``None``. A loss or a gap greater than ``max_gap_seconds`` clears + velocity continuity. Two fresh observations are then needed to forecast. + + A provided uncertainty is a caller's empirical anchor radius in pixels. If + both velocity observations have radii, the forecast radius is current radius + plus horizon times the sum of both radii divided by their actual time gap. + This only propagates those supplied radii through the finite difference. It + excludes acceleration, identity changes, camera motion and timestamp error; + it is not a calibrated probability or an interaction guarantee. If either + radius is unknown, forecast uncertainty remains ``None``. + + Prediction records expire for consumer reuse after + ``max_prediction_age_seconds``. This freshness lifetime does not truncate + the declared forecast horizons: a fresh record can describe a 500 ms future + target while itself becoming stale after 150 ms. A consumer must compare its + current time with ``prediction_use_before_time_seconds`` in the same clock. + No wall-clock, metric speed, physical centroid or robot control is inferred. + """ + + def __init__(self, width: int, height: int, horizons_seconds=(0.1, 0.25, 0.5), + max_gap_seconds=0.15, max_prediction_age_seconds=0.15): + _dimensions(width, height) + if type(horizons_seconds) not in (tuple, list) or not horizons_seconds: + raise ValueError('Provide at least one forecast horizon.') + horizons = [_number(h, 'Forecast horizon', positive=True) + for h in horizons_seconds] + if any(b <= a for a, b in zip(horizons, horizons[1:])): + raise ValueError('Forecast horizons must be unique and increasing.') + self.width, self.height = width, height + self.horizons_seconds = tuple(horizons) + self.max_gap_seconds = _number(max_gap_seconds, 'Maximum evidence gap', positive=True) + self.max_prediction_age_seconds = _number( + max_prediction_age_seconds, 'Maximum prediction age', positive=True) + self._last_update = None + self._previous = None + self._ever_observed = False + + def update(self, time_seconds: float, center_xy_pixels: list[float] | None, + *, support_count: int = 0, status: str = 'tracked', + uncertainty_pixels: float | None = None) -> dict: + """Return a fresh, strict JSON-compatible observation/estimate/forecast row. + + The observed anchor and current estimate are equal in this unsmoothed + baseline, but are stored separately to preserve their meanings. Each + velocity carries the two actual observations used to derive it. Forecast + coordinates stay un-clipped and carry ``out_of_frame`` explicitly. + Invalid updates leave estimator state unchanged. + """ + t = _number(time_seconds, 'Acquisition time', minimum=0) + if self._last_update is not None and t <= self._last_update: + raise ValueError('Acquisition times must strictly increase.') + if type(status) is not str or status not in ('tracked', 'lost'): + raise ValueError('Tracking status must be tracked or lost.') + count = _support(support_count) + radius = (None if uncertainty_pixels is None else + _number(uncertainty_pixels, 'Anchor uncertainty radius', minimum=0)) + center = None if center_xy_pixels is None else _anchor( + center_xy_pixels, self.width, self.height) + if status == 'lost' and center is not None: + raise ValueError('Lost tracking cannot supply an observed anchor.') + if center is None and radius is not None: + raise ValueError('A missing anchor cannot supply an uncertainty radius.') + + row = { + 'kind': KIND, 'schema_version': 1, 'algorithm': ALGORITHM, + 'acquired_time_seconds': t, + 'status': 'lost' if center is None else 'tracked', + 'loss_reason': ('tracking_lost' if status == 'lost' else 'missing_observation') + if center is None else None, + 'continuity': 'lost', + 'support_count': count, + 'image_size_pixels': [self.width, self.height], + 'coordinate_frame': 'image', + 'anchor_semantics': 'tracked-image-feature-anchor-not-physical-centroid', + 'units': dict(UNITS), + 'observed_anchor_xy_pixels': None if center is None else list(center), + 'estimated_center_xy_pixels': None if center is None else list(center), + 'anchor_uncertainty_radius_pixels': radius, + 'velocity_xy_pixels_per_second': None, + 'speed_pixels_per_second': None, + 'velocity_evidence': None, + 'forecasts': [], + 'uncertainty_semantics': UNCERTAINTY_SEMANTICS, + 'uncertainty_basis': 'supplied-anchor-radii-propagated-through-finite-difference', + 'prediction_use_before_time_seconds': None, + 'estimator': {'horizons_seconds': list(self.horizons_seconds), + 'max_gap_seconds': self.max_gap_seconds, + 'max_prediction_age_seconds': self.max_prediction_age_seconds}, + } + if center is not None: + previous = self._previous + row['continuity'] = 'loss_reset' if self._ever_observed else 'initial' + if previous is not None: + previous_t, previous_center, previous_radius = previous + dt = t - previous_t + if dt > self.max_gap_seconds: + row['continuity'] = 'gap_reset' + else: + row['continuity'] = 'continued' + velocity = [_number((b-a)/dt, 'Derived image velocity') + for a, b in zip(previous_center, center)] + speed = _number(math.hypot(*velocity), 'Derived image speed', minimum=0) + row['velocity_xy_pixels_per_second'] = velocity + row['speed_pixels_per_second'] = speed + row['velocity_evidence'] = { + 'start_time_seconds': previous_t, 'end_time_seconds': t, + 'start_anchor_xy_pixels': list(previous_center), + 'end_anchor_xy_pixels': list(center), 'delta_time_seconds': dt, + } + for horizon in self.horizons_seconds: + target = _number(t+horizon, 'Forecast target time', minimum=0) + if target <= t: + raise ValueError('Acquisition clock cannot resolve the forecast horizon.') + forecast = [_number(c+v*horizon, 'Forecast coordinate') + for c, v in zip(center, velocity)] + envelope = None + if radius is not None and previous_radius is not None: + envelope = _number(radius+horizon*(radius+previous_radius)/dt, + 'Forecast uncertainty radius', minimum=0) + row['forecasts'].append({ + 'target_time_seconds': target, 'horizon_seconds': horizon, + 'center_xy_pixels': forecast, + 'uncertainty_radius_pixels': envelope, + 'out_of_frame': not (0 <= forecast[0] < self.width + and 0 <= forecast[1] < self.height), + }) + expiry = _number(t+self.max_prediction_age_seconds, + 'Prediction expiry time', minimum=0) + if expiry <= t: + raise ValueError('Acquisition clock cannot resolve prediction freshness.') + row['prediction_use_before_time_seconds'] = expiry + + # Commit only after all derived numeric values have passed validation. + self._last_update = t + if center is None: + self._previous = None + else: + self._previous = (t, list(center), radius) + self._ever_observed = True + return row + + +def _strict_equal(actual, expected): + """JSON equality with exact types, so True cannot impersonate 1 or 1.0.""" + if type(actual) is not type(expected): + return False + if type(expected) is dict: + return set(actual) == set(expected) and all( + _strict_equal(actual[k], value) for k, value in expected.items()) + if type(expected) is list: + return len(actual) == len(expected) and all( + _strict_equal(a, b) for a, b in zip(actual, expected)) + return actual == expected + + +def validate_motion_rows(rows, width, height): + """Validate strict baseline rows and return the unchanged list. + + This stateless validator causally regenerates every row from its actual + observed anchors, optional radii and embedded estimator configuration. It + rejects extra fields, conflicting units/evidence, forged forecasts, changed + configuration, fabricated estimates through losses and non-monotonic clocks. + It validates internal contract consistency; it cannot authenticate whether a + camera really observed an anchor or establish independent prediction accuracy. + """ + _dimensions(width, height) + if type(rows) is not list: + raise ValueError('Motion rows must be a list.') + estimator = None + for index, row in enumerate(rows): + if type(row) is not dict: + raise ValueError(f'Motion row {index} must be a data-only object.') + config = row.get('estimator') + if type(config) is not dict or set(config) != { + 'horizons_seconds', 'max_gap_seconds', 'max_prediction_age_seconds'}: + raise ValueError(f'Motion row {index} has invalid estimator configuration.') + try: + if estimator is None: + estimator = MotionEstimator(width, height, **config) + reason = row.get('loss_reason') + input_status = 'lost' if reason == 'tracking_lost' else 'tracked' + expected = estimator.update(row.get('acquired_time_seconds'), + row.get('observed_anchor_xy_pixels'), support_count=row.get('support_count'), + status=input_status, uncertainty_pixels=row.get('anchor_uncertainty_radius_pixels')) + except (TypeError, ValueError, OverflowError) as error: + raise ValueError(f'Motion row {index} is invalid: {error}') from None + if not _strict_equal(row, expected): + raise ValueError(f'Motion row {index} does not match its causal evidence or strict schema.') + return rows diff --git a/src/dvidia_training/motion_benchmark.py b/src/dvidia_training/motion_benchmark.py new file mode 100644 index 0000000..caab87d --- /dev/null +++ b/src/dvidia_training/motion_benchmark.py @@ -0,0 +1,448 @@ +"""Authored image-motion tracking/forecast benchmark; not detector accuracy. + +Future reference trajectories are evaluator-only. The video analyzer receives +only the permitted manifest, first-frame human-authored seed and cadence. +Original DVIDIA fixture generation and scoring code: MIT. +""" +from __future__ import annotations + +from bisect import bisect_left +from datetime import datetime, timezone +from fractions import Fraction +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import subprocess +import time + +from .footage_data import canonical + +FPS = 30 +WIDTH, HEIGHT = 640, 360 +FRAMES = 120 +HORIZONS = (0.1, 0.25, 0.5) +CASES = ('stationary', 'constant_velocity', 'stop_start', 'direction_change', + 'brief_occlusion', 'low_texture') + + +def _hash(path): + result = hashlib.sha256() + with Path(path).open('rb') as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b''): + result.update(chunk) + return result.hexdigest() + + +def _write(path, value): + Path(path).write_bytes(canonical(value)) + + +def _finite(value): + return type(value) in (float, int) and math.isfinite(value) + + +def _point(value): + return (type(value) in (tuple, list) and len(value) == 2 + and all(_finite(v) for v in value)) + + +def distribution(values): + """Linear interpolation between sorted samples; nulls for zero samples.""" + values = sorted(values) + if any(not _finite(value) for value in values): + raise ValueError('Scores must be finite numbers.') + def percentile(q): + position = (len(values)-1)*q + lower = int(position) + upper = min(lower+1, len(values)-1) + return values[lower]+(values[upper]-values[lower])*(position-lower) + return {'count': len(values), 'mean': sum(values)/len(values) if values else None, + 'p50': percentile(.5) if values else None, + 'p95': percentile(.95) if values else None, + 'max': values[-1] if values else None} + + +def _nearest(references, times, target, tolerance): + index = bisect_left(times, target) + candidates = [i for i in (index-1, index) if 0 <= i < len(times)] + candidates = [i for i in candidates if abs(times[i]-target) <= tolerance+1e-12] + if not candidates: + return None + # Resolve only floating-point arithmetic ties. Do not widen the declared + # half-frame tolerance to accommodate a sparse or differently timed grid. + index = min(candidates, key=lambda i: (round(abs(times[i]-target), 12), times[i])) + return references[index] + + +def score_motion(rows, references, *, fps=FPS, horizons=HORIZONS): + """Score only emitted image anchors/forecasts against independent frames. + + References contain frame_index, source_pts_seconds, anchor_xy_pixels, + visible, velocity_scoreable and velocity_xy_pixels_per_second. Missing + sampled rows remain coverage gaps. Missing future references and abstentions + have explicit denominators and never become zero endpoint errors. + """ + if not _finite(fps) or fps <= 0 or not references: + raise ValueError('A positive frame rate and nonempty references are required.') + times = [ref['source_pts_seconds'] for ref in references] + if (any(not _finite(t) for t in times) + or any(b <= a for a, b in zip(times, times[1:]))): + raise ValueError('Reference acquisition times must strictly increase.') + if len({ref['frame_index'] for ref in references}) != len(references): + raise ValueError('Reference frame identities must be unique.') + if any(not _point(ref.get('anchor_xy_pixels')) or type(ref.get('visible')) is not bool + for ref in references): + raise ValueError('References require finite image anchors and visibility.') + if any(not _finite(h) or h <= 0 for h in horizons) or len(set(horizons)) != len(horizons): + raise ValueError('Horizons must be unique positive seconds.') + row_times = [row['acquired_time_seconds'] for row in rows] + if (any(not _finite(t) for t in row_times) + or any(b <= a for a, b in zip(row_times, row_times[1:]))): + raise ValueError('Observation acquisition times must strictly increase.') + tolerance = 1/(2*fps) + current_time_tolerance = 2e-6 + matched, tracked_visible = set(), set() + anchor_errors, velocity_errors = [], [] + lost_visible = false_tracking = unmatched_current = 0 + lost_rows = sum(row.get('status') != 'tracked' for row in rows) + first_lost = None + reacquired_after_loss = 0 + horizon_scores = {str(h): {'horizon_seconds': h, 'attempted_rows': len(rows), + 'future_reference_available': 0, 'terminal_censored': 0, + 'future_reference_gap': 0, 'current_reference_unmatched': 0, + 'lost_abstentions': 0, 'velocity_warmup_abstentions': 0, + 'missing_forecast_abstentions': 0, 'constant_velocity': [], + 'hold_current_paired': [], 'hold_current_all_tracked': [], + 'reference_time_offsets': [], 'region_radii': [], 'region_covered': 0} + for h in horizons} + for row in rows: + t = row['acquired_time_seconds'] + ref = _nearest(references, times, t, current_time_tolerance) + tracked = row.get('status') == 'tracked' + center = row.get('estimated_center_xy_pixels') + velocity = row.get('velocity_xy_pixels_per_second') + if tracked and not _point(center): + raise ValueError('Tracked observations require a finite image anchor.') + if velocity is not None and not _point(velocity): + raise ValueError('Velocity must be null or a finite pixel/s vector.') + if not tracked and first_lost is None: + first_lost = t + elif tracked and first_lost is not None: + reacquired_after_loss += 1 + first_lost = None + if ref is None: + unmatched_current += 1 + else: + matched.add(ref['frame_index']) + if tracked and ref['visible']: + tracked_visible.add(ref['frame_index']) + anchor_errors.append(math.dist(center, ref['anchor_xy_pixels'])) + if ref.get('velocity_scoreable') and _point(velocity): + truth_velocity = ref.get('velocity_xy_pixels_per_second') + if not _point(truth_velocity): + raise ValueError('Scoreable reference velocity must be finite.') + velocity_errors.append(math.dist(velocity, truth_velocity)) + elif tracked: + false_tracking += 1 + elif ref['visible']: + lost_visible += 1 + for h in horizons: + score = horizon_scores[str(h)] + target = t+h + if target > times[-1]+1e-12: + score['terminal_censored'] += 1 + continue + future = _nearest(references, times, target, tolerance) + if future is None: + score['future_reference_gap'] += 1 + continue + score['future_reference_available'] += 1 + score['reference_time_offsets'].append(future['source_pts_seconds']-target) + if ref is None: + score['current_reference_unmatched'] += 1 + continue + if not tracked: + score['lost_abstentions'] += 1 + continue + hold_error = math.dist(center, future['anchor_xy_pixels']) + score['hold_current_all_tracked'].append(hold_error) + if velocity is None: + score['velocity_warmup_abstentions'] += 1 + continue + predictions = [p for p in row.get('forecasts', []) + if abs(p.get('horizon_seconds', -1)-h) <= 1e-8] + if not predictions: + score['missing_forecast_abstentions'] += 1 + continue + if len(predictions) != 1: + raise ValueError('A horizon must have at most one forecast.') + prediction = predictions[0] + if (not _point(prediction.get('center_xy_pixels')) + or not _finite(prediction.get('target_time_seconds')) + or abs(prediction['target_time_seconds']-target) > 1e-8): + raise ValueError('Forecast must target its declared causal horizon.') + error = math.dist(prediction['center_xy_pixels'], future['anchor_xy_pixels']) + score['constant_velocity'].append(error) + score['hold_current_paired'].append(hold_error) + radius = prediction.get('uncertainty_radius_pixels') + if radius is not None: + if not _finite(radius) or radius < 0: + raise ValueError('Uncertainty radius must be null or nonnegative.') + score['region_radii'].append(radius) + score['region_covered'] += error <= radius + visible_refs = sum(ref['visible'] for ref in references) + for score in horizon_scores.values(): + for key in ('constant_velocity', 'hold_current_paired', 'hold_current_all_tracked'): + score[key] = distribution(score[key]) + offsets = score.pop('reference_time_offsets') + score['reference_time_offset_seconds'] = distribution(offsets) + score['reference_max_absolute_time_offset_seconds'] = max(map(abs, offsets), default=None) + radii = score.pop('region_radii') + score['uncertainty'] = {'radius_pixels': distribution(radii), + 'covered': score.pop('region_covered'), 'evaluated': len(radii), + 'calibrated': False} + scored = score['constant_velocity']['count'] + eligible = score['future_reference_available'] + score['forecast_answer_coverage'] = scored/eligible if eligible else None + score['endpoint_error_units'] = 'pixels' + return {'kind': 'dvidia.motion-scores.v1', 'units': {'position': 'pixels', 'velocity': 'pixels/second'}, + 'reference_matching': {'rule': 'nearest actual reference PTS; earlier frame on a half-frame tie', + 'maximum_time_tolerance_seconds': tolerance, 'fps': fps, + 'current_reference_rule': 'same acquisition PTS within numerical serialization tolerance; off-grid rows unmatched', + 'current_time_tolerance_seconds': current_time_tolerance, + 'terminal_rule': 'targets after the last reference frame are censored, never rounded backward', + 'observations': 'unsampled references remain gaps; no extra tracking is run for scoring'}, + 'tracking': {'reference_frames': len(references), 'reference_visible_frames': visible_refs, + 'attempted_observation_rows': len(rows), 'matched_reference_frames': len(matched), + 'unsampled_reference_frames': len(references)-len(matched), + 'unmatched_observation_rows': unmatched_current, + 'tracked_visible_reference_frames': len(tracked_visible), + 'track_coverage_visible': len(tracked_visible)/visible_refs if visible_refs else None, + 'lost_rows': lost_rows, 'lost_visible_rows': lost_visible, + 'tracked_during_reference_occlusion_rows': false_tracking, + 'reacquisitions_after_reported_loss': reacquired_after_loss, + 'automatic_reacquisition_supported': False, + 'anchor_error_pixels': distribution(anchor_errors), + 'velocity_error_pixels_per_second': distribution(velocity_errors), + 'velocity_scope': 'visible frames with available estimate; authored discontinuity neighborhoods excluded'}, + 'forecasts': horizon_scores, + 'claim_scope': 'authored fixed-camera image-anchor tracking and passive forecasting only'} + + +def _trajectory(case, index): + if case in ('stationary', 'low_texture'): + x = 160 + elif case in ('constant_velocity', 'brief_occlusion'): + x = 100+3*index + elif case == 'stop_start': + x = 100+3*min(index, 40)+3*max(0, index-80) + elif case == 'direction_change': + x = 100+3*min(index, 60)-3*max(0, index-60) + else: + raise ValueError('Unsupported authored fixture case.') + return x, 180 + + +def _frame_timestamps(video): + result = subprocess.run(['ffprobe', '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'frame=best_effort_timestamp:stream=time_base', '-of', 'json', str(video)], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=30, check=False) + if result.returncode or len(result.stdout) > 1024*1024: + raise ValueError('Could not establish generated frame identities.') + decoded = json.loads(result.stdout) + timebase = Fraction(decoded['streams'][0]['time_base']) + times = [float(int(row['best_effort_timestamp'])*timebase) for row in decoded['frames']] + if (len(times) != FRAMES or times[0] != 0 + or any(b <= a for a, b in zip(times, times[1:]))): + raise ValueError('Generated video does not have the declared zero-origin frame grid.') + if any(abs(t-index/FPS) > 1e-6 for index, t in enumerate(times)): + raise ValueError('Generated video frame timing differs from the frozen recipe.') + return times + + +def generate_fixtures(directory): + """Encode own fixtures; write evaluator references before any tracking.""" + import numpy as np + directory = Path(directory) + directory.mkdir(parents=True) + texture_seed = 4702 + random = np.random.default_rng(texture_seed) + texture = random.integers(40, 235, size=(64, 80, 3), dtype=np.uint8) + texture[::8, :, :] = 245 + texture[:, ::8, :] = 25 + episodes, fixture_receipts = [], [] + for case in CASES: + video = directory/(case+'.mp4') + raw = directory/(case+'.rgb') + raw_hash = hashlib.sha256() + with raw.open('wb') as handle: + for index in range(FRAMES): + frame = np.full((HEIGHT, WIDTH, 3), 230, dtype=np.uint8) + x, y = _trajectory(case, index) + visible = not (case == 'brief_occlusion' and 45 <= index < 65) + if visible: + patch = np.full((64, 80, 3), 120, dtype=np.uint8) if case == 'low_texture' else texture + frame[y-32:y+32, x-40:x+40] = patch + data = frame.tobytes() + raw_hash.update(data) + handle.write(data) + command = ['ffmpeg', '-hide_banner', '-nostdin', '-loglevel', 'error', + '-y', '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-s', f'{WIDTH}x{HEIGHT}', + '-r', str(FPS), '-i', str(raw), '-an', '-c:v', 'libx264', '-preset', 'fast', + '-crf', '18', '-pix_fmt', 'yuv420p', '-threads', '1', '-map_metadata', '-1', + '-movflags', '+faststart', str(video)] + try: + encoded = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=60, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ValueError('Authored fixture encoding requires FFmpeg and a completed 60-second encoding budget.') from exc + finally: + raw.unlink(missing_ok=True) + if encoded.returncode: + raise ValueError('Authored fixture encoding failed.') + times = _frame_timestamps(video) + refs = [] + boundaries = {'stop_start': (40, 80), 'direction_change': (60,)}.get(case, ()) + for index, stamp in enumerate(times): + current = _trajectory(case, index) + velocity = None + if index: + previous = _trajectory(case, index-1) + dt = stamp-times[index-1] + velocity = [(current[d]-previous[d])/dt for d in (0, 1)] + refs.append({'frame_index': index, 'source_pts_seconds': stamp, + 'anchor_xy_pixels': list(current), 'velocity_xy_pixels_per_second': velocity, + 'velocity_scoreable': index > 0 and not any(abs(index-b) <= 2 for b in boundaries), + 'visible': not (case == 'brief_occlusion' and 45 <= index < 65)}) + reference_path = directory/(case+'.reference.json') + _write(reference_path, {'kind': 'dvidia.authored-motion-reference.v1', 'rows': refs, + 'role': 'evaluator_only_never_supplied_to_tracker', 'license': 'MIT', + 'anchor_semantics': 'initial manually seeded image region center plus authored translation'}) + x, y = _trajectory(case, 0) + seed = [(x-36)/WIDTH, (y-28)/HEIGHT, (x+36)/WIDTH, (y+28)/HEIGHT] + episodes.append({'id': case, 'path': video.name, 'title': 'Authored '+case.replace('_', ' '), + 'session_id': 'dvidia-motion-authoring-v1', 'source_recording_id': case, + 'kind': 'synthetic', 'credit': 'DVIDIA authored geometric fixture', 'license': 'MIT', + 'source_url': 'https://github.com/Dvidia-Inference/dvidia-training', + 'permission': 'local_analysis', 'sha256': _hash(video)}) + fixture_receipts.append({'id': case, 'video_path': video.name, 'video_sha256': _hash(video), + 'video_bytes': video.stat().st_size, 'raw_rgb_sha256': raw_hash.hexdigest(), + 'raw_rgb_bytes': FRAMES*WIDTH*HEIGHT*3, 'reference_path': reference_path.name, + 'reference_sha256': _hash(reference_path), 'seed_box_xyxy_normalized': seed, + 'frame_count': len(times), 'source_duration_seconds': FRAMES/FPS}) + manifest = {'kind': 'dvidia.observation-source.v1', + 'task': 'authored image-anchor tracking and passive motion forecasts', 'episodes': episodes} + _write(directory/'source.json', manifest) + return fixture_receipts + + +def _version(command): + try: + output = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10, check=False) + return output.stdout.decode('utf-8', errors='replace').splitlines()[0][:200] + except (OSError, IndexError, subprocess.TimeoutExpired): + return 'unavailable' + + +def _peak_memory(): + try: + import resource + peak = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss + return {'bytes': int(peak if platform.system() == 'Darwin' else peak*1024), + 'scope': 'process-lifetime resident high-water mark including fixture generation; not a stage cap'} + except (ImportError, AttributeError): + return {'bytes': None, 'scope': 'unavailable on this platform'} + + +def run_benchmark(output): + """Run all cases in a new directory, preserving every failed attempt.""" + from .motion_video import analyze_video + output = Path(output).resolve() + if output.exists(): + raise ValueError('Choose a new benchmark directory; prior evidence is never overwritten.') + output.mkdir(parents=True) + fixture_started = time.monotonic() + fixtures = generate_fixtures(output/'inputs') + generation_seconds = time.monotonic()-fixture_started + protocol = {'kind': 'dvidia.motion-benchmark-protocol.v1', + 'created_at': datetime.now(timezone.utc).isoformat(), 'cases': list(CASES), + 'fixture_recipe': {'width': WIDTH, 'height': HEIGHT, 'fps': FPS, 'frames': FRAMES, + 'texture_rng': 'numpy.default_rng(4702)', 'object_size_pixels': [80, 64], + 'seed_inset_pixels': 4, 'background_rgb': [230, 230, 230], + 'encoder': 'libx264 fast crf18 yuv420p single-thread; no metadata', + 'trajectory_code_sha256': _hash(__file__)}, + 'horizons_seconds': list(HORIZONS), 'sample_hz': FPS, + 'sources_manifest_sha256': _hash(output/'inputs/source.json'), + 'fixtures': fixtures, 'reference_access': 'scorer only; never a tracker argument', + 'baseline_comparison': 'hold current on the identical evaluable anchored rows with a constant-velocity forecast', + 'hardware': {'system': platform.system(), 'machine': platform.machine(), + 'processor': platform.processor() or None, 'python': platform.python_version(), + 'os_release': platform.release(), 'logical_cpu_count': os.cpu_count()}, + 'software': {'ffmpeg': _version(['ffmpeg', '-version']), 'ffprobe': _version(['ffprobe', '-version'])}, + 'claim_scope': 'authored synthetic image tracking and passive forecasting; no detection, real-world generalization or robot control'} + _write(output/'protocol.json', protocol) + started = time.monotonic() + results = [] + for fixture in fixtures: + case = fixture['id'] + run_dir = output/'cases'/case + attempted = time.monotonic() + try: + receipt = analyze_video(output/'inputs/source.json', case, run_dir, + seed_box=fixture['seed_box_xyxy_normalized'], label='authored textured region', sample_hz=FPS) + if receipt.get('status') != 'completed': + results.append({'id': case, 'status': 'failed', 'run_path': f'cases/{case}/run.json', + 'run_sha256': _hash(run_dir/'run.json'), + 'errors': receipt.get('errors', []), + 'error': receipt.get('error', 'Analyzer failed; see retained run receipt'), + 'elapsed_seconds': time.monotonic()-attempted}) + continue + rows = receipt.get('rows') + if rows is None: + rows_doc = json.loads((run_dir/'rows.json').read_bytes()) + rows = rows_doc if type(rows_doc) is list else rows_doc['rows'] + reference_raw = (output/'inputs'/fixture['reference_path']).read_bytes() + if hashlib.sha256(reference_raw).hexdigest() != fixture['reference_sha256']: + raise ValueError('Evaluator reference differs from the frozen fixture protocol.') + references = json.loads(reference_raw)['rows'] + score = score_motion(rows, references) + _write(run_dir/'scores.json', score) + results.append({'id': case, 'status': 'completed', 'run_path': f'cases/{case}/run.json', + 'run_sha256': _hash(run_dir/'run.json'), 'scores_path': f'cases/{case}/scores.json', + 'scores_sha256': _hash(run_dir/'scores.json'), 'scores': score, + 'elapsed_seconds': time.monotonic()-attempted, 'timings': receipt.get('timings', {})}) + except Exception as exc: + run_dir.mkdir(parents=True, exist_ok=True) + error = str(exc)[:300] if isinstance(exc, ValueError) else type(exc).__name__ + if not (run_dir/'run.json').exists(): + _write(run_dir/'run.json', {'kind': 'dvidia.motion-run.v1', 'status': 'failed', 'error': error}) + results.append({'id': case, 'status': 'failed', 'run_path': f'cases/{case}/run.json', + 'run_sha256': _hash(run_dir/'run.json'), + 'error': error, 'elapsed_seconds': time.monotonic()-attempted}) + elapsed = time.monotonic()-started + completed = sum(row['status'] == 'completed' for row in results) + report = {'kind': 'dvidia.motion-benchmark.v1', 'protocol_sha256': _hash(output/'protocol.json'), + 'attempted': len(fixtures), 'completed': completed, 'failed': len(fixtures)-completed, + 'cases': results, 'timing': {'batch_elapsed_seconds': elapsed, + 'fixture_generation_seconds': generation_seconds, + 'attempted_source_duration_seconds': len(fixtures)*FRAMES/FPS, + 'completed_source_duration_seconds': completed*FRAMES/FPS, + 'batch_processing_time_source_duration_ratio': elapsed/(len(fixtures)*FRAMES/FPS) + if completed == len(fixtures) else None, + 'ratio_availability': 'all cases must complete; failed batches cannot appear faster through early failure', + 'scope': 'batch intake, probing, streaming decode, causal tracking/forecast, writes and scoring; fixture generation/provisioning excluded', + 'live_sensor_to_action_latency_measured': False}, 'memory': _peak_memory(), + 'limitations': ['Six authored fixtures are engineering integration evidence, not held-out real footage.', + 'Image anchors and pixel/s have no calibrated physical scale.', + 'Seeded sparse tracking is not object detection or semantic task understanding.', + 'Offline source-duration ratio does not establish live latency or deadline guarantees.', + 'No automatic reacquisition or robot commands; losses abstain.', + 'Uncertainty radii are heuristic and uncalibrated.'], + 'robot_skill_qualified': False, 'trained_model': False} + _write(output/'report.json', report) + return report diff --git a/src/dvidia_training/motion_cli.py b/src/dvidia_training/motion_cli.py new file mode 100644 index 0000000..1cfea4e --- /dev/null +++ b/src/dvidia_training/motion_cli.py @@ -0,0 +1,75 @@ +"""Local image-motion analysis, causal forecasts, benchmark and replay.""" +from __future__ import annotations + +import argparse +from pathlib import Path + +from .footage_data import canonical + + +def _box(value): + try: + values = [float(v) for v in value.split(',')] + from .observations import box + return box(values) + except (ValueError, TypeError): + raise argparse.ArgumentTypeError('Use normalized x1,y1,x2,y2, for example 0.2,0.2,0.5,0.5.') from None + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + analyze = commands.add_parser('analyze', help='Track one manually seeded region in permitted local footage.') + analyze.add_argument('manifest', type=Path) + analyze.add_argument('--episode', required=True) + analyze.add_argument('--box', required=True, type=_box) + analyze.add_argument('--label', default='object') + analyze.add_argument('--sample-hz', type=float, default=30.0) + analyze.add_argument('--output', type=Path, required=True) + benchmark = commands.add_parser('benchmark', help='Run authored synthetic tracking/forecast fixtures, not detector accuracy.') + benchmark.add_argument('--output', type=Path, required=True) + inspect = commands.add_parser('inspect', help='Validate hashes, source media and causal rows.') + inspect.add_argument('directory', type=Path) + studio = commands.add_parser('studio', help='Read-only local video replay with forecast overlays.') + studio.add_argument('directory', type=Path) + studio.add_argument('--port', type=int, default=8286) + args = parser.parse_args(argv) + try: + if args.command == 'analyze': + from .motion_video import analyze_video + result = analyze_video(args.manifest, args.episode, args.output, + seed_box=args.box, label=args.label, sample_hz=args.sample_hz) + print(canonical({'status': result['status'], 'frames': len(result['rows']), + 'claim_scope': 'manually-seeded fixed-camera image motion', + 'output': str(args.output)}).decode()) + if result['status'] != 'completed': + return 1 + elif args.command == 'benchmark': + from .motion_benchmark import run_benchmark + report = run_benchmark(args.output) + print(canonical({'kind': report['kind'], 'cases': len(report['cases']), + 'scope': 'authored synthetic tracking and prediction fixtures', + 'output': str(args.output)}).decode()) + if any(c['status'] != 'completed' for c in report['cases']): + return 1 + elif args.command == 'inspect': + from .motion_studio import load_motion_bundle + report, episodes = load_motion_bundle(args.directory) + result = {'integrity': 'verified', 'independent_real_motion_accuracy': 'not_assessed', + 'metric_speed_available': False, 'robot_control_qualified': False, + 'episodes': [{'id': eid, 'status': e['run']['status'], + 'frames': len(e['run']['rows'])} for eid, e in episodes.items()], + 'scope': 'authored synthetic fixtures' if report else 'permitted local recording'} + print(canonical(result).decode()) + else: + from .motion_studio import serve + serve(args.directory, args.port) + except (ValueError, OSError, RuntimeError) as error: + # Provider bodies, credentials and local path traces are never receipts. + message = str(error) if isinstance(error, (ValueError, RuntimeError)) else 'Local evidence could not be accessed.' + parser.exit(2, f'Motion lab: {message}\n') + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/dvidia_training/motion_studio.html b/src/dvidia_training/motion_studio.html new file mode 100644 index 0000000..d7df521 --- /dev/null +++ b/src/dvidia_training/motion_studio.html @@ -0,0 +1,105 @@ + + + + +DVIDIA · Motion Lab + + + +
DVIDIA
MOTION LAB · v0.2 ALPHA
+
+
Observation → motion → prediction

See movement. Measure what comes next.

Replay a selected image region and compare short forecasts with what actually happened. Every prediction uses only observations available at that moment.

LOCAL REPLAY · CPU · READ ONLY
+ +
+
+
+
Loading evidence
+
+
+ +
+
Observed anchor & historyPredicted endpointUncalibrated envelope
+
Loading—
+
+
+ +
+ +
+ +
+ + diff --git a/src/dvidia_training/motion_studio.py b/src/dvidia_training/motion_studio.py new file mode 100644 index 0000000..2514018 --- /dev/null +++ b/src/dvidia_training/motion_studio.py @@ -0,0 +1,281 @@ +"""Read-only, loopback replay of hash-bound image-motion evidence.""" +from __future__ import annotations + +from contextlib import contextmanager +from copy import deepcopy +from fractions import Fraction +import hashlib +from http.server import ThreadingHTTPServer +import os +from pathlib import Path +import re +import threading +from urllib.parse import urlsplit + +from .footage_data import _json, _local_file, _read, canonical +from .motion import validate_motion_rows +from .observation_studio import ObservationHandler +from .observations import box, digest, identity + +MAX_JSON_BYTES = 16 * 1024 * 1024 + + +def _document(path): + path = Path(path) + if path.is_symlink() or not path.is_file(): + raise ValueError('Expected a local, regular evidence file.') + return _json(_read(path, MAX_JSON_BYTES), maximum=MAX_JSON_BYTES) + + +def _file_digest(path): + with Path(path).open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def _bound_file(root, name, expected): + path = _local_file(root, name) + if type(expected) is not str or not re.fullmatch('[a-f0-9]{64}', expected) or _file_digest(path) != expected: + raise ValueError('Evidence file differs from its recorded hash.') + return path + + +def load_motion_run(path): + """Validate declared files, causal rows and the copied source before replay. + + Hashes establish internal consistency, not source authenticity or independent + motion accuracy. Trusted local bundles remain the supported input scope. + """ + path = Path(path) + if path.is_symlink(): + raise ValueError('Use a regular motion receipt.') + path = path.resolve(strict=True) + run = _document(path) + root = path.parent + if type(run) is not dict or run.get('kind') != 'dvidia.motion-run.v1': + raise ValueError('Expected a dvidia.motion-run.v1 receipt.') + if run.get('status') not in ('completed', 'failed'): + raise ValueError('A motion receipt requires a completed or failed status.') + protocol = run.get('protocol') + rows, frames = run.get('rows'), run.get('frames') + if type(protocol) is not dict or type(rows) is not list or type(frames) is not list: + raise ValueError('Missing motion protocol, rows or timestamp evidence.') + if not 0 <= len(rows) <= 1800 or len(frames) != len(rows): + raise ValueError('Motion frame counts exceed the replay envelope.') + for filename, document, key in (('protocol.json', protocol, 'protocol_sha256'), + ('rows.json', rows, 'rows_sha256'), + ('frames.json', frames, 'frames_sha256')): + if run.get(key) != digest(document) or _document(_local_file(root, filename)) != document: + raise ValueError('Motion evidence bytes differ from the recorded receipt.') + size = protocol.get('analysis_image_size_pixels') + if size is None and not rows and run['status'] == 'failed': + pass # Intake may fail before the original image can be probed. + else: + if (type(size) is not list or len(size) != 2 or any(type(n) is not int or not 1 <= n <= 640 for n in size)): + raise ValueError('Unsupported analysis-image dimensions.') + validate_motion_rows(rows, *size) + seed = run.get('seed') + if (type(seed) is not dict or seed.get('kind') != 'human_authored_image_region' + or digest(seed) != run.get('seed_sha256') or digest(seed) != protocol.get('seed_sha256')): + raise ValueError('The human seed differs from the motion protocol.') + box(seed.get('normalized_xyxy')) + if type(seed.get('label')) is not str or not 1 <= len(seed['label']) <= 160: + raise ValueError('The human seed requires a bounded text label.') + if type(run.get('source')) is not dict: + raise ValueError('A motion run requires a source receipt.') + if run.get('episode_id') != protocol.get('episode_id'): + raise ValueError('Episode identity differs from its motion protocol.') + if (run.get('frame_count') != len(rows) or type(run.get('frame_count')) is not int + or run.get('tracked_frame_count') != sum(row['status'] == 'tracked' for row in rows)): + raise ValueError('Motion coverage counts differ from the retained evidence.') + if any(row['estimator'] != protocol.get('estimator') for row in rows): + raise ValueError('Motion estimator differs from the declared protocol.') + seed_evidence = run.get('seed_evidence') + if seed_evidence is not None: + if type(seed_evidence) is not dict: + raise ValueError('Malformed seed evidence.') + _bound_file(root, seed_evidence.get('path'), seed_evidence.get('sha256')) + for index, (frame, row) in enumerate(zip(frames, rows, strict=True)): + if (type(frame) is not dict or frame.get('processed_index') != index + or type(frame.get('processed_index')) is not int + or frame.get('source_pts_seconds') != row['acquired_time_seconds']): + raise ValueError('Motion rows do not match their source presentation times.') + pts, base = frame.get('source_pts_integer'), frame.get('source_time_base') + if (type(pts) is not int or type(base) is not list or len(base) != 2 + or any(type(n) is not int or n <= 0 for n in base) + or float(pts * Fraction(*base)) != frame['source_pts_seconds']): + raise ValueError('Motion timestamps differ from the retained integer presentation clock.') + media = run.get('media') + if media is not None: + if (type(media) is not dict or type(media.get('bytes')) is not int + or not 0 < media['bytes'] <= 128 * 1024 * 1024 + or type(media.get('sha256')) is not str + or not re.fullmatch('[a-f0-9]{64}', media['sha256'])): + raise ValueError('Invalid motion media receipt.') + video = _local_file(root, media.get('path')) + if video.stat().st_size != media['bytes']: + raise ValueError('Recorded motion media size changed.') + if _file_digest(video) != media['sha256']: + raise ValueError('Recorded motion media bytes changed.') + source = run.get('source') + if (type(source) is not dict or source.get('sha256') != media['sha256'] + or source.get('bytes') != media['bytes'] or protocol.get('source_sha256') != media['sha256']): + raise ValueError('Motion media no longer matches its declared original.') + elif run['status'] == 'completed': + raise ValueError('A completed motion run requires copied source media.') + if run['status'] == 'completed' and not rows: + raise ValueError('A completed motion run requires decoded evidence.') + return run + + +def load_motion_bundle(directory): + root = Path(directory).resolve(strict=True) + if not root.is_dir(): + raise ValueError('Use an analysis or benchmark directory.') + report_path, run_path = root / 'report.json', root / 'run.json' + report, entries = None, [] + if run_path.is_file(): + run = load_motion_run(run_path) + entries = [('episode', run_path, run, None)] + elif report_path.is_file(): + report = _document(report_path) + if type(report) is not dict or report.get('kind') != 'dvidia.motion-benchmark.v1': + raise ValueError('Unsupported motion benchmark report.') + cases = report.get('cases') + if type(cases) is not list or not 1 <= len(cases) <= 50: + raise ValueError('A motion benchmark requires bounded attempted cases.') + from .motion_benchmark import score_motion + protocol = _document(_bound_file(root, 'protocol.json', report.get('protocol_sha256'))) + if type(protocol) is not dict or protocol.get('kind') != 'dvidia.motion-benchmark-protocol.v1': + raise ValueError('Unsupported benchmark protocol.') + fixtures = protocol.get('fixtures') + if type(fixtures) is not list or len(fixtures) != len(cases): + raise ValueError('Benchmark attempts differ from the frozen fixture protocol.') + fixture_map = {identity(fixture.get('id')): fixture for fixture in fixtures if type(fixture) is dict} + if len(fixture_map) != len(cases): + raise ValueError('Malformed or duplicate benchmark fixtures.') + _bound_file(root, 'inputs/source.json', protocol.get('sources_manifest_sha256')) + seen = set() + for case in cases: + if type(case) is not dict: + raise ValueError('Malformed motion case.') + eid = identity(case.get('id')) + if eid in seen: + raise ValueError('Duplicate motion case identifier.') + seen.add(eid) + if eid not in fixture_map: + raise ValueError('Case differs from the frozen benchmark protocol.') + target = _bound_file(root, case.get('run_path'), case.get('run_sha256')) + run = load_motion_run(target) + if run['status'] != case.get('status'): + raise ValueError('Case status differs from its motion receipt.') + fixture = fixture_map[eid] + if (run.get('episode_id') != eid or run.get('source', {}).get('sha256') != fixture.get('video_sha256') + or run.get('seed', {}).get('normalized_xyxy') != fixture.get('seed_box_xyxy_normalized')): + raise ValueError('Benchmark run differs from its frozen source or seed.') + inputs = root / 'inputs' + _bound_file(inputs, fixture.get('video_path'), fixture.get('video_sha256')) + reference = _document(_bound_file(inputs, fixture.get('reference_path'), fixture.get('reference_sha256'))) + if (type(reference) is not dict or reference.get('kind') != 'dvidia.authored-motion-reference.v1' + or type(reference.get('rows')) is not list or not 1 <= len(reference['rows']) <= 1800): + raise ValueError('Unsupported or unbounded authored motion reference.') + scores = None + if run['status'] == 'completed': + scores = _document(_bound_file(root, case.get('scores_path'), case.get('scores_sha256'))) + if scores != case.get('scores') or score_motion(run['rows'], reference.get('rows'), + fps=protocol['fixture_recipe']['fps'], horizons=protocol['horizons_seconds']) != scores: + raise ValueError('Benchmark scores do not match the retained evaluator evidence.') + entries.append((eid, target, run, scores)) + completed = sum(entry[2]['status'] == 'completed' for entry in entries) + if (report.get('attempted') != len(entries) or report.get('completed') != completed + or report.get('failed') != len(entries)-completed): + raise ValueError('Benchmark completion counts differ from the retained attempts.') + else: + raise ValueError('No motion run or benchmark report found.') + episodes = {} + for eid, path, run, scores in entries: + episodes[eid] = {'run_path': path, 'run': run, 'scores': scores} + return report, episodes + + +class MotionServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, address, directory): + if address[0] != '127.0.0.1': + raise ValueError('The motion lab binds only to 127.0.0.1.') + self.report, self.episodes = load_motion_bundle(directory) + self.lock = threading.RLock() + self.media_signatures = {} + super().__init__(address, MotionHandler) + + @contextmanager + def media(self, eid): + entry = self.episodes[eid] + media = entry['run']['media'] + path = _local_file(entry['run_path'].parent, media['path']) + with path.open('rb') as handle: + def signature(): + s = os.fstat(handle.fileno()) + return s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns + before = signature() + if before[2] != media['bytes']: + raise ValueError('Motion source media changed.') + with self.lock: + if self.media_signatures.get(eid) != before: + h = hashlib.file_digest(handle, 'sha256').hexdigest() + if signature() != before or h != media['sha256']: + raise ValueError('Motion source media changed.') + self.media_signatures[eid] = before + handle.seek(0) + yield path, handle, before[2] + + +class MotionHandler(ObservationHandler): + server_version = 'DvidiaMotion/0.2-alpha' + + def do_GET(self): + if not self.origin_ok(): + return self.send(403, {'error': 'Use the motion lab from its loopback address.'}) + path = urlsplit(self.path).path + try: + if path == '/': + return self.send(200, Path(__file__).with_name('motion_studio.html').read_bytes(), 'text/html; charset=utf-8') + if path == '/api/episodes': + episodes = [] + for eid, entry in self.server.episodes.items(): + run = entry['run'] + episodes.append({'id': eid, 'title': eid.replace('-', ' ').replace('_', ' '), + 'status': run['status'], 'frame_count': len(run['rows'])}) + return self.send(200, {'episodes': episodes, 'benchmark': self.server.report}) + match = re.fullmatch(r'/api/episodes/([A-Za-z0-9_-]+)(/video)?', path) + if match and match[1] in self.server.episodes: + eid = match[1] + entry = self.server.episodes[eid] + if match[2]: + if entry['run'].get('media') is None: + return self.send(404, {'error': 'No media retained for this failed attempt.'}) + with self.server.media(eid) as (video, handle, size): + return self.stream_video(video, handle, size) + return self.send(200, {'run': deepcopy(entry['run']), 'scores': entry['scores'], + 'video_url': f'/api/episodes/{eid}/video'}) + return self.send(404, {'error': 'No such motion resource.'}) + except (BrokenPipeError, ConnectionResetError): + return + except (ValueError, OSError, KeyError, TypeError): + return self.send(409, {'error': 'Motion evidence changed or is inconsistent. Inspect the bundle.'}) + + def do_POST(self): + return self.send(405, {'error': 'Motion replay is read-only.'}, extra={'Allow': 'GET, HEAD'}) + + +def serve(directory, port=8286): + if type(port) is not int or not 1024 <= port <= 65535: + raise ValueError('Use an unprivileged loopback port.') + server = MotionServer(('127.0.0.1', port), directory) + print(f'Motion lab: http://127.0.0.1:{port}', flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() diff --git a/src/dvidia_training/motion_video.py b/src/dvidia_training/motion_video.py new file mode 100644 index 0000000..bd8b6cb --- /dev/null +++ b/src/dvidia_training/motion_video.py @@ -0,0 +1,469 @@ +"""Causal, CPU-only image motion from one manually seeded local video region. + +The tracked quantity is an image anchor: the seed-box center translated by the +robust displacement of its initial features. It is not a physical centroid, +object detector, camera-motion compensation or robot control signal. +""" +from __future__ import annotations + +from datetime import datetime, timezone +from fractions import Fraction +import math +import os +from pathlib import Path +import platform +import queue +import re +import shutil +import signal +import subprocess +import threading +import time + +from .footage_data import _local_file, canonical, probe_video +from .observation_pipeline import validate_manifest, validate_timebase +from .observations import box, digest, file_hash, identity, number, text + + +MAX_IMAGE_SIDE = 640 +MAX_DECODE_SECONDS = 120 +MAX_STDERR_BYTES = 2 * 1024 * 1024 +MAX_STDERR_LINE_BYTES = 8192 +MAX_TIMESTAMP_QUEUE = 16 +MAX_TRACK_GAP_SECONDS = 0.25 +MIN_FEATURES = 6 +FEATURE_RECIPE = { + 'maximum_initial_features': 100, 'quality_level': 0.01, + 'minimum_feature_distance_pixels': 5, 'feature_block_size': 7, + 'lk_window_pixels': 21, 'lk_pyramid_levels': 3, + 'maximum_forward_backward_error_pixels': 1.5, + 'minimum_features': MIN_FEATURES, 'minimum_valid_fraction': 0.5, + 'minimum_consensus_fraction': 0.7, 'minimum_consensus_radius_pixels': 2.0, + 'maximum_consensus_radius_pixels': 4.0, + 'maximum_track_gap_seconds': MAX_TRACK_GAP_SECONDS, + 'maximum_step_pixels': 'max(32, 0.10 * maximum analysis image side)', + 'lost_track_policy': 'terminal; no replenishment, reacquisition or interpolation', +} +LIMITATIONS = [ + 'Fixed-camera scope; camera movement is not compensated or detected.', + 'Human-authored seed and label; no automatic semantic object detection.', + 'Image anchor from initial feature translation, not a physical object centroid.', + 'Units are analysis-image pixels and pixels/second, not metric 3D motion.', + 'Sparse optical flow can drift, fail under deformation or follow background features.', + 'Reported tracking uncertainty is an uncalibrated heuristic, not a probability guarantee.', + 'Lost tracks remain lost; no predictions bridge occlusion or large timestamp gaps.', + 'Offline causal replay; inference uses no later frames, but live robot latency is unmeasured.', + 'Local-analysis permission does not confer publication or redistribution rights.', + 'Resource settings are requests and input bounds; peak RAM/thread usage is unmeasured.', + 'No physical interaction, actuator command, depth or robot safety qualification.', +] +_PTS = re.compile(rb'\bn:\s*(\d+)\s+pts:\s*(-?\d+)\s+pts_time:([-+0-9.eE]+)') +_TIMEBASE = re.compile(rb'\bconfig in time_base:\s*(\d+)/(\d+)') + + +class MotionVideoError(ValueError): + """A bounded, user-readable motion intake or decoding error.""" + + +def _dimensions(width, height): + scale = min(1.0, MAX_IMAGE_SIDE / max(width, height)) + return max(1, int(width * scale)), max(1, int(height * scale)) + + +def _stop_process(process): + """Stop only the child process group created for this invocation.""" + try: + if os.name == 'posix': + os.killpg(process.pid, signal.SIGKILL) + elif process.poll() is None: + process.kill() + except ProcessLookupError: + pass + except PermissionError: + # Some hosted sandboxes forbid group signals but allow terminating an + # owned direct child. Never signal an unrelated group or process. + if process.poll() is None: + process.kill() + + +def _decoded_frames(video, width, height, *, sample_hz, max_frames, duration, + timeout_seconds=MAX_DECODE_SECONDS): + """Stream bounded raw frames with their selected *input* presentation times. + + FFmpeg showinfo runs before rawvideo output. A draining stderr thread binds + one metadata row to each raw frame; a watchdog stops blocked decoding. PTS + are never reconstructed from nominal frame rate or OpenCV position values. + """ + interval = 1.0 / sample_hz + filters = ("select='isnan(prev_selected_t)+gte(t-prev_selected_t," + f"{interval:.12f}-0.000000001)'," + f"trim=end_frame={max_frames},scale={width}:{height},setsar=1,showinfo") + command = ['ffmpeg', '-hide_banner', '-nostdin', '-nostats', '-loglevel', 'info', + '-xerror', '-threads', '1', '-filter_threads', '1', + '-protocol_whitelist', 'file', '-format_whitelist', 'mov,matroska,webm,avi', + '-copyts', '-noautorotate', '-i', str(video), '-map', '0:v:0', + '-an', '-sn', '-dn', '-t', str(duration), '-vf', filters, + '-frames:v', str(max_frames), '-fps_mode', 'vfr', + '-threads', '1', '-pix_fmt', 'bgr24', '-f', 'rawvideo', 'pipe:1'] + try: + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + start_new_session=(os.name == 'posix')) + except OSError as exc: + raise MotionVideoError('FFmpeg is unavailable for bounded local decoding.') from exc + metadata = queue.Queue(maxsize=MAX_TIMESTAMP_QUEUE) + errors, ended, timed_out = [], threading.Event(), threading.Event() + stderr_bytes = [0] + + def consume_metadata(): + source_timebase = None + try: + while line := process.stderr.readline(MAX_STDERR_LINE_BYTES + 1): + stderr_bytes[0] += len(line) + if len(line) > MAX_STDERR_LINE_BYTES or stderr_bytes[0] > MAX_STDERR_BYTES: + errors.append('Decoder diagnostics exceeded the bounded metadata budget.') + _stop_process(process) + return + clock = _TIMEBASE.search(line) + if clock: + numerator, denominator = int(clock.group(1)), int(clock.group(2)) + if numerator <= 0 or denominator <= 0: + errors.append('Decoder time base must be a positive rational clock.') + _stop_process(process) + return + clock_value = Fraction(numerator, denominator) + if source_timebase is not None and source_timebase != clock_value: + errors.append('Decoder presentation time base changed within the selected stream.') + _stop_process(process) + return + source_timebase = clock_value + match = _PTS.search(line) + if match: + if source_timebase is None: + errors.append('Decoder presented a frame before declaring its rational time base.') + _stop_process(process) + return + # pts_time text is rounded by FFmpeg; preserve the actual + # integer PTS clock rather than that formatted approximation. + integer_pts = int(match.group(2)) + stamp = float(integer_pts * source_timebase) + if not math.isfinite(stamp): + errors.append('Decoder reported a nonfinite presentation timestamp.') + _stop_process(process) + return + try: + metadata.put((int(match.group(1)), stamp, integer_pts, + [source_timebase.numerator, source_timebase.denominator]), timeout=1) + except queue.Full: + errors.append('Decoder presentation timestamps exceeded the bounded queue.') + _stop_process(process) + return + except (OSError, ValueError): + errors.append('Decoder metadata could not be read.') + _stop_process(process) + finally: + ended.set() + + def expire(): + timed_out.set() + _stop_process(process) + + reader = threading.Thread(target=consume_metadata, daemon=True, + name='dvidia-motion-timestamps') + watchdog = threading.Timer(timeout_seconds, expire) + watchdog.daemon = True + reader.start() + watchdog.start() + frame_bytes, count, last_stamp = width * height * 3, 0, None + try: + while count < max_frames: + raw = bytearray() + while len(raw) < frame_bytes: + block = process.stdout.read(frame_bytes - len(raw)) + if not block: + break + raw.extend(block) + if timed_out.is_set(): + raise MotionVideoError('Local video decoding exceeded its 120-second watchdog.') + if errors: + raise MotionVideoError(errors[0]) + if not raw: + break + if len(raw) != frame_bytes: + raise MotionVideoError('Decoder returned an incomplete raw video frame.') + try: + index, stamp, integer_pts, source_timebase = metadata.get(timeout=2) + except queue.Empty as exc: + raise MotionVideoError('A decoded frame has no bounded presentation timestamp.') from exc + if (index != count or not 0 <= stamp <= duration + 0.05 + or (count == 0 and stamp != 0) + or (last_stamp is not None and stamp <= last_stamp)): + raise MotionVideoError('Selected input timestamps must begin at zero and increase strictly.') + yield count, stamp, bytes(raw), {'source_pts_integer': integer_pts, + 'source_time_base': source_timebase} + count += 1 + last_stamp = stamp + try: + returncode = process.wait(timeout=5) + except subprocess.TimeoutExpired as exc: + raise MotionVideoError('Local decoder did not exit after its bounded frame stream.') from exc + reader.join(timeout=2) + if timed_out.is_set(): + raise MotionVideoError('Local video decoding exceeded its 120-second watchdog.') + if errors: + raise MotionVideoError(errors[0]) + if returncode or not ended.is_set(): + raise MotionVideoError('Local video decoding failed; no external media was fetched.') + if count == 0 or not metadata.empty(): + raise MotionVideoError('Decoded frame and presentation timestamp counts do not match.') + finally: + watchdog.cancel() + _stop_process(process) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + reader.join(timeout=2) + process.stdout.close() + process.stderr.close() + + +class _FeatureAnchor: + """Initial-region LK translation, with explicit terminal track loss.""" + + def __init__(self, cv2, np, width, height, seed_box): + self.cv2, self.np = cv2, np + self.width, self.height = width, height + self.seed_box = seed_box + self.center = np.array([(seed_box[0] + seed_box[2]) * width / 2, + (seed_box[1] + seed_box[3]) * height / 2], dtype='float64') + self.previous_gray, self.points, self.previous_time = None, None, None + self.lost_reason = None + self.uncertainty = 1.0 + + def _lost(self, reason, *, valid=0, inliers=0): + self.lost_reason = reason + self.points = None + return None, 0, None, {'state': 'lost', 'reason': reason, + 'valid_feature_count': valid, 'inlier_feature_count': inliers} + + def update(self, stamp, image): + cv2, np = self.cv2, self.np + if self.lost_reason: + return self._lost(self.lost_reason) + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + if self.previous_gray is None: + mask = np.zeros((self.height, self.width), dtype='uint8') + x1, y1 = int(math.ceil(self.seed_box[0] * self.width)), int(math.ceil(self.seed_box[1] * self.height)) + x2, y2 = int(math.floor(self.seed_box[2] * self.width)), int(math.floor(self.seed_box[3] * self.height)) + if x2 <= x1 or y2 <= y1: + return self._lost('seed_region_has_no_analysis_pixels') + mask[y1:y2, x1:x2] = 255 + self.points = cv2.goodFeaturesToTrack(gray, maxCorners=100, qualityLevel=0.01, + minDistance=5, blockSize=7, mask=mask) + if self.points is None or len(self.points) < MIN_FEATURES: + return self._lost('insufficient_initial_features', + valid=0 if self.points is None else len(self.points)) + self.previous_gray, self.previous_time = gray, stamp + support = len(self.points) + return self.center.tolist(), support, self.uncertainty, { + 'state': 'seeded', 'reason': None, 'valid_feature_count': support, + 'inlier_feature_count': support, 'forward_backward_median_pixels': None, + 'displacement_dispersion_pixels': None} + if stamp - self.previous_time > MAX_TRACK_GAP_SECONDS: + return self._lost('timestamp_gap_exceeds_tracking_envelope') + options = {'winSize': (21, 21), 'maxLevel': 3, + 'criteria': (cv2.TERM_CRITERIA_EPS | cv2.TERM_CRITERIA_COUNT, 30, 0.01)} + forward, ok_forward, _ = cv2.calcOpticalFlowPyrLK(self.previous_gray, gray, self.points, None, **options) + if forward is None or ok_forward is None: + return self._lost('forward_flow_unavailable') + backward, ok_backward, _ = cv2.calcOpticalFlowPyrLK(gray, self.previous_gray, forward, None, **options) + if backward is None or ok_backward is None: + return self._lost('backward_flow_unavailable') + old, new, back = self.points.reshape(-1, 2), forward.reshape(-1, 2), backward.reshape(-1, 2) + fb = np.linalg.norm(old - back, axis=1) + valid = ((ok_forward.reshape(-1) == 1) & (ok_backward.reshape(-1) == 1) + & np.isfinite(new).all(axis=1) & np.isfinite(back).all(axis=1) + & (fb <= 1.5) & (new[:, 0] >= 0) & (new[:, 0] < self.width) + & (new[:, 1] >= 0) & (new[:, 1] < self.height)) + valid_count = int(valid.sum()) + if valid_count < MIN_FEATURES or valid_count < 0.5 * len(old): + return self._lost('insufficient_forward_backward_support', valid=valid_count) + displacement = new[valid] - old[valid] + median = np.median(displacement, axis=0) + residual = np.linalg.norm(displacement - median, axis=1) + median_residual = float(np.median(residual)) + inlier = residual <= max(2.0, min(4.0, 3.0 * median_residual)) + inlier_count = int(inlier.sum()) + if inlier_count < MIN_FEATURES or inlier_count < 0.7 * valid_count: + return self._lost('ambiguous_feature_motion', valid=valid_count, inliers=inlier_count) + shift = np.median(displacement[inlier], axis=0) + if float(np.linalg.norm(shift)) > max(32, 0.1 * max(self.width, self.height)): + return self._lost('feature_step_exceeds_tracking_envelope', valid=valid_count, + inliers=inlier_count) + center = self.center + shift + if not (0 <= center[0] < self.width and 0 <= center[1] < self.height): + return self._lost('image_anchor_left_frame', valid=valid_count, inliers=inlier_count) + fb_median = float(np.median(fb[valid][inlier])) + dispersion = float(np.median(np.linalg.norm(displacement[inlier] - shift, axis=1))) + self.uncertainty += fb_median + dispersion / math.sqrt(inlier_count) + self.center = center + self.points = new[valid][inlier].astype('float32').reshape(-1, 1, 2) + self.previous_gray, self.previous_time = gray, stamp + return center.tolist(), inlier_count, self.uncertainty, { + 'state': 'tracked', 'reason': None, 'valid_feature_count': valid_count, + 'inlier_feature_count': inlier_count, 'forward_backward_median_pixels': fb_median, + 'displacement_dispersion_pixels': dispersion} + + +def analyze_video(manifest_path, episode_id, output, *, seed_box, label='object', + sample_hz=30.0, max_duration_seconds=60, max_frames=1800): + """Analyze one explicitly permitted source; preserve a receipt on attempt failure. + + Invalid arguments/manifests and an existing output are rejected before any + output is created. Once intake begins, processing failures return a failed + receipt with partial rows; no failed attempt is overwritten or hidden. + """ + started = time.monotonic() + episode_id = identity(episode_id) + seed_box = list(box(seed_box)) + label = text(label, 'Seed label', 160) + sample_hz = number(sample_hz, 1, 30, 'Sampling rate') + max_duration_seconds = number(max_duration_seconds, 0.01, 60, 'Maximum duration') + if type(max_frames) is not int or not 1 <= max_frames <= 1800: + raise ValueError('Maximum frames must be an integer between one and 1800.') + root, manifest = validate_manifest(manifest_path) + sources = [row for row in manifest['episodes'] if row['id'] == episode_id] + if len(sources) != 1: + raise ValueError('Select exactly one episode id present in the source manifest.') + source_row = sources[0] + source_path = _local_file(root, source_row['path']) + output = Path(output).resolve() + if output.exists(): + raise ValueError('Choose a new run directory; previous motion evidence is never overwritten.') + output.mkdir(parents=True, exist_ok=False) + seed = {'kind': 'human_authored_image_region', 'normalized_xyxy': seed_box, + 'label': label, 'anchor_semantics': 'initial_seed_center_plus_robust_feature_translation'} + protocol = {'kind': 'dvidia.motion-protocol.v1', 'created_at': datetime.now(timezone.utc).isoformat(), + 'algorithm': 'manual_seed_sparse_lk_translation', 'manifest_sha256': digest(manifest), + 'episode_id': episode_id, 'seed_sha256': digest(seed), + 'execution': 'local_offline_causal_replay', 'geometry': 'analysis_image_coordinates_only', + 'sample_hz_requested': sample_hz, 'max_duration_seconds': max_duration_seconds, + 'max_frames': max_frames, 'max_image_side': MAX_IMAGE_SIDE, + 'decoder_watchdog_seconds': MAX_DECODE_SECONDS, + 'ffmpeg_threads_requested': 1, 'ffmpeg_filter_threads_requested': 1, + 'opencv_threads_requested': 1, 'worker_count': 1, + 'tracking': FEATURE_RECIPE, + 'estimator': {'horizons_seconds': [0.1, 0.25, 0.5], 'max_gap_seconds': 0.15, + 'max_prediction_age_seconds': 0.15}, + 'cadence_note': 'Velocity needs two actual observations no more than 0.15 seconds apart; low source/sample cadence can prevent every forecast.', + 'processing_budget': {'seconds': 150, 'enforcement': 'cooperative check between processed frames; native tracking call is not preempted'}, + 'timebase': 'FFmpeg showinfo integer input PTS multiplied by declared rational time base; zero origin required', + 'video_rotation': 'automatic rotation disabled; seed coordinates use decoded source raster', + 'hardware': {'platform': platform.system(), 'machine': platform.machine()}, + 'implementation_sha256': {'motion_video.py': file_hash(Path(__file__)), + 'observation_pipeline.py': file_hash(Path(__file__).with_name('observation_pipeline.py'))}, + 'limitations': LIMITATIONS} + source = {key: value for key, value in source_row.items() if key != 'path'} + rows, frames, failures, media, seed_evidence = [], [], [], None, None + timings = {'manifest_validation_seconds': time.monotonic() - started, + 'intake_seconds': 0.0, 'dependency_setup_seconds': 0.0, + 'decode_wait_seconds': 0.0, 'tracking_estimation_seconds': 0.0, + 'seed_evidence_write_seconds': 0.0} + stage = 'intake' + (output / 'protocol.json').write_bytes(canonical(protocol)) + try: + stage_started = time.monotonic() + actual_hash = file_hash(source_path) + if actual_hash != source_row['sha256']: + raise MotionVideoError('Source hash differs from the frozen source manifest.') + info = probe_video(source_path) + if info['duration_seconds'] > max_duration_seconds or max(info['width'], info['height']) > 4096: + raise MotionVideoError('Source exceeds the selected duration or 4096-pixel source-side limit.') + validate_timebase(source_path, info) + target = output / ('original' + source_path.suffix.lower()) + shutil.copyfile(source_path, target) + if file_hash(target) != actual_hash: + raise MotionVideoError('Original changed during local intake.') + width, height = _dimensions(info['width'], info['height']) + source.update(bytes=target.stat().st_size, timing=info) + media = {'path': target.name, 'sha256': actual_hash, 'bytes': target.stat().st_size} + protocol.update(original_image_size_pixels=[info['width'], info['height']], + analysis_image_size_pixels=[width, height], + source_to_analysis_scale_xy=[width / info['width'], height / info['height']], + source_sha256=actual_hash) + timings['intake_seconds'] = time.monotonic() - stage_started + stage = 'dependency_setup' + stage_started = time.monotonic() + try: + import cv2 + import numpy as np + except ImportError as exc: + raise MotionVideoError('Install the observations extra to enable CPU optical flow.') from exc + from .motion import MotionEstimator + cv2.setNumThreads(1) + protocol.update(opencv_threads_reported=cv2.getNumThreads(), + software={'opencv': cv2.__version__, 'numpy': np.__version__}) + motion_path = Path(__file__).with_name('motion.py') + protocol['implementation_sha256']['motion.py'] = file_hash(motion_path) + estimator = MotionEstimator(width, height, **protocol['estimator']) + tracker = _FeatureAnchor(cv2, np, width, height, seed_box) + timings['dependency_setup_seconds'] = time.monotonic() - stage_started + (output / 'protocol.json').write_bytes(canonical(protocol)) + stage = 'decode_and_track' + stream = _decoded_frames(target, width, height, sample_hz=sample_hz, + max_frames=max_frames, duration=info['duration_seconds']) + try: + while True: + decode_started = time.monotonic() + try: + index, stamp, raw, clock_evidence = next(stream) + except StopIteration: + timings['decode_wait_seconds'] += time.monotonic() - decode_started + break + timings['decode_wait_seconds'] += time.monotonic() - decode_started + stage_started = time.monotonic() + image = np.frombuffer(raw, dtype='uint8').reshape(height, width, 3) + center, support, uncertainty, tracking = tracker.update(stamp, image) + row = estimator.update(stamp, center, support_count=support, + status='tracked' if center is not None else 'lost', + uncertainty_pixels=uncertainty) + rows.append(row) + frames.append({'processed_index': index, 'source_pts_seconds': stamp, + **clock_evidence, 'tracking': tracking}) + timings['tracking_estimation_seconds'] += time.monotonic() - stage_started + if index == 0: + evidence_started = time.monotonic() + if not cv2.imwrite(str(output / 'seed.png'), image): + raise MotionVideoError('Could not retain the first decoded seed evidence image.') + seed_evidence = {'path': 'seed.png', 'sha256': file_hash(output / 'seed.png')} + timings['seed_evidence_write_seconds'] += time.monotonic() - evidence_started + if time.monotonic() - started > 150: + raise MotionVideoError('Local motion replay exceeded its 150-second processing budget.') + finally: + stream.close() + except Exception as exc: + message = str(exc) if isinstance(exc, MotionVideoError) else f'Local {stage} failed ({type(exc).__name__}).' + failures.append({'stage': stage, 'error': message[:300]}) + (output / 'protocol.json').write_bytes(canonical(protocol)) + (output / 'rows.json').write_bytes(canonical(rows)) + (output / 'frames.json').write_bytes(canonical(frames)) + timings['elapsed_seconds'] = time.monotonic() - started + receipt = {'kind': 'dvidia.motion-run.v1', 'status': 'failed' if failures else 'completed', + 'episode_id': episode_id, 'task': manifest['task'], 'source': source, + 'seed': seed, 'seed_sha256': digest(seed), 'seed_evidence': seed_evidence, + 'protocol': protocol, 'protocol_sha256': digest(protocol), + 'rows': rows, 'rows_sha256': digest(rows), 'media': media, + 'frames': frames, 'frames_sha256': digest(frames), + 'timings': timings, 'errors': failures, 'limitations': LIMITATIONS, + 'frame_count': len(rows), 'tracked_frame_count': sum(row['status'] == 'tracked' for row in rows), + 'coverage': {'frame_limit_reached': len(rows) >= max_frames, + 'last_observed_source_pts_seconds': frames[-1]['source_pts_seconds'] if frames else None, + 'full_source_coverage_status': 'not_assessed', + 'note': 'Only selected frames were analyzed; reaching the frame cap may leave the remainder unobserved.'}, + 'accuracy_status': 'not_assessed', 'real_time_status': 'not_assessed', + 'robot_skill_qualified': False, + 'timing_scope': 'manifest validation, intake, dependency setup, streaming decode waits, tracking and evidence writes; excludes final run.json write', + 'timing_note': 'Decode and tracking overlap; decode_wait_seconds measures consumer wait, not total decoder compute.'} + (output / 'run.json').write_bytes(canonical(receipt)) + return receipt diff --git a/tests/browser/motion_replay.cjs b/tests/browser/motion_replay.cjs new file mode 100644 index 0000000..385415a --- /dev/null +++ b/tests/browser/motion_replay.cjs @@ -0,0 +1,42 @@ +/* Read-only browser acceptance on the authored motion benchmark, no private footage. */ +const browserTypes=require(process.env.PLAYWRIGHT_MODULE||'playwright'); +const assert=require('node:assert/strict'),fs=require('node:fs'),path=require('node:path'); +const base=process.env.MOTION_URL||'http://127.0.0.1:8286'; +const out=path.resolve(process.env.MOTION_OUTPUT||'runs/motion-ui-20261010');fs.mkdirSync(out,{recursive:true}); +(async()=>{ + // DOM assertions need Playwright's injected evaluator. This affects only the + // isolated test browser; the server's production CSP remains unchanged. + const browser=await browserTypes[process.env.PLAYWRIGHT_BROWSER||'chromium'].launch({headless:true,...(process.env.PLAYWRIGHT_CHANNEL?{channel:process.env.PLAYWRIGHT_CHANNEL}:{}),...(process.env.PLAYWRIGHT_EXECUTABLE_PATH?{executablePath:process.env.PLAYWRIGHT_EXECUTABLE_PATH}:{})}),context=await browser.newContext({viewport:{width:1440,height:1000},bypassCSP:true}),page=await context.newPage(); + const errors=[],checks=[];page.on('pageerror',error=>errors.push(error.message)); + async function seek(t){await page.locator('#timeline').fill(String(t));await page.waitForFunction(t=>Math.abs(document.querySelector('#video').currentTime-t)<.002,t);} + async function select(id){await page.locator('#episode').selectOption(id);await page.waitForFunction(id=>document.querySelector('#video').src.endsWith('/'+id+'/video')&&document.querySelector('#video').readyState>=1,id);} + try{ + await page.goto(base);await page.waitForFunction(()=>document.querySelector('#video').readyState>=1&&document.querySelector('#score-rows').children.length===3); + assert.equal(await page.locator('#episode').inputValue(),'constant_velocity');assert.equal(await page.locator('#error').isVisible(),false); + await seek(.02);assert.match(await page.locator('#state').textContent(),/warming up/);assert.equal(await page.locator('#speed').textContent(),'—'); + await seek(.051);const first=(await(await context.request.get(base+'/api/episodes/constant_velocity')).json()).run; + const row=first.rows.filter(r=>r.acquired_time_seconds<=.051).at(-1);assert.equal(await page.locator('#speed').textContent(),row.speed_pixels_per_second.toFixed(1));assert.match(await page.locator('#age').textContent(),/17\.7 ms/); + checks.push('Playhead uses only the latest past observation; no velocity appears before two actual samples'); + await page.getByRole('button',{name:'500 ms',exact:true}).click();assert.equal(await page.getByRole('button',{name:'500 ms',exact:true}).getAttribute('aria-pressed'),'true'); + await page.locator('#envelope').uncheck();assert.equal(await page.locator('#envelope').isChecked(),false);await page.locator('#envelope').check(); + assert.match(await page.locator('#score-rows').textContent(),/500 ms/);assert.match(await page.locator('#dimensions').textContent(),/640 × 360/); + checks.push('Forecast horizon and heuristic envelope controls work; dimensions and image units are explicit'); + const downloadPromise=page.waitForEvent('download');await page.getByRole('button',{name:'Export run receipt'}).click();const download=await downloadPromise;await download.saveAs(path.join(out,'receipt-export.json'));assert.deepEqual(JSON.parse(fs.readFileSync(path.join(out,'receipt-export.json'),'utf8')),first); + checks.push('Downloaded receipt is the exact served evidence rather than UI-generated predictions'); + await select('brief_occlusion');await seek(2.5);assert.match(await page.locator('#state').textContent(),/Tracking lost/);assert.equal(await page.locator('#speed').textContent(),'—');assert.equal(await page.locator('#anchor').textContent(),'—'); + await select('low_texture');await seek(.5);assert.match(await page.locator('#state').textContent(),/Tracking lost/);assert.match(await page.locator('#score-rows').textContent(),/0 \/ 117/); + checks.push('Occlusion and low texture visibly abstain instead of displaying invented motion'); + await select('constant_velocity');await seek(1.2);await page.screenshot({path:path.join(out,'desktop.png'),fullPage:true}); + for(const size of [{width:390,height:844},{width:320,height:740},{width:844,height:390}]){ + await page.setViewportSize(size);assert.equal(await page.evaluate(()=>document.documentElement.scrollWidth<=innerWidth),true); + const undersized=await page.locator('button:visible,select:visible').evaluateAll(nodes=>nodes.filter(n=>n.getBoundingClientRect().height<43.5).map(n=>n.id));assert.deepEqual(undersized,[]); + await page.screenshot({path:path.join(out,`${size.width}x${size.height}.png`),fullPage:true}); + } + checks.push('Desktop, 390px, 320px and landscape layouts have no page overflow and retain 44px controls'); + // Deliberately remove later API observations in a separate UI-only replay to + // exercise expiration without changing any served source evidence on disk. + await page.route('**/api/episodes/constant_velocity',async route=>{const data=await(await route.fetch()).json();data.run.rows=data.run.rows.slice(0,2);await route.fulfill({json:data})}); + await select('stationary');await select('constant_velocity');await seek(.4);assert.match(await page.locator('#state').textContent(),/expired/);assert.equal(await page.locator('#speed').textContent(),'—');checks.push('An expired observation withholds forecasts at later playhead times'); + assert.deepEqual(errors,[]);fs.writeFileSync(path.join(out,'acceptance.json'),JSON.stringify({scope:'Authored synthetic replay workflow checks; not model accuracy or live latency',checks,errors},null,2)+'\n');console.log(JSON.stringify({checks,errors},null,2)); + }finally{await browser.close()} +})().catch(error=>{console.error(error);process.exitCode=1}); diff --git a/tests/test_motion.py b/tests/test_motion.py new file mode 100644 index 0000000..3bc2b76 --- /dev/null +++ b/tests/test_motion.py @@ -0,0 +1,227 @@ +"""Causal image-motion contract, including variable frame times and loss resets.""" +from copy import deepcopy +import json +import math +import unittest + +from dvidia_training.motion import MotionEstimator, validate_motion_rows + + +class MotionEstimatorTest(unittest.TestCase): + def test_two_actual_observations_use_variable_acquisition_times(self): + estimator = MotionEstimator(640, 480) + first = estimator.update(0, [100, 80], support_count=12, uncertainty_pixels=1) + second = estimator.update(.04, [104, 82], support_count=11, uncertainty_pixels=2) + third = estimator.update(.13, [113, 86.5], support_count=10, uncertainty_pixels=2) + self.assertIsNone(first['velocity_xy_pixels_per_second']) + self.assertEqual(first['forecasts'], []) + for row in (second, third): + self.assertAlmostEqual(row['velocity_xy_pixels_per_second'][0], 100) + self.assertAlmostEqual(row['velocity_xy_pixels_per_second'][1], 50) + self.assertAlmostEqual(row['speed_pixels_per_second'], math.hypot(100, 50)) + for forecast in row['forecasts']: + h = forecast['horizon_seconds'] + self.assertAlmostEqual(forecast['target_time_seconds'], row['acquired_time_seconds']+h) + self.assertAlmostEqual(forecast['center_xy_pixels'][0], row['observed_anchor_xy_pixels'][0]+100*h) + self.assertAlmostEqual(forecast['center_xy_pixels'][1], row['observed_anchor_xy_pixels'][1]+50*h) + self.assertEqual(third['velocity_evidence']['start_time_seconds'], .04) + self.assertEqual(third['velocity_evidence']['end_time_seconds'], .13) + self.assertEqual(first['observed_anchor_xy_pixels'], first['estimated_center_xy_pixels']) + self.assertEqual(second['units']['velocity'], 'image_pixels_per_second') + self.assertIn('not-physical-centroid', second['anchor_semantics']) + + def test_missing_and_explicit_loss_clear_continuity(self): + estimator = MotionEstimator(100, 100) + rows = [estimator.update(0, [20, 20]), estimator.update(.05, [21, 20]), + estimator.update(.07, None), estimator.update(.1, [24, 20]), + estimator.update(.12, [25, 20]), estimator.update(.13, None, status='lost'), + estimator.update(.15, [27, 20])] + for index in (2, 5): + self.assertEqual(rows[index]['status'], 'lost') + for key in ('observed_anchor_xy_pixels', 'estimated_center_xy_pixels', + 'velocity_xy_pixels_per_second', 'speed_pixels_per_second', + 'prediction_use_before_time_seconds'): + self.assertIsNone(rows[index][key]) + self.assertEqual(rows[index]['forecasts'], []) + for index in (3, 6): + self.assertEqual(rows[index]['continuity'], 'loss_reset') + self.assertIsNone(rows[index]['velocity_xy_pixels_per_second']) + self.assertEqual(rows[index]['forecasts'], []) + self.assertIsNotNone(rows[4]['velocity_xy_pixels_per_second']) + self.assertEqual(rows[2]['loss_reason'], 'missing_observation') + self.assertEqual(rows[5]['loss_reason'], 'tracking_lost') + self.assertIs(validate_motion_rows(rows, 100, 100), rows) + + def test_gap_reset_needs_two_fresh_observations(self): + estimator = MotionEstimator(100, 100, max_gap_seconds=.15) + estimator.update(0, [10, 10]) + second = estimator.update(.15, [11, 10]) + self.assertEqual(second['continuity'], 'continued') + third = estimator.update(.301, [50, 30]) + self.assertEqual(third['continuity'], 'gap_reset') + self.assertIsNone(third['velocity_xy_pixels_per_second']) + fourth = estimator.update(.35, [51, 30]) + self.assertEqual(fourth['continuity'], 'continued') + self.assertEqual(fourth['velocity_evidence']['start_time_seconds'], .301) + + def test_future_observations_cannot_change_any_prefix(self): + samples = [(0, [10, 20]), (.03, [12, 22]), (.08, [13, 24]), + (.1, None), (.12, [15, 25]), (.14, [16, 26])] + def replay(sequence): + estimator = MotionEstimator(100, 100) + return [estimator.update(t, p) for t, p in sequence] + whole = replay(samples) + for length in range(1, len(samples)+1): + self.assertEqual(replay(samples[:length]), whole[:length]) + modified = replay(samples[:-1]+[(.14, [90, 90])]) + self.assertEqual(modified[:-1], whole[:-1]) + + def test_forecasts_are_unclipped_and_mark_out_of_frame(self): + estimator = MotionEstimator(100, 100) + estimator.update(0, [89, 10]) + row = estimator.update(.05, [99, 10]) + self.assertEqual(row['velocity_xy_pixels_per_second'], [200.0, 0.0]) + self.assertEqual(row['forecasts'][0]['center_xy_pixels'], [119.0, 10.0]) + self.assertTrue(all(f['out_of_frame'] for f in row['forecasts'])) + estimator = MotionEstimator(100, 100) + estimator.update(0, [11, 10]) + row = estimator.update(.05, [1, 10]) + self.assertEqual(row['forecasts'][0]['center_xy_pixels'], [-19.0, 10.0]) + + def test_radius_propagation_is_declared_not_probability(self): + estimator = MotionEstimator(100, 100) + estimator.update(0, [20, 20], uncertainty_pixels=1) + row = estimator.update(.1, [21, 20], uncertainty_pixels=2) + self.assertAlmostEqual(row['forecasts'][0]['uncertainty_radius_pixels'], 5) + self.assertEqual(row['uncertainty_semantics'], 'empirical-envelope-not-calibrated-probability') + estimator = MotionEstimator(100, 100) + estimator.update(0, [20, 20]) + row = estimator.update(.1, [21, 20], uncertainty_pixels=2) + self.assertTrue(all(f['uncertainty_radius_pixels'] is None for f in row['forecasts'])) + + def test_freshness_lifetime_does_not_truncate_forecast_horizon(self): + estimator = MotionEstimator(100, 100, max_prediction_age_seconds=.02) + estimator.update(0, [10, 10]) + row = estimator.update(.1, [11, 10]) + self.assertAlmostEqual(row['prediction_use_before_time_seconds'], .12) + self.assertAlmostEqual(row['forecasts'][-1]['target_time_seconds'], .6) + + def test_output_and_input_do_not_alias_estimator_state(self): + estimator = MotionEstimator(100, 100) + point = [10, 10] + row = estimator.update(0, point) + point[0] = 99 + row['observed_anchor_xy_pixels'][0] = 99 + row['estimated_center_xy_pixels'][0] = 99 + row['estimator']['horizons_seconds'][0] = 99 + next_row = estimator.update(.1, [11, 10]) + self.assertEqual(next_row['velocity_evidence']['start_anchor_xy_pixels'], [10.0, 10.0]) + self.assertEqual(next_row['velocity_xy_pixels_per_second'], [10.0, 0.0]) + self.assertEqual(next_row['forecasts'][0]['horizon_seconds'], .1) + json.dumps(next_row, allow_nan=False) + + def test_numeric_inputs_reject_booleans_nan_infinity_and_overflow(self): + invalid = [True, False, float('nan'), float('inf'), -float('inf'), '1', 10**1000] + for value in invalid: + with self.subTest(value=str(value)[:30]): + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(value, [10, 10]) + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, [value, 10]) + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, [10, 10], uncertainty_pixels=value) + with self.assertRaises(ValueError): + MotionEstimator(100, 100, horizons_seconds=[value]) + for time in (-1,): + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(time, [10, 10]) + for point in ([100, 0], [0, 100], [-1, 0], (1, 1), [1], [1, 2, 3]): + with self.subTest(point=point), self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, point) + + def test_constructor_and_tracking_metadata_are_strict(self): + for dimensions in [(True, 100), (100, False), (0, 100), (100, 1.5), (-1, 100)]: + with self.subTest(dimensions=dimensions), self.assertRaises(ValueError): + MotionEstimator(*dimensions) + for horizons in ([], (), None, [.1, .1], [.5, .1], [0], [-1]): + with self.subTest(horizons=horizons), self.assertRaises(ValueError): + MotionEstimator(100, 100, horizons_seconds=horizons) + for keyword in ('max_gap_seconds', 'max_prediction_age_seconds'): + for value in (0, -1, True, float('nan')): + with self.subTest(keyword=keyword, value=value), self.assertRaises(ValueError): + MotionEstimator(100, 100, **{keyword: value}) + for status in ('pending', True, None): + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, [10, 10], status=status) + for count in (True, -1, 1.0, float('nan')): + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, [10, 10], support_count=count) + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, [10, 10], status='lost') + with self.assertRaises(ValueError): + MotionEstimator(100, 100).update(0, None, uncertainty_pixels=1) + + def test_invalid_update_is_atomic_and_time_must_increase(self): + estimator = MotionEstimator(100, 100) + estimator.update(.1, [10, 10]) + for time in (.1, .05): + with self.assertRaises(ValueError): + estimator.update(time, [11, 10]) + with self.assertRaises(ValueError): + estimator.update(.2, [100, 10]) + row = estimator.update(.2, [11, 10]) + self.assertEqual(row['velocity_evidence']['start_time_seconds'], .1) + self.assertEqual(row['velocity_xy_pixels_per_second'], [10.0, 0.0]) + + def test_derived_overflow_rejected_without_mutation(self): + estimator = MotionEstimator(100, 100) + estimator.update(0, [0, 0]) + with self.assertRaises(ValueError): + estimator.update(1e-320, [1, 0]) + row = estimator.update(.1, [1, 0]) + self.assertEqual(row['velocity_xy_pixels_per_second'], [10.0, 0.0]) + + +class MotionValidationTest(unittest.TestCase): + def fixture(self): + estimator = MotionEstimator(100, 100) + return [estimator.update(0, [10, 10], uncertainty_pixels=1), + estimator.update(.1, [11, 10], uncertainty_pixels=1), + estimator.update(.2, None), estimator.update(.3, [12, 10])] + + def test_json_round_trip_and_empty_stream_are_valid(self): + rows = json.loads(json.dumps(self.fixture(), allow_nan=False)) + self.assertIs(validate_motion_rows(rows, 100, 100), rows) + self.assertEqual(validate_motion_rows([], 100, 100), []) + + def test_rejects_extra_fields_bad_units_and_forged_forecasts(self): + mutations = [ + lambda r: r[1].update(extra='hidden-command'), + lambda r: r[1]['units'].update(velocity='metres_per_second'), + lambda r: r[1]['forecasts'][0]['center_xy_pixels'].__setitem__(0, 0), + lambda r: r[1]['velocity_evidence'].update(start_time_seconds=0.05), + lambda r: r[1].update(velocity_xy_pixels_per_second=[999.0, 0.0]), + lambda r: r[2].update(estimated_center_xy_pixels=[12.0, 10.0]), + lambda r: r[3].update(velocity_xy_pixels_per_second=[10.0, 0.0]), + lambda r: r[1]['estimator'].update(max_gap_seconds=.2), + lambda r: r[1].update(schema_version=True), + lambda r: r[1].update(acquired_time_seconds=True), + lambda r: r[1]['forecasts'][0].update(out_of_frame=0), + lambda r: r[1].update(speed_pixels_per_second=float('nan')), + lambda r: r[1].update(acquired_time_seconds=0), + ] + for mutate in mutations: + rows = deepcopy(self.fixture()); mutate(rows) + with self.subTest(mutation=mutate), self.assertRaises(ValueError): + validate_motion_rows(rows, 100, 100) + + def test_rejects_wrong_geometry_and_non_objects(self): + with self.assertRaises(ValueError): + validate_motion_rows(self.fixture(), 101, 100) + for rows in (None, {}, (), [None], [{}]): + with self.subTest(rows=rows), self.assertRaises(ValueError): + validate_motion_rows(rows, 100, 100) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_motion_benchmark.py b/tests/test_motion_benchmark.py new file mode 100644 index 0000000..c4b10b1 --- /dev/null +++ b/tests/test_motion_benchmark.py @@ -0,0 +1,260 @@ +"""Scoring denominators, authored references and causal pipeline separation.""" +from copy import deepcopy +import json +from pathlib import Path +import shutil +import tempfile +import unittest +from unittest.mock import patch + +from dvidia_training.motion import MotionEstimator +from dvidia_training import motion_benchmark as benchmark + + +def reference_rows(count=11, fps=10): + return [{'frame_index': index, 'source_pts_seconds': index/fps, + 'anchor_xy_pixels': [10+10*index, 100], 'visible': True, + 'velocity_scoreable': index > 0, + 'velocity_xy_pixels_per_second': [10*fps, 0] if index else None} + for index in range(count)] + + +def estimated_rows(references): + estimator = MotionEstimator(640, 360) + return [estimator.update(ref['source_pts_seconds'], ref['anchor_xy_pixels'], + support_count=15, uncertainty_pixels=1) + for ref in references] + + +class MotionScoresTest(unittest.TestCase): + def test_perfect_constant_velocity_pairs_baseline_and_counts_terminal(self): + refs = reference_rows() + scores = benchmark.score_motion(estimated_rows(refs), refs, fps=10) + first = scores['forecasts']['0.1'] + self.assertEqual(first['attempted_rows'], 11) + self.assertEqual(first['future_reference_available'], 10) + self.assertEqual(first['terminal_censored'], 1) + self.assertEqual(first['velocity_warmup_abstentions'], 1) + self.assertEqual(first['constant_velocity']['count'], 9) + self.assertEqual(first['hold_current_paired']['count'], 9) + self.assertAlmostEqual(first['constant_velocity']['mean'], 0) + self.assertAlmostEqual(first['hold_current_paired']['mean'], 10) + self.assertAlmostEqual(first['forecast_answer_coverage'], .9) + self.assertEqual(scores['tracking']['track_coverage_visible'], 1) + + def test_untracked_frames_are_abstentions_not_zero_errors(self): + refs = reference_rows() + estimator = MotionEstimator(640, 360) + rows = [estimator.update(r['source_pts_seconds'], None, status='lost') for r in refs] + scores = benchmark.score_motion(rows, refs, fps=10) + self.assertEqual(scores['tracking']['lost_rows'], 11) + self.assertEqual(scores['tracking']['track_coverage_visible'], 0) + first = scores['forecasts']['0.1'] + self.assertEqual(first['lost_abstentions'], 10) + self.assertEqual(first['constant_velocity']['count'], 0) + self.assertIsNone(first['constant_velocity']['p50']) + self.assertEqual(first['forecast_answer_coverage'], 0) + + def test_unsampled_frames_remain_coverage_gaps(self): + refs = reference_rows() + rows = estimated_rows(refs) + del rows[5] + scores = benchmark.score_motion(rows, refs, fps=10) + self.assertEqual(scores['tracking']['unsampled_reference_frames'], 1) + self.assertAlmostEqual(scores['tracking']['track_coverage_visible'], 10/11) + + def test_current_off_grid_rows_cannot_borrow_nearest_frame_truth(self): + refs = reference_rows() + rows = estimated_rows(refs) + for row in rows: + row['acquired_time_seconds'] += .001 + for forecast in row['forecasts']: + forecast['target_time_seconds'] += .001 + scores = benchmark.score_motion(rows, refs, fps=10) + tracking = scores['tracking'] + self.assertEqual(tracking['unmatched_observation_rows'], 11) + self.assertEqual(tracking['unsampled_reference_frames'], 11) + self.assertEqual(tracking['anchor_error_pixels']['count'], 0) + self.assertEqual(scores['forecasts']['0.1']['constant_velocity']['count'], 0) + + def test_future_reference_gap_is_explicit_and_does_not_round_across_gap(self): + refs = reference_rows() + rows = estimated_rows(refs) + del refs[5] + scores = benchmark.score_motion(rows, refs, fps=10) + first = scores['forecasts']['0.1'] + self.assertEqual(first['future_reference_gap'], 1) + self.assertEqual(first['current_reference_unmatched'], 1) + self.assertEqual(scores['tracking']['unmatched_observation_rows'], 1) + self.assertEqual(first['constant_velocity']['count'], 7) + + def test_half_frame_matching_uses_earlier_identity_and_discloses_offset(self): + refs = reference_rows() + scores = benchmark.score_motion(estimated_rows(refs), refs, fps=10) + quarter = scores['forecasts']['0.25'] + self.assertAlmostEqual(quarter['constant_velocity']['mean'], 5) + self.assertAlmostEqual(quarter['reference_max_absolute_time_offset_seconds'], .05) + self.assertAlmostEqual(quarter['reference_time_offset_seconds']['mean'], -.05) + self.assertEqual(quarter['terminal_censored'], 3) + + def test_scoring_does_not_change_any_estimator_output_or_reference(self): + refs = reference_rows() + rows = estimated_rows(refs) + originals = deepcopy((rows, refs)) + benchmark.score_motion(rows, refs, fps=10) + self.assertEqual((rows, refs), originals) + changed = deepcopy(refs) + changed[-1]['anchor_xy_pixels'] = [500, 200] + before = benchmark.score_motion(rows, refs, fps=10) + after = benchmark.score_motion(rows, changed, fps=10) + self.assertNotEqual(before['forecasts'], after['forecasts']) + self.assertEqual(rows, originals[0]) + + def test_velocity_discontinuity_neighborhood_is_excluded(self): + refs = reference_rows() + rows = estimated_rows(refs) + refs[5]['velocity_scoreable'] = False + refs[5]['velocity_xy_pixels_per_second'] = [999999, 0] + scores = benchmark.score_motion(rows, refs, fps=10) + velocity = scores['tracking']['velocity_error_pixels_per_second'] + self.assertEqual(velocity['count'], 9) + self.assertAlmostEqual(velocity['mean'], 0) + + def test_tracked_occlusion_not_counted_as_visible_tracking_accuracy(self): + refs = reference_rows() + rows = estimated_rows(refs) + refs[5]['visible'] = False + scores = benchmark.score_motion(rows, refs, fps=10) + tracking = scores['tracking'] + self.assertEqual(tracking['reference_visible_frames'], 10) + self.assertEqual(tracking['anchor_error_pixels']['count'], 10) + self.assertEqual(tracking['tracked_during_reference_occlusion_rows'], 1) + + def test_uncertainty_is_evaluated_without_claiming_calibration(self): + refs = reference_rows() + scores = benchmark.score_motion(estimated_rows(refs), refs, fps=10) + uncertainty = scores['forecasts']['0.1']['uncertainty'] + self.assertFalse(uncertainty['calibrated']) + self.assertEqual(uncertainty['covered'], 9) + self.assertEqual(uncertainty['evaluated'], 9) + + def test_invalid_time_identity_forecast_and_nonfinite_data_rejected(self): + refs = reference_rows() + rows = estimated_rows(refs) + invalid_refs = deepcopy(refs) + invalid_refs[5]['frame_index'] = invalid_refs[4]['frame_index'] + with self.assertRaises(ValueError): + benchmark.score_motion(rows, invalid_refs, fps=10) + invalid_rows = deepcopy(rows) + invalid_rows[2]['acquired_time_seconds'] = invalid_rows[1]['acquired_time_seconds'] + with self.assertRaises(ValueError): + benchmark.score_motion(invalid_rows, refs, fps=10) + for field, value in [('target_time_seconds', 10), ('center_xy_pixels', [float('nan'), 1])]: + invalid_rows = deepcopy(rows) + invalid_rows[1]['forecasts'][0][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + benchmark.score_motion(invalid_rows, refs, fps=10) + + def test_distribution_documents_interpolation_and_no_sample_nulls(self): + self.assertEqual(benchmark.distribution([])['count'], 0) + self.assertIsNone(benchmark.distribution([])['mean']) + self.assertEqual(benchmark.distribution([0, 10])['p50'], 5) + self.assertEqual(benchmark.distribution([0, 10])['p95'], 9.5) + + +def fake_fixtures(directory): + directory = Path(directory) + directory.mkdir() + benchmark._write(directory/'source.json', {'kind': 'synthetic-manifest-test', 'episodes': []}) + result = [] + for case in benchmark.CASES: + reference = directory/(case+'.reference.json') + benchmark._write(reference, {'rows': reference_rows()}) + result.append({'id': case, 'reference_path': reference.name, + 'reference_sha256': benchmark._hash(reference), + 'seed_box_xyxy_normalized': [.1, .1, .2, .2]}) + return result + + +class BenchmarkAttemptTest(unittest.TestCase): + def test_failed_attempts_preserve_receipts_and_cannot_appear_fast(self): + calls = [] + def failure(manifest, episode_id, output, **kwargs): + calls.append((episode_id, kwargs)) + self.assertEqual(set(kwargs), {'seed_box', 'label', 'sample_hz'}) + self.assertNotIn('reference', Path(manifest).read_text()) + output.mkdir(parents=True) + receipt = {'kind': 'dvidia.motion-run.v1', 'status': 'failed', + 'errors': [{'stage': 'decode', 'message': 'controlled failure'}]} + benchmark._write(output/'run.json', receipt) + return receipt + with tempfile.TemporaryDirectory() as temp, \ + patch.object(benchmark, 'generate_fixtures', fake_fixtures), \ + patch('dvidia_training.motion_video.analyze_video', failure): + output = Path(temp)/'new-run' + report = benchmark.run_benchmark(output) + self.assertEqual(report['attempted'], 6) + self.assertEqual(report['failed'], 6) + self.assertEqual(len(calls), 6) + self.assertEqual(report['timing']['completed_source_duration_seconds'], 0) + self.assertIsNone(report['timing']['batch_processing_time_source_duration_ratio']) + for row in report['cases']: + self.assertTrue((output/row['run_path']).exists()) + self.assertEqual(row['errors'][0]['stage'], 'decode') + + def test_changed_evaluator_reference_invalidates_case_without_rewriting_run(self): + def complete(manifest, episode_id, output, **kwargs): + output.mkdir(parents=True) + if episode_id == benchmark.CASES[0]: + reference = Path(manifest).parent/(episode_id+'.reference.json') + reference.write_text('{}') + receipt = {'kind': 'dvidia.motion-run.v1', 'status': 'completed', + 'rows': estimated_rows(reference_rows())} + benchmark._write(output/'run.json', receipt) + return receipt + with tempfile.TemporaryDirectory() as temp, \ + patch.object(benchmark, 'generate_fixtures', fake_fixtures), \ + patch('dvidia_training.motion_video.analyze_video', complete): + output = Path(temp)/'new-run' + report = benchmark.run_benchmark(output) + self.assertEqual(report['completed'], 5) + self.assertEqual(report['failed'], 1) + first = report['cases'][0] + self.assertIn('reference differs', first['error']) + self.assertEqual(json.loads((output/first['run_path']).read_bytes())['status'], 'completed') + self.assertIsNone(report['timing']['batch_processing_time_source_duration_ratio']) + + +@unittest.skipUnless(shutil.which('ffmpeg') and shutil.which('ffprobe'), 'FFmpeg required') +class AuthoredFixtureTest(unittest.TestCase): + def test_generated_media_references_and_manual_seeds_are_independent(self): + with tempfile.TemporaryDirectory() as temp: + root = Path(temp)/'inputs' + fixtures = benchmark.generate_fixtures(root) + self.assertEqual(len(fixtures), 6) + manifest = json.loads((root/'source.json').read_bytes()) + for fixture, episode in zip(fixtures, manifest['episodes'], strict=True): + self.assertEqual(fixture['video_sha256'], benchmark._hash(root/fixture['video_path'])) + self.assertEqual(episode['license'], 'MIT') + self.assertNotIn('reference', json.dumps(episode)) + self.assertNotIn('seed', json.dumps(episode)) + ref = json.loads((root/fixture['reference_path']).read_bytes()) + self.assertEqual(ref['role'], 'evaluator_only_never_supplied_to_tracker') + self.assertEqual(len(ref['rows']), 120) + self.assertEqual(ref['rows'][0]['source_pts_seconds'], 0) + x1, y1, x2, y2 = fixture['seed_box_xyxy_normalized'] + self.assertAlmostEqual((x1+x2)*320, ref['rows'][0]['anchor_xy_pixels'][0]) + self.assertAlmostEqual((y1+y2)*180, ref['rows'][0]['anchor_xy_pixels'][1]) + self.assertEqual(list(root.glob('*.rgb')), []) + + def test_existing_benchmark_evidence_is_never_overwritten(self): + with tempfile.TemporaryDirectory() as temp: + marker = Path(temp)/'report.json' + marker.write_text('preserved') + with self.assertRaises(ValueError): + benchmark.run_benchmark(temp) + self.assertEqual(marker.read_text(), 'preserved') + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_motion_studio.py b/tests/test_motion_studio.py new file mode 100644 index 0000000..98f0b89 --- /dev/null +++ b/tests/test_motion_studio.py @@ -0,0 +1,422 @@ +"""Real generated motion evidence, read-only replay and tamper rejection.""" +from copy import deepcopy +import http.client +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import threading +import unittest + +from dvidia_training.footage_data import canonical +from dvidia_training.motion_benchmark import score_motion +from dvidia_training.motion_studio import MotionServer, load_motion_bundle, load_motion_run +from dvidia_training.motion_video import analyze_video +from dvidia_training.observations import digest, file_hash + +try: + import numpy as np + import cv2 + HAS_CV = True +except ImportError: + HAS_CV = False + + +def write_document(path, value): + Path(path).write_bytes(canonical(value)) + + +def write_run_parts(directory, run): + for filename, key in (('protocol.json', 'protocol'), ('rows.json', 'rows'), + ('frames.json', 'frames')): + write_document(directory/filename, run[key]) + run[key+'_sha256'] = digest(run[key]) + write_document(directory/'run.json', run) + + +@unittest.skipUnless(HAS_CV and shutil.which('ffmpeg') and shutil.which('ffprobe'), + 'Requires observations extra and local FFmpeg/ffprobe.') +class MotionStudioTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.base_temp = tempfile.TemporaryDirectory() + cls.base = Path(cls.base_temp.name) + width, height, count = 160, 120, 12 + texture = np.random.default_rng(4711).integers(30, 230, (40, 48, 3), dtype='uint8') + frames = [] + for index in range(count): + frame = np.full((height, width, 3), 20, dtype='uint8') + frame[30:70, 20+2*index:68+2*index] = texture + frames.append(frame.tobytes()) + cls.video = cls.base/'authored.mp4' + result = subprocess.run(['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', + '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', '160x120', '-r', '30', '-i', 'pipe:0', + '-an', '-c:v', 'libx264', '-crf', '0', '-pix_fmt', 'yuv420p', '-threads', '1', + '-fps_mode', 'vfr', str(cls.video)], input=b''.join(frames), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=20, check=False) + if result.returncode: + raise RuntimeError('Could not generate own motion-studio fixture.') + cls.manifest = {'kind': 'dvidia.observation-source.v1', 'task': 'Authored replay fixture', + 'episodes': [{'id': 'fixture', 'path': cls.video.name, + 'title': 'Own neutral textured rectangle', 'session_id': 'studio-test-session', + 'source_recording_id': 'studio-test-recording', 'kind': 'synthetic', + 'credit': 'DVIDIA authored engineering fixture', 'license': 'MIT', + 'source_url': 'https://example.invalid/dvidia-authored-fixture', + 'permission': 'local_analysis', 'sha256': file_hash(cls.video)}]} + write_document(cls.base/'source.json', cls.manifest) + cls.seed = [24/width, 34/height, 64/width, 66/height] + cls.receipt = analyze_video(cls.base/'source.json', 'fixture', cls.base/'complete', + seed_box=cls.seed) + if cls.receipt['status'] != 'completed' or len(cls.receipt['rows']) != count: + raise RuntimeError('Own fixture did not produce a complete motion run.') + wrong = deepcopy(cls.manifest) + wrong['episodes'][0]['sha256'] = '0'*64 + write_document(cls.base/'wrong-source.json', wrong) + cls.failed_receipt = analyze_video(cls.base/'wrong-source.json', 'fixture', cls.base/'failed', + seed_box=cls.seed) + if cls.failed_receipt['status'] != 'failed' or cls.failed_receipt['rows']: + raise RuntimeError('Own wrong-hash fixture did not preserve its failed intake.') + # One-case benchmark envelope around the genuine generated run. The + # authored trajectory is evaluator-only and never enters analyze_video. + cls.benchmark = cls.base/'benchmark' + (cls.benchmark/'inputs').mkdir(parents=True) + shutil.copyfile(cls.video, cls.benchmark/'inputs'/cls.video.name) + shutil.copyfile(cls.base/'source.json', cls.benchmark/'inputs/source.json') + shutil.copytree(cls.base/'complete', cls.benchmark/'cases/fixture') + references = [{'frame_index': index, 'source_pts_seconds': row['acquired_time_seconds'], + 'anchor_xy_pixels': [44+2*index, 50], 'visible': True, + 'velocity_scoreable': index > 0, + 'velocity_xy_pixels_per_second': [60, 0] if index else None} + for index, row in enumerate(cls.receipt['rows'])] + reference_path = cls.benchmark/'inputs/fixture.reference.json' + write_document(reference_path, {'kind': 'dvidia.authored-motion-reference.v1', + 'role': 'evaluator_only_never_supplied_to_tracker', 'rows': references}) + scores = score_motion(cls.receipt['rows'], references) + write_document(cls.benchmark/'cases/fixture/scores.json', scores) + benchmark_protocol = {'kind': 'dvidia.motion-benchmark-protocol.v1', + 'fixture_recipe': {'fps': 30}, 'horizons_seconds': [.1, .25, .5], + 'sources_manifest_sha256': file_hash(cls.benchmark/'inputs/source.json'), + 'fixtures': [{'id': 'fixture', 'video_path': cls.video.name, + 'video_sha256': file_hash(cls.video), 'reference_path': reference_path.name, + 'reference_sha256': file_hash(reference_path), + 'seed_box_xyxy_normalized': cls.seed}]} + write_document(cls.benchmark/'protocol.json', benchmark_protocol) + write_document(cls.benchmark/'report.json', {'kind': 'dvidia.motion-benchmark.v1', + 'protocol_sha256': file_hash(cls.benchmark/'protocol.json'), + 'attempted': 1, 'completed': 1, 'failed': 0, + 'cases': [{'id': 'fixture', 'status': 'completed', 'run_path': 'cases/fixture/run.json', + 'run_sha256': file_hash(cls.benchmark/'cases/fixture/run.json'), + 'scores_path': 'cases/fixture/scores.json', + 'scores_sha256': file_hash(cls.benchmark/'cases/fixture/scores.json'), + 'scores': scores}]}) + + @classmethod + def tearDownClass(cls): + cls.base_temp.cleanup() + + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.run_dir = self.root/'run' + shutil.copytree(self.base/'complete', self.run_dir) + + def read_run(self): + return json.loads((self.run_dir/'run.json').read_bytes()) + + def copy_benchmark(self): + destination = self.root/'benchmark' + shutil.copytree(self.benchmark, destination) + return destination + + def read_report(self, directory): + return json.loads((directory/'report.json').read_bytes()) + + def update_benchmark_protocol(self, directory, mutate): + protocol = json.loads((directory/'protocol.json').read_bytes()) + mutate(protocol) + write_document(directory/'protocol.json', protocol) + report = self.read_report(directory) + report['protocol_sha256'] = file_hash(directory/'protocol.json') + write_document(directory/'report.json', report) + + def test_actual_generated_run_and_bundle_load_preserve_scopes(self): + run = load_motion_run(self.run_dir/'run.json') + self.assertEqual(run, self.receipt) + self.assertEqual(run['protocol']['analysis_image_size_pixels'], [160, 120]) + self.assertEqual(run['frame_count'], 12) + self.assertEqual(run['accuracy_status'], 'not_assessed') + self.assertFalse(run['robot_skill_qualified']) + report, episodes = load_motion_bundle(self.run_dir) + self.assertIsNone(report) + self.assertEqual(set(episodes), {'episode'}) + self.assertEqual(episodes['episode']['run'], run) + self.assertIsNone(episodes['episode']['scores']) + + def test_changed_sidecar_protocol_rows_and_frames_are_rejected(self): + for name, field, value in [('protocol', 'sample_hz_requested', 20), + ('rows', 'support_count', 999), + ('frames', 'source_pts_seconds', .01)]: + path = self.run_dir/(name+'.json') + original = path.read_bytes() + changed = json.loads(original) + if name == 'protocol': + changed[field] = value + else: + changed[0][field] = value + write_document(path, changed) + with self.subTest(sidecar=name), self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + path.write_bytes(original) + + def test_changed_receipt_hash_is_rejected(self): + run = self.read_run() + run['rows_sha256'] = '0'*64 + write_document(self.run_dir/'run.json', run) + with self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + + def test_forged_forecast_is_rejected_even_with_recomputed_hashes(self): + run = self.read_run() + run['rows'][1]['forecasts'][0]['center_xy_pixels'][0] += 10 + write_run_parts(self.run_dir, run) + with self.assertRaisesRegex(ValueError, 'causal|schema|evidence'): + load_motion_run(self.run_dir/'run.json') + + def test_source_timestamp_mismatch_rejected_even_when_frames_rehashed(self): + run = self.read_run() + run['frames'][1]['source_pts_seconds'] += .001 + write_run_parts(self.run_dir, run) + with self.assertRaisesRegex(ValueError, 'presentation|timestamp|source'): + load_motion_run(self.run_dir/'run.json') + + def test_seed_receipt_hash_and_protocol_binding_are_checked(self): + run = self.read_run() + run['seed']['label'] = 'Unrelated seed' + write_document(self.run_dir/'run.json', run) + with self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + run['seed_sha256'] = digest(run['seed']) + write_document(self.run_dir/'run.json', run) + with self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + + def test_same_size_seed_evidence_change_is_detected(self): + run = self.read_run() + image = self.run_dir/run['seed_evidence']['path'] + original = image.read_bytes() + image.write_bytes(bytes([original[0] ^ 1])+original[1:]) + self.assertEqual(image.stat().st_size, len(original)) + with self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + + def test_same_size_source_media_change_is_detected(self): + run = self.read_run() + video = self.run_dir/run['media']['path'] + original = video.read_bytes() + video.write_bytes(bytes([original[0] ^ 1])+original[1:]) + self.assertEqual(video.stat().st_size, run['media']['bytes']) + with self.assertRaisesRegex(ValueError, 'media|source'): + load_motion_run(self.run_dir/'run.json') + + def test_missing_evidence_file_is_rejected(self): + (self.run_dir/'frames.json').unlink() + with self.assertRaises((ValueError, FileNotFoundError)): + load_motion_run(self.run_dir/'run.json') + + def test_failed_preprobe_receipt_with_no_dimensions_or_rows_is_replayable(self): + failed_dir = self.root/'failed' + shutil.copytree(self.base/'failed', failed_dir) + failed = load_motion_run(failed_dir/'run.json') + self.assertEqual(failed['status'], 'failed') + self.assertEqual(failed['rows'], []) + self.assertEqual(failed['frames'], []) + self.assertIsNone(failed['media']) + self.assertNotIn('analysis_image_size_pixels', failed['protocol']) + report, episodes = load_motion_bundle(failed_dir) + self.assertIsNone(report) + self.assertEqual(episodes['episode']['run']['errors'][0]['stage'], 'intake') + + def test_completed_receipt_cannot_silently_drop_decoded_evidence(self): + run = self.read_run() + run['rows'], run['frames'] = [], [] + write_run_parts(self.run_dir, run) + with self.assertRaises(ValueError): + load_motion_run(self.run_dir/'run.json') + + def test_benchmark_bundle_recomputes_real_run_scores_from_retained_references(self): + directory = self.copy_benchmark() + report, episodes = load_motion_bundle(directory) + self.assertEqual((report['attempted'], report['completed'], report['failed']), (1, 1, 0)) + self.assertEqual(set(episodes), {'fixture'}) + self.assertEqual(episodes['fixture']['scores'], report['cases'][0]['scores']) + self.assertEqual(episodes['fixture']['run']['source']['sha256'], file_hash(self.video)) + + def test_benchmark_score_tamper_rejected_even_with_matching_score_hash_and_report(self): + directory = self.copy_benchmark() + report = self.read_report(directory) + report['cases'][0]['scores']['forecasts']['0.1']['constant_velocity']['mean'] = 999 + path = directory/report['cases'][0]['scores_path'] + write_document(path, report['cases'][0]['scores']) + report['cases'][0]['scores_sha256'] = file_hash(path) + write_document(directory/'report.json', report) + with self.assertRaisesRegex(ValueError, 'scores|evaluator'): + load_motion_bundle(directory) + + def test_changed_benchmark_reference_rejected_before_replay(self): + directory = self.copy_benchmark() + path = directory/'inputs/fixture.reference.json' + reference = json.loads(path.read_bytes()) + reference['rows'][5]['anchor_xy_pixels'][0] += 20 + write_document(path, reference) + with self.assertRaisesRegex(ValueError, 'hash|evidence'): + load_motion_bundle(directory) + # Updating the reference's hash alone cannot retain stale evaluator scores. + self.update_benchmark_protocol(directory, lambda p: p['fixtures'][0].update( + reference_sha256=file_hash(path))) + with self.assertRaisesRegex(ValueError, 'scores|evaluator'): + load_motion_bundle(directory) + + def test_benchmark_case_run_hash_and_completion_counts_are_checked(self): + directory = self.copy_benchmark() + original = (directory/'report.json').read_bytes() + report = self.read_report(directory) + report['cases'][0]['run_sha256'] = '0'*64 + write_document(directory/'report.json', report) + with self.assertRaises(ValueError): + load_motion_bundle(directory) + (directory/'report.json').write_bytes(original) + report = self.read_report(directory) + report['attempted'] = 2 + write_document(directory/'report.json', report) + with self.assertRaisesRegex(ValueError, 'counts|attempts'): + load_motion_bundle(directory) + + def test_benchmark_outer_case_identity_matches_its_actual_run(self): + directory = self.copy_benchmark() + self.update_benchmark_protocol(directory, lambda p: p['fixtures'][0].update(id='other')) + report = self.read_report(directory) + report['cases'][0]['id'] = 'other' + write_document(directory/'report.json', report) + with self.assertRaises(ValueError): + load_motion_bundle(directory) + + def test_benchmark_outer_fixture_source_matches_the_source_actually_analyzed(self): + directory = self.copy_benchmark() + input_video = directory/'inputs'/self.video.name + original = input_video.read_bytes() + input_video.write_bytes(bytes([original[0] ^ 1])+original[1:]) + self.update_benchmark_protocol(directory, lambda p: p['fixtures'][0].update( + video_sha256=file_hash(input_video))) + with self.assertRaises(ValueError): + load_motion_bundle(directory) + + def start_server(self, directory=None): + server = MotionServer(('127.0.0.1', 0), directory or self.run_dir) + worker = threading.Thread(target=server.serve_forever, daemon=True) + worker.start() + def stop(): + server.shutdown() + server.server_close() + worker.join(timeout=3) + self.addCleanup(stop) + return server + + def request(self, server, method, path, *, headers=None, body=None): + connection = http.client.HTTPConnection('127.0.0.1', server.server_port, timeout=5) + try: + connection.request(method, path, body=body, headers=headers or {}) + response = connection.getresponse() + return response.status, dict(response.getheaders()), response.read() + finally: + connection.close() + + def test_loopback_get_head_and_exact_media_ranges(self): + server = self.start_server() + status, headers, body = self.request(server, 'GET', '/') + self.assertEqual(status, 200) + self.assertIn('text/html', headers['Content-Type']) + self.assertTrue(body) + self.assertEqual(headers['Cache-Control'], 'no-store') + status, _, body = self.request(server, 'GET', '/api/episodes') + self.assertEqual(status, 200) + listing = json.loads(body) + self.assertEqual(listing['episodes'][0]['id'], 'episode') + status, _, body = self.request(server, 'GET', '/api/episodes/episode') + self.assertEqual(status, 200) + episode = json.loads(body) + self.assertEqual(episode['run']['rows'], self.receipt['rows']) + self.assertEqual(episode['video_url'], '/api/episodes/episode/video') + original = (self.run_dir/self.receipt['media']['path']).read_bytes() + status, headers, body = self.request(server, 'GET', episode['video_url']) + self.assertEqual(status, 200) + self.assertEqual(body, original) + self.assertEqual(headers['Accept-Ranges'], 'bytes') + status, headers, body = self.request(server, 'GET', episode['video_url'], + headers={'Range': 'bytes=4-19'}) + self.assertEqual(status, 206) + self.assertEqual(body, original[4:20]) + self.assertEqual(headers['Content-Range'], f'bytes 4-19/{len(original)}') + self.assertEqual(self.request(server, 'GET', episode['video_url'], + headers={'Range': 'bytes=-8'})[2], original[-8:]) + status, headers, body = self.request(server, 'HEAD', episode['video_url']) + self.assertEqual(status, 200) + self.assertEqual(body, b'') + self.assertEqual(int(headers['Content-Length']), len(original)) + for requested in ('bytes=999999-', 'bytes=0-2,5-9', 'bytes=-0', 'bytes=20-4'): + with self.subTest(requested=requested): + self.assertEqual(self.request(server, 'GET', episode['video_url'], + headers={'Range': requested})[0], 416) + self.assertEqual(self.request(server, 'GET', '/api/episodes/missing')[0], 404) + self.assertEqual(self.request(server, 'GET', '/unknown')[0], 404) + + def test_foreign_host_origin_and_cross_site_are_rejected(self): + server = self.start_server() + for headers in ({'Host': 'evil.invalid'}, {'Origin': 'https://evil.invalid'}, + {'Sec-Fetch-Site': 'cross-site'}): + with self.subTest(headers=headers): + self.assertEqual(self.request(server, 'GET', '/api/episodes', headers=headers)[0], 403) + self.assertEqual(self.request(server, 'GET', '/api/episodes/episode/video', headers=headers)[0], 403) + expected = f'http://127.0.0.1:{server.server_port}' + self.assertEqual(self.request(server, 'GET', '/api/episodes', headers={'Origin': expected})[0], 200) + with self.assertRaises(ValueError): + MotionServer(('0.0.0.0', 0), self.run_dir) + + def test_post_is_read_only_and_does_not_change_bundle(self): + server = self.start_server() + original = {file.name: file.read_bytes() for file in self.run_dir.iterdir()} + status, headers, body = self.request(server, 'POST', '/api/episodes/episode', + headers={'Content-Type': 'application/json'}, body=b'{"rows":[]}') + self.assertEqual(status, 405) + self.assertEqual(headers['Allow'], 'GET, HEAD') + self.assertIn('read-only', json.loads(body)['error']) + self.assertEqual({file.name: file.read_bytes() for file in self.run_dir.iterdir()}, original) + + def test_server_detects_same_size_media_change_after_first_read(self): + server = self.start_server() + path = '/api/episodes/episode/video' + self.assertEqual(self.request(server, 'GET', path)[0], 200) + video = self.run_dir/self.receipt['media']['path'] + original = video.read_bytes() + video.write_bytes(bytes([original[0] ^ 1])+original[1:]) + self.assertEqual(video.stat().st_size, len(original)) + status, _, body = self.request(server, 'GET', path) + self.assertEqual(status, 409) + self.assertIn('inconsistent', json.loads(body)['error']) + + def test_failed_intake_is_visible_without_media(self): + failed_dir = self.root/'failed' + shutil.copytree(self.base/'failed', failed_dir) + server = self.start_server(failed_dir) + status, _, body = self.request(server, 'GET', '/api/episodes') + self.assertEqual(status, 200) + self.assertEqual(json.loads(body)['episodes'][0]['status'], 'failed') + status, _, body = self.request(server, 'GET', '/api/episodes/episode') + self.assertEqual(status, 200) + self.assertEqual(json.loads(body)['run']['errors'][0]['stage'], 'intake') + self.assertEqual(self.request(server, 'GET', '/api/episodes/episode/video')[0], 404) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_motion_video.py b/tests/test_motion_video.py new file mode 100644 index 0000000..573a13f --- /dev/null +++ b/tests/test_motion_video.py @@ -0,0 +1,274 @@ +"""Actual local decoding, causal feature tracking and visible failure checks.""" +from __future__ import annotations + +import json +from fractions import Fraction +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import patch + +from dvidia_training import motion_video as mv +from dvidia_training.motion import validate_motion_rows +from dvidia_training.observations import digest, file_hash + +try: + import cv2 + import numpy as np + HAS_CV = True +except ImportError: + HAS_CV = False + + +def manifest_for(directory, video): + manifest = {'kind': 'dvidia.observation-source.v1', 'task': 'Authored motion fixture', + 'episodes': [{'id': 'fixture', 'path': video.name, + 'title': 'Generated neutral textured rectangle', 'session_id': 'generated-session', + 'source_recording_id': 'generated-source', 'kind': 'synthetic', + 'credit': 'DVIDIA authored test fixture', 'license': 'MIT', + 'source_url': 'https://example.invalid/authored-test-fixture', + 'permission': 'local_analysis', 'sha256': file_hash(video)}]} + target = directory / 'source.json' + target.write_text(json.dumps(manifest)) + return target + + +@unittest.skipUnless(HAS_CV and shutil.which('ffmpeg') and shutil.which('ffprobe'), + 'Requires observations extra and local FFmpeg/ffprobe.') +class MotionVideoTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + + def video(self, *, count=12, fps=30, moving=True, textured=True, + pts_filter=None, offset=None): + width, height = 160, 120 + rng = np.random.default_rng(31415) + texture = rng.integers(30, 230, (40, 48, 3), dtype='uint8') + images = [] + for index in range(count): + image = np.full((height, width, 3), 20, dtype='uint8') + x = 20 + (index * 2 if moving else 0) + if textured: + image[30:70, x:x+48] = texture + images.append(image.tobytes()) + video = self.root / 'fixture.mp4' + command = ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', + '-f', 'rawvideo', '-pix_fmt', 'bgr24', '-s', f'{width}x{height}', + '-r', str(fps), '-i', 'pipe:0', '-an'] + if pts_filter: + command.extend(['-vf', pts_filter]) + command.extend(['-c:v', 'libx264', '-crf', '0', '-pix_fmt', 'yuv420p', + '-fps_mode', 'vfr']) + if offset is not None: + command.extend(['-output_ts_offset', str(offset)]) + command.append(str(video)) + result = subprocess.run(command, input=b''.join(images), stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=20, check=False) + self.assertEqual(result.returncode, 0, result.stderr.decode()[-1000:]) + return video + + def analyze(self, video, name='run', **kwargs): + return mv.analyze_video(manifest_for(self.root, video), 'fixture', self.root/name, + seed_box=[24/160, 34/120, 64/160, 66/120], **kwargs) + + def test_actual_decode_tracks_translation_and_preserves_data_hashes(self): + video = self.video() + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'completed', receipt['errors']) + self.assertEqual(receipt['frame_count'], 12) + self.assertEqual(receipt['tracked_frame_count'], 12) + self.assertFalse(receipt['robot_skill_qualified']) + self.assertEqual(receipt['accuracy_status'], 'not_assessed') + self.assertEqual(receipt['seed']['kind'], 'human_authored_image_region') + self.assertEqual(receipt['protocol']['analysis_image_size_pixels'], [160, 120]) + self.assertEqual(receipt['protocol']['original_image_size_pixels'], [160, 120]) + self.assertEqual(file_hash(self.root/'run'/'original.mp4'), file_hash(video)) + self.assertEqual(receipt['media']['bytes'], video.stat().st_size) + self.assertEqual(receipt['protocol_sha256'], digest(receipt['protocol'])) + self.assertEqual(receipt['rows_sha256'], digest(receipt['rows'])) + self.assertEqual(receipt['frames_sha256'], digest(receipt['frames'])) + self.assertEqual(receipt['seed_sha256'], digest(receipt['seed'])) + for name, key in [('protocol', 'protocol'), ('rows', 'rows'), ('frames', 'frames')]: + self.assertEqual(json.loads((self.root/'run'/f'{name}.json').read_text()), receipt[key]) + validate_motion_rows(receipt['rows'], 160, 120) + for frame, row in zip(receipt['frames'], receipt['rows'], strict=True): + self.assertEqual(frame['source_pts_seconds'], row['acquired_time_seconds']) + self.assertEqual(frame['source_pts_seconds'], float(frame['source_pts_integer'] * + Fraction(*frame['source_time_base']))) + self.assertAlmostEqual(receipt['rows'][-1]['acquired_time_seconds'], 11/30, places=12) + for row in receipt['rows'][1:]: + self.assertAlmostEqual(row['velocity_xy_pixels_per_second'][0], 60, delta=2) + self.assertAlmostEqual(row['velocity_xy_pixels_per_second'][1], 0, delta=2) + self.assertAlmostEqual(receipt['rows'][-1]['observed_anchor_xy_pixels'][0], 66, delta=0.5) + self.assertEqual(set(p.name for p in (self.root/'run').iterdir()), + {'original.mp4', 'seed.png', 'protocol.json', 'rows.json', 'frames.json', 'run.json'}) + + def test_actual_irregular_pts_are_not_reconstructed_from_nominal_rate(self): + video = self.video(count=4, fps=10, + pts_filter="setpts='if(eq(N,0),0,if(eq(N,1),1,if(eq(N,2),5,6)))'") + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'completed', receipt['errors']) + stamps = [row['acquired_time_seconds'] for row in receipt['rows']] + self.assertEqual(len(stamps), 4) + for actual, expected in zip(stamps, [0, 0.1, 0.5, 0.6], strict=True): + self.assertAlmostEqual(actual, expected, places=5) + self.assertEqual(receipt['rows'][2]['status'], 'lost') + self.assertEqual(receipt['frames'][2]['tracking']['reason'], + 'timestamp_gap_exceeds_tracking_envelope') + self.assertEqual(receipt['rows'][3]['status'], 'lost') + self.assertEqual(receipt['rows'][3]['forecasts'], []) + + def test_low_texture_loss_is_visible_and_never_reacquires(self): + video = self.video(count=6, textured=False) + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'completed', receipt['errors']) + self.assertEqual(receipt['tracked_frame_count'], 0) + self.assertTrue(all(row['status'] == 'lost' for row in receipt['rows'])) + self.assertTrue(all(row['observed_anchor_xy_pixels'] is None and not row['forecasts'] + for row in receipt['rows'])) + self.assertEqual(receipt['frames'][0]['tracking']['reason'], 'insufficient_initial_features') + validate_motion_rows(receipt['rows'], 160, 120) + + def test_overwrite_rejected_and_original_receipt_preserved(self): + video = self.video(count=3) + self.analyze(video) + original = (self.root/'run'/'run.json').read_bytes() + with self.assertRaisesRegex(ValueError, 'never overwritten'): + self.analyze(video) + self.assertEqual((self.root/'run'/'run.json').read_bytes(), original) + + def test_changed_original_produces_failed_intake_receipt_without_paths(self): + video = self.video(count=3) + manifest = manifest_for(self.root, video) + data = json.loads(manifest.read_text()) + data['episodes'][0]['sha256'] = '0'*64 + manifest.write_text(json.dumps(data)) + receipt = mv.analyze_video(manifest, 'fixture', self.root/'run', + seed_box=[0.1, 0.1, 0.5, 0.5]) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['errors'][0]['stage'], 'intake') + self.assertEqual(receipt['rows'], []) + self.assertIsNone(receipt['media']) + self.assertNotIn(str(self.root), json.dumps(receipt)) + self.assertTrue((self.root/'run'/'run.json').is_file()) + + def test_nonzero_original_timeline_is_rejected(self): + video = self.video(count=3, offset=1) + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['errors'][0]['stage'], 'intake') + self.assertEqual(receipt['rows'], []) + + def test_invalid_limits_and_unknown_episode_create_no_output(self): + video = self.video(count=3) + manifest = manifest_for(self.root, video) + for extra in ({'sample_hz': 31}, {'sample_hz': True}, {'max_duration_seconds': 61}, + {'max_frames': 1801}, {'max_frames': True}, + {'seed_box': [0.5, 0.1, 0.1, 0.5]}): + args = {'seed_box': [0.1, 0.1, 0.5, 0.5], **extra} + with self.subTest(extra=extra), self.assertRaises(ValueError): + mv.analyze_video(manifest, 'fixture', self.root/'invalid', **args) + with self.assertRaises(ValueError): + mv.analyze_video(manifest, 'other', self.root/'invalid', seed_box=[0, 0, 1, 1]) + self.assertFalse((self.root/'invalid').exists()) + + def test_frame_cap_does_not_create_false_framerate_or_extra_frames(self): + video = self.video(count=12) + receipt = self.analyze(video, max_frames=4) + self.assertEqual(receipt['status'], 'completed', receipt['errors']) + self.assertEqual(receipt['frame_count'], 4) + self.assertEqual(receipt['protocol']['max_frames'], 4) + self.assertEqual(receipt['frames'][-1]['processed_index'], 3) + + def test_decoder_failure_retains_partial_rows(self): + video = self.video(count=3) + original = mv._decoded_frames + + def interrupted(*args, **kwargs): + stream = original(*args, **kwargs) + try: + yield next(stream) + raise mv.MotionVideoError('Authored decoder failure fixture.') + finally: + stream.close() + + with patch.object(mv, '_decoded_frames', interrupted): + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['frame_count'], 1) + self.assertEqual(receipt['rows_sha256'], digest(receipt['rows'])) + self.assertEqual(receipt['errors'][0]['stage'], 'decode_and_track') + self.assertTrue((self.root/'run'/'seed.png').is_file()) + + def test_no_reacquisition_even_if_later_texture_appears(self): + tracker = mv._FeatureAnchor(cv2, np, 160, 120, [0.1, 0.1, 0.8, 0.8]) + blank = np.full((120, 160, 3), 30, dtype='uint8') + center, _, _, _ = tracker.update(0.0, blank) + self.assertIsNone(center) + texture = np.random.default_rng(41).integers(0, 256, (120, 160, 3), dtype='uint8') + center, _, _, state = tracker.update(0.03, texture) + self.assertIsNone(center) + self.assertEqual(state['reason'], 'insufficient_initial_features') + + def test_ambiguous_flow_clusters_produce_terminal_loss(self): + tracker = mv._FeatureAnchor(cv2, np, 160, 120, [0.1, 0.1, 0.8, 0.8]) + image = np.random.default_rng(42).integers(0, 256, (120, 160, 3), dtype='uint8') + center, _, _, _ = tracker.update(0.0, image) + self.assertIsNotNone(center) + # Authored optical-flow evidence: equally supported opposed motions. + tracker.points = np.array([[40 + i, 50] for i in range(12)], + dtype='float32').reshape(-1, 1, 2) + old = tracker.points.copy() + forward = old.copy() + forward[:6, :, 0] += 8 + forward[6:, :, 0] -= 8 + ok = np.ones((12, 1), dtype='uint8') + with patch.object(cv2, 'calcOpticalFlowPyrLK', + side_effect=[(forward, ok, None), (old, ok, None)]): + center, support, _, state = tracker.update(0.03, image) + self.assertIsNone(center) + self.assertEqual(support, 0) + self.assertEqual(state['reason'], 'ambiguous_feature_motion') + + def test_decoder_watchdog_stops_blocked_owned_process(self): + original_popen = subprocess.Popen + processes = [] + + def blocked(*args, **kwargs): + child = original_popen([sys.executable, '-c', 'import time; time.sleep(30)'], + **kwargs) + processes.append(child) + return child + + with patch.object(mv.subprocess, 'Popen', side_effect=blocked): + with self.assertRaisesRegex(mv.MotionVideoError, 'watchdog'): + list(mv._decoded_frames(self.root/'never-read.mp4', 16, 16, + sample_hz=30, max_frames=2, duration=1, timeout_seconds=0.05)) + self.assertEqual(len(processes), 1) + self.assertIsNotNone(processes[0].poll()) + + def test_excess_decoder_diagnostics_are_bounded_and_child_stopped(self): + original_popen = subprocess.Popen + processes = [] + + def excessive(*args, **kwargs): + child = original_popen([sys.executable, '-c', + 'import sys,time; sys.stderr.write("x"*9000); sys.stderr.flush(); time.sleep(30)'], + **kwargs) + processes.append(child) + return child + + with patch.object(mv.subprocess, 'Popen', side_effect=excessive): + with self.assertRaisesRegex(mv.MotionVideoError, 'bounded metadata budget'): + list(mv._decoded_frames(self.root/'never-read.mp4', 16, 16, + sample_hz=30, max_frames=2, duration=1, timeout_seconds=2)) + self.assertIsNotNone(processes[0].poll()) + + +if __name__ == '__main__': + unittest.main() From f8e7767e3fb69ffd509fca296f3c40ec7cf43bfe Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 10 Oct 2026 22:21:05 -0400 Subject: [PATCH 2/2] Validate video display geometry before motion intake and replay --- docs/motion-observations.md | 18 ++-- src/dvidia_training/motion_studio.py | 5 + src/dvidia_training/motion_video.py | 152 ++++++++++++++++++++++++++- tests/test_motion_studio.py | 77 ++++++++++++++ tests/test_motion_video.py | 125 ++++++++++++++++++++++ 5 files changed, 367 insertions(+), 10 deletions(-) diff --git a/docs/motion-observations.md b/docs/motion-observations.md index 582d1f6..fa4f3ce 100644 --- a/docs/motion-observations.md +++ b/docs/motion-observations.md @@ -64,8 +64,9 @@ reports an unsupported video format instead of substituting a different source. ## Analyze your permitted footage -Use a stationary camera and a textured, clearly visible target for the first -trial. Keep original frame times and the complete motion/outcome. The input +Use a stationary camera, an unrotated square-pixel recording and a textured, +clearly visible target for the first trial. Keep original frame times and the +complete motion/outcome. The input uses the existing strict [`dvidia.observation-source.v1` manifest](observations.md#supply-permitted-videos), including original SHA-256, session/recording groups, credit, license and an explicit `local_analysis` permission declaration. The declaration records the @@ -83,10 +84,15 @@ Choose one listed episode and manually specify its first-frame region: `--box` is normalized `x1,y1,x2,y2`, from 0 to 1, in the decoded source raster. It is a human-authored region, not a model detection or proof of physical object -identity. Automatic rotation is disabled; a phone recording's rotation metadata -is not applied to this coordinate system. The first decoded image is retained -as `seed.png` for inspection. Box drawing/correction and automatic semantic -object selection remain subsequent interface work. +identity. Intake and replay reject nonzero rotation, nonidentity display +matrices and explicit non-square pixel aspect ratios so browser overlays cannot +silently use different coordinates from the tracker. Undefined aspect metadata +uses the square-pixel raster default; it does not establish physical calibration. +Automatic rotation is disabled. If normalization is needed, keep the original, +record the normalized copy's new hash and choose new seed coordinates for it. +The first decoded image is retained as `seed.png` for inspection. Box +drawing/correction and automatic semantic object selection remain subsequent +interface work. ## What the baseline computes diff --git a/src/dvidia_training/motion_studio.py b/src/dvidia_training/motion_studio.py index 2514018..733bbf0 100644 --- a/src/dvidia_training/motion_studio.py +++ b/src/dvidia_training/motion_studio.py @@ -120,6 +120,11 @@ def load_motion_run(path): if (type(source) is not dict or source.get('sha256') != media['sha256'] or source.get('bytes') != media['bytes'] or protocol.get('source_sha256') != media['sha256']): raise ValueError('Motion media no longer matches its declared original.') + from .motion_video import validate_display_geometry + display_geometry = validate_display_geometry(video) + if protocol.get('display_geometry') is not None or source.get('display_geometry') is not None: + if protocol.get('display_geometry') != display_geometry or source.get('display_geometry') != display_geometry: + raise ValueError('Motion display geometry differs from the recorded source.') elif run['status'] == 'completed': raise ValueError('A completed motion run requires copied source media.') if run['status'] == 'completed' and not rows: diff --git a/src/dvidia_training/motion_video.py b/src/dvidia_training/motion_video.py index bd8b6cb..9c5d71f 100644 --- a/src/dvidia_training/motion_video.py +++ b/src/dvidia_training/motion_video.py @@ -20,7 +20,7 @@ import threading import time -from .footage_data import _local_file, canonical, probe_video +from .footage_data import _json, _local_file, canonical, probe_video from .observation_pipeline import validate_manifest, validate_timebase from .observations import box, digest, file_hash, identity, number, text @@ -30,6 +30,8 @@ MAX_STDERR_BYTES = 2 * 1024 * 1024 MAX_STDERR_LINE_BYTES = 8192 MAX_TIMESTAMP_QUEUE = 16 +MAX_GEOMETRY_BYTES = 64 * 1024 +MAX_GEOMETRY_SECONDS = 10 MAX_TRACK_GAP_SECONDS = 0.25 MIN_FEATURES = 6 FEATURE_RECIPE = { @@ -46,6 +48,7 @@ } LIMITATIONS = [ 'Fixed-camera scope; camera movement is not compensated or detected.', + 'Unrotated square-pixel display geometry only; explicit aspect/rotation/transforms are checked before decoding.', 'Human-authored seed and label; no automatic semantic object detection.', 'Image anchor from initial feature translation, not a physical object centroid.', 'Units are analysis-image pixels and pixels/second, not metric 3D motion.', @@ -86,6 +89,143 @@ def _stop_process(process): process.kill() +def _display_geometry_from_metadata(metadata): + """Allow only the browser's identity raster display geometry. + + Absent/undefined SAR means no aspect override: a browser displays the coded + raster with square pixels. Explicit non-square ratios or display transforms + cannot be silently erased by the motion decoder's setsar/noautorotate flags. + """ + if type(metadata) is not dict: + raise MotionVideoError('Display geometry metadata must be a data-only object.') + streams = metadata.get('streams') + if type(streams) is not list or len(streams) != 1 or type(streams[0]) is not dict: + raise MotionVideoError('Could not establish one primary video display geometry.') + stream = streams[0] + sar = stream.get('sample_aspect_ratio') + undefined = sar is None or sar in ('N/A', '0:1') + if not undefined: + if type(sar) is not str or not re.fullmatch(r'[0-9]{1,10}:[0-9]{1,10}', sar): + raise MotionVideoError('Sample aspect ratio is unknown; use a square-pixel source.') + numerator, denominator = (int(value) for value in sar.split(':')) + if numerator <= 0 or denominator <= 0 or numerator != denominator: + raise MotionVideoError('Non-square sample aspect ratio is unsupported; source/display anchors would differ.') + + def check_rotation(value): + if type(value) not in (str, int, float) or (type(value) is str and len(value) > 80): + raise MotionVideoError('Unknown display rotation is unsupported.') + try: + angle = float(value) + except (ValueError, OverflowError) as exc: + raise MotionVideoError('Unknown display rotation is unsupported.') from exc + if not math.isfinite(angle) or angle != 0: + raise MotionVideoError('Rotated video is unsupported; use an unrotated source with identity display geometry.') + + def check_tags(value): + if type(value) is not dict: + raise MotionVideoError('Display rotation tags must be a data-only object.') + for key, angle in value.items(): + if key.lower() in ('rotate', 'rotation'): + check_rotation(angle) + + check_tags(stream.get('tags', {})) + container = metadata.get('format', {}) + if type(container) is not dict: + raise MotionVideoError('Display container metadata must be a data-only object.') + check_tags(container.get('tags', {})) + side_data = stream.get('side_data_list', []) + if type(side_data) is not list or len(side_data) > 64: + raise MotionVideoError('Video display side data exceeds the supported envelope.') + matrix_count = 0 + identity_matrix = [65536, 0, 0, 0, 65536, 0, 0, 0, 1073741824] + for entry in side_data: + if type(entry) is not dict: + raise MotionVideoError('Video display side data must contain data-only objects.') + if 'rotation' in entry: + check_rotation(entry['rotation']) + if entry.get('side_data_type') == 'Display Matrix' or 'displaymatrix' in entry: + matrix_count += 1 + matrix = entry.get('displaymatrix') + if type(matrix) is not str or len(matrix) > 2048: + raise MotionVideoError('Unknown display matrix is unsupported; identity geometry is required.') + lines = matrix.strip().splitlines() + if len(lines) != 3: + raise MotionVideoError('Unknown display matrix is unsupported; identity geometry is required.') + values = [] + for row_index, line in enumerate(lines): + match = re.fullmatch(r'\s*([0-9a-fA-F]{8}):\s*(-?[0-9]{1,12})\s+(-?[0-9]{1,12})\s+(-?[0-9]{1,12})\s*', line) + if match is None or int(match.group(1), 16) != row_index: + raise MotionVideoError('Unknown display matrix is unsupported; identity geometry is required.') + values.extend(int(match.group(index)) for index in (2, 3, 4)) + if values != identity_matrix or matrix_count > 1: + raise MotionVideoError('Nonidentity display matrix is unsupported; reflected, rotated or transformed overlays would misalign.') + return {'effective_sample_aspect_ratio': [1, 1], 'declared_sample_aspect_ratio': sar, + 'sample_aspect_ratio_basis': 'unspecified-raster-default' if undefined else 'explicit-square-pixels', + 'display_rotation_degrees': 0, 'display_matrix': 'explicit-identity' if matrix_count else 'absent-identity-default', + 'geometry_status': 'identity_display_geometry'} + + +def _probe_display_geometry(video, *, timeout_seconds=MAX_GEOMETRY_SECONDS): + """Read at most 64 KiB of local display metadata within a 10-second watchdog.""" + command = ['ffprobe', '-v', 'error', '-protocol_whitelist', 'file', + '-format_whitelist', 'mov,matroska,webm,avi', '-select_streams', 'v:0', + '-show_entries', 'stream=sample_aspect_ratio:stream_tags=rotate,rotation,ROTATE,ROTATION:' + 'stream_side_data=side_data_type,displaymatrix,rotation:format_tags=rotate,rotation,ROTATE,ROTATION', + '-of', 'json', str(video)] + try: + process = subprocess.Popen(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, start_new_session=(os.name == 'posix')) + except OSError as exc: + raise MotionVideoError('ffprobe is unavailable for the local display geometry check.') from exc + timed_out = threading.Event() + + def expire(): + timed_out.set() + _stop_process(process) + + watchdog = threading.Timer(timeout_seconds, expire) + watchdog.daemon = True + watchdog.start() + try: + raw = process.stdout.read(MAX_GEOMETRY_BYTES + 1) + if timed_out.is_set(): + raise MotionVideoError('Local display geometry check exceeded its 10-second watchdog.') + if len(raw) > MAX_GEOMETRY_BYTES: + raise MotionVideoError('Display geometry metadata exceeds its 64 KiB limit.') + try: + result = process.wait(timeout=2) + except subprocess.TimeoutExpired as exc: + raise MotionVideoError('Local display geometry probe did not exit after its bounded output.') from exc + if result: + raise MotionVideoError('Could not inspect local display geometry; no external media was fetched.') + try: + metadata = _json(raw, maximum=MAX_GEOMETRY_BYTES) + except ValueError as exc: + raise MotionVideoError('Display geometry probe did not return bounded finite JSON metadata.') from exc + return _display_geometry_from_metadata(metadata) + finally: + watchdog.cancel() + _stop_process(process) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + pass + process.stdout.close() + + +def validate_display_geometry(video): + """Return bounded identity-display metadata or reject the local clip. + + This public intake/replay guard does not rewrite media. Absent/undefined SAR + is recorded as the square-pixel raster default; explicit non-square aspect, + nonzero rotation or a nonidentity/unknown display matrix is rejected. + """ + video = Path(video) + if video.is_symlink() or not video.is_file() or not 0 < video.stat().st_size <= 128 * 1024 * 1024: + raise MotionVideoError('Display geometry requires a bounded, regular local video file.') + return _probe_display_geometry(video.resolve()) + + def _decoded_frames(video, width, height, *, sample_hz, max_frames, duration, timeout_seconds=MAX_DECODE_SECONDS): """Stream bounded raw frames with their selected *input* presentation times. @@ -359,7 +499,10 @@ def analyze_video(manifest_path, episode_id, output, *, seed_box, label='object' 'cadence_note': 'Velocity needs two actual observations no more than 0.15 seconds apart; low source/sample cadence can prevent every forecast.', 'processing_budget': {'seconds': 150, 'enforcement': 'cooperative check between processed frames; native tracking call is not preempted'}, 'timebase': 'FFmpeg showinfo integer input PTS multiplied by declared rational time base; zero origin required', - 'video_rotation': 'automatic rotation disabled; seed coordinates use decoded source raster', + 'video_rotation': 'identity display geometry required; automatic rotation disabled; seed coordinates use decoded source raster', + 'display_geometry_policy': 'explicit non-square SAR and nonzero rotation/nonidentity display matrices rejected; absent SAR uses square-pixel raster default', + 'display_geometry_probe_limits': {'maximum_stdout_bytes': MAX_GEOMETRY_BYTES, + 'watchdog_seconds': MAX_GEOMETRY_SECONDS}, 'hardware': {'platform': platform.system(), 'machine': platform.machine()}, 'implementation_sha256': {'motion_video.py': file_hash(Path(__file__)), 'observation_pipeline.py': file_hash(Path(__file__).with_name('observation_pipeline.py'))}, @@ -381,17 +524,18 @@ def analyze_video(manifest_path, episode_id, output, *, seed_box, label='object' if info['duration_seconds'] > max_duration_seconds or max(info['width'], info['height']) > 4096: raise MotionVideoError('Source exceeds the selected duration or 4096-pixel source-side limit.') validate_timebase(source_path, info) + display_geometry = validate_display_geometry(source_path) target = output / ('original' + source_path.suffix.lower()) shutil.copyfile(source_path, target) if file_hash(target) != actual_hash: raise MotionVideoError('Original changed during local intake.') width, height = _dimensions(info['width'], info['height']) - source.update(bytes=target.stat().st_size, timing=info) + source.update(bytes=target.stat().st_size, timing=info, display_geometry=display_geometry) media = {'path': target.name, 'sha256': actual_hash, 'bytes': target.stat().st_size} protocol.update(original_image_size_pixels=[info['width'], info['height']], analysis_image_size_pixels=[width, height], source_to_analysis_scale_xy=[width / info['width'], height / info['height']], - source_sha256=actual_hash) + source_sha256=actual_hash, display_geometry=display_geometry) timings['intake_seconds'] = time.monotonic() - stage_started stage = 'dependency_setup' stage_started = time.monotonic() diff --git a/tests/test_motion_studio.py b/tests/test_motion_studio.py index 98f0b89..607252b 100644 --- a/tests/test_motion_studio.py +++ b/tests/test_motion_studio.py @@ -220,6 +220,83 @@ def test_same_size_source_media_change_is_detected(self): with self.assertRaisesRegex(ValueError, 'media|source'): load_motion_run(self.run_dir/'run.json') + def test_display_geometry_descriptor_cannot_disagree_with_retained_media(self): + for fields in (('source',), ('protocol',), ('source', 'protocol')): + run = deepcopy(self.receipt) + for field in fields: + run[field]['display_geometry']['display_rotation_degrees'] = 180 + write_run_parts(self.run_dir, run) + with self.subTest(fields=fields), self.assertRaisesRegex(ValueError, 'display geometry'): + load_motion_run(self.run_dir/'run.json') + + def test_older_identity_receipt_without_geometry_descriptors_remains_replayable(self): + run = self.read_run() + run['protocol'].pop('display_geometry', None) + run['source'].pop('display_geometry', None) + write_run_parts(self.run_dir, run) + loaded = load_motion_run(self.run_dir/'run.json') + self.assertEqual(loaded['rows'], self.receipt['rows']) + self.assertNotIn('display_geometry', loaded['protocol']) + self.assertNotIn('display_geometry', loaded['source']) + report, episodes = load_motion_bundle(self.run_dir) + self.assertIsNone(report) + self.assertEqual(episodes['episode']['run']['media']['sha256'], self.receipt['media']['sha256']) + + def rotated_video(self, source, angle): + """Require real nonidentity metadata; support pre/post-FFmpeg 9 recipes.""" + target = self.root/f'rotated-{angle}.mp4' + commands = [ + ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', '-y', + '-i', str(source), '-map', '0', '-c', 'copy', + '-metadata:s:v:0', f'rotate={angle}', str(target)], + ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', '-y', + '-display_rotation', str(angle), '-i', str(source), + '-map', '0', '-c', 'copy', str(target)], + ] + for command in commands: + result = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=20, check=False) + if result.returncode: + continue + proof = subprocess.run(['ffprobe', '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'stream_tags=rotate:stream_side_data=side_data_type,displaymatrix,rotation', + '-of', 'json', str(target)], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10, check=False) + if proof.returncode or len(proof.stdout) > 65536: + continue + metadata = json.loads(proof.stdout) + streams = metadata.get('streams', []) + if len(streams) != 1: + continue + stream = streams[0] + angles = [entry.get('rotation', 0) for entry in stream.get('side_data_list', [])] + if 'rotate' in stream.get('tags', {}): + angles.append(stream['tags']['rotate']) + if any(float(value) % 360 != 0 for value in angles): + return target + self.skipTest('Installed FFmpeg cannot create inspectable stream-copy rotation metadata.') + + def test_older_rotated_receipt_cannot_bypass_startup_geometry_check_by_rebinding_hashes(self): + run = self.read_run() + media_path = self.run_dir/run['media']['path'] + rotated = self.rotated_video(media_path, 90) + shutil.copyfile(rotated, media_path) + changed_hash, changed_size = file_hash(media_path), media_path.stat().st_size + run['media'].update(sha256=changed_hash, bytes=changed_size) + run['source'].update(sha256=changed_hash, bytes=changed_size) + run['protocol']['source_sha256'] = changed_hash + run['protocol'].pop('display_geometry', None) + run['source'].pop('display_geometry', None) + write_run_parts(self.run_dir, run) + # Every retained hash is coherent. The media's actual display transform, + # rather than a stale/missing descriptor, must reject replay startup. + self.assertEqual(run['media']['sha256'], file_hash(media_path)) + self.assertEqual(run['protocol_sha256'], digest(run['protocol'])) + with self.assertRaisesRegex(ValueError, 'Rotated video|display geometry'): + load_motion_run(self.run_dir/'run.json') + with self.assertRaisesRegex(ValueError, 'Rotated video|display geometry'): + MotionServer(('127.0.0.1', 0), self.run_dir) + def test_missing_evidence_file_is_rejected(self): (self.run_dir/'frames.json').unlink() with self.assertRaises((ValueError, FileNotFoundError)): diff --git a/tests/test_motion_video.py b/tests/test_motion_video.py index 573a13f..fc23481 100644 --- a/tests/test_motion_video.py +++ b/tests/test_motion_video.py @@ -87,6 +87,8 @@ def test_actual_decode_tracks_translation_and_preserves_data_hashes(self): self.assertEqual(receipt['seed']['kind'], 'human_authored_image_region') self.assertEqual(receipt['protocol']['analysis_image_size_pixels'], [160, 120]) self.assertEqual(receipt['protocol']['original_image_size_pixels'], [160, 120]) + self.assertEqual(receipt['protocol']['display_geometry']['geometry_status'], 'identity_display_geometry') + self.assertEqual(receipt['source']['display_geometry']['effective_sample_aspect_ratio'], [1, 1]) self.assertEqual(file_hash(self.root/'run'/'original.mp4'), file_hash(video)) self.assertEqual(receipt['media']['bytes'], video.stat().st_size) self.assertEqual(receipt['protocol_sha256'], digest(receipt['protocol'])) @@ -164,6 +166,88 @@ def test_nonzero_original_timeline_is_rejected(self): self.assertEqual(receipt['errors'][0]['stage'], 'intake') self.assertEqual(receipt['rows'], []) + def rotated_video(self, source, angle): + """Create real track rotation, accounting for FFmpeg metadata changes.""" + target = self.root / f'rotation-{angle}.mp4' + commands = [ + ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', '-y', + '-i', str(source), '-map', '0', '-c', 'copy', + '-metadata:s:v:0', f'rotate={angle}', str(target)], + ['ffmpeg', '-hide_banner', '-loglevel', 'error', '-nostdin', '-y', + '-display_rotation', str(angle), '-i', str(source), + '-map', '0', '-c', 'copy', str(target)], + ] + for command in commands: + result = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=20, check=False) + if result.returncode: + continue + proof = subprocess.run(['ffprobe', '-v', 'error', '-select_streams', 'v:0', + '-show_entries', 'stream_tags=rotate:stream_side_data=side_data_type,displaymatrix,rotation', + '-of', 'json', str(target)], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10, check=False) + if proof.returncode or len(proof.stdout) > 65536: + continue + stream = json.loads(proof.stdout)['streams'][0] + angles = [entry.get('rotation', 0) for entry in stream.get('side_data_list', [])] + if 'rotate' in stream.get('tags', {}): + angles.append(stream['tags']['rotate']) + if any(float(value) != 0 for value in angles): + return target + self.skipTest(f'Installed FFmpeg cannot encode inspectable {angle}-degree stream-copy rotation metadata.') + + def assert_geometry_intake_rejected(self, video, message): + receipt = self.analyze(video) + self.assertEqual(receipt['status'], 'failed') + self.assertEqual(receipt['errors'][0]['stage'], 'intake') + self.assertIn(message, receipt['errors'][0]['error']) + self.assertEqual(receipt['rows'], []) + self.assertEqual(receipt['frames'], []) + self.assertIsNone(receipt['media']) + self.assertFalse((self.root/'run'/'seed.png').exists()) + self.assertFalse(any((self.root/'run').glob('original.*'))) + self.assertTrue((self.root/'run'/'run.json').is_file()) + + def test_actual_90_degree_metadata_rotation_is_rejected_before_decode(self): + source = self.video(count=3) + self.assert_geometry_intake_rejected(self.rotated_video(source, 90), 'Rotated video') + + def test_actual_180_degree_metadata_rotation_is_rejected_before_decode(self): + source = self.video(count=3) + self.assert_geometry_intake_rejected(self.rotated_video(source, 180), 'Rotated video') + + def test_actual_anamorphic_sample_aspect_ratio_is_rejected_before_decode(self): + video = self.video(count=3, pts_filter='setsar=2/1') + self.assert_geometry_intake_rejected(video, 'Non-square sample aspect ratio') + + def test_display_matrix_requires_exact_identity_even_if_rotation_is_zero(self): + identity = '\n00000000: 65536 0 0\n00000001: 0 65536 0\n00000002: 0 0 1073741824\n' + matrix = {'side_data_type': 'Display Matrix', 'displaymatrix': identity, 'rotation': 0} + metadata = {'streams': [{'sample_aspect_ratio': '1:1', 'side_data_list': [matrix]}]} + self.assertEqual(mv._display_geometry_from_metadata(metadata)['display_matrix'], 'explicit-identity') + for value in (identity.replace('65536 0 0', '-65536 0 0', 1), + identity.replace('65536 0 0', '65536 5 0', 1), + identity.replace('0 0 1073741824', '100 0 1073741824'), + 'unknown display matrix'): + with self.subTest(value=value), self.assertRaises(mv.MotionVideoError): + mv._display_geometry_from_metadata({'streams': [{'sample_aspect_ratio': '1:1', + 'side_data_list': [{**matrix, 'displaymatrix': value}]}]}) + + def test_rotation_tags_and_unknown_display_metadata_cannot_claim_identity(self): + for tags in ({'rotate': '90'}, {'rotation': '180'}, {'ROTATE': 'nan'}, {'rotate': True}): + with self.subTest(tags=tags), self.assertRaises(mv.MotionVideoError): + mv._display_geometry_from_metadata({'streams': [{'sample_aspect_ratio': '1:1', 'tags': tags}]}) + with self.assertRaises(mv.MotionVideoError): + mv._display_geometry_from_metadata({'streams': [{'sample_aspect_ratio': '1:1', + 'side_data_list': [{'side_data_type': 'Display Matrix', 'rotation': 0}]}]}) + for ratio in ('16:15', '4:3', '1:0', 'invalid', True): + with self.subTest(ratio=ratio), self.assertRaises(mv.MotionVideoError): + mv._display_geometry_from_metadata({'streams': [{'sample_aspect_ratio': ratio}]}) + for ratio in (None, 'N/A', '0:1'): + source = {} if ratio is None else {'sample_aspect_ratio': ratio} + self.assertEqual(mv._display_geometry_from_metadata({'streams': [source]})['sample_aspect_ratio_basis'], + 'unspecified-raster-default') + def test_invalid_limits_and_unknown_episode_create_no_output(self): video = self.video(count=3) manifest = manifest_for(self.root, video) @@ -269,6 +353,47 @@ def excessive(*args, **kwargs): sample_hz=30, max_frames=2, duration=1, timeout_seconds=2)) self.assertIsNotNone(processes[0].poll()) + def test_display_geometry_probe_watchdog_stops_blocked_owned_child(self): + original_popen = subprocess.Popen + processes = [] + + def blocked(*args, **kwargs): + child = original_popen([sys.executable, '-c', 'import time; time.sleep(30)'], **kwargs) + processes.append(child) + return child + + with patch.object(mv.subprocess, 'Popen', side_effect=blocked): + with self.assertRaisesRegex(mv.MotionVideoError, '10-second watchdog'): + mv._probe_display_geometry(self.root/'never-read.mp4', timeout_seconds=0.05) + self.assertIsNotNone(processes[0].poll()) + + def test_display_geometry_probe_stdout_is_bounded_and_child_stopped(self): + original_popen = subprocess.Popen + processes = [] + + def oversized(*args, **kwargs): + child = original_popen([sys.executable, '-c', + 'import sys,time; sys.stdout.write("x"*70000); sys.stdout.flush(); time.sleep(30)'], **kwargs) + processes.append(child) + return child + + with patch.object(mv.subprocess, 'Popen', side_effect=oversized): + with self.assertRaisesRegex(mv.MotionVideoError, '64 KiB limit'): + mv._probe_display_geometry(self.root/'never-read.mp4', timeout_seconds=2) + self.assertIsNotNone(processes[0].poll()) + + def test_public_display_guard_rejects_missing_files_and_symlinks(self): + with self.assertRaisesRegex(mv.MotionVideoError, 'regular local video'): + mv.validate_display_geometry(self.root/'missing.mp4') + video = self.video(count=3) + link = self.root/'linked.mp4' + try: + link.symlink_to(video) + except (OSError, NotImplementedError): + self.skipTest('This platform does not permit creating a symlink fixture.') + with self.assertRaisesRegex(mv.MotionVideoError, 'regular local video'): + mv.validate_display_geometry(link) + if __name__ == '__main__': unittest.main()