From d355a4a284cf9bcc4f48dd375ca072d2da188dad Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 10 Oct 2026 07:29:14 -0400 Subject: [PATCH 1/3] Add local video observation and review pilot --- README.md | 12 + benchmarks/observation-20261010/README.md | 100 ++++ .../observation-20261010/episode-summary.json | 80 +++ .../hardened-protocol.json | 1 + .../observation-20261010/hardened-run.json | 1 + .../initial-protocol.json | 1 + .../observation-20261010/initial-run.json | 1 + .../observation-20261010/ui-validation.json | 22 + docs/observation-models.md | 31 ++ docs/observation-pilot-plan.md | 63 +++ docs/observation-ui.md | 80 +++ docs/observations.md | 122 ++++ pyproject.toml | 6 +- scripts/download_observation_models.py | 71 +++ src/dvidia_training/observation_models.py | 236 ++++++++ src/dvidia_training/observation_pipeline.py | 233 ++++++++ src/dvidia_training/observation_studio.html | 522 ++++++++++++++++++ src/dvidia_training/observation_studio.py | 221 ++++++++ src/dvidia_training/observations.py | 208 +++++++ tests/test_observation_models.py | 36 ++ tests/test_observations.py | 231 ++++++++ .../object_detection_yolox-LICENSE.txt | 201 +++++++ .../palm_detection_mediapipe-LICENSE.txt | 202 +++++++ 23 files changed, 2680 insertions(+), 1 deletion(-) create mode 100644 benchmarks/observation-20261010/README.md create mode 100644 benchmarks/observation-20261010/episode-summary.json create mode 100644 benchmarks/observation-20261010/hardened-protocol.json create mode 100644 benchmarks/observation-20261010/hardened-run.json create mode 100644 benchmarks/observation-20261010/initial-protocol.json create mode 100644 benchmarks/observation-20261010/initial-run.json create mode 100644 benchmarks/observation-20261010/ui-validation.json create mode 100644 docs/observation-models.md create mode 100644 docs/observation-pilot-plan.md create mode 100644 docs/observation-ui.md create mode 100644 docs/observations.md create mode 100644 scripts/download_observation_models.py create mode 100644 src/dvidia_training/observation_models.py create mode 100644 src/dvidia_training/observation_pipeline.py create mode 100644 src/dvidia_training/observation_studio.html create mode 100644 src/dvidia_training/observation_studio.py create mode 100644 src/dvidia_training/observations.py create mode 100644 tests/test_observation_models.py create mode 100644 tests/test_observations.py create mode 100644 third_party/observation-models/object_detection_yolox-LICENSE.txt create mode 100644 third_party/observation-models/palm_detection_mediapipe-LICENSE.txt diff --git a/README.md b/README.md index 7e2635f..9f9213c 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,18 @@ Open [the collection planner](http://127.0.0.1:8270/coverage) in the same local The [episode tape recorder](docs/episode-tape.md) preserves the existing simulated arm's measured state, requested/applied commands, contacts and sensor ages. Recording needs the optional arm extra; validation and inspection need only Python's standard library. These tools are available from the current checkout and do not train a new policy or change the older Hugging Face pilot. +## Review what is visible in a video + +The new [local observation lab](docs/observations.md) uses optional CPU object and +palm detectors to propose timestamped image observations. Open the original video, +inspect its boxes and timeline, correct labels, and save a separate review revision. +Raw evidence stays unchanged. No account, cloud inference, or GPU is needed. + +The [first feasibility run](benchmarks/observation-20261010/README.md) completed +six public excerpts. This establishes the integration, not accuracy: labels can +be wrong, outcomes remain unknown, and there is no 3D reconstruction, automatic +action understanding, or human-video-to-robot control bridge. + ## Try the pipeline Generate an explicitly synthetic example, then train and verify its output: diff --git a/benchmarks/observation-20261010/README.md b/benchmarks/observation-20261010/README.md new file mode 100644 index 0000000..ba8379c --- /dev/null +++ b/benchmarks/observation-20261010/README.md @@ -0,0 +1,100 @@ +# Local observation feasibility — 10 October 2026 + +**Engineering integration passed on six public excerpts; accuracy not assessed.** +This is not a cup-placement benchmark, 3D reconstruction or trained robot skill. +The [plan](../../docs/observation-pilot-plan.md) was written before the model run. + +## Inputs and execution + +The fixed order was shirt, stir, pack, onion, melon, jeans, selected from existing +local public examples before examining model output. Five are attributed +EgoAnnotate excerpts; stir is a CoMind excerpt. Licenses and public source links +are preserved in both protocol files. Exact excerpts were hash-verified. The +full upstream originals were not downloaded or hash-verified for this pilot. +No raw media, sampled frames or private user metadata are included in this report. + +Six excerpts total 19 seconds, 2,753,287 bytes and 38 sampled frames. They are not +six independent complete attempts; unknown session grouping remains one +conservative shared group. No train/test split or learning run took place. +Publisher titles were never fed to the detector. + +| Attempt | Complete / attempted | Processing time | Failures | +| --- | --- | ---: | ---: | +| Initial recipe v1 | 6 / 6 | 4.835 seconds | 0 | +| Hardened recipe v1.1 | 6 / 6 | 4.635 seconds | 0 | + +Timing covers intake, copy/hash, frame decoding, detection, association and +episode writes. It excludes model provisioning/loading, final receipt writing, +UI startup and human review. Single runs, warm machine; this is neither a p95 +latency measurement nor a production throughput guarantee. Peak RAM, energy and +dollar cost were not measured. No paid inference API was called. + +Runtime: Python 3.12.9, OpenCV 5.0.0.93, NumPy 2.5.3, macOS ARM64, four CPU threads, +one inference worker. Both pinned ONNX models total approximately 40 MB. OpenCV +5 printed that target selection is unsupported by its new graph engine; this +build uses the OpenCV CPU backend and no GPU path was requested or measured. + +## What changed after the first run + +Review identified a possible browser-clock shift for videos with nonzero start +timestamps. Recipe v1.1 rejects nonzero container/video/decoded starts rather +than moving boxes relative to the unchanged original. The initial artifacts are +preserved alongside the subsequent run, which processed the same six inputs. +All six inputs have zero starts and were accepted in both runs. + +The review server was also hardened against same-size media replacement, +malformed review IDs and oversized run metadata. These are workflow fixes, +not improvements in detector quality. + +## Observed limitations + +- The three-frame preliminary adapter check included a saucepan incorrectly + labeled cup and storage jars labeled cups. Hands were missed in the stir + sample. These are qualitative findings, not scored accuracy estimates. +- COCO labels do not cover every task object; palm rectangles do not cover full + hands or prove contact. Two frames/second can miss short actions. +- Association uses same-class image overlap. It is not optical flow and can + fragment or switch identities. Overlap proposals do not establish grasping. +- Outcomes remain unknown. Model scores are uncalibrated, and the pilot creates + no automatic natural-language action captions or robot commands. +- The intended permission-cleared cup-placement cohort and independent reference + annotations have not been collected. Accuracy, false-completion rate and + correction time remain unmeasured. No model ranking is established. + +## Evidence + +- `initial-protocol.json`, `initial-run.json`: original frozen inputs and receipt. +- `hardened-protocol.json`, `hardened-run.json`: later timing policy and receipt. +- `episode-summary.json`: durations, proposal counts, timing and excerpt hashes. +- Local ignored run directories retain media, frames and raw episode annotations. + Run and source hashes bind the receipts to those local artifacts; this report + alone cannot reproduce a visual inspection without obtaining permitted clips. + +Browser interaction testing uses a separate copied run. Test review edits do not +count as independent human annotations and do not alter the original run's +unreviewed status. + +## Implementation validation + +The repository suite passed 307 tests, followed by 24 focused observation tests +including four additional detector-admission/suppression tests. These exercise +real FFmpeg decoding, timestamps, source tampering, failed attempts, malformed +reviews, revision conflicts, local-origin enforcement and video byte ranges. + +Fifteen browser checks passed on an isolated review copy: playback, timestamped +overlays, correction save/reload, preservation of raw labels, manual events, +export, unsaved-edit protection, completion validation and 390/320px layouts. +There were no console/page errors. See `ui-validation.json`. Headless browser +checks are not physical-phone or Safari qualification. + +The wheel built successfully; its final HTML, command entry points and both +upstream license files were verified. Raw footage/model files are Git-ignored. +Production uploads, accounts, storage and website deployment were not modified. + +## Next decision + +Collect 30–50 complete cup-placement attempts with permission and explicit +session/person grouping, including failures, occlusion and recovery. Establish +blinded reference labels first. Then compare object-only, temporal caption-only, +combined proposals and manual annotation at matched quality and total cost. +Only after that should an app integration claim reliable task labeling. diff --git a/benchmarks/observation-20261010/episode-summary.json b/benchmarks/observation-20261010/episode-summary.json new file mode 100644 index 0000000..a6cf4a6 --- /dev/null +++ b/benchmarks/observation-20261010/episode-summary.json @@ -0,0 +1,80 @@ +[ + { + "id": "shirt", + "duration_seconds": 3.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 6, + "elapsed_seconds": 0.6693273328710347, + "frames": 6, + "inference_seconds": 0.46699687489308417 + }, + "proposals": 7, + "source_sha256": "8392c62f4e9590a15745311746114c2b6355e24c60fa87fef68e834f20762113" + }, + { + "id": "stir", + "duration_seconds": 4.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 8, + "elapsed_seconds": 1.1173061658628285, + "frames": 8, + "inference_seconds": 0.6056890829931945 + }, + "proposals": 25, + "source_sha256": "4ffe78850d4e6475546a521e0eb425f7a34c632679183cf112aacb70ec1b0861" + }, + { + "id": "pack", + "duration_seconds": 3.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 6, + "elapsed_seconds": 0.6751292911358178, + "frames": 6, + "inference_seconds": 0.4420628750231117 + }, + "proposals": 7, + "source_sha256": "87a67e5b10f124a65df7d669086d4a9b2cb7ae25a5d758594b9b3aa7b0938f4c" + }, + { + "id": "onion", + "duration_seconds": 3.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 6, + "elapsed_seconds": 0.7398202081676573, + "frames": 6, + "inference_seconds": 0.43661770783364773 + }, + "proposals": 10, + "source_sha256": "100e25698ee99e99dc648c41d9864144e312f0fce7c07dc9a7b06b487b9a2a5f" + }, + { + "id": "melon", + "duration_seconds": 3.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 6, + "elapsed_seconds": 0.7281540830153972, + "frames": 6, + "inference_seconds": 0.4237409171182662 + }, + "proposals": 14, + "source_sha256": "f32cf75d28602d169b4f302830ba633a351ac890596887c54c8e928efcf2650e" + }, + { + "id": "jeans", + "duration_seconds": 3.0, + "metrics": { + "accuracy_status": "not_assessed", + "detected_frames": 6, + "elapsed_seconds": 0.6997090000659227, + "frames": 6, + "inference_seconds": 0.4299595837946981 + }, + "proposals": 9, + "source_sha256": "95845362951b3f78b7b731691df6d472bf176ced4b07ecef4d99d0b8a21da3e2" + } +] diff --git a/benchmarks/observation-20261010/hardened-protocol.json b/benchmarks/observation-20261010/hardened-protocol.json new file mode 100644 index 0000000..f424e21 --- /dev/null +++ b/benchmarks/observation-20261010/hardened-protocol.json @@ -0,0 +1 @@ +{"created_at":"2026-10-10T11:22:03.270961+00:00","execution":"local_offline","hardware":{"machine":"arm64","platform":"Darwin"},"kind":"dvidia.observation-protocol.v1","limits":{"cpu_threads":4,"workers":1},"manifest_sha256":"6e4e3a0b1b12eeb32cafcba0668d2614c80b022a91eafc7740018be880eddb29","model":{"backend":"opencv","coordinates":"normalized_image_xyxy","cpu_threads":10,"device":"cpu","hand_box_semantics":"Palm detector rectangle; not full hand segmentation or observed contact.","hand_detector_enabled":true,"hand_threshold":0.5,"implementation_sha256":"e1c6939779ab39a32497e00c35892a98eef59c3dd512135f1ac6cd4f897d1a22","implementation_version":1,"kind":"opencv-zoo-yolox-palm","limitations":["COCO labels can be wrong or absent, especially for small and occluded objects.","Palm boxes do not establish finger contact, grasp, force, depth, task success or robot actions.","Object identity and temporal relationships require a separate tracking stage."],"max_detections":30,"max_nms_candidates":1000,"models":[{"bytes":35858002,"directory":"object_detection_yolox","download_url":"https://media.githubusercontent.com/media/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/object_detection_yolox_2022nov.onnx","license":"Apache-2.0","license_bytes":11371,"license_sha256":"0ec3668d3274bcf29e8a29e9576d5a2cd96fc78d3c5bec4387355a796e5d9088","license_url":"https://raw.githubusercontent.com/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/LICENSE","name":"object_detection_yolox_2022nov.onnx","revision":"47534e27c9851bb1128ccc0102f1145e27f23f98","role":"objects","sha256":"c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063","source":"https://github.com/opencv/opencv_zoo/blob/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/object_detection_yolox_2022nov.onnx"},{"bytes":3905734,"directory":"palm_detection_mediapipe","download_url":"https://media.githubusercontent.com/media/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/palm_detection_mediapipe_2023feb.onnx","license":"Apache-2.0","license_bytes":11358,"license_sha256":"cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30","license_url":"https://raw.githubusercontent.com/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/LICENSE","name":"palm_detection_mediapipe_2023feb.onnx","revision":"47534e27c9851bb1128ccc0102f1145e27f23f98","role":"palms","sha256":"78ff51c38496b7fc8b8ebdb6cc8c1abb02fa6c38427c6848254cdaba57fcce7c","source":"https://github.com/opencv/opencv_zoo/blob/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/palm_detection_mediapipe_2023feb.onnx"}],"nms_iou":{"objects":0.5,"palms":0.3},"runtime":{"numpy":"2.5.3","opencv":"5.0.0"},"score_kind":"uncalibrated_model_evidence","threshold":0.35},"recipe":{"automatic_outcome":"unknown","claim_scope":"sampled_2d_observations","id":"dvidia-observations-v1.1","max_duration_seconds":60,"max_frames":120,"max_image_side":640,"max_track_gap_seconds":0.8,"minimum_association_iou":0.2,"samples_per_second":2,"timebase":"zero-origin video and container; nonzero starts rejected"},"sources":[{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"shirt","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"8392c62f4e9590a15745311746114c2b6355e24c60fa87fef68e834f20762113","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010079.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010079.mp4","title":"Starting to fold a white shirt"},{"credit":"Alexey Gavryushin, Dingxi Zhang, Zhao Huang, Alexandros Delitzas, Jiaqi Chen, Ben Ellis, Cedric Z\u00f6llner, Manthan Patel, Manuel Kaufmann, Marc Pollefeys, and Xi Wang; CoMind (2026); excerpt starts 1s, original full hash not verified","id":"stir","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"4ffe78850d4e6475546a521e0eb425f7a34c632679183cf112aacb70ec1b0861","source_recording_id":"https://comind.ethz.ch/","source_url":"https://comind.ethz.ch/","title":"Stirring vegetables in a pan"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"pack","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"87a67e5b10f124a65df7d669086d4a9b2cb7ae25a5d758594b9b3aa7b0938f4c","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010087.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010087.mp4","title":"Moving food from a tray toward a container"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"onion","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"100e25698ee99e99dc648c41d9864144e312f0fce7c07dc9a7b06b487b9a2a5f","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010072.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010072.mp4","title":"Slicing an onion on a pink plate"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"melon","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"f32cf75d28602d169b4f302830ba633a351ac890596887c54c8e928efcf2650e","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010076.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010076.mp4","title":"Cutting a watermelon wedge"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"jeans","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"95845362951b3f78b7b731691df6d472bf176ced4b07ecef4d99d0b8a21da3e2","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010081.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010081.mp4","title":"Folding grey jeans"}]} \ No newline at end of file diff --git a/benchmarks/observation-20261010/hardened-run.json b/benchmarks/observation-20261010/hardened-run.json new file mode 100644 index 0000000..30e5c2a --- /dev/null +++ b/benchmarks/observation-20261010/hardened-run.json @@ -0,0 +1 @@ +{"accuracy_status":"not_assessed","attempted":6,"completed":6,"elapsed_seconds":4.635012916056439,"episodes":[{"episode_sha256":"26b67bb490a0853482817f840715a17ae827830f32898b5fed60ccbcd51fd1a0","id":"shirt"},{"episode_sha256":"8b18fbda42aa44fa008fd763c865a045a416947b00464126b2e31b1c2c1c5ac8","id":"stir"},{"episode_sha256":"bb76a1996112fcb0bed37b988a4698676788b068d77e33ac4bc8cd080f6042da","id":"pack"},{"episode_sha256":"b94f9d15c4ecfc8c843d6c2d0e22568dbe32337cfe03091b7ef1bb529e1ee449","id":"onion"},{"episode_sha256":"7be44932308e08032322cfff7d9ee264335c016266c36015b0a991fb3cef9dab","id":"melon"},{"episode_sha256":"dee34f090cd5697889a6f904e30be6669085bf4db79273c56af3e4959151168e","id":"jeans"}],"failures":[],"geometry":"image_coordinates_only","kind":"dvidia.observation-run.v1","protocol_sha256":"3cd74c261157101b459563ff2b449c847194055fa024e60a6ed56aed94cf1b9c","robot_skill_qualified":false,"sessions":{"unknown-shared-publisher-session":["shirt","stir","pack","onion","melon","jeans"]},"timing_scope":"intake, decode, inference, association and per-episode writes; excludes model setup and final receipt"} \ No newline at end of file diff --git a/benchmarks/observation-20261010/initial-protocol.json b/benchmarks/observation-20261010/initial-protocol.json new file mode 100644 index 0000000..4425dd8 --- /dev/null +++ b/benchmarks/observation-20261010/initial-protocol.json @@ -0,0 +1 @@ +{"created_at":"2026-10-10T11:14:17.377906+00:00","execution":"local_offline","hardware":{"machine":"arm64","platform":"Darwin"},"kind":"dvidia.observation-protocol.v1","limits":{"cpu_threads":4,"workers":1},"manifest_sha256":"6e4e3a0b1b12eeb32cafcba0668d2614c80b022a91eafc7740018be880eddb29","model":{"backend":"opencv","coordinates":"normalized_image_xyxy","cpu_threads":10,"device":"cpu","hand_box_semantics":"Palm detector rectangle; not full hand segmentation or observed contact.","hand_detector_enabled":true,"hand_threshold":0.5,"implementation_sha256":"89cf6a3058661ec03acf78e3eaeab1182f235b0c2fe0d03b406fbaaafb29c2a4","implementation_version":1,"kind":"opencv-zoo-yolox-palm","limitations":["COCO labels can be wrong or absent, especially for small and occluded objects.","Palm boxes do not establish finger contact, grasp, force, depth, task success or robot actions.","Object identity and temporal relationships require a separate tracking stage."],"max_detections":30,"max_nms_candidates":1000,"models":[{"bytes":35858002,"directory":"object_detection_yolox","download_url":"https://media.githubusercontent.com/media/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/object_detection_yolox_2022nov.onnx","license":"Apache-2.0","license_url":"https://raw.githubusercontent.com/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/LICENSE","name":"object_detection_yolox_2022nov.onnx","revision":"47534e27c9851bb1128ccc0102f1145e27f23f98","role":"objects","sha256":"c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063","source":"https://github.com/opencv/opencv_zoo/blob/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox/object_detection_yolox_2022nov.onnx"},{"bytes":3905734,"directory":"palm_detection_mediapipe","download_url":"https://media.githubusercontent.com/media/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/palm_detection_mediapipe_2023feb.onnx","license":"Apache-2.0","license_url":"https://raw.githubusercontent.com/opencv/opencv_zoo/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/LICENSE","name":"palm_detection_mediapipe_2023feb.onnx","revision":"47534e27c9851bb1128ccc0102f1145e27f23f98","role":"palms","sha256":"78ff51c38496b7fc8b8ebdb6cc8c1abb02fa6c38427c6848254cdaba57fcce7c","source":"https://github.com/opencv/opencv_zoo/blob/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe/palm_detection_mediapipe_2023feb.onnx"}],"nms_iou":{"objects":0.5,"palms":0.3},"runtime":{"numpy":"2.5.3","opencv":"5.0.0"},"score_kind":"uncalibrated_model_evidence","threshold":0.35},"recipe":{"automatic_outcome":"unknown","claim_scope":"sampled_2d_observations","id":"dvidia-observations-v1","max_duration_seconds":60,"max_frames":120,"max_image_side":640,"max_track_gap_seconds":0.8,"minimum_association_iou":0.2,"samples_per_second":2},"sources":[{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"shirt","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"8392c62f4e9590a15745311746114c2b6355e24c60fa87fef68e834f20762113","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010079.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010079.mp4","title":"Starting to fold a white shirt"},{"credit":"Alexey Gavryushin, Dingxi Zhang, Zhao Huang, Alexandros Delitzas, Jiaqi Chen, Ben Ellis, Cedric Z\u00f6llner, Manthan Patel, Manuel Kaufmann, Marc Pollefeys, and Xi Wang; CoMind (2026); excerpt starts 1s, original full hash not verified","id":"stir","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"4ffe78850d4e6475546a521e0eb425f7a34c632679183cf112aacb70ec1b0861","source_recording_id":"https://comind.ethz.ch/","source_url":"https://comind.ethz.ch/","title":"Stirring vegetables in a pan"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"pack","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"87a67e5b10f124a65df7d669086d4a9b2cb7ae25a5d758594b9b3aa7b0938f4c","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010087.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010087.mp4","title":"Moving food from a tray toward a container"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"onion","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"100e25698ee99e99dc648c41d9864144e312f0fce7c07dc9a7b06b487b9a2a5f","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010072.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010072.mp4","title":"Slicing an onion on a pink plate"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"melon","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"f32cf75d28602d169b4f302830ba633a351ac890596887c54c8e928efcf2650e","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010076.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010076.mp4","title":"Cutting a watermelon wedge"},{"credit":"Taher Panbiharwala and Zainab Barwaniwala; EgoAnnotate v1; excerpt starts 0s, original full hash not verified","id":"jeans","kind":"public","license":"CC BY 4.0","permission":"local_analysis","session_id":"unknown-shared-publisher-session","sha256":"95845362951b3f78b7b731691df6d472bf176ced4b07ecef4d99d0b8a21da3e2","source_recording_id":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010081.mp4","source_url":"https://huggingface.co/datasets/TaherPanbiharwala/EgoAnnotate/blob/156d13adaee4110cf63e54fa940ec16ff199d12f/videos/GX010081.mp4","title":"Folding grey jeans"}]} \ No newline at end of file diff --git a/benchmarks/observation-20261010/initial-run.json b/benchmarks/observation-20261010/initial-run.json new file mode 100644 index 0000000..5a57c5e --- /dev/null +++ b/benchmarks/observation-20261010/initial-run.json @@ -0,0 +1 @@ +{"accuracy_status":"not_assessed","attempted":6,"completed":6,"elapsed_seconds":4.835139082977548,"episodes":[{"episode_sha256":"e5225fd8209e6a27f1b906d02cd642f52d4b21693a5d1ec671146aacaa7282b7","id":"shirt"},{"episode_sha256":"a4b1323797769802f7b945185748814038c1ef391fb0f94cf77a84cf5748a24e","id":"stir"},{"episode_sha256":"4ab53292050295dbd3fa3615b499b42a61013c7751fd992ba725266b241cec7b","id":"pack"},{"episode_sha256":"a4a4cf5b23ae679a712f79a522824b347560d5f0c1a6dcc246316244af6f900e","id":"onion"},{"episode_sha256":"a5cc8dd002f65cde1639771952fa0099415e22a2103e44cd1bfba7147e8e62ca","id":"melon"},{"episode_sha256":"dc0ace7b027e2260e791b278d9241c8b42f2e9243ce7d0ab9e47a00a25db734d","id":"jeans"}],"failures":[],"geometry":"image_coordinates_only","kind":"dvidia.observation-run.v1","protocol_sha256":"71629e04edd72e1daccb968e011f68db07ae87cf9fa083976cb1a73744a32926","robot_skill_qualified":false,"sessions":{"unknown-shared-publisher-session":["shirt","stir","pack","onion","melon","jeans"]},"timing_scope":"intake, decode, inference, association and per-episode writes; excludes model setup and final receipt"} \ No newline at end of file diff --git a/benchmarks/observation-20261010/ui-validation.json b/benchmarks/observation-20261010/ui-validation.json new file mode 100644 index 0000000..e39c612 --- /dev/null +++ b/benchmarks/observation-20261010/ui-validation.json @@ -0,0 +1,22 @@ +{ + "scope": "Local isolated UI QA copy; not model accuracy", + "checks": [ + "Desktop loads real episode and unknown outcome without horizontal overflow", + "Video plays, sampled overlays render, and box visibility toggles", + "Corrected approval persists after reload while raw label and unknown outcome remain unchanged", + "A new blank manual event can be removed; a labeled manual event can be approved and saved", + "Downloaded review matches persisted revision and original source hash", + "Episode navigation protects unsaved changes, preserves Keep reviewing, and honors explicit discard", + "Structured model and source provenance render as readable text", + "390px phone layout has no horizontal overflow and visible controls have 44px targets", + "320px phone layout has no horizontal overflow and visible controls have 44px targets", + "No browser console or page errors", + "Accepted events preserve decimal timing, including 0.25s\u20130.50s", + "Overlay counts match timestamp-bounded real evidence at 12 sample and gap boundaries", + "Pending events block completion without writing an invalid revision", + "390px: event selection reveals editor, provenance follows review, no overflow, 44px controls", + "320px: event selection reveals editor, provenance follows review, no overflow, 44px controls" + ], + "console_errors": [], + "screenshots": "Kept locally with ignored QA run; no source frames redistributed" +} diff --git a/docs/observation-models.md b/docs/observation-models.md new file mode 100644 index 0000000..b63cca9 --- /dev/null +++ b/docs/observation-models.md @@ -0,0 +1,31 @@ +# Observation model provenance + +Weights are downloaded explicitly into ignored local storage, never committed. +The observation adapter's pre/postprocessing is adapted from OpenCV Zoo; that +file is Apache-2.0 and retains upstream attribution. The rest of this repository +retains its existing license. Original licenses are included in +[`third_party/observation-models`](../third_party/observation-models/). + +Pinned upstream revision: `47534e27c9851bb1128ccc0102f1145e27f23f98`. + +| Role | Exact artifact | Bytes | SHA-256 | +| --- | --- | ---: | --- | +| COCO object detection | `object_detection_yolox_2022nov.onnx` | 35,858,002 | `c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063` | +| Palm detection | `palm_detection_mediapipe_2023feb.onnx` | 3,905,734 | `78ff51c38496b7fc8b8ebdb6cc8c1abb02fa6c38427c6848254cdaba57fcce7c` | + +The official [YOLOX directory](https://github.com/opencv/opencv_zoo/tree/47534e27c9851bb1128ccc0102f1145e27f23f98/models/object_detection_yolox) +and [MediaPipe palm directory](https://github.com/opencv/opencv_zoo/tree/47534e27c9851bb1128ccc0102f1145e27f23f98/models/palm_detection_mediapipe) +each state that all files in the directory are licensed under Apache 2.0. +The downloaded license text is also pinned and hash-checked by the downloader. +YOLOX's included license attributes Megvii Inc., 2021–2022. + +DVIDIA modifications include local weight verification, bounded deterministic +suppression, clipped normalized image boxes, explicit palm-only semantics, +metadata, runtime limits and validation of model output. No remote Python code +is downloaded or executed. Source data licensing remains independent from model +licensing. These license records do not establish rights to any user's footage. + +No SpatialLM, MANO, depth model or language model weights are part of this pilot. +The exact detector recipes and runtime versions are recorded in each run. The +first reports preserve the adapter implementation hash as it existed at run time; +later source or notice changes do not retroactively change those receipts. diff --git a/docs/observation-pilot-plan.md b/docs/observation-pilot-plan.md new file mode 100644 index 0000000..5967f7b --- /dev/null +++ b/docs/observation-pilot-plan.md @@ -0,0 +1,63 @@ +# DVIDIA observation pilot v1 + +Planned 10 October 2026, before model execution. This is an engineering feasibility +pilot, not a trained robot skill, production deployment, or accuracy benchmark. + +## Deliverables + +1. Local, bounded video decoding with actual sampled presentation timestamps, + original/frame hashes, source rights and session provenance. +2. Optional local detector and temporal caption adapters. DVIDIA owns the + orchestration, association, annotation contract and review workflow; pretrained + components retain their own licenses. No SpatialLM weights are used. +3. A mobile-friendly video/timeline review with editable proposals, explicit + outcomes, immutable review revisions and optimistic concurrency. +4. A runnable pilot receipt with time, resource scope, failures and limitations. + An independent reference-label evaluator must report not assessed when no + references exist; model agreement is not human review. + +## Fixed experiment scope + +- First candidate task: move a cup to a marked surface. Available public sample + footage may instead establish adapter feasibility; record task mismatches. +- Begin with five existing attributed EgoAnnotate excerpts (shirt, pack, + onion, melon, jeans) and one CoMind excerpt (stir), without selecting by model outputs. These are + excerpts, not independent complete cup-placement attempts. +- At most six initial clips, 60 seconds per clip, 128 MiB per clip, 120 frames + per clip, four CPU threads, one inference worker, no concurrent model jobs. +- Decode at most two samples/second, maximum image side 640. Proposals reference + actual decoded timestamps. No off-device media transfer or paid compute. +- The first run uses object/palm detections and 2D overlap proposals. Temporal + language captions, if added, receive a separate frozen recipe: at most three + windows per clip and four frames per window, maximum 96 generated tokens per + window. Record model/load/inference time separately. +- Download only official model artifacts and registry packages, capped at 2 GiB + of model weights for this first run. Inference is offline after preparation. +- Model failure remains a failed attempt. Never fabricate detections, timestamps, + grasp states, successful completion, metric geometry, contact or robot actions. +- Freeze the recipe before the initial run; adjustments produce another recipe + and result directory. Preserve the original attempt and failures. + +## Proposed cup cohort (not collected) + +Collect 30–50 complete permission-cleared attempts from several people and +setups, including occluded cups, wrong targets, drops, incomplete attempts and +recoveries. Group by original/session/person/object before a development/test +split. Obtain independent references for object boxes, event boundaries and +observable outcome, blinded to proposals. A marked target and known-size +reference enable later geometry checks; no metric coordinates are implied now. + +Compare detector-only proposals, caption-only proposals, their combination and +manual review at matched quality. Report detection precision/recall and IoU, +boundary timing error, missed/extra events, false completion, correction time, +abstention and full cost. Keep model scores separate from measured accuracy. +Physical skill qualification and action learning require separate experiments. + +## Gates + +The engineering gate is readable real video, hash-bound finite annotations, +successful correction/reload/export, offline execution, explicit unknowns, +resource limits and tests. The accuracy gate remains open until independent +references exist. Commercial release additionally requires review of exact +model/data licenses and evaluation on consented DVIDIA footage. Repeated scans +may later support a 3D view, but v1 supplies no 3D reconstruction. diff --git a/docs/observation-ui.md b/docs/observation-ui.md new file mode 100644 index 0000000..dfa4deb --- /dev/null +++ b/docs/observation-ui.md @@ -0,0 +1,80 @@ +# Local observation review + +`src/dvidia_training/observation_studio.html` is the dependency-free review page +served by the observation pilot. It makes no network requests outside its local +origin and does not include sample observations or fallback fixture data. + +The video and timeline share one clock. Bounding boxes use normalized +`[x1, y1, x2, y2]` coordinates and follow the displayed video area, including +letterboxing. A box appears only within 0.25 seconds of its sampled timestamp; +there is no tracking or interpolation claim. Box labels always come from raw +model output, even after an event is reviewed. The separate green timeline and +accepted list represent the review draft. + +Reviewers can select an observation, correct its label and boundaries, approve +or reject it, and add a missed event. Original observations remain visible and +unchanged. Newly added unsaved events can be removed; persisted events retain +their review history and can be rejected in a later revision. Observed outcome +defaults to `unknown`. The other choices are `visible_completion` and +`incomplete`, describing a person's visual assessment rather than robot success. + +Saving posts the complete draft with the current base revision and fetches the +episode again to confirm persistence. A reviewer must explicitly mark completion, +and every event must have a decision before a review is marked complete. +Conflicts or other save errors preserve the local draft. Previous/next navigation +offers to save, discard, or keep reviewing when changes exist. The browser also +warns before closing a page with unsaved edits. Downloads contain the last saved +review, episode ID, and original source hash; downloading is disabled while a +draft is unsaved. + +## HTTP contract + +- `GET /api/episodes`: `{ "episodes": [{ "id": "..." }], "failures": [] }`. + Optional failures remain visible above the review workspace. +- `GET /api/episodes/{id}`: the episode document below. +- `POST /api/episodes/{id}/review`: `{ "base_revision": 0, "events": [], + "outcome": "unknown", "status": "in_review" }`; returns the saved review. +- The episode's `video_url` must resolve to the same origin. The server must + support browser video byte ranges. + +An episode has `id`, optional `title`, `video_url`, `duration_seconds`, +`observations`, and `review`. Each observation has `id`, `label`, `start_seconds`, +`end_seconds`, optional numeric `confidence`, and a `boxes` array. Each box has +`time_seconds`, normalized `xyxy`, and an optional `label`. + +The review has `revision`, `status`, `outcome`, and `events`. Review events carry +`id`, `observation_id`, `label`, `start_seconds`, `end_seconds`, and `decision` +(`pending`, `approved`, or `rejected`). New manual events use a `manual-` UUID +and `observation_id: null`. Missing review entries are shown as pending. + +Optional provenance: `source.{sha256,kind,credit,license}`, +`recipe.{id,model,notes}`, and +`metrics.{elapsed_seconds,frames,detected_frames}`. Missing values are labeled +as not supplied. Model metadata accepts a name or an object with `kind` and a +`models` list of `role`, `name`, and `license` entries. Source kind distinguishes +public, synthetic, and authorized user footage. No accuracy or robot competence +metric is inferred. + +## Accessibility and responsive behavior + +Controls support keyboard input, visible focus, explicit names, and a minimum +44px target. Checkboxes use their full labeled area as the target. The video +never autoplays. Review/save feedback uses a polite status region; server errors +use an alert. Raw event chips expose label and exact timing to screen readers. +The layout stacks at tablet and phone widths, supports 320px-wide screens, and +honors reduced-motion preferences. Selecting an event on a stacked layout brings +the editor into view; accepted event timing retains decimal seconds. Provenance +follows the review panels in both the visual and document order. There are no +third-party assets or scripts. + +## Interface verification + +An isolated copy of the six-episode pilot was checked in a headless Chromium +browser on October 10, 2026. Checks exercised real playback, sampled overlays, +label correction and approval with save/reload, adding/removing a new event, +manual-event persistence, saved-review download including the original source +hash, and unsaved navigation. Desktop, 390px, and 320px layouts were captured; +phone layouts had no horizontal overflow and their visible controls met the +44px target. Browser console/page errors were absent. Local receipts and +screenshots are in the ignored `runs/observation-ui-qa-v1/` directory. This +verifies interface behavior only, not model accuracy or human review quality. diff --git a/docs/observations.md b/docs/observations.md new file mode 100644 index 0000000..8b06d66 --- /dev/null +++ b/docs/observations.md @@ -0,0 +1,122 @@ +# Local video observation lab + +This is a runnable research feature in DVIDIA Training. It does not replace +the upload flow or activate a production analysis service. The implementation +owns intake, timestamps, proposal association, reviews and provenance. The two +optional pretrained detectors retain their own licenses; no new foundation +model has been trained and no SpatialLM weights are included. + +## Setup + +From this repository checkout, with Python 3.12+ and FFmpeg/ffprobe installed: + +```sh +python3.12 -m venv .venv +.venv/bin/python -m pip install '.[observations]' +.venv/bin/python scripts/download_observation_models.py --download --output runs/observation-models +``` + +The explicit provisioning command downloads about 40 MB from pinned official +OpenCV Zoo artifacts and verifies sizes and SHA-256 hashes. Inference never +downloads anything. Install software and models before disconnecting from the +network. The tested runtime is Python 3.12.9, OpenCV 5.0.0.93, NumPy 2.5.3 on +macOS ARM64. Other platforms have not been benchmarked. Model loading and +software installation are excluded from the published six-clip processing time. + +## Supply permitted videos + +Keep the following manifest next to your own files. Replace every example value, +including the hash; it is deliberately not a runnable dataset. Do not add raw +recordings, private metadata or local paths to Git. + +```json +{ + "kind": "dvidia.observation-source.v1", + "task": "Move a cup onto a marked surface", + "episodes": [{ + "id": "attempt-001", + "path": "attempt-001.mp4", + "title": "Cup placement, first attempt", + "session_id": "participant-a-session-1", + "source_recording_id": "consented-original-001", + "kind": "user_authorized", + "credit": "Consenting contributor; local analysis only", + "license": "Private; local analysis permission recorded", + "source_url": "local-authorized-recording", + "permission": "local_analysis", + "sha256": "REPLACE_WITH_THE_ORIGINAL_FILE_SHA256" + }] +} +``` + +The permission field records a declaration; it does not obtain consent or grant +rights. Use the same session/person/original groups for related clips. Unknown +grouping must remain unknown, not become a new independent sample per excerpt. +Publisher titles are display metadata, never model inputs. + +Inputs are limited to 50 clips per run, 60 seconds and 128 MiB per clip, 512 MiB +per batch and 4096 pixels per source side. Formats follow the existing footage +reader. For this pilot, video and container timestamps must start at zero; +offset recordings fail with a saved reason rather than silently shifting boxes. +An explicit normalized derivative can be supplied separately with its provenance; +the tool never rewrites the original. + +## Analyze, review, inspect + +```sh +.venv/bin/dvidia-observe analyze /path/to/source.json --models runs/observation-models --output runs/observations-001 +.venv/bin/dvidia-observation-studio runs/observations-001 --port 8280 +``` + +Open http://127.0.0.1:8280. Choose an episode, play or scrub, select an observation, +correct its description/timing and approve or reject it. Add a missed event when +needed. Save a draft or explicitly finish the review. Source proposals remain +separate; edits create append-only revisions with stale-tab conflict detection. +Download exports the last saved review. Outcomes default to unknown. + +```sh +.venv/bin/dvidia-observe inspect runs/observations-001 +``` + +This verifies the source/protocol/episode hashes and revision chain, then reports +review progress. Without independent references it returns `accuracy: null` and +`accuracy_status: not_assessed`. A review is an assessment by the local operator, +not an authenticated identity or independent accuracy measurement. + +The lab listens only on 127.0.0.1, checks the local origin for writes, supports +bounded video byte ranges, and performs no uploads. It serves no arbitrary files +or model URLs. Run it only with bundles you trust; local users can read files in +the same account. Hashes detect accidental changes but are not digital signatures. + +## What the observations mean + +- YOLOX suggests COCO object labels and image rectangles. Scores are uncalibrated. +- A MediaPipe palm detector suggests palm rectangles, not full hand segmentation. +- Greedy same-class image-overlap association joins nearby detections; it does + not prove physical identity and splits gaps. No boxes fill an occlusion. +- Hand/object box overlap yields an explicit 2D proximity hypothesis. It cannot + establish touching, grasping, force, depth, task completion or robot commands. +- Sampling is two frames/second, up to 120 frames and 640 pixels per side. The UI + displays boxes only within 0.25 seconds of a sampled timestamp. This is a + recorded evidence viewer, not a smooth live hand tracker. + +The run stores copied originals, sampled frames, raw proposals, failures, recipe, +model hashes, source rights and review revisions. Allow scratch space beyond the +input size. Four CPU threads and one inference worker are used; this is not a +hard RAM/container limit. Do not install it on a production database machine +without a separate resource-constrained deployment test. + +## Next gates + +1. Collect the [planned cup-placement cohort](observation-pilot-plan.md), including + complete successes, failures, occlusions and recoveries from several setups. +2. Obtain independent human references before assessing labels or timing. +3. Add a separate temporal caption recipe to describe observable actions; compare + it with detector-only and combined proposals, including correction effort. +4. Add depth/geometry only with explicit scale and validation; then test a + separately supervised robot-action bridge and closed-loop safety/competence. +5. Integrate queued analysis into the app only after quality, cost, retries, + permissions and mobile review are validated on consented DVIDIA recordings. + +See [model licenses](observation-models.md), [UI contract](observation-ui.md) and +the [actual feasibility report](../benchmarks/observation-20261010/README.md). diff --git a/pyproject.toml b/pyproject.toml index 251455b..ecfcc9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,8 @@ name = "dvidia-training" version = "0.1.0" description = "Local CPU training from Skillspace footage, with optional simulation movement candidates" readme = "README.md" -license = "MIT" +license = "MIT AND Apache-2.0" +license-files = ["LICENSE", "third_party/observation-models/*LICENSE.txt"] requires-python = ">=3.12" dependencies = ["numpy==2.5.3"] classifiers = [ @@ -18,6 +19,7 @@ classifiers = [ [project.optional-dependencies] arm = ["mujoco==3.15.0"] +observations = ["opencv-python==5.0.0.93"] [project.urls] Homepage = "https://research.dvidia.org/papers/skillspace-footage-training/" @@ -34,6 +36,8 @@ dvidia-skillspace-coverage = "dvidia_training.coverage:main" dvidia-episode-tape = "dvidia_training.episode_tape:main" dvidia-reaction = "dvidia_training.reaction_benchmark:main" dvidia-motor-memory = "dvidia_training.motor_benchmark:main" +dvidia-observe = "dvidia_training.observation_pipeline:main" +dvidia-observation-studio = "dvidia_training.observation_studio:main" [tool.setuptools.packages.find] where = ["src"] diff --git a/scripts/download_observation_models.py b/scripts/download_observation_models.py new file mode 100644 index 0000000..000ebac --- /dev/null +++ b/scripts/download_observation_models.py @@ -0,0 +1,71 @@ +"""Explicitly download only the two pinned, official observation model assets. + +Run with --download and --output PATH. Ordinary detector construction never +downloads. The complete ONNX payload is under 40 MB; every byte count and SHA +is checked before an atomic rename. No fetched code is executed. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +import sys +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'src')) +from dvidia_training.observation_models import MODEL_ASSETS, model_metadata + + +def fetch(url, path, maximum, expected_sha=None): + if path.is_symlink() or path.exists(): + raise ValueError(f'Refusing to replace an existing artifact: {path.name}') + part = path.with_name(path.name + '.partial') + digest, count, created = sha256(), 0, False + try: + with part.open('xb') as output: + created = True + with urllib.request.urlopen(url, timeout=30) as response: + while chunk := response.read(min(1024 * 1024, maximum + 1 - count)): + count += len(chunk) + if count > maximum: + raise ValueError('Official artifact exceeded its size cap.') + digest.update(chunk) + output.write(chunk) + if expected_sha and (count != maximum or digest.hexdigest() != expected_sha): + raise ValueError('Official model size or SHA-256 did not match its reviewed Git LFS pointer.') + part.rename(path) + except BaseException: + if created: + part.unlink(missing_ok=True) + raise + return {'name': path.name, 'bytes': count, 'sha256': digest.hexdigest(), 'source': url} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--download', action='store_true', help='Explicitly authorize the bounded network downloads.') + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + if not args.download: + parser.error('No files downloaded. Pass --download to provision the reviewed models.') + root = args.output.absolute() + if any(p.is_symlink() for p in (root, *root.parents)): + raise ValueError('Output path may not contain symlinks.') + if root.exists() and any(root.iterdir()): + raise ValueError('Use a new or empty directory; existing model assets are never overwritten.') + root.mkdir(parents=True, exist_ok=True) + receipt = {'kind': 'dvidia.observation-model-assets', 'models': [], 'licenses': []} + for asset in MODEL_ASSETS: + record = model_metadata(asset) + downloaded = fetch(record['download_url'], root / asset['name'], asset['bytes'], asset['sha256']) + receipt['models'].append({**record, 'downloaded_sha256': downloaded['sha256']}) + receipt['licenses'].append(fetch(record['license_url'], root / (asset['directory'] + '-LICENSE.txt'), + asset['license_bytes'], asset['license_sha256'])) + print(f"Verified {asset['name']}: {asset['bytes']} bytes", flush=True) + (root / 'manifest.json').write_text(json.dumps(receipt, indent=2) + '\n') + print(f'Model provenance saved to {root / "manifest.json"}', flush=True) + + +if __name__ == '__main__': + main() diff --git a/src/dvidia_training/observation_models.py b/src/dvidia_training/observation_models.py new file mode 100644 index 0000000..7f70ea9 --- /dev/null +++ b/src/dvidia_training/observation_models.py @@ -0,0 +1,236 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Offline CPU image observations with pinned OpenCV Zoo ONNX models. + +Pre/postprocessing follows the reviewed OpenCV Zoo YOLOX and MediaPipe palm +references at ZOO_REVISION (Apache-2.0; YOLOX copyright Megvii Inc. 2021–2022). +Adaptations copyright DVIDIA contributors 2026. This implementation adds local hash verification, normalized clipped boxes, +bounded deterministic suppression, and explicit palm-only hand semantics. +No downloaded Python code is imported and no network access occurs here. +""" +from __future__ import annotations + +from hashlib import sha256 +import math +from pathlib import Path + + +ZOO_REVISION = '47534e27c9851bb1128ccc0102f1145e27f23f98' +ZOO_URL = 'https://github.com/opencv/opencv_zoo' +MODEL_ASSETS = ( + {'name': 'object_detection_yolox_2022nov.onnx', + 'directory': 'object_detection_yolox', 'role': 'objects', + 'bytes': 35858002, + 'license_bytes': 11371, + 'license_sha256': '0ec3668d3274bcf29e8a29e9576d5a2cd96fc78d3c5bec4387355a796e5d9088', + 'sha256': 'c5c2d13e59ae883e6af3b45daea64af4833a4951c92d116ec270d9ddbe998063'}, + {'name': 'palm_detection_mediapipe_2023feb.onnx', + 'directory': 'palm_detection_mediapipe', 'role': 'palms', + 'bytes': 3905734, + 'license_bytes': 11358, + 'license_sha256': 'cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30', + 'sha256': '78ff51c38496b7fc8b8ebdb6cc8c1abb02fa6c38427c6848254cdaba57fcce7c'}, +) +CLASSES = ( + 'person', 'bicycle', 'car', 'motorcycle', 'airplane', 'bus', 'train', + 'truck', 'boat', 'traffic light', 'fire hydrant', 'stop sign', + 'parking meter', 'bench', 'bird', 'cat', 'dog', 'horse', 'sheep', 'cow', + 'elephant', 'bear', 'zebra', 'giraffe', 'backpack', 'umbrella', 'handbag', + 'tie', 'suitcase', 'frisbee', 'skis', 'snowboard', 'sports ball', 'kite', + 'baseball bat', 'baseball glove', 'skateboard', 'surfboard', 'tennis racket', + 'bottle', 'wine glass', 'cup', 'fork', 'knife', 'spoon', 'bowl', 'banana', + 'apple', 'sandwich', 'orange', 'broccoli', 'carrot', 'hot dog', 'pizza', + 'donut', 'cake', 'chair', 'couch', 'potted plant', 'bed', 'dining table', + 'toilet', 'tv', 'laptop', 'mouse', 'remote', 'keyboard', 'cell phone', + 'microwave', 'oven', 'toaster', 'sink', 'refrigerator', 'book', 'clock', + 'vase', 'scissors', 'teddy bear', 'hair drier', 'toothbrush', +) + + +def model_metadata(asset): + relative = f"models/{asset['directory']}/{asset['name']}" + return {**asset, 'revision': ZOO_REVISION, 'license': 'Apache-2.0', + 'source': f'{ZOO_URL}/blob/{ZOO_REVISION}/{relative}', + 'download_url': f'https://media.githubusercontent.com/media/opencv/opencv_zoo/{ZOO_REVISION}/{relative}', + 'license_url': f"https://raw.githubusercontent.com/opencv/opencv_zoo/{ZOO_REVISION}/models/{asset['directory']}/LICENSE"} + + +def _fraction(value, name): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or not 0 < value < 1: + raise ValueError(f'{name} must be a finite number between zero and one.') + return float(value) + + +def _iou(left, right): + overlap = max(0., min(left[2], right[2]) - max(left[0], right[0])) * max(0., min(left[3], right[3]) - max(left[1], right[1])) + union = (left[2] - left[0]) * (left[3] - left[1]) + (right[2] - right[0]) * (right[3] - right[1]) - overlap + return overlap / union if union > 0 else 0. + + +def _suppress(rows, threshold, limit): + kept = [] + for row in sorted(rows, key=lambda r: (-r['score'], r['label'], r['xyxy']))[:1000]: + if any(old['label'] == row['label'] and _iou(old['xyxy'], row['xyxy']) > threshold for old in kept): + continue + kept.append(row) + if len(kept) >= limit: + break + return kept + + +class OpenCVDetector: + """A frame detector; scores are uncalibrated and hand boxes cover palms. + + Both exact model files must be provisioned separately, unless include_hands + is false. Construction rejects changed bytes before loading any network. + Instances are intended for serial use in one worker, with at most four CPU + threads. detect accepts a local image path and returns normalized xyxy boxes. + """ + + def __init__(self, weights_dir, *, threshold=0.35, hand_threshold=0.5, + max_detections=30, include_hands=True): + self.threshold = _fraction(threshold, 'threshold') + self.hand_threshold = _fraction(hand_threshold, 'hand_threshold') + if type(max_detections) is not int or not 1 <= max_detections <= 100: + raise ValueError('max_detections must be an integer from 1 to 100.') + if type(include_hands) is not bool: + raise ValueError('include_hands must be boolean.') + self.max_detections = max_detections + root = Path(weights_dir).expanduser().resolve(strict=True) + assets = MODEL_ASSETS if include_hands else MODEL_ASSETS[:1] + paths = [] + for asset in assets: + path = root / asset['name'] + if path.is_symlink() or not path.is_file() or path.stat().st_size != asset['bytes']: + raise ValueError(f"Missing or invalid pinned local model: {asset['name']}") + with path.open('rb') as stream: + digest = sha256() + for chunk in iter(lambda: stream.read(1024 * 1024), b''): + digest.update(chunk) + if digest.hexdigest() != asset['sha256']: + raise ValueError(f"Model SHA-256 mismatch: {asset['name']}") + paths.append(path) + try: + import cv2 + import numpy as np + except ImportError as exc: + raise RuntimeError('Observation detection requires a separately provisioned OpenCV and NumPy environment.') from exc + self.cv, self.np = cv2, np + cv2.setNumThreads(4) + self.networks = [] + for path in paths: + net = cv2.dnn.readNetFromONNX(str(path)) + net.setPreferableBackend(cv2.dnn.DNN_BACKEND_OPENCV) + net.setPreferableTarget(cv2.dnn.DNN_TARGET_CPU) + self.networks.append(net) + grids, strides = [], [] + for stride in (8, 16, 32): + size = 640 // stride + y, x = np.mgrid[:size, :size] + grid = np.stack((x, y), axis=-1).reshape(-1, 2) + grids.append(grid) + strides.append(np.full((len(grid), 1), stride)) + self.grid = np.concatenate(grids).astype(np.float32) + self.strides = np.concatenate(strides).astype(np.float32) + # The released palm model has 2 anchors/cell at 24x24 and 6 at 12x12. + self.palm_anchors = np.asarray([ + ((x + .5) / size, (y + .5) / size) + for size, repeats in ((24, 2), (12, 6)) + for y in range(size) for x in range(size) for _ in range(repeats) + ], dtype=np.float32) + self.metadata = { + 'kind': 'opencv-zoo-yolox-palm', 'implementation_version': 1, + 'implementation_sha256': sha256(Path(__file__).read_bytes()).hexdigest(), + 'models': [model_metadata(a) for a in assets], + 'device': 'cpu', 'backend': 'opencv', 'cpu_threads': cv2.getNumThreads(), + 'runtime': {'opencv': cv2.__version__, 'numpy': np.__version__}, + 'threshold': self.threshold, 'hand_threshold': self.hand_threshold, + 'nms_iou': {'objects': .5, 'palms': .3}, + 'max_detections': max_detections, 'max_nms_candidates': 1000, + 'hand_detector_enabled': include_hands, + 'score_kind': 'uncalibrated_model_evidence', + 'coordinates': 'normalized_image_xyxy', + 'hand_box_semantics': 'Palm detector rectangle; not full hand segmentation or observed contact.', + 'limitations': [ + 'COCO labels can be wrong or absent, especially for small and occluded objects.', + 'Palm boxes do not establish finger contact, grasp, force, depth, task success or robot actions.', + 'Object identity and temporal relationships require a separate tracking stage.', + ], + } + + def _row(self, label, score, box, width, height): + np = self.np + if not math.isfinite(float(score)) or not 0 <= score <= 1 or not np.isfinite(box).all(): + raise ValueError('Detector produced nonfinite or invalid evidence.') + normalized = np.clip(np.asarray(box, dtype=np.float64) / [width, height, width, height], 0., 1.).tolist() + if normalized[2] <= normalized[0] or normalized[3] <= normalized[1]: + return None + return {'label': label, 'score': float(score), 'xyxy': normalized} + + def _objects(self, image): + cv, np = self.cv, self.np + h, w = image.shape[:2] + ratio = min(640 / h, 640 / w) + resized = cv.resize(cv.cvtColor(image, cv.COLOR_BGR2RGB), (max(1, int(w * ratio)), max(1, int(h * ratio)))) + padded = np.full((640, 640, 3), 114., dtype=np.float32) + padded[:resized.shape[0], :resized.shape[1]] = resized + net = self.networks[0] + net.setInput(np.transpose(padded, (2, 0, 1))[None]) + outputs = net.forward(net.getUnconnectedOutLayersNames()) + if len(outputs) != 1 or outputs[0].shape != (1, 8400, 85) or not np.isfinite(outputs[0]).all(): + raise ValueError('Unexpected YOLOX output.') + data = outputs[0][0] + scores = data[:, 4:5] * data[:, 5:] + classes = scores.argmax(axis=1) + best = scores[np.arange(len(scores)), classes] + selected = np.flatnonzero(best >= self.threshold) + rows = [] + with np.errstate(over='raise', invalid='raise'): + centers = (data[selected, :2] + self.grid[selected]) * self.strides[selected] + sizes = np.exp(data[selected, 2:4]) * self.strides[selected] + boxes = np.concatenate((centers - sizes / 2, centers + sizes / 2), axis=1) / ratio + for index, box in zip(selected, boxes, strict=True): + row = self._row(CLASSES[int(classes[index])], best[index], box, w, h) + if row is not None: + rows.append(row) + return _suppress(rows, .5, self.max_detections) + + def _palms(self, image): + cv, np = self.cv, self.np + h, w = image.shape[:2] + ratio = min(192 / h, 192 / w) + nw, nh = max(1, int(w * ratio)), max(1, int(h * ratio)) + left, top = (192 - nw) // 2, (192 - nh) // 2 + resized = cv.resize(cv.cvtColor(image, cv.COLOR_BGR2RGB), (nw, nh)) + padded = np.zeros((192, 192, 3), dtype=np.float32) + padded[top:top + nh, left:left + nw] = resized / 255. + net = self.networks[1] + net.setInput(padded[None]) + outputs = net.forward(net.getUnconnectedOutLayersNames()) + by_shape = {array.shape: array for array in outputs} + if set(by_shape) != {(1, 2016, 18), (1, 2016, 1)} or any(not np.isfinite(a).all() for a in outputs): + raise ValueError('Unexpected palm detector output.') + delta = by_shape[(1, 2016, 18)][0, :, :4] + logits = by_shape[(1, 2016, 1)][0, :, 0].astype(np.float64) + scores = 1. / (1. + np.exp(-np.clip(logits, -100., 100.))) + selected = np.flatnonzero(scores >= self.hand_threshold) + centers = delta[selected, :2] + self.palm_anchors[selected] * 192 + sizes = delta[selected, 2:4] + boxes = (np.concatenate((centers - sizes / 2, centers + sizes / 2), axis=1) - [left, top, left, top]) / ratio + rows = [] + for index, box in zip(selected, boxes, strict=True): + row = self._row('hand', scores[index], box, w, h) + if row is not None: + rows.append(row) + return _suppress(rows, .3, self.max_detections) + + def detect(self, image_path): + path = Path(image_path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 50 * 1024 * 1024: + raise ValueError('Expected a bounded local image file.') + image = self.cv.imread(str(path), self.cv.IMREAD_COLOR) + if image is None or image.size == 0 or image.shape[0] * image.shape[1] > 16_000_000: + raise ValueError('Image is undecodable or exceeds 16 million pixels.') + rows = self._objects(image) + if len(self.networks) == 2: + rows.extend(self._palms(image)) + return sorted(rows, key=lambda r: (-r['score'], r['label'], r['xyxy']))[:self.max_detections] diff --git a/src/dvidia_training/observation_pipeline.py b/src/dvidia_training/observation_pipeline.py new file mode 100644 index 0000000..220aadc --- /dev/null +++ b/src/dvidia_training/observation_pipeline.py @@ -0,0 +1,233 @@ +"""Offline video -> timestamped detections -> editable observation proposals.""" +from __future__ import annotations + +import argparse +from datetime import datetime, timezone +import json +from pathlib import Path +import platform +import re +import shutil +import subprocess +import time + +from .footage_data import _json, _local_file, _read, probe_video, canonical +from .observations import (associate, digest, evaluate_reviews, file_hash, identity, + number, propose_relations, text) + +MAX_BYTES = 128 * 1024 * 1024 +RECIPE = {'id': 'dvidia-observations-v1.1', 'samples_per_second': 2, + 'max_frames': 120, 'max_duration_seconds': 60, 'max_image_side': 640, + 'max_track_gap_seconds': 0.8, 'minimum_association_iou': 0.2, + 'claim_scope': 'sampled_2d_observations', 'automatic_outcome': 'unknown', + 'timebase': 'zero-origin video and container; nonzero starts rejected'} + + +def validate_manifest(path): + path = Path(path).resolve(strict=True) + if path.stat().st_size > 1024*1024: + raise ValueError('Manifest exceeds 1 MiB.') + manifest = _json(_read(path, 1024*1024)) + if (type(manifest) is not dict or set(manifest) != {'kind', 'task', 'episodes'} + or manifest['kind'] != 'dvidia.observation-source.v1'): + raise ValueError('Expected a dvidia.observation-source.v1 manifest.') + text(manifest['task'], 'Task') + episodes = manifest['episodes'] + if type(episodes) is not list or not 1 <= len(episodes) <= 50: + raise ValueError('Use between one and fifty explicitly permitted videos.') + ids, total = set(), 0 + for row in episodes: + required = {'id', 'path', 'title', 'session_id', 'source_recording_id', 'kind', + 'credit', 'license', 'source_url', 'permission', 'sha256'} + if type(row) is not dict or set(row) != required: + raise ValueError('Each source must include the documented provenance fields.') + eid = identity(row['id']) + if eid in ids: + raise ValueError('Duplicate episode id.') + ids.add(eid) + for key in ('title', 'session_id', 'source_recording_id', 'credit', 'license', 'source_url'): + text(row[key], key, 1200) + if row['kind'] not in ('public', 'synthetic', 'user_authorized') or row['permission'] != 'local_analysis': + raise ValueError('Every source requires an explicit local-analysis permission declaration.') + if type(row['sha256']) is not str or not re.fullmatch('[a-f0-9]{64}', row['sha256']): + raise ValueError('Every source requires an original SHA-256.') + video = _local_file(path.parent, row['path']) + total += video.stat().st_size + if not 0 < video.stat().st_size <= MAX_BYTES or total > 512*1024*1024: + raise ValueError('Footage exceeds the 128 MiB/clip or 512 MiB/batch limit.') + return path.parent, manifest + + +def validate_timebase(video, info): + """Do not silently shift boxes relative to the original browser media clock.""" + result = subprocess.run(['ffprobe', '-v', 'error', '-protocol_whitelist', 'file', + '-format_whitelist', 'mov,matroska,webm,avi', '-show_entries', 'format=start_time', + '-of', 'json', str(video)], stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10, check=False) + if result.returncode or len(result.stdout) > 1024*1024: + raise ValueError('Could not establish the original video timeline.') + container = _json(result.stdout).get('format', {}) + start = float(container.get('start_time', 'nan')) + number(start, 0, 0, 'Container start time') + number(info['start_seconds'], 0, 0, 'Video start time') + + +def extract_frames(video, output, duration): + """One bounded decode; actual input PTS survive selection and scaling.""" + output.mkdir() + filters = ("select='isnan(prev_selected_t)+gte(t-prev_selected_t,0.5)'," + "scale=640:640:force_original_aspect_ratio=decrease,showinfo") + command = ['ffmpeg', '-hide_banner', '-nostdin', '-nostats', '-loglevel', 'info', + '-xerror', '-threads', '1', '-filter_threads', '1', + '-protocol_whitelist', 'file', '-format_whitelist', 'mov,matroska,webm,avi', + '-copyts', '-i', str(video), '-map', '0:v:0', '-an', '-sn', '-dn', + '-vf', filters, '-frames:v', '120', '-fps_mode', 'vfr', + '-threads', '1', str(output/'frame-%04d.png')] + result = subprocess.run(command, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=120, check=False) + if result.returncode: + raise ValueError('Video decoding failed; the original is unchanged.') + times = [float(v) for v in re.findall(rb'\bn:\s*\d+\s+pts:\s*-?\d+\s+pts_time:([-+0-9.eE]+)', result.stderr)] + files = sorted(output.glob('frame-*.png')) + if not files or len(times) != len(files) or len(files) > 120: + raise ValueError('Could not bind every sampled frame to a presentation timestamp.') + first_pts = times[0] + number(first_pts, 0, 0, 'First decoded timestamp') + rows = [] + for index, (stamp, file) in enumerate(zip(times, files, strict=True)): + t = stamp-first_pts + number(t, 0, duration+0.05, 'Decoded time') + if index and stamp <= times[index-1]: + raise ValueError('Decoded timestamps must strictly increase.') + rows.append({'id': f'frame-{index+1}', 'time_seconds': t, + 'source_pts_seconds': stamp, 'path': 'frames/'+file.name, + 'sha256': file_hash(file)}) + return rows + + +def run(manifest_path, output, detector): + started = time.monotonic() + root, manifest = validate_manifest(manifest_path) + output = Path(output).resolve() + if output.exists(): + raise ValueError('Choose a new run directory; previous evidence is never overwritten.') + output.mkdir(parents=True) + metadata = json.loads(canonical(detector.metadata)) + # Freeze the declared recipe and inputs before observing any model output. + protocol = {'kind': 'dvidia.observation-protocol.v1', 'recipe': RECIPE, + 'model': metadata, 'manifest_sha256': digest(manifest), + 'sources': [{k: v for k, v in row.items() if k != 'path'} for row in manifest['episodes']], + 'created_at': datetime.now(timezone.utc).isoformat(), + 'hardware': {'platform': platform.system(), 'machine': platform.machine()}, + 'execution': 'local_offline', 'limits': {'cpu_threads': 4, 'workers': 1}} + (output/'protocol.json').write_bytes(canonical(protocol)) + entries, failed, seen_hashes, sessions = [], [], {}, {} + for row in manifest['episodes']: + episode_started = time.monotonic() + directory = output/row['id'] + directory.mkdir() + try: + source = _local_file(root, row['path']) + actual_hash = file_hash(source) + if actual_hash != row['sha256']: + raise ValueError('Source hash differs from the frozen manifest.') + info = probe_video(source) + if info['duration_seconds'] > 60 or max(info['width'], info['height']) > 4096: + raise ValueError('Pilot inputs must be <=60 seconds and <=4096 pixels per side.') + validate_timebase(source, info) + target = directory/('original'+source.suffix.lower()) + shutil.copyfile(source, target) + if file_hash(target) != actual_hash: + raise ValueError('Original changed during intake.') + frames = extract_frames(target, directory/'frames', info['duration_seconds']) + inference_started = time.monotonic() + for frame in frames: + frame['detections'] = detector.detect(directory/frame['path']) + inference_seconds = time.monotonic()-inference_started + observations = associate(frames, info['duration_seconds']) + observations.extend(propose_relations(frames, info['duration_seconds'])) + if len(observations) > 1200: + raise ValueError('Too many fragmented proposals; keep this failed run for inspection.') + source_meta = {key: row[key] for key in ('kind', 'credit', 'license', 'source_url', + 'session_id', 'source_recording_id', 'permission')} + source_meta.update(sha256=actual_hash, bytes=target.stat().st_size, + video_path=target.name, timing=info, + duplicate_of=seen_hashes.get(actual_hash)) + seen_hashes.setdefault(actual_hash, row['id']) + sessions.setdefault(row['session_id'], []).append(row['id']) + episode = {'kind': 'dvidia.observation-episode.v1', 'id': row['id'], + 'title': row['title'], 'task': manifest['task'], 'source': source_meta, + 'duration_seconds': info['duration_seconds'], 'recipe': {**RECIPE, 'model': metadata}, + 'video_url': f"/api/episodes/{row['id']}/video", 'observations': observations, + 'frames': frames, 'metrics': {'elapsed_seconds': time.monotonic()-episode_started, + 'inference_seconds': inference_seconds, 'frames': len(frames), + 'detected_frames': sum(bool(f['detections']) for f in frames), + 'accuracy_status': 'not_assessed'}} + (directory/'episode.json').write_bytes(canonical(episode)) + entries.append({'id': row['id'], 'episode_sha256': digest(episode)}) + except Exception as exc: + # Do not expose original machine paths, provider bodies or credentials. + message = str(exc) if isinstance(exc, ValueError) else type(exc).__name__ + failure = {'id': row['id'], 'error': message[:500], 'status': 'failed', + 'elapsed_seconds': time.monotonic()-episode_started} + (directory/'failure.json').write_bytes(canonical(failure)) + failed.append(failure) + receipt = {'kind': 'dvidia.observation-run.v1', 'protocol_sha256': digest(protocol), + 'episodes': entries, 'failures': failed, 'attempted': len(manifest['episodes']), + 'completed': len(entries), 'elapsed_seconds': time.monotonic()-started, + 'timing_scope': 'intake, decode, inference, association and per-episode writes; excludes model setup and final receipt', + 'accuracy_status': 'not_assessed', 'sessions': sessions, + 'robot_skill_qualified': False, 'geometry': 'image_coordinates_only'} + (output/'run.json').write_bytes(canonical(receipt)) + return receipt + + +def load_episodes(directory, *, verify_media=True): + directory = Path(directory).resolve(strict=True) + receipt = _json(_read(_local_file(directory, 'run.json'), 2*1024*1024)) + if (type(receipt) is not dict or receipt.get('kind') != 'dvidia.observation-run.v1' + or type(receipt.get('episodes')) is not list or len(receipt['episodes']) > 50): + raise ValueError('Unsupported observation run.') + protocol = _json(_read(_local_file(directory, 'protocol.json'), 2*1024*1024)) + if digest(protocol) != receipt.get('protocol_sha256'): + raise ValueError('Protocol was modified after the run.') + episodes = {} + for item in receipt['episodes']: + eid = identity(item['id']) + path = _local_file(directory, eid+'/episode.json') + episode = _json(_read(path, 8*1024*1024), maximum=8*1024*1024) + if digest(episode) != item['episode_sha256'] or episode['id'] != eid or eid in episodes: + raise ValueError('Episode evidence was modified or duplicated.') + video = _local_file(path.parent, episode['source']['video_path']) + if verify_media and file_hash(video) != episode['source']['sha256']: + raise ValueError('Video does not match its observation source.') + episodes[eid] = episode + return receipt, episodes + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + analyze = commands.add_parser('analyze') + analyze.add_argument('manifest', type=Path) + analyze.add_argument('--output', required=True, type=Path) + analyze.add_argument('--models', required=True, type=Path) + inspect = commands.add_parser('inspect') + inspect.add_argument('directory', type=Path) + args = parser.parse_args(argv) + if args.command == 'analyze': + from .observation_models import OpenCVDetector + detector = OpenCVDetector(args.models) + result = run(args.manifest, args.output, detector) + else: + from .observations import latest_review + result, episodes = load_episodes(args.directory) + for eid, episode in episodes.items(): + episode['review'] = latest_review(args.directory/eid, episode) + result = {**result, 'review': evaluate_reviews(list(episodes.values()))} + print(json.dumps(result, indent=2, allow_nan=False)) + return 1 if result.get('failures') else 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/dvidia_training/observation_studio.html b/src/dvidia_training/observation_studio.html new file mode 100644 index 0000000..40b9684 --- /dev/null +++ b/src/dvidia_training/observation_studio.html @@ -0,0 +1,522 @@ + + + + + + + Observation lab · DVIDIA + + + +
+
+
+ +
DVIDIA
Applied physical AI
+
+
Local review · Footage stays here
+
+
+
+
From footage to reviewed evidence

Observation lab

Connecting to your local episodes…

+ +
+ + +
+
+
+
+ +
Your footage, ready for a closer look.

Loading the local observation record.

+ + +
+
+ + 0:00 / 0:00 + + + +
+
+
+

One shared timeline

No sampled boxes yet
+
+
+
Raw model
+
Accepted
+
0:00———
+
+

Boxes are sampled detections. Hand boxes mark detected palms; overlap does not prove a grasp.

+
+
+

Raw observations 0

Original model output. Select an event to review.

+
+

No observations loaded.

+ +
+
+ +
Source, analysis and limitations

Accuracy has not been measured on this episode. An accepted visual event is not evidence of robot skill or task success.

+
+
+
DVIDIA · Observation pilotHuman review required · Accuracy not measured · Robot capability not evaluated
+
+

Keep your review changes?

Your current edits have not been saved to a revision.

+ + + diff --git a/src/dvidia_training/observation_studio.py b/src/dvidia_training/observation_studio.py new file mode 100644 index 0000000..f044459 --- /dev/null +++ b/src/dvidia_training/observation_studio.py @@ -0,0 +1,221 @@ +"""Loopback-only observation review, with private media and append-only revisions.""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +from copy import deepcopy +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import mimetypes +import os +from pathlib import Path +import re +import threading +from urllib.parse import urlsplit + +from .footage_data import _json, _local_file, canonical +from .observation_pipeline import load_episodes +from .observations import latest_review, validate_review + + +class ObservationServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, address, directory): + if address[0] != '127.0.0.1': + raise ValueError('The observation lab binds only to 127.0.0.1.') + self.directory = Path(directory).resolve(strict=True) + self.receipt, self.episodes = load_episodes(self.directory) + self.lock = threading.RLock() + self.media_signatures = {} + super().__init__(address, ObservationHandler) + + @contextmanager + def media(self, eid, *, verify=False): + source = self.episodes[eid]['source'] + path = _local_file(self.directory/eid, source['video_path']) + with path.open('rb') as handle: + def signature(): + s = os.fstat(handle.fileno()) + return (s.st_dev, s.st_ino, s.st_size, s.st_mtime_ns, s.st_ctime_ns) + before = signature() + if before[2] != source['bytes']: + raise ValueError('Source media changed after validation.') + with self.lock: + if verify or self.media_signatures.get(eid) != before: + digest = hashlib.sha256() + while chunk := handle.read(1024*1024): + digest.update(chunk) + if signature() != before or digest.hexdigest() != source['sha256']: + raise ValueError('Source media changed after validation.') + self.media_signatures[eid] = before + handle.seek(0) + yield path, handle, before[2] + + def episode(self, eid): + result = deepcopy(self.episodes[eid]) + result['review'] = latest_review(self.directory/eid, result) + return result + + def save(self, eid, payload): + with self.lock: + episode = self.episode(eid) + with self.media(eid, verify=True): + pass + review = validate_review(payload, episode, episode['review']) + target = self.directory/eid/f"review-{review['revision']:06d}.json" + # Exclusive creation plus a same-directory atomic link provides + # no-overwrite safety even if another server reviews this run. + temporary = target.with_suffix(f'.{os.getpid()}.{threading.get_ident()}.tmp') + try: + with temporary.open('xb') as handle: + handle.write(canonical(review)) + handle.flush() + os.fsync(handle.fileno()) + try: + os.link(temporary, target) + except FileExistsError as exc: + raise RuntimeError('This review changed. Reload before saving.') from exc + finally: + temporary.unlink(missing_ok=True) + return review + + +class ObservationHandler(BaseHTTPRequestHandler): + server_version = 'DvidiaObservation/1' + + def log_message(self, *_): + pass + + def origin_ok(self, *, write=False): + expected = f'127.0.0.1:{self.server.server_port}' + origin = self.headers.get('Origin') + return (self.headers.get('Host') == expected + and self.headers.get('Sec-Fetch-Site') not in ('cross-site',) + and (origin == 'http://'+expected if write else origin in (None, 'http://'+expected))) + + def send(self, status, data, content_type='application/json; charset=utf-8', extra=None): + raw = data if type(data) is bytes else canonical(data) + self.send_response(status) + self.send_header('Content-Type', content_type) + self.send_header('Content-Length', str(len(raw))) + self.send_header('Cache-Control', 'no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cross-Origin-Resource-Policy', 'same-origin') + self.send_header('Content-Security-Policy', "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; media-src 'self'; connect-src 'self'; object-src 'none'; base-uri 'none'; frame-ancestors 'none'") + for key, value in (extra or {}).items(): + self.send_header(key, value) + self.end_headers() + if self.command != 'HEAD': + self.wfile.write(raw) + + def video(self, eid): + with self.server.media(eid) as (path, handle, size): + return self.stream_video(path, handle, size) + + def stream_video(self, path, handle, size): + start, end, status = 0, size-1, 200 + requested = self.headers.get('Range') + if requested: + match = re.fullmatch(r'bytes=(\d*)-(\d*)', requested) + if not match or not any(match.groups()): + return self.send(416, {'error': 'Unsupported byte range.'}, extra={'Content-Range': f'bytes */{size}'}) + left, right = match.groups() + if left: + start, end = int(left), min(int(right), size-1) if right else size-1 + else: + start, end = max(0, size-int(right)), size-1 + if start > end or start >= size or not left and int(right) == 0: + return self.send(416, {'error': 'Range is outside the video.'}, extra={'Content-Range': f'bytes */{size}'}) + status = 206 + self.send_response(status) + self.send_header('Content-Type', mimetypes.guess_type(path.name)[0] or 'application/octet-stream') + self.send_header('Content-Length', str(end-start+1)) + self.send_header('Accept-Ranges', 'bytes') + self.send_header('Cache-Control', 'private, no-store') + self.send_header('X-Content-Type-Options', 'nosniff') + self.send_header('Cross-Origin-Resource-Policy', 'same-origin') + if status == 206: + self.send_header('Content-Range', f'bytes {start}-{end}/{size}') + self.end_headers() + if self.command == 'HEAD': + return + handle.seek(start) + remaining = end-start+1 + while remaining: + chunk = handle.read(min(64*1024, remaining)) + if not chunk: + break + self.wfile.write(chunk) + remaining -= len(chunk) + + def do_HEAD(self): + self.do_GET() + + def do_GET(self): + if not self.origin_ok(): + return self.send(403, {'error': 'Use this lab from its loopback address.'}) + path = urlsplit(self.path).path + try: + if path == '/': + return self.send(200, Path(__file__).with_name('observation_studio.html').read_bytes(), 'text/html; charset=utf-8') + if path == '/api/episodes': + rows = [] + for eid in self.server.episodes: + e = self.server.episode(eid) + rows.append({'id': eid, 'title': e['title'], 'duration_seconds': e['duration_seconds'], + 'video_url': e['video_url'], 'status': e['review']['status']}) + return self.send(200, {'episodes': rows, 'failures': self.server.receipt['failures']}) + match = re.fullmatch(r'/api/episodes/([A-Za-z0-9_-]+)(/video)?', path) + if match and match[1] in self.server.episodes: + return self.video(match[1]) if match[2] else self.send(200, self.server.episode(match[1])) + return self.send(404, {'error': 'No such observation resource.'}) + except (BrokenPipeError, ConnectionResetError): + return + except (ValueError, OSError, KeyError): + return self.send(409, {'error': 'Source or review evidence changed. Inspect the run before continuing.'}) + + def do_POST(self): + if not self.origin_ok(write=True): + return self.send(403, {'error': 'Review changes require the same local origin.'}) + path = urlsplit(self.path).path + match = re.fullmatch(r'/api/episodes/([A-Za-z0-9_-]+)/review', path) + if not match or match[1] not in self.server.episodes: + return self.send(404, {'error': 'No such episode.'}) + try: + length = int(self.headers.get('Content-Length', '0')) + if (self.headers.get_content_type() != 'application/json' or self.headers.get('Transfer-Encoding') + or not 0 < length <= 1024*1024): + return self.send(400, {'error': 'Use bounded JSON with a content length.'}) + self.connection.settimeout(10) + raw = self.rfile.read(length) + if len(raw) != length: + raise ValueError('Incomplete review request.') + payload = _json(raw, maximum=1024*1024) + review = self.server.save(match[1], payload) + return self.send(200, {'review': review}) + except RuntimeError as exc: + return self.send(409, {'error': str(exc)}) + except (ValueError, OSError) as exc: + return self.send(400, {'error': str(exc) if isinstance(exc, ValueError) else 'Could not store the review.'}) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('directory', type=Path) + parser.add_argument('--port', type=int, default=8280) + args = parser.parse_args(argv) + if not 1024 <= args.port <= 65535: + parser.error('Use an unprivileged local port.') + server = ObservationServer(('127.0.0.1', args.port), args.directory) + print(f'Observation lab: http://127.0.0.1:{args.port}', flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + finally: + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/src/dvidia_training/observations.py b/src/dvidia_training/observations.py new file mode 100644 index 0000000..2084fc1 --- /dev/null +++ b/src/dvidia_training/observations.py @@ -0,0 +1,208 @@ +"""DVIDIA-owned, data-only observation and append-only review contracts.""" +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +import math +from pathlib import Path +import re + +from .footage_data import _json, _read, canonical + +ID = re.compile(r'^[A-Za-z0-9][A-Za-z0-9_-]{0,95}$') +OUTCOMES = ('unknown', 'visible_completion', 'incomplete') + + +def digest(value): + return hashlib.sha256(canonical(value)).hexdigest() + + +def file_hash(path): + h = hashlib.sha256() + with Path(path).open('rb') as handle: + while chunk := handle.read(1024 * 1024): + h.update(chunk) + return h.hexdigest() + + +def number(value, low, high, name): + if type(value) not in (int, float) or not math.isfinite(value) or not low <= value <= high: + raise ValueError(f'{name} must be finite and between {low} and {high}.') + return value + + +def text(value, name, limit=300): + if (type(value) is not str or not value.strip() or len(value) > limit + or any(ord(c) < 32 or 0xD800 <= ord(c) <= 0xDFFF for c in value)): + raise ValueError(f'{name} must be bounded plain text.') + return value.strip() + + +def identity(value): + if type(value) is not str or not ID.fullmatch(value): + raise ValueError('Invalid observation identifier.') + return value + + +def box(value): + if type(value) is not list or len(value) != 4: + raise ValueError('Boxes require four normalized xyxy coordinates.') + for coordinate in value: + number(coordinate, 0, 1, 'Box coordinate') + if value[0] >= value[2] or value[1] >= value[3]: + raise ValueError('Boxes must have positive width and height.') + return value + + +def iou(a, b): + intersection = max(0, min(a[2], b[2])-max(a[0], b[0])) * max(0, min(a[3], b[3])-max(a[1], b[1])) + union = (a[2]-a[0])*(a[3]-a[1]) + (b[2]-b[0])*(b[3]-b[1]) - intersection + return intersection / union if union else 0.0 + + +def associate(frames, duration, *, max_gap=0.8, minimum_iou=0.2): + """Greedy same-class IoU association. No optical flow or physical identity claim. + + Only fresh detections extend a track; no boxes are interpolated through + occlusion. Sequence snippets split after a gap to avoid hiding uncertainty. + """ + tracks = [] + last_time = -1.0 + for frame in frames: + t = number(frame['time_seconds'], 0, duration, 'Frame time') + if t <= last_time: + raise ValueError('Frame times must increase strictly.') + last_time = t + rows = frame['detections'] + if type(rows) is not list or len(rows) > 60: + raise ValueError('Too many detections in a frame.') + used = set() + for row in sorted(rows, key=lambda x: -x['score']): + label = text(row['label'], 'Object label', 64) + score = number(row['score'], 0, 1, 'Detector score') + bounds = box(row['xyxy']) + candidates = [(iou(track['boxes'][-1]['xyxy'], bounds), n) + for n, track in enumerate(tracks) + if n not in used and track['object_label'] == label + and 0 < t-track['boxes'][-1]['time_seconds'] <= max_gap] + best = max(candidates, default=(0, -1)) + if best[0] >= minimum_iou: + n = best[1] + else: + n = len(tracks) + tracks.append({'id': f'track-{n+1}', 'object_label': label, 'boxes': []}) + used.add(n) + tracks[n]['boxes'].append({'time_seconds': t, 'xyxy': bounds, + 'label': label, 'confidence': score}) + for track in tracks: + times = [b['time_seconds'] for b in track['boxes']] + # A sampled observation spans only a short evidence interval, not the gap + # to a future detection or the full duration of a physical action. + track.update(label=track['object_label'].capitalize()+' visible', + start_seconds=times[0], end_seconds=min(duration, times[-1]+min(max_gap, 0.5)), + confidence=sum(b['confidence'] for b in track['boxes'])/len(times), + evidence='detector', review_state='proposed') + return tracks + + +def propose_relations(frames, duration): + """2D proximity hypotheses, never contact, grasp, depth or task completion.""" + events = [] + for frame in frames: + hands = [d for d in frame['detections'] if d['label'] == 'hand'] + for row in frame['detections']: + if row['label'] not in ('cup', 'bottle', 'bowl'): + continue + a = row['xyxy'] + if any(iou(a, h['xyxy']) > 0.02 for h in hands): + t = frame['time_seconds'] + events.append({'id': f'relation-{len(events)+1}', + 'label': f"Hand near {row['label']} · image overlap", + 'start_seconds': t, 'end_seconds': min(duration, t+0.5), + 'boxes': [], 'evidence': '2d-overlap-heuristic', + 'review_state': 'proposed'}) + return events + + +def initial_review(episode): + return {'revision': 0, 'status': 'unreviewed', 'outcome': 'unknown', + 'events': [{'id': o['id'], 'observation_id': o['id'], 'label': o['label'], + 'start_seconds': o['start_seconds'], 'end_seconds': o['end_seconds'], + 'decision': 'pending'} for o in episode['observations']]} + + +def validate_review(payload, episode, current): + if type(payload) is not dict or set(payload) != {'base_revision', 'events', 'outcome', 'status'}: + raise ValueError('Expected base_revision, events, outcome and status.') + if type(payload['base_revision']) is not int or payload['base_revision'] < 0: + raise ValueError('A nonnegative base revision is required.') + if payload['base_revision'] != current['revision']: + raise RuntimeError('This review changed in another tab. Reload before saving.') + if payload['outcome'] not in OUTCOMES or payload['status'] not in ('in_review', 'reviewed'): + raise ValueError('Choose a supported review status and visible outcome.') + events = payload['events'] + if type(events) is not list or len(events) > 1500: + raise ValueError('Reviews support at most 1500 events.') + observation_ids = {o['id'] for o in episode['observations']} + seen, seen_sources, clean = set(), set(), [] + for event in events: + if type(event) is not dict or set(event) != {'id', 'observation_id', 'label', 'start_seconds', 'end_seconds', 'decision'}: + raise ValueError('An event contains unsupported fields.') + eid = identity(event['id']) + oid = event['observation_id'] + if eid in seen: + raise ValueError('Duplicate event identifier.') + seen.add(eid) + if oid is not None: + identity(oid) + if oid not in observation_ids or oid in seen_sources or eid != oid: + raise ValueError('A proposal must retain its unique original identity.') + seen_sources.add(oid) + elif not eid.startswith('manual-'): + raise ValueError('New events require a manual- identifier.') + start = number(event['start_seconds'], 0, episode['duration_seconds'], 'Start') + end = number(event['end_seconds'], 0, episode['duration_seconds'], 'End') + if start >= end: + raise ValueError('Event end must be later than its start.') + if event['decision'] not in ('pending', 'approved', 'rejected'): + raise ValueError('Invalid event decision.') + if payload['status'] == 'reviewed' and event['decision'] == 'pending': + raise ValueError('Resolve all pending suggestions before finishing review.') + clean.append({**event, 'label': text(event['label'], 'Event label')}) + if seen_sources != observation_ids: + raise ValueError('Keep every original proposal; reject it rather than deleting its history.') + return {'revision': current['revision']+1, 'status': payload['status'], + 'outcome': payload['outcome'], 'events': clean, + 'source_sha256': episode['source']['sha256'], 'episode_id': episode['id'], + 'annotation_sha256': digest(episode['observations']), + 'parent_sha256': digest(current), 'review_origin': 'local_operator', + 'created_at': datetime.now(timezone.utc).isoformat()} + + +def latest_review(directory, episode): + current = initial_review(episode) + for path in sorted(Path(directory).glob('review-*.json')): + if path.is_symlink() or path.stat().st_size > 2*1024*1024: + raise ValueError('Unsafe review revision.') + row = _json(_read(path, 2*1024*1024)) + if (type(row) is not dict or row.get('parent_sha256') != digest(current) + or row.get('source_sha256') != episode['source']['sha256'] + or row.get('annotation_sha256') != digest(episode['observations']) + or row.get('episode_id') != episode['id']): + raise ValueError('Review history does not match the source and prior revision.') + checked = validate_review({'base_revision': current['revision'], 'events': row['events'], + 'outcome': row['outcome'], 'status': row['status']}, episode, current) + if row.get('revision') != checked['revision']: + raise ValueError('Review revisions are not consecutive.') + current = row + return current + + +def evaluate_reviews(episodes): + """Review progress is not an independently referenced accuracy score.""" + reviewed = sum(e.get('review', {}).get('status') == 'reviewed' for e in episodes) + return {'episodes': len(episodes), 'reviewed_episodes': reviewed, + 'accuracy': None, 'accuracy_status': 'not_assessed', + 'reason': 'No independent blinded reference annotations supplied.', + 'robot_skill_qualified': False, 'geometry': 'image_coordinates_only'} diff --git a/tests/test_observation_models.py b/tests/test_observation_models.py new file mode 100644 index 0000000..feed4fd --- /dev/null +++ b/tests/test_observation_models.py @@ -0,0 +1,36 @@ +"""Offline contract tests for suppression and weight admission; no fake accuracy.""" +from pathlib import Path +import tempfile +import unittest + +from dvidia_training.observation_models import OpenCVDetector, MODEL_ASSETS, _suppress, _fraction + + +class DetectorContractTest(unittest.TestCase): + def test_overlapping_same_class_suppressed_but_distinct_class_retained(self): + rows = [{'label': 'cup', 'score': .9, 'xyxy': [.1,.1,.5,.5]}, + {'label': 'cup', 'score': .8, 'xyxy': [.11,.11,.51,.51]}, + {'label': 'hand', 'score': .7, 'xyxy': [.1,.1,.5,.5]}] + self.assertEqual([r['label'] for r in _suppress(rows,.5,30)], ['cup','hand']) + + def test_detections_are_bounded_and_ordered(self): + rows = [{'label': 'cup', 'score': score, 'xyxy': [0,0,.1,.1]} + for score in [.1,.8,.4]] + self.assertEqual(_suppress(rows,.5,1)[0]['score'], .8) + + def test_invalid_thresholds_rejected(self): + for value in [True, float('nan'), 0, 1, '0.5']: + with self.assertRaises(ValueError): _fraction(value,'score') + + def test_missing_and_modified_weights_rejected_before_import(self): + with tempfile.TemporaryDirectory() as directory: + with self.assertRaises(ValueError): OpenCVDetector(directory) + asset = MODEL_ASSETS[0] + with (Path(directory)/asset['name']).open('wb') as output: + output.truncate(asset['bytes']) + with self.assertRaisesRegex(ValueError,'SHA-256 mismatch'): + OpenCVDetector(directory,include_hands=False) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_observations.py b/tests/test_observations.py new file mode 100644 index 0000000..8c8c1a5 --- /dev/null +++ b/tests/test_observations.py @@ -0,0 +1,231 @@ +"""Contract, real FFmpeg intake, and loopback review acceptance checks.""" +from copy import deepcopy +import http.client +import json +from pathlib import Path +import shutil +import subprocess +import tempfile +import threading +import unittest + +from dvidia_training import observations as o +from dvidia_training.observation_pipeline import run, load_episodes, validate_manifest +from dvidia_training.observation_studio import ObservationServer + + +def detection(label='cup', bounds=None): + return {'label': label, 'score': .7, 'xyxy': bounds or [.2, .2, .5, .5]} + + +def episode(): + frames = [{'time_seconds': t, 'detections': [detection()]} for t in [0, .5, 1]] + return {'id': 'fixture', 'source': {'sha256': 'a'*64}, 'duration_seconds': 2, + 'observations': o.associate(frames, 2)} + + +def payload(e, base=0): + return {'base_revision': base, 'events': o.initial_review(e)['events'], + 'outcome': 'unknown', 'status': 'in_review'} + + +class ObservationContractTest(unittest.TestCase): + def test_association_preserves_evidence_and_splits_occlusion(self): + frames = [{'time_seconds': t, 'detections': [detection()]} for t in [0, .5, 2]] + tracks = o.associate(frames, 3) + self.assertEqual(len(tracks), 2) + self.assertEqual(len(tracks[0]['boxes']), 2) + self.assertEqual(tracks[0]['end_seconds'], 1) + self.assertEqual(tracks[1]['start_seconds'], 2) + + def test_same_frame_cannot_merge_two_objects(self): + tracks = o.associate([{'time_seconds': 0, 'detections': [detection(), detection()]}], 1) + self.assertEqual(len(tracks), 2) + + def test_bad_scores_boxes_and_timestamps_rejected(self): + for row in [detection(bounds=[0, 0, float('nan'), 1]), detection(bounds=[.5,.2,.2,.5]), + {**detection(), 'score': True}, {**detection(), 'score': 2}]: + with self.subTest(row=row), self.assertRaises(ValueError): + o.associate([{'time_seconds': 0, 'detections': [row]}], 1) + with self.assertRaises(ValueError): + o.associate([{'time_seconds': 0, 'detections': []}]*2, 1) + + def test_proximity_does_not_claim_grasp_or_success(self): + proposals = o.propose_relations([{'time_seconds': 0, 'detections': [detection(), detection('hand')]}], 1) + self.assertEqual(len(proposals), 1) + self.assertIn('image overlap', proposals[0]['label']) + self.assertEqual(o.initial_review(episode())['outcome'], 'unknown') + + def test_review_keeps_raw_separate_and_binds_source(self): + e = episode(); original = deepcopy(e) + p = payload(e); p['events'][0]['label'] = 'Cup held, needs checking' + result = o.validate_review(p, e, o.initial_review(e)) + self.assertEqual(e, original) + self.assertEqual(result['source_sha256'], e['source']['sha256']) + self.assertEqual(result['revision'], 1) + + def test_stale_review_rejected(self): + e = episode() + with self.assertRaises(RuntimeError): + o.validate_review(payload(e, 1), e, o.initial_review(e)) + + def test_invalid_observation_ids_are_controlled_errors(self): + e = episode() + for value in ([], {}, 3, True): + p = payload(e); p['events'][0]['observation_id'] = value + with self.subTest(value=value), self.assertRaises(ValueError): + o.validate_review(p, e, o.initial_review(e)) + + def test_cannot_drop_or_duplicate_model_proposals(self): + e = episode() + for events in [[], payload(e)['events']*2]: + with self.assertRaises(ValueError): + o.validate_review({**payload(e), 'events': events}, e, o.initial_review(e)) + + def test_reviewed_requires_all_decisions(self): + e = episode(); p = payload(e); p['status'] = 'reviewed' + with self.assertRaises(ValueError): + o.validate_review(p, e, o.initial_review(e)) + p['events'][0]['decision'] = 'rejected' + self.assertEqual(o.validate_review(p, e, o.initial_review(e))['outcome'], 'unknown') + + def test_manual_event_accepted_only_with_valid_interval(self): + e = episode(); p = payload(e) + manual = dict(id='manual-new', observation_id=None, label='Reach', start_seconds=0, + end_seconds=.5, decision='approved') + p['events'].append(manual) + self.assertEqual(len(o.validate_review(p,e,o.initial_review(e))['events']), 2) + for end in [0, 3, float('inf')]: + manual['end_seconds'] = end + with self.assertRaises(ValueError): + o.validate_review(p,e,o.initial_review(e)) + + def test_unknown_accuracy_without_independent_references(self): + self.assertIsNone(o.evaluate_reviews([episode()])['accuracy']) + + def test_changed_review_chain_rejected(self): + with tempfile.TemporaryDirectory() as directory: + e = episode(); first = o.validate_review(payload(e),e,o.initial_review(e)) + target = Path(directory)/'review-000001.json'; target.write_text(json.dumps(first)) + self.assertEqual(o.latest_review(directory,e)['revision'], 1) + first['parent_sha256'] = 'b'*64; target.write_text(json.dumps(first)) + with self.assertRaises(ValueError): + o.latest_review(directory,e) + target.write_text('[]') + with self.assertRaises(ValueError): + o.latest_review(directory,e) + + +class FixtureDetector: + metadata = {'kind': 'synthetic-test-only', 'device': 'cpu'} + def detect(self, _): + return [detection()] + + +@unittest.skipUnless(shutil.which('ffmpeg') and shutil.which('ffprobe'), 'FFmpeg required') +class ObservationAcceptanceTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory(); cls.root = Path(cls.tmp.name) + cls.video = cls.root/'synthetic.mp4' + subprocess.run(['ffmpeg','-v','error','-f','lavfi','-i','color=c=blue:s=160x120:r=10:d=1.2', + '-c:v','libx264','-pix_fmt','yuv420p',str(cls.video)], check=True) + cls.row = dict(id='fixture',path='synthetic.mp4',title='Synthetic fixture',session_id='session1', + source_recording_id='synthetic1',kind='synthetic',credit='DVIDIA software fixture', + license='MIT',source_url='local-synthetic-fixture',permission='local_analysis',sha256=o.file_hash(cls.video)) + cls.manifest = cls.root/'source.json' + cls.manifest.write_text(json.dumps(dict(kind='dvidia.observation-source.v1',task='Test fixture',episodes=[cls.row]))) + cls.output = cls.root/'run' + cls.result = run(cls.manifest, cls.output, FixtureDetector()) + + @classmethod + def tearDownClass(cls): + cls.tmp.cleanup() + + def test_real_decoder_hash_timestamps_and_offline_bundle(self): + receipt, episodes = load_episodes(self.output) + self.assertEqual(receipt['completed'], 1) + e = episodes['fixture'] + self.assertEqual([f['time_seconds'] for f in e['frames']], [0,.5,1]) + self.assertTrue(all(len(f['sha256']) == 64 for f in e['frames'])) + self.assertEqual(e['source']['sha256'], self.row['sha256']) + self.assertNotIn(str(self.root), json.dumps(e)) + + def test_existing_run_not_overwritten(self): + with self.assertRaises(ValueError): + run(self.manifest, self.output, FixtureDetector()) + + def test_nonzero_video_start_keeps_failed_attempt(self): + video = self.root/'offset.mp4' + subprocess.run(['ffmpeg', '-v', 'error', '-i', str(self.video), '-c', 'copy', + '-output_ts_offset', '2', str(video)], check=True) + manifest = self.root/'offset.json' + manifest.write_text(json.dumps(dict(kind='dvidia.observation-source.v1', task='Timing fixture', + episodes=[{**self.row, 'path':video.name, 'sha256':o.file_hash(video)}]))) + result = run(manifest, self.root/'offset-run', FixtureDetector()) + self.assertEqual(result['completed'], 0) + self.assertIn('start time', result['failures'][0]['error']) + + def test_run_metadata_is_bounded_before_reading(self): + copy=self.root/'oversized'; shutil.copytree(self.output,copy) + (copy/'run.json').write_bytes(b' '* (2*1024*1024+1)) + with self.assertRaises(ValueError): load_episodes(copy) + + def test_hash_failure_preserved(self): + manifest = self.root/'bad.json' + manifest.write_text(json.dumps(dict(kind='dvidia.observation-source.v1',task='Test',episodes=[{**self.row,'sha256':'a'*64}]))) + result = run(manifest, self.root/'bad-run', FixtureDetector()) + self.assertEqual(result['completed'],0) + self.assertEqual(len(result['failures']),1) + + def test_unsafe_or_unpermitted_sources_rejected(self): + for index, changes in enumerate([{'permission':'upload'}, {'path':'../synthetic.mp4'}, {'sha256':'bad'}]): + path=self.root/f'invalid{index}.json' + path.write_text(json.dumps(dict(kind='dvidia.observation-source.v1',task='Test',episodes=[{**self.row,**changes}]))) + with self.assertRaises(ValueError): validate_manifest(path) + + def test_tampered_episode_rejected(self): + copy=self.root/'tampered'; shutil.copytree(self.output,copy) + path=copy/'fixture/episode.json'; data=json.loads(path.read_text()); data['title']='Changed'; path.write_text(json.dumps(data)) + with self.assertRaises(ValueError): load_episodes(copy) + + def test_http_review_range_origin_and_reopen(self): + run_copy=self.root/'http'; shutil.copytree(self.output,run_copy) + server=ObservationServer(('127.0.0.1',0),run_copy) + thread=threading.Thread(target=server.serve_forever,daemon=True); thread.start() + def request(method,path,body=None,headers=None): + conn=http.client.HTTPConnection('127.0.0.1',server.server_port,timeout=5) + conn.request(method,path,body=body,headers=headers or {}) + response=conn.getresponse(); data=response.read(); result=(response.status,dict(response.getheaders()),data); conn.close(); return result + try: + status,_,raw=request('GET','/api/episodes/fixture'); self.assertEqual(status,200) + e=json.loads(raw); p=payload(e) + headers={'Content-Type':'application/json','Origin':f'http://127.0.0.1:{server.server_port}'} + self.assertEqual(request('POST','/api/episodes/fixture/review',json.dumps(p),{'Content-Type':'application/json'})[0],403) + self.assertEqual(request('POST','/api/episodes/fixture/review',json.dumps(p),headers)[0],200) + self.assertEqual(request('POST','/api/episodes/fixture/review',json.dumps(p),headers)[0],409) + malformed=payload(e, 1); malformed['events'][0]['observation_id']=[] + self.assertEqual(request('POST','/api/episodes/fixture/review',json.dumps(malformed),headers)[0],400) + self.assertEqual(json.loads(request('GET','/api/episodes/fixture')[2])['review']['revision'],1) + self.assertEqual(o.latest_review(run_copy/'fixture',e)['revision'],1) + self.assertEqual(request('GET','/api/episodes',headers={'Host':'evil.test'})[0],403) + self.assertEqual(request('GET','/api/episodes',headers={'Origin':'https://evil.test'})[0],403) + video=self.video.read_bytes() + status,headers,data=request('GET','/api/episodes/fixture/video',headers={'Range':'bytes=4-19'}) + self.assertEqual((status,data),(206,video[4:20])); self.assertEqual(headers['Content-Length'],'16') + self.assertEqual(request('GET','/api/episodes/fixture/video',headers={'Range':'bytes=-8'})[2],video[-8:]) + self.assertEqual(request('GET','/api/episodes/fixture/video',headers={'Range':'bytes=999999-'})[0],416) + self.assertEqual(request('GET','/api/episodes/fixture/video',headers={'Range':'bytes=0-2,5-9'})[0],416) + self.assertEqual(request('HEAD','/api/episodes/fixture/video')[2],b'') + self.assertEqual(request('GET','/../../source.json')[0],404) + changed=run_copy/'fixture'/e['source']['video_path'] + original=changed.read_bytes(); changed.write_bytes(b'x'+original[1:]) + self.assertEqual(request('GET','/api/episodes/fixture/video')[0],409) + self.assertEqual(request('POST','/api/episodes/fixture/review',json.dumps(payload(e,1)), + {'Content-Type':'application/json','Origin':f'http://127.0.0.1:{server.server_port}'})[0],400) + finally: + server.shutdown(); thread.join(); server.server_close() + + +if __name__ == '__main__': + unittest.main() diff --git a/third_party/observation-models/object_detection_yolox-LICENSE.txt b/third_party/observation-models/object_detection_yolox-LICENSE.txt new file mode 100644 index 0000000..1d4dc76 --- /dev/null +++ b/third_party/observation-models/object_detection_yolox-LICENSE.txt @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2021-2022 Megvii Inc. All rights reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/third_party/observation-models/palm_detection_mediapipe-LICENSE.txt b/third_party/observation-models/palm_detection_mediapipe-LICENSE.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/third_party/observation-models/palm_detection_mediapipe-LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 3cc4a3b03ca6d61479c9f6a86cebf7ccc77fd117 Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 10 Oct 2026 07:46:07 -0400 Subject: [PATCH 2/3] Publish Observation Lab installation, synthetic review demo and provenance --- .github/workflows/ci.yml | 45 ++++- README.md | 6 + benchmarks/observation-20261010/README.md | 7 +- docs/observation-models.md | 3 +- docs/observations.md | 19 +- .../huggingface/observation-lab/README.md | 76 ++++++++ .../observation-lab/generate_demo.py | 121 +++++++++++++ .../huggingface/observation-lab/prepare.py | 165 ++++++++++++++++++ .../observation-lab/templates/README.md | 94 ++++++++++ .../observation-lab/templates/THIRD_PARTY.md | 18 ++ .../observation-lab/templates/demo-adapter.js | 118 +++++++++++++ .../observation-lab/test_adapter.cjs | 78 +++++++++ .../observation-lab/test_prepare.py | 111 ++++++++++++ publishing/huggingface/publish_observation.py | 145 +++++++++++++++ .../huggingface/test_publish_observation.py | 85 +++++++++ src/dvidia_training/observation_models.py | 5 +- 16 files changed, 1088 insertions(+), 8 deletions(-) create mode 100644 publishing/huggingface/observation-lab/README.md create mode 100644 publishing/huggingface/observation-lab/generate_demo.py create mode 100644 publishing/huggingface/observation-lab/prepare.py create mode 100644 publishing/huggingface/observation-lab/templates/README.md create mode 100644 publishing/huggingface/observation-lab/templates/THIRD_PARTY.md create mode 100644 publishing/huggingface/observation-lab/templates/demo-adapter.js create mode 100644 publishing/huggingface/observation-lab/test_adapter.cjs create mode 100644 publishing/huggingface/observation-lab/test_prepare.py create mode 100644 publishing/huggingface/publish_observation.py create mode 100644 publishing/huggingface/test_publish_observation.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0210f7f..8e151b5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,9 +63,52 @@ jobs: dvidia-episode-tape --help dvidia-reaction --help dvidia-motor-memory --help + dvidia-observe --help + dvidia-observation-studio --help dvidia-training-doctor --json - name: Test footage, visual training and loopback studio - run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_footage_benchmark tests.test_training_studio tests.test_doctor tests.test_coverage tests.test_episode_tape tests.test_reaction tests.test_reaction_metrics tests.test_reaction_convergence tests.test_reaction_benchmark tests.test_motor_memory tests.test_motor_metrics tests.test_motor_benchmark -v + run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_footage_benchmark tests.test_training_studio tests.test_doctor tests.test_coverage tests.test_episode_tape tests.test_reaction tests.test_reaction_metrics tests.test_reaction_convergence tests.test_reaction_benchmark tests.test_motor_memory tests.test_motor_metrics tests.test_motor_benchmark tests.test_observations tests.test_observation_models -v + + observations: + name: Observation detectors / Linux / Python 3.12 + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: "3.12" + - name: Install observation runtime + run: | + sudo apt-get update + sudo apt-get install -y ffmpeg + python -m pip install ".[observations]" + python -m pip check + - name: Verify review contracts and explicitly provision pinned detectors + run: | + python -m unittest tests.test_observations tests.test_observation_models -v + python -m unittest discover -s publishing/huggingface -p 'test_publish_observation.py' -v + python -m unittest discover -s publishing/huggingface/observation-lab -p 'test_*.py' -v + node publishing/huggingface/observation-lab/test_adapter.cjs + python scripts/download_observation_models.py --download --output runs/observation-models-ci + - name: Smoke test actual ONNX loading and inference + run: | + python - <<'PY' + import json, tempfile + from pathlib import Path + import cv2, numpy as np + from dvidia_training.observation_models import OpenCVDetector + detector = OpenCVDetector('runs/observation-models-ci') + with tempfile.TemporaryDirectory() as directory: + frame = Path(directory) / 'neutral.png' + assert cv2.imwrite(str(frame), np.full((240, 320, 3), 240, np.uint8)) + rows = detector.detect(frame) + assert isinstance(rows, list) + print(json.dumps({'scope': 'neutral-image model-loading smoke test; no accuracy measurement', + 'proposals': len(rows), 'runtime': detector.metadata})) + PY arm: name: Optional arm / Linux / Python 3.12 diff --git a/README.md b/README.md index 9f9213c..3472d63 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,12 @@ palm detectors to propose timestamped image observations. Open the original vide inspect its boxes and timeline, correct labels, and save a separate review revision. Raw evidence stays unchanged. No account, cloud inference, or GPU is needed. +[Observation Lab release](https://github.com/Dvidia-Inference/dvidia-training/releases/tag/observation-lab-v0.1.0) · [Research paper](https://research.dvidia.org/papers/dvidia-observation-lab/) · [Synthetic review demo](https://huggingface.co/spaces/Dvidia/observation-lab) + +Use the `observation-lab-v0.1.0` tag for this feature. The hosted demo uses +authored synthetic clips and keeps edits in your browser; actual detection runs +locally with the optional observation runtime and explicitly provisioned models. + The [first feasibility run](benchmarks/observation-20261010/README.md) completed six public excerpts. This establishes the integration, not accuracy: labels can be wrong, outcomes remain unknown, and there is no 3D reconstruction, automatic diff --git a/benchmarks/observation-20261010/README.md b/benchmarks/observation-20261010/README.md index ba8379c..864df1b 100644 --- a/benchmarks/observation-20261010/README.md +++ b/benchmarks/observation-20261010/README.md @@ -29,8 +29,11 @@ UI startup and human review. Single runs, warm machine; this is neither a p95 latency measurement nor a production throughput guarantee. Peak RAM, energy and dollar cost were not measured. No paid inference API was called. -Runtime: Python 3.12.9, OpenCV 5.0.0.93, NumPy 2.5.3, macOS ARM64, four CPU threads, -one inference worker. Both pinned ONNX models total approximately 40 MB. OpenCV +Runtime: Python 3.12.9, OpenCV 5.0.0.93, NumPy 2.5.3, macOS ARM64 and one +inference worker. Four OpenCV threads were requested, but the saved model +metadata records `cv2.getNumThreads()` as ten after model loading. Actual peak +thread concurrency was not measured; a four-thread cap is not established. +Both pinned ONNX models total approximately 40 MB. OpenCV 5 printed that target selection is unsupported by its new graph engine; this build uses the OpenCV CPU backend and no GPU path was requested or measured. diff --git a/docs/observation-models.md b/docs/observation-models.md index b63cca9..5338367 100644 --- a/docs/observation-models.md +++ b/docs/observation-models.md @@ -21,7 +21,8 @@ YOLOX's included license attributes Megvii Inc., 2021–2022. DVIDIA modifications include local weight verification, bounded deterministic suppression, clipped normalized image boxes, explicit palm-only semantics, -metadata, runtime limits and validation of model output. No remote Python code +metadata, bounded inputs and validation of model output. Inference time and peak +RAM/thread use are not hard capped. No remote Python code is downloaded or executed. Source data licensing remains independent from model licensing. These license records do not establish rights to any user's footage. diff --git a/docs/observations.md b/docs/observations.md index 8b06d66..7bad3e9 100644 --- a/docs/observations.md +++ b/docs/observations.md @@ -8,6 +8,14 @@ model has been trained and no SpatialLM weights are included. ## Setup +The public Observation Lab release is pinned separately from the older training +pilot. Obtain it with: + +```sh +git clone --branch observation-lab-v0.1.0 https://github.com/Dvidia-Inference/dvidia-training.git +cd dvidia-training +``` + From this repository checkout, with Python 3.12+ and FFmpeg/ffprobe installed: ```sh @@ -102,8 +110,10 @@ the same account. Hashes detect accidental changes but are not digital signature The run stores copied originals, sampled frames, raw proposals, failures, recipe, model hashes, source rights and review revisions. Allow scratch space beyond the -input size. Four CPU threads and one inference worker are used; this is not a -hard RAM/container limit. Do not install it on a production database machine +input size. One inference worker is used and four OpenCV threads are requested; +the six-clip pilot recorded `cv2.getNumThreads()` as ten after loading. Actual +peak thread concurrency was not measured, so no four-thread cap is established. +There is no hard inference-time, RAM or container limit. Do not install it on a production database machine without a separate resource-constrained deployment test. ## Next gates @@ -120,3 +130,8 @@ without a separate resource-constrained deployment test. See [model licenses](observation-models.md), [UI contract](observation-ui.md) and the [actual feasibility report](../benchmarks/observation-20261010/README.md). + +[Try the synthetic review demo](https://huggingface.co/spaces/Dvidia/observation-lab) +or [read the research publication](https://research.dvidia.org/papers/dvidia-observation-lab/). +The hosted demo has authored boxes and browser-local review state; live detection +of permitted footage runs in the installed local tool. diff --git a/publishing/huggingface/observation-lab/README.md b/publishing/huggingface/observation-lab/README.md new file mode 100644 index 0000000..d44b7f1 --- /dev/null +++ b/publishing/huggingface/observation-lab/README.md @@ -0,0 +1,76 @@ +# Publishing the Observation Lab workflow demo + +This preparation package is separate from the original training-pilot publisher. +It stages **only an authored synthetic static Space**, intended for +`Dvidia/observation-lab`. It has no upload function, credential reader or model +dependency. Do not pass its plan to the older three-repository publisher, which +correctly refuses a different destination/layout. + +## Prepare and inspect + +From the training repository, using installed Python3.12+ and FFmpeg/libx264: + +```sh +python3 publishing/huggingface/observation-lab/prepare.py --org Dvidia --output runs/observation-space-publication +python3 -m http.server 8294 --bind 127.0.0.1 --directory runs/observation-space-publication/space +``` + +Open http://127.0.0.1:8294. Check playback, sampled boxes, label/time edits, +approval/rejection, manual events, save/reload, review JSON download and reset. +Check a second clean browser context starts at revision 0, while another tab in +the same browser reports a stale revision instead of overwriting it. Blocked +browser storage should show the temporary-memory notice. + +The preparation script has a fixed source allowlist; it never scans private +`runs/`, user footage, model directories or recursively copies `src/`. It copies +the exact review UI to `source/` and makes a labeled browser-only API derivative. +It generates two 8-second 640×360 schematic clips using stdlib RGB drawing and +FFmpeg/libx264. All demo state stays in browser storage or temporary memory. +Nothing is posted to a server; media requests read only the included fixtures. + +Review `space/README.md`, `index.html`, `demo-adapter.js`, `provenance.json`, +`demo/provenance.json` and the exact `publication-plan.json` inventory/hashes. +The preparation budget is 2 MiB. No model weights, secrets, private clips or +commercial server dependencies are bundled. A static Space has no container; +therefore there is no container/image validation or model throughput claim. + +## Maintainer publication + +Use the official Hub SDK in an isolated maintainer environment and the existing +secure credential path. Create public `Dvidia/observation-lab` with +`repo_type='space'` and `space_sdk='static'`; upload only the reviewed `space/` +directory with the plan's exact inventory. Do not upload the outer staging +directory or local token files. No application token/secret, GPU allocation or +persistent shared review store is needed. + +Before a remote mutation, freeze the reviewed bytes to a separate bounded +snapshot, verify every digest/inventory entry, check destination organization +membership and preserve any differing existing remote files rather than +overwriting them silently. After publication, anonymously fetch every artifact +at the returned exact commit and compare hashes; confirm the live Space starts, +playback/download work and synthetic/local-state labels remain visible. + +The user explicitly authorized this public publication, but this preparation +script performs no remote calls. Actual local ONNX tool installation and model +notices are linked from the demo; its own code/data remain MIT. + +The dedicated publisher validates this single-Space plan, reads the named +Keychain credential through the existing in-memory adapter, refuses differing +existing remote bytes, and anonymously verifies every file at the new commit: + +```sh +python3 publishing/huggingface/publish_observation.py check runs/observation-space-publication/publication-plan.json +runs/huggingface-tools/.venv/bin/python publishing/huggingface/publish_observation.py publish runs/observation-space-publication/publication-plan.json +``` + +Provision the maintainer environment and Keychain entry using the parent +[publishing guide](../README.md). Never paste a token into a command or the plan. + +```sh +python3 -m unittest discover -s publishing/huggingface/observation-lab -p 'test_*.py' +node publishing/huggingface/observation-lab/test_adapter.cjs +``` + +Official references: [Static Spaces](https://huggingface.co/docs/hub/spaces-sdks-static), +[configuration](https://huggingface.co/docs/hub/spaces-config-reference), +[Hub uploads](https://huggingface.co/docs/huggingface_hub/guides/upload). diff --git a/publishing/huggingface/observation-lab/generate_demo.py b/publishing/huggingface/observation-lab/generate_demo.py new file mode 100644 index 0000000..10748cd --- /dev/null +++ b/publishing/huggingface/observation-lab/generate_demo.py @@ -0,0 +1,121 @@ +"""Generate only DVIDIA-authored schematic video and annotation fixtures.""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +import shutil +import subprocess + +WIDTH, HEIGHT, FPS, SECONDS = 640, 360, 20, 8 + + +def rectangle(image, x, y, width, height, color): + row = bytes(color) * width + for yy in range(y, y + height): + image[(yy * WIDTH + x) * 3:(yy * WIDTH + x + width) * 3] = row + + +def object_box(time): + x = int(96 + 344 * min(time / 5.5, 1)) + return [x, 172, x + 64, 236] + + +def frame(time, occluded): + image = bytearray(bytes([245, 244, 239]) * WIDTH * HEIGHT) + rectangle(image, 32, 80, 576, 228, [224, 226, 220]) + for x, y, w, h in ((430, 153, 100, 3), (430, 253, 100, 3), + (430, 153, 3, 103), (527, 153, 3, 103)): + rectangle(image, x, y, w, h, [76, 124, 91]) + x1, y1, x2, y2 = object_box(time) + rectangle(image, x1, y1, x2 - x1, y2 - y1, [61, 106, 191]) + if occluded: + rectangle(image, 245, 108, 142, 173, [195, 194, 183]) + for x in range(248, 385, 12): + rectangle(image, x, 110, 3, 169, [214, 214, 205]) + return image + + +def generate(output): + output = Path(output).absolute() + if any(p.is_symlink() for p in (output, *output.parents)) or output.exists(): + raise ValueError('Use a fresh plain fixture directory.') + ffmpeg = shutil.which('ffmpeg') + if ffmpeg is None: + raise ValueError('Preparing schematic videos requires installed FFmpeg with libx264.') + output.mkdir(parents=True) + version = subprocess.run([ffmpeg, '-version'], check=True, capture_output=True, text=True).stdout.splitlines()[0] + fixtures = [] + for name, occluded in [('visible', False), ('occluded', True)]: + video = output / (name + '.mp4') + args = [ffmpeg, '-v', 'error', '-nostdin', '-threads', '1', '-f', 'rawvideo', + '-pix_fmt', 'rgb24', '-s', f'{WIDTH}x{HEIGHT}', '-r', str(FPS), + '-i', 'pipe:0', '-an', '-c:v', 'libx264', '-threads', '1', + '-pix_fmt', 'yuv420p', '-movflags', '+faststart', '-map_metadata', '-1', str(video)] + process = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE) + try: + for index in range(FPS * SECONDS): + process.stdin.write(frame(index / FPS, occluded)) + process.stdin.close() + error = process.stderr.read() + if process.wait(timeout=30) != 0: + raise ValueError('Synthetic video encoding failed.') + finally: + if process.poll() is None: + process.kill() + process.wait() + source_hash = sha256(video.read_bytes()).hexdigest() + observations = [] + intervals = [(0, 8)] if not occluded else [(0, 1.5), (5, 8)] + for number, (start, end) in enumerate(intervals, 1): + boxes = [] + for index in range(SECONDS * 2): + t = index / 2 + if not start <= t < end: + continue + bounds = object_box(t) + boxes.append({'time_seconds': t, 'xyxy': [bounds[0] / WIDTH, bounds[1] / HEIGHT, + bounds[2] / WIDTH, bounds[3] / HEIGHT], 'label': 'blue shape · authored'}) + observations.append({'id': f'sample-{number}', 'label': 'Blue shape visible · authored example', + 'start_seconds': start, 'end_seconds': end, 'boxes': boxes, + 'evidence': 'authored_schematic_example', 'review_state': 'proposed'}) + observations.append({'id': 'region', 'label': 'Marked region visible · authored example', + 'start_seconds': 0, 'end_seconds': 8, 'boxes': [ + {'time_seconds': i / 2, 'xyxy': [430 / WIDTH, 153 / HEIGHT, 530 / WIDTH, 256 / HEIGHT], + 'label': 'marked region · authored'} for i in range(SECONDS * 2)], + 'evidence': 'authored_schematic_example', 'review_state': 'proposed'}) + episode = {'kind': 'dvidia.observation-demo-episode.v1', 'id': name, + 'title': 'Blue shape crosses an occluder' if occluded else 'Blue shape moves into a marked region', + 'video_url': 'demo/' + name + '.mp4', 'duration_seconds': SECONDS, + 'source': {'kind': 'synthetic', 'sha256': source_hash, 'credit': 'DVIDIA contributors · authored geometric fixture', + 'license': 'MIT', 'session_id': 'synthetic-demo-one-dependent-group', + 'source_recording_id': 'authored-' + name, 'source_url': 'generate_demo.py', + 'notes': 'No person, robot, camera capture, detector or learned action is represented.'}, + 'recipe': {'id': 'dvidia-authored-schematic-demo-v1', + 'model': {'kind': 'Authored fixture · No model ran', 'models': []}, + 'notes': 'Deterministic example annotations derived from the drawing recipe. No live inference, accuracy score, task segmentation or robot skill is demonstrated.'}, + 'observations': observations, + 'review': {'revision': 0, 'status': 'unreviewed', 'outcome': 'unknown', 'events': [ + {'id': o['id'], 'observation_id': o['id'], 'label': o['label'], + 'start_seconds': o['start_seconds'], 'end_seconds': o['end_seconds'], + 'decision': 'pending'} for o in observations]}} + (output / (name + '.json')).write_text(json.dumps(episode, indent=2) + '\n') + fixtures.append({'id': name, 'title': episode['title'], 'duration_seconds': SECONDS, + 'video_url': episode['video_url'], 'status': 'unreviewed'}) + (output / 'episodes.json').write_text(json.dumps({'episodes': fixtures, 'failures': []}, indent=2) + '\n') + files = [{'path': p.name, 'bytes': p.stat().st_size, 'sha256': sha256(p.read_bytes()).hexdigest()} + for p in sorted(output.iterdir()) if p.is_file()] + (output / 'provenance.json').write_text(json.dumps({'kind': 'dvidia.authored-demo-fixtures.v1', + 'generation': 'stdlib RGB rectangle drawing piped to local FFmpeg/libx264; no source footage input', + 'generator_sha256': sha256(Path(__file__).read_bytes()).hexdigest(), 'ffmpeg': version, + 'dimensions': [WIDTH, HEIGHT], 'fps': FPS, 'seconds_per_clip': SECONDS, + 'annotations': 'authored image rectangles every 0.5 seconds; no model inference', + 'occlusion': 'box annotations deliberately absent when the blue shape is behind the drawn panel', + 'independent_source_groups': 1, 'files': files}, indent=2) + '\n') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True, type=Path) + generate(parser.parse_args().output) diff --git a/publishing/huggingface/observation-lab/prepare.py b/publishing/huggingface/observation-lab/prepare.py new file mode 100644 index 0000000..2b417c3 --- /dev/null +++ b/publishing/huggingface/observation-lab/prepare.py @@ -0,0 +1,165 @@ +"""Prepare an explicit synthetic-only static Observation Lab Space; never upload.""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +import re +import shutil +import subprocess + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parents[2] +MAX_TOTAL_BYTES = 2 * 1024 * 1024 + + +def record(path): + raw = path.read_bytes() + return {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + + +def plain(path): + if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file(): + raise ValueError('Preparation accepts only the named plain source files.') + return path + + +def fresh(path): + path = Path(path).absolute() + if any(p.is_symlink() for p in (path, *path.parents)): + raise ValueError('Publication output may not contain symlinks.') + if path.exists() and (not path.is_dir() or any(path.iterdir())): + raise ValueError('Choose a fresh or empty publication directory.') + path.mkdir(parents=True, exist_ok=True) + return path + + +def replace_once(text, old, new): + if text.count(old) != 1: + raise ValueError('Observation UI changed; review the demo adapter before preparing.') + return text.replace(old, new) + + +def stage(namespace, output): + if not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9-]{0,95}', namespace): + raise ValueError('Expected a Hugging Face organization name.') + output = fresh(output) + space = output / 'space' + space.mkdir() + # Explicit inventory only: no private runs, clips, models, credentials or + # recursive source/package copies can enter the public Space. + sources = { + 'README.md': HERE / 'templates' / 'README.md', + 'demo-adapter.js': HERE / 'templates' / 'demo-adapter.js', + 'generate_demo.py': HERE / 'generate_demo.py', + 'LICENSE': ROOT / 'LICENSE', + 'THIRD_PARTY.md': HERE / 'templates' / 'THIRD_PARTY.md', + 'licenses/object_detection_yolox-LICENSE.txt': ROOT / 'third_party' / 'observation-models' / 'object_detection_yolox-LICENSE.txt', + 'licenses/palm_detection_mediapipe-LICENSE.txt': ROOT / 'third_party' / 'observation-models' / 'palm_detection_mediapipe-LICENSE.txt', + 'source/observation_studio.html': ROOT / 'src' / 'dvidia_training' / 'observation_studio.html', + } + source_records = [] + for name, source in sources.items(): + plain(source) + target = space / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + source_records.append({'source_path': source.relative_to(ROOT).as_posix(), + 'published_path': name, **record(source)}) + readme = space / 'README.md' + readme.write_text(readme.read_text().replace('{{HF_ORG}}', namespace)) + ui = (space / 'source' / 'observation_studio.html').read_text() + start = ui.index(' const api = {\n') + end = ui.index(' function error(message)', start) + if ' save(id,body)' not in ui[start:end]: + raise ValueError('Expected the documented Observation API.') + ui = ui[:start] + ' const api = window.DvidiaObservationDemo;\n' + ui[end:] + ui = replace_once(ui, '\n