diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8a7c123..7654be9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,9 +58,10 @@ jobs: ffprobe -version dvidia-train --help dvidia-training-studio --help + dvidia-training-benchmark --help dvidia-training-doctor --json - name: Test footage, visual training and loopback studio - run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_training_studio tests.test_doctor -v + run: python -m unittest tests.test_footage_data tests.test_footage_train tests.test_footage_pipeline tests.test_footage_benchmark tests.test_training_studio tests.test_doctor -v arm: name: Optional arm / Linux / Python 3.12 diff --git a/README.md b/README.md index ec5ff54..578a85e 100644 --- a/README.md +++ b/README.md @@ -78,6 +78,8 @@ Related recording/session/shoe-pair IDs and identical video hashes stay in the s This adds MuJoCo 3.15.0 for the existing bounded simulation placement adapter, native telemetry examples and `dvidia-skill-capsule`. A movement candidate still needs an explicit compatible task contract and independent closed-loop qualification. It is not a physical robot driver or a learned shoe policy. See [the movement data contract](docs/skillspace-format.md#movement-supervision). +To measure a candidate before improving it, use the [sample-count and simulation benchmark](docs/benchmarks.md). It compares nested sets of whole recordings, keeps development/test footage fixed, freezes candidate weights before evaluation, and reports new-layout outcomes alongside matched open-jaw controls. It runs locally on a CPU and retains failures and qualification receipts. Install from the current source checkout to get `dvidia-training-benchmark`; the frozen v0.1.0 release predates this tool. + ## Public pilot on Hugging Face [Recorded demo](https://huggingface.co/spaces/Dvidia/dvidia-training) · [Synthetic datasets](https://huggingface.co/datasets/Dvidia/dvidia-training-examples) · [Pilot models and measurements](https://huggingface.co/Dvidia/dvidia-training-pilot) diff --git a/benchmarks/method-review-2026-10-08.md b/benchmarks/method-review-2026-10-08.md new file mode 100644 index 0000000..41bd4b1 --- /dev/null +++ b/benchmarks/method-review-2026-10-08.md @@ -0,0 +1,15 @@ +# Benchmark method review — 8 October 2026 + +A bounded methodological critique ran through Pi using the exact configured model `xai/grok-4.7`, with local tools, MCP, extensions, skills, automatic workspace context and session saving disabled. Only a public description of the current pilot and proposed benchmark was supplied. This was model feedback, not an independently executed experiment or a literature review. + +The implementation retains these useful review points: + +- One nested 2/4/7 group order mixes source count with source composition. Its results are a bounded pilot, not a demonstration sufficiency law. +- Counts share fixed scenes and development/test recordings. Repeated conditions do not create independent new scenes. +- Keep nominal and actuator-stress outcomes separate; preserve execution errors in planned counts. +- Retain the matched open-jaw control and distinguish authored phases, gripper logic and privileged observations from the learned movement head. +- Bind source-scene exclusion to exact evidence bytes and state the limits of supplier declarations. + +The model also suggested controls for gradient update budgets, which do not apply to this closed-form ridge fitter. A scene bootstrap alone would not establish a causal law relating demonstration count to competence. Suggestions to truncate rollouts or remove the gripper supervisor were not adopted: the current full native exact-scene qualification predicate remains unchanged. + +Prompt SHA256: `578f2be766c4c33c4a497c49bd01c55473328098aee7a71d396fda1e46c5b434`. Response SHA256: `bdba3374152a3b8c67becf929a82c4572b78eb3c68d49b9611565a0a29b1aa9e`. These hashes identify the local review inputs and response; they do not independently reproduce model execution. diff --git a/benchmarks/sample-count-19047/README.md b/benchmarks/sample-count-19047/README.md new file mode 100644 index 0000000..026e22d --- /dev/null +++ b/benchmarks/sample-count-19047/README.md @@ -0,0 +1,77 @@ +# Frozen sample-count pilot — 8 October 2026 + +This offline CPU experiment fits and installs three movement candidates from +the public synthetic native Skillspace, then evaluates them on twelve new +object/target layouts. The 2/4/7 conditions use nested whole recording groups; +the two development recordings and one test recording remain fixed. All models +are frozen before any benchmark physics is evaluated. This is one composition +order, not a law relating video count to acquired skill. + +| Training groups | Aligned training actions | Nominal qualification | Actuator-stress qualification | Nominal mean target error, including failures | Movement test MSE, rad² | +| --- | ---: | ---: | ---: | ---: | ---: | +| 2 | 1,135 | 8/8 | 3/4 | 0.677 mm | 1.120 × 10⁻⁶ | +| 4 | 2,281 | 7/8 | 3/4 | 31.641 mm | 1.109 × 10⁻⁶ | +| 7 | 4,029 | 8/8 | 3/4 | 0.267 mm | 1.325 × 10⁻⁶ | + +The authored teacher completes 8/8 nominal and 3/4 stress scenes. Every one of +the 36 matched open-jaw controls fails; there are no execution errors. All four +conditions fail stress scene 11. The four-group candidate additionally fails +nominal scene 04, which its teacher and the other two candidates complete. +Failures remain in all planned denominators and distance summaries. + +The seven-group candidate's nominal maximum target error is 0.483 mm. Its eight +successful nominal episodes simulate 106.263 seconds in 7.813 seconds of native +episode wall time, or **13.60 simulated seconds per wall second**. The complete +benchmark takes **102.727 seconds** on an Apple M5 host with 24 GiB RAM and no +GPU use, including input copies, fitting, installation, 84 native episodes and +artifact output. Process startup, top-level imports and final report writing +are excluded. This is a one-host timing observation, not a minimum hardware +requirement, speedup over another implementation, or physical-accuracy result. + +Pipeline timers for the 2/4/7 fits are 0.592/0.840/1.252 seconds, with their +narrower scope recorded in the report. The movement test has one recording and +621 correlated labels. Its prediction error does not rank closed-loop success: +the four-group candidate has the lowest test MSE yet an additional task failure. +More recordings did not monotonically improve execution in this pilot. + +## Inspect and reproduce + +- [Exact report](report.json), [frozen protocol](protocol.json), + [protocol hash](protocol.sha256), and [frozen model identities](models-frozen.json). +- Installed-candidate payloads: [2 groups](groups-002.skill-capsule.json), + [4 groups](groups-004.skill-capsule.json), [7 groups](groups-007.skill-capsule.json). + These are simulation candidates, bound to the existing runtime and adapter. + Downloading one does not authorize a physical arm or an untested scene. +- [Full evidence download](https://github.com/Dvidia-Inference/dvidia-training/releases/tag/benchmark-2026-10-08) + includes native traces, controls, qualification receipts and replays. + [Archive size and SHA256](archive.json) and the + [individual file manifest](evidence-manifest.json) identify the exact bytes. +- [Reproduction commands and method](../../docs/benchmarks.md) and + [Grok 4.7 method review](../method-review-2026-10-08.md). + +Source data came from an anonymous, hash-verified download of the public +[DVIDIA dataset](https://huggingface.co/datasets/Dvidia/dvidia-training-examples/tree/d351a32923d54e3166e8ff780fd827c80174a903). +Training and evaluation then ran with network access restricted. The source +archive contains ten synthetic native recordings, totaling 116.5 seconds, with +aligned numerical simulator labels. These are not five-second human shoe clips. +Repeating this seed is a repeat of observed scenes; subsequent learner changes +need a fresh, frozen evaluation suite. + +## What this establishes + +The product can now measure whether a fitted, portable candidate survives +installation and new-layout simulation qualification, preserving both successful +and failed cases. Exact layouts from all ten source recordings are excluded +relative to their hash-bound native declarations. That checks consistency, not +independent capture history or out-of-distribution transfer. + +The RGB predictor does not control the arm. Movement uses privileged simulator +state and an authored phase/waypoint/gripper/recovery scaffold. No raw +human-video action bridge, shoe skill, sufficient demonstration count, contact +accuracy on hardware, or physical robot qualification follows from these results. + +The next optimization targets are measurable: investigate the four-group +closed-loop failure, distinguish movement-head behavior from the shared actuator +stress limit, then test any revised learner across new composition orders and +fresh layouts. Human footage also needs a validated observation-to-action bridge +for a defined arm and gripper before it can enter this movement benchmark. diff --git a/benchmarks/sample-count-19047/archive.json b/benchmarks/sample-count-19047/archive.json new file mode 100644 index 0000000..f1da968 --- /dev/null +++ b/benchmarks/sample-count-19047/archive.json @@ -0,0 +1,8 @@ +{ + "schema_version": 1, + "filename": "sample-count-19047-evidence.zip", + "bytes": 44732405, + "sha256": "bea158d27b754ea547693330ddbfacb15b905e32ba8f574341ea1ad8eba192ba", + "files": 274, + "scope": "Full selected native traces and qualification evidence, plus frozen plans/candidates. No human recordings." +} diff --git a/benchmarks/sample-count-19047/evidence-manifest.json b/benchmarks/sample-count-19047/evidence-manifest.json new file mode 100644 index 0000000..a7577ee --- /dev/null +++ b/benchmarks/sample-count-19047/evidence-manifest.json @@ -0,0 +1,1375 @@ +{ + "excluded": "Source/subset media copies and training intermediates are omitted. Public source data and fitted candidate capsules remain sufficient for reproduction; native episode traces, controls, receipts and replay files are included.", + "files": [ + { + "bytes": 14879, + "path": "candidates/groups-002/candidate.skill-capsule.json", + "sha256": "c998cab9a704fb37f5e60493eafc52c20b285d579065c3e4fbeaf182079c95d3" + }, + { + "bytes": 14899, + "path": "candidates/groups-004/candidate.skill-capsule.json", + "sha256": "32c62628ed40064de57e3bcb3e32c041b16b268a48dd59307c32f778c3b5ec4d" + }, + { + "bytes": 14866, + "path": "candidates/groups-007/candidate.skill-capsule.json", + "sha256": "712eb4ed16939294b0898d8e6f39500201f0decf31c18122575de593887bf15a" + }, + { + "bytes": 622, + "path": "grounding.json", + "sha256": "c53d3966399eb54ae7975829b43db64d16e2c7f5847d5d0171d04170db0154f6" + }, + { + "bytes": 3552, + "path": "models-frozen.json", + "sha256": "1ca16dbd8d0aa26f36139056b98e7a808800a7edf5b259fc1ffc0f93fff51cc0" + }, + { + "bytes": 14202, + "path": "protocol.json", + "sha256": "1e6d475aed2b505ec1ed4ef3990c723b7f9171863647bbfeb01610ba9c8c8eec" + }, + { + "bytes": 65, + "path": "protocol.sha256", + "sha256": "c914c02b5c2287356bc1bd8207465edd78d4e97bd5b40f0f1b27733dd8e8f7eb" + }, + { + "bytes": 2071723, + "path": "qualification/groups-002/scene-00/controls.json", + "sha256": "cefd52b5fede1fac0876e0a4676ae3b6d2da8aa9a01da69380ee97e82e226213" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-00/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1349, + "path": "qualification/groups-002/scene-00/qualification.json", + "sha256": "6c61546e70cdb4fbea9f387d032c500f4821cd77dbca8bf884b0e8a1945aaea8" + }, + { + "bytes": 322650, + "path": "qualification/groups-002/scene-00/replay.html", + "sha256": "8618afa365d1bcf89901ffd1546e6553ade2c879e0d9cc284a049ca3e89926a4" + }, + { + "bytes": 1558357, + "path": "qualification/groups-002/scene-00/run.json", + "sha256": "2ea48e69f2b806eef2d2103e47232f1e3aab66af02f99e67e09b4356898c495f" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-00/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2056970, + "path": "qualification/groups-002/scene-01/controls.json", + "sha256": "08f37516c2bca2d664b549e6a5ec25daed0bb34be192432426f5d2bad0991b2a" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-01/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-002/scene-01/qualification.json", + "sha256": "7ef28c255fe8db99926290168727f9bc362f584ee5d2ea2715265167ef31d58a" + }, + { + "bytes": 304679, + "path": "qualification/groups-002/scene-01/replay.html", + "sha256": "516a6c56252008265534e5019a56a13346983750bf237d3447196de8bccf5a0e" + }, + { + "bytes": 1454287, + "path": "qualification/groups-002/scene-01/run.json", + "sha256": "129f051574ef638b40cfbaa75ad8992cc13a0fc359a7e2a25f6aa2c010559edf" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-01/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2066684, + "path": "qualification/groups-002/scene-02/controls.json", + "sha256": "bc28203a76311f47ebea5645d524d5700e9e6b7ca720acb39aedb3e1525757e0" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-02/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-002/scene-02/qualification.json", + "sha256": "aa85ea04f09c80b77fd16429274f0bac0df2dde2af0aded2e574c1422da91a9c" + }, + { + "bytes": 254958, + "path": "qualification/groups-002/scene-02/replay.html", + "sha256": "de52c7fe99cfbad3c0eebe22f5e86d80bec6e6ea26d2c91eeab763835e7f337c" + }, + { + "bytes": 1185409, + "path": "qualification/groups-002/scene-02/run.json", + "sha256": "b38fc07b8c68999008701d5575d28c75754afecdc09c17585d09a632f5987843" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-02/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2059689, + "path": "qualification/groups-002/scene-03/controls.json", + "sha256": "fb70ead92577d6b01cd7d8eea63c5d6564f8a01e4bb08aa2dc8c7ee480188da8" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-03/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1353, + "path": "qualification/groups-002/scene-03/qualification.json", + "sha256": "536c564471171daac4f95a6d56d8f0651ca10b52d48ab590ad74b4878bfe3d81" + }, + { + "bytes": 298958, + "path": "qualification/groups-002/scene-03/replay.html", + "sha256": "6c6c85c6c1c68ab35207fd195fc2fca475e0b9c09a78f1a921c101dc80c699ad" + }, + { + "bytes": 1423482, + "path": "qualification/groups-002/scene-03/run.json", + "sha256": "d77ee5924c69cf211c536912778e1f6eee58016191c3d7e836a183cd6da52a79" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-03/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2072364, + "path": "qualification/groups-002/scene-04/controls.json", + "sha256": "b6679b4cdcc80b4998988302558ce3bca7d2d17972095873eb7384a5a960aa28" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-04/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-002/scene-04/qualification.json", + "sha256": "d63b9a94ea037136f52bbf41287acba910a9760d6e3fed34b3e7815bc3c6311a" + }, + { + "bytes": 337017, + "path": "qualification/groups-002/scene-04/replay.html", + "sha256": "d01371f319f1213647155260772839fbe75579e860182b52c51b53d85d0ad572" + }, + { + "bytes": 1629060, + "path": "qualification/groups-002/scene-04/run.json", + "sha256": "dbfc7a84b91aa7113963f88333afca45a03561775177567438969d84802d12c1" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-04/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2073451, + "path": "qualification/groups-002/scene-05/controls.json", + "sha256": "75dc0f69c589b3d336a2259fa0a5a03b2912da3e1b7615e3fd72b74910bc6eac" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-05/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1352, + "path": "qualification/groups-002/scene-05/qualification.json", + "sha256": "b85d312080eafa81706ae6374599c91c6775d8a346077d24efe254dfa81ea6f1" + }, + { + "bytes": 304172, + "path": "qualification/groups-002/scene-05/replay.html", + "sha256": "3c62807e7a6f21a66c074c67ec3c5e596be6ec504a3a8e6d2d3de31f85d905e9" + }, + { + "bytes": 1454044, + "path": "qualification/groups-002/scene-05/run.json", + "sha256": "88a37b87bbd2a1eb12387ebcd1bcf7cdf950604e4ad74ef85314f5471c3b8a98" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-05/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2069235, + "path": "qualification/groups-002/scene-06/controls.json", + "sha256": "e78d338b2501728c7f0b32023c29de4ca40ec910a3df988d498c053a33bb71e9" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-06/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-002/scene-06/qualification.json", + "sha256": "4ce1e4c3f8d27cdff8beed9921bbee876ed79a0b5a8a20b9b3970c5746a10f68" + }, + { + "bytes": 408191, + "path": "qualification/groups-002/scene-06/replay.html", + "sha256": "833e3776cccf4ad1500ee6a0370852ae83e478dd4ba7acf5e417248618500d8d" + }, + { + "bytes": 2019770, + "path": "qualification/groups-002/scene-06/run.json", + "sha256": "8ffdd8efc8b6f402412d9454f381b4f0fd9e0df017c6490b6097b786da885a70" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-06/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2077188, + "path": "qualification/groups-002/scene-07/controls.json", + "sha256": "b27b043c9ade3aeb07a8532fb3f2847746ccf48f5af5a692db4a85a6d6d2a929" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-07/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-002/scene-07/qualification.json", + "sha256": "85b7443171c5418d0e732e5441dfeacc8e1034940aa87416ee0d4dcd79735a8a" + }, + { + "bytes": 297825, + "path": "qualification/groups-002/scene-07/replay.html", + "sha256": "54c49f7614e6f63070d9d4b1abd9521d0f8100c7c903c9a6b58b842914f1f62c" + }, + { + "bytes": 1413960, + "path": "qualification/groups-002/scene-07/run.json", + "sha256": "57884463749f3631a43e16efca0df186ed7d57aef5bcfb344e4ae8d216fc4ffd" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-07/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2067264, + "path": "qualification/groups-002/scene-08/controls.json", + "sha256": "c412f8be2410a210bf0d12f972eb2851e821af4dafb55e4627b17461698a65d0" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-08/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-002/scene-08/qualification.json", + "sha256": "e1ebe7eec2b9eafa1d9d3bff7988c3a31ba7279bff509377780bf7ed21868fb6" + }, + { + "bytes": 376749, + "path": "qualification/groups-002/scene-08/replay.html", + "sha256": "2cf07ccf43dbe8287f08f7d2a0c21a13cf140525e126710238af75c3aff10754" + }, + { + "bytes": 1845829, + "path": "qualification/groups-002/scene-08/run.json", + "sha256": "03f5ecd1dbe6b71f6262bfd83e700c7d6e4c0a7b09594173a67c86e007f8412d" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-08/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2056527, + "path": "qualification/groups-002/scene-09/controls.json", + "sha256": "2610efc58e5d28fa449ce77e786ef0ab2e1e2cd38ec7d882b758b40b285c2e09" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-09/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1343, + "path": "qualification/groups-002/scene-09/qualification.json", + "sha256": "ffaf83d638843399db91fcb2f328dabf67316583ac1b120f9622160c2642aae2" + }, + { + "bytes": 375990, + "path": "qualification/groups-002/scene-09/replay.html", + "sha256": "c2323a5717a79730de06aff00711f1f449943d666bc36b3cd3db5d8bcad77f1f" + }, + { + "bytes": 1851568, + "path": "qualification/groups-002/scene-09/run.json", + "sha256": "96290e1d032375145f983304600ec6288bbde116d28ac9c287bfb6c0c2fd0e06" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-09/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2074050, + "path": "qualification/groups-002/scene-10/controls.json", + "sha256": "33e17c008c73c3653ca32772955cd1e4d9765ec2d4c408c5707b725c4f6d398e" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-10/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1349, + "path": "qualification/groups-002/scene-10/qualification.json", + "sha256": "45daa7bf20b659408aa083f6a13e62a809daf01535d2651b474d76773c24d512" + }, + { + "bytes": 383856, + "path": "qualification/groups-002/scene-10/replay.html", + "sha256": "58dfe833082b944c4413e2e58cb1833f73996c0c97675c5f3dea55a44bd31215" + }, + { + "bytes": 1891265, + "path": "qualification/groups-002/scene-10/run.json", + "sha256": "b9a7903ae06bcf9cad16c5b2cdf5c73735c3b38e47a9b3e28a48339c81b7a3c1" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-10/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 1837944, + "path": "qualification/groups-002/scene-11/controls.json", + "sha256": "b151f39b2d42c0c247a5bd8fff95381dbbc59f27e180462cf640f5cc9e5e3038" + }, + { + "bytes": 682, + "path": "qualification/groups-002/scene-11/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 121, + "path": "qualification/groups-002/scene-11/qualification.json", + "sha256": "f8c940795aa9160d1df2428c430c97f2753179956db46c0eeafc6845245d4c19" + }, + { + "bytes": 383886, + "path": "qualification/groups-002/scene-11/replay.html", + "sha256": "bade3901da17cbc2cbd080a6526e95f4a0b11ec14d797e1bd70fe68d1680fe83" + }, + { + "bytes": 1838809, + "path": "qualification/groups-002/scene-11/run.json", + "sha256": "8eed7ca6dc3c512ed1cecc99476d2fd4c3be34e5fcc46038a981b64ba6f282ea" + }, + { + "bytes": 332, + "path": "qualification/groups-002/scene-11/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2072117, + "path": "qualification/groups-004/scene-00/controls.json", + "sha256": "b914265832128c7ecd9200c4ac1ee7004a7cb7f6704efec3cfc87d0be7862489" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-00/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1346, + "path": "qualification/groups-004/scene-00/qualification.json", + "sha256": "7a896b5d26466adca4344b4f3d0b5033d0beea7e20ec6c632e6532d320ef23d9" + }, + { + "bytes": 322544, + "path": "qualification/groups-004/scene-00/replay.html", + "sha256": "8bee090e2fe991ae773a1b64025c23ad10cf969e2924fc0a04023828abde8db4" + }, + { + "bytes": 1555930, + "path": "qualification/groups-004/scene-00/run.json", + "sha256": "6678a5226c0e6e27c534e2c4781e20c418d0709f56eb4597578fa760e768d00f" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-00/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2057385, + "path": "qualification/groups-004/scene-01/controls.json", + "sha256": "089c3668bb297dbbfbdbf140b540a83aff75fb5eb29ff0a02d0c4410eb416def" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-01/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-004/scene-01/qualification.json", + "sha256": "a11a0dd69ac037295fff4b60776623d077ce292bbf41d5f7844aab53f321416c" + }, + { + "bytes": 307011, + "path": "qualification/groups-004/scene-01/replay.html", + "sha256": "e22aa1a954744b5471f5d57658771f2a7f30c51c8a04c94b696b3d9e1121ae26" + }, + { + "bytes": 1463558, + "path": "qualification/groups-004/scene-01/run.json", + "sha256": "a2c5993a31827dd2b263706ac7fba1d83d25b3e162e7ea5188126aa146a56d48" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-01/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2066166, + "path": "qualification/groups-004/scene-02/controls.json", + "sha256": "e3dbe635d37a8e939321bd8cfc809bd0fbe786d661585a850ca8cc4d0973469e" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-02/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-004/scene-02/qualification.json", + "sha256": "2e7710661ef3f09c4d28a9a9379e7354bbd4e360cc28b3f0c88ccae4936176af" + }, + { + "bytes": 252966, + "path": "qualification/groups-004/scene-02/replay.html", + "sha256": "f0e7c3cded1c1d713bf35d99f6ea3c9d2ace42209557ee8a06cfec45128f074c" + }, + { + "bytes": 1176599, + "path": "qualification/groups-004/scene-02/run.json", + "sha256": "ed7216919a0880fdd416d31c53a8bb504384f00cfa8943e658549e579b557e49" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-02/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2062046, + "path": "qualification/groups-004/scene-03/controls.json", + "sha256": "d41e015a4c3160e8676ec5f62442ec8830d47002a36aab39d6b14b0032d08619" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-03/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1353, + "path": "qualification/groups-004/scene-03/qualification.json", + "sha256": "4676f058fa9ffa8508104c08f09cb0c41923530b9e27724a4758e4ac17810b94" + }, + { + "bytes": 296785, + "path": "qualification/groups-004/scene-03/replay.html", + "sha256": "e4b1fc07c284a47a27ef4a1d5d4d785ee89e2b85ed208b8e5014b7b92832471e" + }, + { + "bytes": 1416007, + "path": "qualification/groups-004/scene-03/run.json", + "sha256": "1d1c3a5884e5c24fa92c5e8ea424829303e276677a2b42255e146e01ec1315d0" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-03/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2071341, + "path": "qualification/groups-004/scene-04/controls.json", + "sha256": "d588b1c30710198694dc142bc6918f9bedeb988f7e4e8137c59024b412812640" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-04/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 121, + "path": "qualification/groups-004/scene-04/qualification.json", + "sha256": "f8c940795aa9160d1df2428c430c97f2753179956db46c0eeafc6845245d4c19" + }, + { + "bytes": 416377, + "path": "qualification/groups-004/scene-04/replay.html", + "sha256": "ab25644a03cf001eebd74d1b1afd6be767bd602f3d09a529042691fa2ea95f96" + }, + { + "bytes": 2072264, + "path": "qualification/groups-004/scene-04/run.json", + "sha256": "83bc56dd512180f0391b0945bb056c24cbce68295cf580635a72d96441d1b995" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-04/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2077795, + "path": "qualification/groups-004/scene-05/controls.json", + "sha256": "cbca07ba41428048916277c3c69665f891415b9c0fa568b21a5c722b75f11afc" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-05/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1352, + "path": "qualification/groups-004/scene-05/qualification.json", + "sha256": "a34386fafa808448e4a72972476d59d6a8955f2ac1405661000faaae2705739d" + }, + { + "bytes": 314923, + "path": "qualification/groups-004/scene-05/replay.html", + "sha256": "ee88c1dc8a6e6632bae4cfb98f0397d4227af02ac287b722b9bd43a639c05cc0" + }, + { + "bytes": 1505055, + "path": "qualification/groups-004/scene-05/run.json", + "sha256": "b4b9c99229cfd77283b8b366076f78f055f68bdc528dd68bdf40b5687d8f7afc" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-05/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2068569, + "path": "qualification/groups-004/scene-06/controls.json", + "sha256": "efcf1022483a471ab88961c73f9d3983262ffed41111377d4e2f12bc045dfd18" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-06/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-004/scene-06/qualification.json", + "sha256": "7f7100c9ea6b96147fca8d685a38d2885f0a96809a3d6d6be72c3b6d580b588a" + }, + { + "bytes": 410157, + "path": "qualification/groups-004/scene-06/replay.html", + "sha256": "038a452f2e40601228ef4d5a8788325c63af31e8305708c9ef3cbf0edda08e19" + }, + { + "bytes": 2033007, + "path": "qualification/groups-004/scene-06/run.json", + "sha256": "c3879463dc39c658a97843e5168cc9d0f672f508fa68f03e705ab7c6c64fddb4" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-06/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2074725, + "path": "qualification/groups-004/scene-07/controls.json", + "sha256": "d7256b7a463baaedc86eb9eb1feb13c650304eeb37481d7432740c3cb34f1916" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-07/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-004/scene-07/qualification.json", + "sha256": "071bffb058d44b9980a341ec1d3474b32cad9a19a675473da4bdc6b7f4f841af" + }, + { + "bytes": 316796, + "path": "qualification/groups-004/scene-07/replay.html", + "sha256": "095e96247a0b37f98286cd4da5bb80b563a77ebd681a5877f3423f96a321d836" + }, + { + "bytes": 1519557, + "path": "qualification/groups-004/scene-07/run.json", + "sha256": "355f607c34fc0492bb0a102e2e7b319855fb17c68253ee114038d257af6d1d2a" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-07/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2067444, + "path": "qualification/groups-004/scene-08/controls.json", + "sha256": "a950e9e406f0ce597d7598163e04d528d83d6737ef4c503ade1f1980df19797d" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-08/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-004/scene-08/qualification.json", + "sha256": "167c4f45dcdaff58a95433877438968743cfd707bd399a3fa5f0f02a83f72838" + }, + { + "bytes": 372485, + "path": "qualification/groups-004/scene-08/replay.html", + "sha256": "db334bd588c8cf88abb8e40086a907bcb3672931085baca3b5d0f4a784648263" + }, + { + "bytes": 1832066, + "path": "qualification/groups-004/scene-08/run.json", + "sha256": "29d60b162fa70984406feecf8c3e94563ad4435e0f1b7d330065b63440f2aaac" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-08/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2056261, + "path": "qualification/groups-004/scene-09/controls.json", + "sha256": "33820a39489ec3bd10b7e7482541280cd2f3bea7173933d9ed8d31d1f5242513" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-09/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1342, + "path": "qualification/groups-004/scene-09/qualification.json", + "sha256": "c36b8a977490e58e3a2f320fcb5b5a3f2cdc44b8de27c5a169c979e4c05994a0" + }, + { + "bytes": 375798, + "path": "qualification/groups-004/scene-09/replay.html", + "sha256": "6c4f4c07e36c0403fa1e8cbae133882dd0b23232c99deb8c8d13798da6c333e4" + }, + { + "bytes": 1846475, + "path": "qualification/groups-004/scene-09/run.json", + "sha256": "64a6490908fbe6b03a637f55b3c446ffd40e05713c570511ad4d6887b25191f7" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-09/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2073408, + "path": "qualification/groups-004/scene-10/controls.json", + "sha256": "e41fc3983c87c7dcb36db8c31a46b3275c38ca4155bb0be133392b53963f4059" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-10/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-004/scene-10/qualification.json", + "sha256": "3e978a32739395657a71fdbef09bb29b3ccaf08c1fdb1dd6682a4f47392df25a" + }, + { + "bytes": 385866, + "path": "qualification/groups-004/scene-10/replay.html", + "sha256": "ab15f8bf7f7f2c88318fc84552aedb3c385a76cb3f6b929eb4e0b1375e388558" + }, + { + "bytes": 1895831, + "path": "qualification/groups-004/scene-10/run.json", + "sha256": "438a963e82ebc507513c7c03b147f847082eec204fdb45c22728ab2f7ee0dac9" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-10/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 1837943, + "path": "qualification/groups-004/scene-11/controls.json", + "sha256": "1ddeecc519266692901f21a30d63a6dbd7d5aa751ad58ddb43a46ac4a9b814a7" + }, + { + "bytes": 682, + "path": "qualification/groups-004/scene-11/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 121, + "path": "qualification/groups-004/scene-11/qualification.json", + "sha256": "f8c940795aa9160d1df2428c430c97f2753179956db46c0eeafc6845245d4c19" + }, + { + "bytes": 383887, + "path": "qualification/groups-004/scene-11/replay.html", + "sha256": "65c4e860dc0652dcb2a4eab6825b2a28cc6c7a7afc4e59b2ec932649e6a6f9c7" + }, + { + "bytes": 1838810, + "path": "qualification/groups-004/scene-11/run.json", + "sha256": "638ba7b30c363a89ad3b0489977bf61268a3b643c279c4839643494ccac1e846" + }, + { + "bytes": 332, + "path": "qualification/groups-004/scene-11/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2071508, + "path": "qualification/groups-007/scene-00/controls.json", + "sha256": "932c4a6c6440f4b0a96803a1bf22c0430ede07bf3ea864816b592693e8a1ce20" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-00/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-007/scene-00/qualification.json", + "sha256": "338375b7725e3d60a10f691188d00e4bd4cafef3447fa3d8a611b56f8f949c73" + }, + { + "bytes": 330980, + "path": "qualification/groups-007/scene-00/replay.html", + "sha256": "798c4684e320c9d7fd573d8b19eb7c0a45ddadfe1e7d8021d306f435038b156d" + }, + { + "bytes": 1597422, + "path": "qualification/groups-007/scene-00/run.json", + "sha256": "98a635d30e3b71fbf4dbbf0920fbd59d1ed6a777e9c236a1279f3e170847d2e4" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-00/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2059667, + "path": "qualification/groups-007/scene-01/controls.json", + "sha256": "59bbc4dc64e898d1c572398e590be16989e9cab5ed445f9d9d73429e2603ac1b" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-01/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-007/scene-01/qualification.json", + "sha256": "e68be936449ba42555251bbd00ded2c40e660c5a8d14b212f675417ab2b3cdae" + }, + { + "bytes": 305088, + "path": "qualification/groups-007/scene-01/replay.html", + "sha256": "d324104c1749e458acde4f93418114b40fa38756c00452d2b1669a37148a4db7" + }, + { + "bytes": 1459903, + "path": "qualification/groups-007/scene-01/run.json", + "sha256": "c178ff19f08e6397d14e07bf60f0ed59ebde5da847084c36028653ded2b4ca28" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-01/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2068355, + "path": "qualification/groups-007/scene-02/controls.json", + "sha256": "a5e1202ebb68ff24dd4a3ee587c822fd0f661f34fe93262a5508edb5464931d3" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-02/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-007/scene-02/qualification.json", + "sha256": "b11a0ee48c0afd85b8fe2cdf279d8be5c8f1dc7434f452b9464aa4344563d1e0" + }, + { + "bytes": 255153, + "path": "qualification/groups-007/scene-02/replay.html", + "sha256": "336cf73a474e4a1e38a7480af1ce4426c82d2145086beece26edece9e7de6898" + }, + { + "bytes": 1188070, + "path": "qualification/groups-007/scene-02/run.json", + "sha256": "2887b3b28d3d9ba2b735d1ae654285d46b6f247f2cd28adaaaec2efb1757e7ec" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-02/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2059985, + "path": "qualification/groups-007/scene-03/controls.json", + "sha256": "49954d9530daa30270dc2b932dcf296a614fed1c854b21673b681179a9a66fa4" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-03/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1351, + "path": "qualification/groups-007/scene-03/qualification.json", + "sha256": "a143ade7ea56ece7a08b2555cd2ca02b3dbd0bfabb14728f67b5a9c186d0ede0" + }, + { + "bytes": 305305, + "path": "qualification/groups-007/scene-03/replay.html", + "sha256": "c12cc83eab66516ec5da638ca4d87d934fc64cda10cba2c6889769d2ea37877d" + }, + { + "bytes": 1463078, + "path": "qualification/groups-007/scene-03/run.json", + "sha256": "ad9c527bc70a5b8c47275f43efc82b4c3a1e76ee61fbf0f1ee995b253eb2dc47" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-03/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2072585, + "path": "qualification/groups-007/scene-04/controls.json", + "sha256": "ed29cbe7b5b2484cc0b482de78e8ae345e70a4582c96a5b3ba8301a442f5549e" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-04/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-007/scene-04/qualification.json", + "sha256": "66e0c469aa37858af6cb4f6bb723600662806463ee1980d424693eff03b7e9e7" + }, + { + "bytes": 341271, + "path": "qualification/groups-007/scene-04/replay.html", + "sha256": "02cfeb1239c80d9578511296a3ecaa288e24939ba8e553a644afa699b4400a7f" + }, + { + "bytes": 1653310, + "path": "qualification/groups-007/scene-04/run.json", + "sha256": "8fc4b1b80322161aeb303a3691f11c29e25630eae5396fa484a015fb5683f60a" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-04/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2079200, + "path": "qualification/groups-007/scene-05/controls.json", + "sha256": "647f06b7ad31ee4a1b58d6449cd550f8be9d8824167a1770c8f957ed7f1d2837" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-05/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1353, + "path": "qualification/groups-007/scene-05/qualification.json", + "sha256": "1dfdb144818199693af23b6c2c4855ef5430d9b4fa5ae25d42a4369dd0847dbe" + }, + { + "bytes": 312703, + "path": "qualification/groups-007/scene-05/replay.html", + "sha256": "1e66be42b7897cf31b8f86ee19c6ee202137f6a9399af060d31b38fee7da7bd6" + }, + { + "bytes": 1495777, + "path": "qualification/groups-007/scene-05/run.json", + "sha256": "3674609b01b4d0581fdbff16f319420adedb041a88c92f2e465f86e0f9703bc8" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-05/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2067822, + "path": "qualification/groups-007/scene-06/controls.json", + "sha256": "dabbfbe3c524b400e82001cbf171dde9efa49d3a324214b016f457a5b753ecd0" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-06/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1346, + "path": "qualification/groups-007/scene-06/qualification.json", + "sha256": "597d65836023fbaba825096b1f8ec0a63b9fd8dbd90cad8df14a0c0a0640b208" + }, + { + "bytes": 410147, + "path": "qualification/groups-007/scene-06/replay.html", + "sha256": "b24f3611a6c7f3c5a6ccdcbbf197c731fa7acc2e88df3c7f5a0d0c05ec051fb8" + }, + { + "bytes": 2030061, + "path": "qualification/groups-007/scene-06/run.json", + "sha256": "2f307b728541e20ef049e548d90bb032b16f7ebfd05c87e61589a93f45d02bd0" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-06/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2078377, + "path": "qualification/groups-007/scene-07/controls.json", + "sha256": "8198c574e78b24be5b138c7c91a8f242b35f97902914b57c8bd7018d51749d42" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-07/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1348, + "path": "qualification/groups-007/scene-07/qualification.json", + "sha256": "b40f4e41c975c031d542fd881a5c0769629d4a93a256508916397b3edbaca47e" + }, + { + "bytes": 308250, + "path": "qualification/groups-007/scene-07/replay.html", + "sha256": "503359e4918860c084a1bca4997cd285f1bdc0be225ca06c1a20903eb25255bc" + }, + { + "bytes": 1469479, + "path": "qualification/groups-007/scene-07/run.json", + "sha256": "25183e905bd31cf0cbe5170c9e76f2a0c1532b50049d59b04f6a0103e5effa8c" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-07/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2067830, + "path": "qualification/groups-007/scene-08/controls.json", + "sha256": "2f15686b6e36ff16fe092c6a32df19485d02fa58d331622c91ea78919d1298e3" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-08/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1345, + "path": "qualification/groups-007/scene-08/qualification.json", + "sha256": "df0bbd010522370f1dffb2c9ca759a9a50ac26b15921caedfe4096cb3103495a" + }, + { + "bytes": 380972, + "path": "qualification/groups-007/scene-08/replay.html", + "sha256": "aacc8a434d4a3dbf05cdce402a1654d08381b15955e83101edad0037cb902bcf" + }, + { + "bytes": 1873451, + "path": "qualification/groups-007/scene-08/run.json", + "sha256": "b5e3c06caa6d9e5ee7da0ec1c02d8a0dcedafa20acb551f6fa52c6ebdb5b30c5" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-08/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2055471, + "path": "qualification/groups-007/scene-09/controls.json", + "sha256": "0fa12a7dd854eaec20abe3824712f1d072b2d3e411977a6e78a82112f81fe828" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-09/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1344, + "path": "qualification/groups-007/scene-09/qualification.json", + "sha256": "d09c65549f49d86ab819d508b1d7a598a2e8551089b1913d699c6dedff4ae89f" + }, + { + "bytes": 377855, + "path": "qualification/groups-007/scene-09/replay.html", + "sha256": "32b3e7bc3da10fb5a1a1545be617457e8041856776362c915e422e8af1dd3fc9" + }, + { + "bytes": 1857420, + "path": "qualification/groups-007/scene-09/run.json", + "sha256": "78e9120f5dd1b02bdd594c96c2b5784397c4f38b73dd7f2ef46f22900fc86896" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-09/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 2073383, + "path": "qualification/groups-007/scene-10/controls.json", + "sha256": "d00a46b5836b3bb1592ea9418bf3e70ac297541a7feb7cf078a22ef1825f313a" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-10/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 1347, + "path": "qualification/groups-007/scene-10/qualification.json", + "sha256": "4e50666605ada930dc5f1d376db73f535b9d5040ec6c644cbadae70c6ed8f85f" + }, + { + "bytes": 383807, + "path": "qualification/groups-007/scene-10/replay.html", + "sha256": "0dce50003f63178ad225948a82eec30de431f221fe7312fc8f96a0e6643af30d" + }, + { + "bytes": 1887668, + "path": "qualification/groups-007/scene-10/run.json", + "sha256": "393b792e689121b35f2da1b4359a015c0306bc7baee531a000a334309a6763e8" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-10/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 1837943, + "path": "qualification/groups-007/scene-11/controls.json", + "sha256": "52df85b684f672b69f5453db6c24754208d737e2af3acf14e96244c164b892aa" + }, + { + "bytes": 682, + "path": "qualification/groups-007/scene-11/grounding.json", + "sha256": "c0b2a3b81322e71f1fc4294d1798300d74ecff599e30099bf06be96748e02b86" + }, + { + "bytes": 121, + "path": "qualification/groups-007/scene-11/qualification.json", + "sha256": "f8c940795aa9160d1df2428c430c97f2753179956db46c0eeafc6845245d4c19" + }, + { + "bytes": 383883, + "path": "qualification/groups-007/scene-11/replay.html", + "sha256": "d13c5d3f100a05ba5be116f44136ac63482feee3f761346ce0c9d32544d1ec6a" + }, + { + "bytes": 1838806, + "path": "qualification/groups-007/scene-11/run.json", + "sha256": "63684c85d5de1f16e244dc230df60000a0b5fd5c8ed9dcb8f0edbbfeef1c1805" + }, + { + "bytes": 332, + "path": "qualification/groups-007/scene-11/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 54705, + "path": "report.json", + "sha256": "e8035534485bfd7473c215c368213c687584573013b07a4e75bc428c668e6d08" + }, + { + "bytes": 332, + "path": "task.skill.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-00/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 318352, + "path": "teacher/scene-00/replay.html", + "sha256": "ba8abb03ca2a718afc0b6f66f6d9092c333903338177310335a2e6c0e2991448" + }, + { + "bytes": 1544391, + "path": "teacher/scene-00/run.json", + "sha256": "ebafa69b8c4ef07d78c17b0559a20816e6c458620c6a8428f184edf559783358" + }, + { + "bytes": 332, + "path": "teacher/scene-00/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-01/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 288104, + "path": "teacher/scene-01/replay.html", + "sha256": "44500d38aaaf6480e229e75a53ca6aba5e7f0728215e5496b2ed6d119258b1d7" + }, + { + "bytes": 1371974, + "path": "teacher/scene-01/run.json", + "sha256": "5d0c97218f74b4b99df66dbd115afea34fedc8d3af9ee1ba057c00bcc4287a8d" + }, + { + "bytes": 332, + "path": "teacher/scene-01/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-02/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 252767, + "path": "teacher/scene-02/replay.html", + "sha256": "46bdcf604aae4d7461e8cdff2530a4db940c26f99363b2fb93f002932099c65b" + }, + { + "bytes": 1179531, + "path": "teacher/scene-02/run.json", + "sha256": "9f5b9b08358a3e4864174f79ba4040e0ff5a56b63fa7582d2d3b1ee364c2e91a" + }, + { + "bytes": 332, + "path": "teacher/scene-02/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-03/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 286335, + "path": "teacher/scene-03/replay.html", + "sha256": "d4ee408b8099de5ad4812aed9e37acf178840444e64b025a09f2ad5e4a4b05cb" + }, + { + "bytes": 1367593, + "path": "teacher/scene-03/run.json", + "sha256": "737ba8f6f132a7eba07783312258969798cea0e060b7d3c6b667e9b76d35c6be" + }, + { + "bytes": 332, + "path": "teacher/scene-03/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-04/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 324328, + "path": "teacher/scene-04/replay.html", + "sha256": "5ca471bf9fae18e5ef995186678401597e7e70e6c071b6b28ab44d8e1a8ad605" + }, + { + "bytes": 1575151, + "path": "teacher/scene-04/run.json", + "sha256": "9beb46c32051c12c5fa94aa5052ca9c9464a82e47c74a8e3a35b981cfc6570c9" + }, + { + "bytes": 332, + "path": "teacher/scene-04/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-05/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 299989, + "path": "teacher/scene-05/replay.html", + "sha256": "0a63f9209d848a1c422c4bc1fea257e2bbdecce752e3003d35b9f6ac809cb809" + }, + { + "bytes": 1436826, + "path": "teacher/scene-05/run.json", + "sha256": "b021249022adbf472d2825df19665ae120018af4f317a6caa3b3571d694c568a" + }, + { + "bytes": 332, + "path": "teacher/scene-05/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-06/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 374256, + "path": "teacher/scene-06/replay.html", + "sha256": "ce63685a1298ee4a81abd168f22da141f49de23833cd2237ba8b7c6a5fe01ace" + }, + { + "bytes": 1846170, + "path": "teacher/scene-06/run.json", + "sha256": "483f6d04352832fe2d83d0e9250b6be55c2c2e01461101dc20939c58745846f8" + }, + { + "bytes": 332, + "path": "teacher/scene-06/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-07/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 282966, + "path": "teacher/scene-07/replay.html", + "sha256": "f2929018cb59302d8f7966512b47d01926f622f31b098a076f67995fb9962a7d" + }, + { + "bytes": 1340392, + "path": "teacher/scene-07/run.json", + "sha256": "06a34cde44b51e6d48552204bb25acdc5189b44f6dcab0f413021b555f2e3f99" + }, + { + "bytes": 332, + "path": "teacher/scene-07/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-08/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 358126, + "path": "teacher/scene-08/replay.html", + "sha256": "8b2a65b4c7c2d68e57d5fb0059c485d94f7b7d7be4f43fe0df07bc38d1abea43" + }, + { + "bytes": 1755964, + "path": "teacher/scene-08/run.json", + "sha256": "2aeebafd3ee94911dbfc0756588eec8854eed2a38f63bef7d1aa8421b194cf74" + }, + { + "bytes": 332, + "path": "teacher/scene-08/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-09/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 361505, + "path": "teacher/scene-09/replay.html", + "sha256": "73025b817a5f2f186ddfd872f958d59d0183863ce7d91511737eb0638ba8e081" + }, + { + "bytes": 1773906, + "path": "teacher/scene-09/run.json", + "sha256": "4452515aeb9ae4bef0f7c80d64d383cd27db35b752109d09b58ceaaee2b51ecc" + }, + { + "bytes": 332, + "path": "teacher/scene-09/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-10/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 352262, + "path": "teacher/scene-10/replay.html", + "sha256": "6c8b01b5a0905d7cca215b6e5f0ffa2da04e0da24206b80601bb76abafd99b68" + }, + { + "bytes": 1728606, + "path": "teacher/scene-10/run.json", + "sha256": "72da716780021ba47876226410ecc29ff63ab96ad5bdf0ee9a490b65c1f03779" + }, + { + "bytes": 332, + "path": "teacher/scene-10/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + }, + { + "bytes": 682, + "path": "teacher/scene-11/grounding.json", + "sha256": "4bf12f83253bf796bbc9244e2abfa9602038d384a82022d85258d2ededf60db8" + }, + { + "bytes": 382823, + "path": "teacher/scene-11/replay.html", + "sha256": "889e396ae601be71d15836160575f2c5dfed5ce8b3a6c99bea5a5215f65c1c78" + }, + { + "bytes": 1837746, + "path": "teacher/scene-11/run.json", + "sha256": "62c0810b39d07a3d95620a9ccee3910010860c019213545075acfe805249f3c1" + }, + { + "bytes": 332, + "path": "teacher/scene-11/source.json", + "sha256": "23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15" + } + ], + "kind": "dvidia.frozen-simulation-benchmark-evidence", + "physical_robot_ready": false, + "protocol_sha256": "1e6d475aed2b505ec1ed4ef3990c723b7f9171863647bbfeb01610ba9c8c8eec", + "schema_version": 1, + "scope": "simulation-only" +} diff --git a/benchmarks/sample-count-19047/groups-002.skill-capsule.json b/benchmarks/sample-count-19047/groups-002.skill-capsule.json new file mode 100644 index 0000000..5ea92c1 --- /dev/null +++ b/benchmarks/sample-count-19047/groups-002.skill-capsule.json @@ -0,0 +1 @@ +{"compatibility":{"actions":[{"name":"joint_targets","semantics":"absolute_joint_position","shape":[6],"units":"rad"},{"name":"gripper_width","semantics":"total_symmetric_jaw_gap","shape":[],"units":"m"}],"adapter_id":"contact-pick-place-box-v0","arm_count":1,"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","control_hz":50.0,"coordinates":"world_metres","gripper":{"gap_limits_m":[0.0,0.08],"independent_commands":1,"jaw_joints":["jaw_left","jaw_right"],"kind":"symmetric_parallel_jaw","maximum_jaw_actuator_force_n":15.0},"independent_commands":7,"joint_limits_rad":[[-2.6,2.6],[-2.2,1.6],[-2.7,2.7],[-3.0,3.0],[-2.8,2.8],[-3.1,3.1]],"joint_order":["j0","j1","j2","j3","j4","j5"],"maximum_joint_actuator_torque_nm":40.0,"object_profile":"upright_axis_aligned_rigid_box_30_to_50_mm","observation_origin":"privileged_native_simulator_state","observations":[{"frame":"joint","name":"joint_position","shape":[6],"units":"rad"},{"frame":"joint","name":"joint_velocity","shape":[6],"units":"rad/s"},{"frame":"joint","name":"joint_actuator_torque","shape":[6],"units":"N*m"},{"frame":"world","name":"end_effector_position","shape":[3],"units":"m"},{"frame":"world","name":"end_effector_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"end_effector_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_position","shape":[3],"units":"m"},{"frame":"world","name":"object_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"object_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_angular_velocity","shape":[3],"units":"rad/s"},{"frame":"world","name":"target_position","shape":[3],"units":"m"},{"frame":"object","name":"object_size","shape":[3],"units":"m"},{"frame":"gripper","name":"gripper_width","shape":[],"units":"m"},{"frame":"gripper","name":"grasp_contacts","order":["left","right"],"shape":[2],"units":"count"},{"frame":"gripper","name":"pad_normal_forces","order":["left","right"],"shape":[2],"units":"N"},{"frame":"contact","name":"effective_sliding_friction","order":["jaw_object","table_object"],"shape":[2],"units":"coefficient"},{"frame":"contact","name":"table_contacts","shape":[],"units":"count"}],"physical_robot_joints":8,"supervisor_revision":"dimension-aware-contact-rates-recovery-round2","target_rate_limits":{"jaw_gap_m_s":0.16,"joint_rad_s":3.0},"tool_orientation":"fixed_identity_rotation","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29],"raised_tool_radial_max_m":0.56}},"format":"dvidia.skill-capsule","grounding":{"adapter_id":"contact-pick-place-box-v0","arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","frame":"world","language_aliases":["Set a cup on the mark."],"object":{"friction":0.8,"kind":"box","mass":0.04,"position":[0.42,-0.04,0.31],"size":[0.04,0.04,0.04]},"procedure":["approach","descend","close","lift","transfer","place","release","retreat","verify"],"provenance":"authored_simulation_adapter","schema_version":1,"table_height":0.29,"target":{"dwell_seconds":0.25,"position":[0.54,0.1,0.31],"position_tolerance":0.025},"task_type":"place_cup","units":"m","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29]}},"movement_head":{"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","centers":[[0.8776489496231079,0.9002423882484436,-0.9438050389289856,0.00016362832684535533,0.47778889536857605,-0.879067063331604,0.08985759317874908,-0.5103289484977722,0.4555756747722626,-1.2435067892074585,-0.8455549478530884,-0.24184942245483398],[-2.2943685054779053,0.6307700276374817,-1.6828137636184692,-0.00041817352757789195,1.6438560485839844,2.2985544204711914,-0.3356776833534241,2.7438924312591553,-1.6882293224334717,0.6029743552207947,2.26117205619812,-1.4849523305892944],[0.06053687259554863,-0.5996294021606445,1.4750016927719116,-0.00017458273214288056,-1.4070764780044556,-0.05405457690358162,0.0491330511868,-0.4970721900463104,0.6835838556289673,0.7984893918037415,-0.040607843548059464,1.1637314558029175],[-0.8680028319358826,-0.8713129758834839,-0.13348820805549622,0.0001792853872757405,0.8898569941520691,0.8574585318565369,-0.09203817695379257,0.6684794425964355,-1.3411413431167603,0.7216993570327759,0.777559220790863,-1.0634286403656006],[-1.6680657863616943,-0.21539434790611267,-0.7951662540435791,-8.220457675633952e-05,1.2000449895858765,1.6650028228759766,-0.27545809745788574,1.742316484451294,-1.5047229528427124,0.6339955925941467,1.572011113166809,-1.2620916366577148],[0.25997769832611084,-0.8617539405822754,0.5030556917190552,0.000347282737493515,0.07454896718263626,-0.26185762882232666,0.06286489963531494,-0.5025957822799683,0.7539874911308289,0.4951666593551636,-0.21260033547878265,-0.24625661969184875],[-0.0403517410159111,-1.0965999364852905,0.41455721855163574,-3.496095450827852e-05,0.35452741384506226,0.03774440661072731,0.02680579200387001,-0.27438315749168396,-0.9450993537902832,0.7751712203025818,0.0012820051051676273,-0.5824522972106934],[0.955508291721344,1.5091526508331299,-0.4049511253833771,-0.00011778425687225536,-0.723637580871582,-0.953830897808075,0.08508166670799255,-0.5111517310142517,0.6624951958656311,-1.503186583518982,-0.927710235118866,1.120854139328003]],"context_mean":[-0.07536272704601288,-1.2713303565979004,1.432302713394165,2.1556945739575895e-06,-0.16093745827674866,0.07384335249662399,-0.0013792915269732475,0.03438752517104149,-0.06603991985321045,0.06357423961162567,0.08509783446788788,-0.09797351807355881],"context_scale":[0.2049875557422638,0.15528517961502075,0.24257291853427887,0.019999999552965164,0.1880050152540207,0.20382720232009888,0.019999999552965164,0.06742531061172485,0.06861884146928787,0.04183868691325188,0.09107182919979095,0.05650162696838379],"dataset_sha256":"678630ea95cd0cd31e0d84141fc1743432a756d4c8b0e31c7abc85627f8f8135","feature_contract":"q6_object_minus_tcp3_target_minus_tcp3__capped_goal_delta3_upright_orientation_delta3_v1","kernel_width":1.4146959473295806,"kind":"dvidia.numpy-rbf-movement-head","physical_robot_ready":false,"schema_version":1,"scope":"simulation-only","status":"candidate","train_cases":["example-02","example-04"],"training":{"algorithm":"normalized-rbf-ridge","regularization":0.01,"sample_count":1135,"seed":19047},"weights":[[[-0.00035793078131973743,0.025796838104724884,-0.03578739985823631,1.3060398487141356e-06,0.010110934264957905,0.00013486829993780702],[0.011819134466350079,0.0041123018600046635,-0.0052633401937782764,5.3432531785801984e-06,0.001953264931216836,-0.008745918050408363],[9.3720322183799e-05,0.001554427552036941,-0.021944187581539154,1.2796414921467658e-05,0.01955007202923298,-0.0010209460742771626],[-0.0006547769298776984,-0.006456151604652405,0.004943931475281715,1.9018333432541112e-06,-0.0019184055272489786,0.0011736826272681355],[0.0008630575030110776,0.007914726622402668,-0.016170432791113853,0.00010411304538138211,0.04030706360936165,-0.009324164129793644],[0.0006685047410428524,-0.005001420620828867,0.009852655231952667,-0.0001321948366239667,-0.011372310109436512,0.00960939284414053]],[[0.0066077797673642635,0.01973445527255535,-0.033145882189273834,6.471127562690526e-06,0.010486110113561153,-0.005455422680824995],[0.010471472516655922,-0.014824769459664822,0.023654861375689507,3.2555919915466802e-06,-0.008297748863697052,-0.009280356578528881],[0.0024226047098636627,0.010890938341617584,-0.020797863602638245,-7.567929060314782e-06,0.00846375897526741,5.403043905971572e-05],[-0.0012506373459473252,-0.0015350551111623645,0.003571572480723262,6.9954144237271976e-06,0.000995390466414392,0.00450423127040267],[-0.001248557004146278,-0.0010886323871091008,0.002577422186732292,5.13462600792991e-06,0.0027785440906882286,0.006895033642649651],[0.004093390889465809,0.01201966218650341,-0.025720778852701187,-6.549875251948833e-05,0.020878702402114868,0.020275535061955452]],[[0.012019992806017399,0.0052588931284844875,-0.004748051520437002,2.5234741769963875e-05,-0.004870225675404072,-0.014920124784111977],[0.021194661036133766,-0.008342882618308067,0.011396365240216255,2.8859061785624363e-05,-0.005957444664090872,-0.01858723722398281],[0.00011805663234554231,-0.012392069213092327,-0.007842631079256535,4.743587396660587e-06,0.019658401608467102,-0.00010877343447646126],[-0.0009313125046901405,-0.008759242482483387,0.024372972548007965,1.4022742107044905e-05,-0.011489546857774258,0.0003447291674092412],[0.0012591631384566426,-0.003734281985089183,-0.0030844612047076225,3.389206176507287e-05,0.0423266775906086,-0.001054361928254366],[0.0021592658013105392,0.00064765231218189,-0.001554616610519588,-6.646366091445088e-05,0.003138229250907898,0.0024080516304820776]],[[-0.0006579267210327089,0.03141685947775841,-0.03153495863080025,-7.455169907188974e-06,0.001731321681290865,0.0012179811019450426],[0.014278235845267773,-0.006717505864799023,0.007789896335452795,6.641962045250693e-06,-0.0002008140436373651,-0.011086390353739262],[-0.00037881589378230274,0.0015802917769178748,-0.023279879242181778,-1.4124623248790158e-06,0.018484536558389664,-0.0010639737593010068],[-0.00039740235661156476,0.0026977057568728924,-0.0023539315443485975,3.3065023785638914e-07,0.0014839554205536842,-0.0009674204047769308],[-0.0008102926076389849,0.004567250143736601,-0.005156110972166061,-2.2043759599910118e-05,0.011652565561234951,-0.006289478857070208],[0.0017698705196380615,-0.0031389258801937103,0.004114392679184675,-7.622472185175866e-05,-0.008432005532085896,0.011367355473339558]],[[0.009372883476316929,0.025242790579795837,-0.034872543066740036,-4.66565052192891e-06,0.008147464133799076,-0.006591231096535921],[0.013088185340166092,-0.006959886755794287,0.009824192151427269,2.7346021624907735e-07,-0.002614235272631049,-0.009211231023073196],[-0.0005560615099966526,0.003100123954936862,-0.0043767583556473255,1.1852989700855687e-05,0.0014910789905115962,-0.00109204207547009],[0.00039897338137961924,0.00031326539465226233,-0.0013404404744505882,-1.8070079477183754e-06,0.0014489960158243775,0.0006812568753957748],[0.0005500030238181353,0.0006613831501454115,-0.0020648811478167772,-3.4197089462395525e-06,0.00255695846863091,0.0005789369461126626],[0.0009429063647985458,0.0014578051632270217,-0.0011333376169204712,-1.0375977581134066e-05,0.00026482620160095394,0.0025657350197434425]],[[-0.003873704932630062,0.023417523130774498,-0.02851535752415657,-2.1160343749215826e-05,0.005447694566100836,0.004588181618601084],[0.016345638781785965,-0.0013300672871991992,0.002145356498658657,2.9865692340536043e-05,-0.001109867007471621,-0.01313429418951273],[-0.00026837087352760136,-0.002213731175288558,-0.017348550260066986,-3.6025787267135456e-05,0.01747175306081772,0.0014190258225426078],[-0.001830826047807932,-0.0032387597020715475,0.008739694952964783,-1.1368397281330545e-05,-0.0016764973988756537,0.0016526636900380254],[-0.001307085738517344,0.001099423854611814,-0.007085293065756559,-0.000244848255533725,0.03355870768427849,0.00948810763657093],[-0.00027984409825876355,0.001084754359908402,-0.0045678336173295975,-0.0003019434225279838,0.009705690667033195,0.022072115913033485]],[[0.0030995921697467566,0.020136931911110878,-0.01868724264204502,2.6752970370580442e-05,-0.002293798141181469,-0.00457734102383256],[0.015665851533412933,0.002551544923335314,-0.003500512568280101,2.6652591259335168e-05,0.0022119951900094748,-0.014256409369409084],[4.3688320147339255e-05,-0.0024626529775559902,-0.01692996546626091,-1.0118972568307072e-05,0.017875012010335922,-0.0013936908217146993],[-0.000903592212125659,0.0063213761895895,-0.004351282492280006,8.058430012169993e-07,0.002388500142842531,-0.0014948323369026184],[-0.0019074593437835574,0.013471073471009731,-0.01836227811872959,-0.00012541147589217871,0.030205857008695602,-0.00883916113525629],[-0.0006533630657941103,-0.008871552534401417,0.01149720698595047,-0.00019936688477173448,-0.013203742913901806,0.029230697080492973]],[[-0.00040898568113334477,0.00657676113769412,-0.014729762449860573,1.3599883459392004e-05,0.004122150130569935,-0.0014084450667724013],[0.01010164525359869,0.003644155338406563,-0.000890198687557131,-2.428036623314256e-06,-0.0037837696727365255,-0.007265910971909761],[-2.3966922526597045e-05,-0.009774231351912022,-0.00670110248029232,-1.2862097946708673e-06,0.01579868234694004,0.00013039432815276086],[-0.0008222601027227938,-0.0040444317273795605,-0.001713198609650135,4.184965746389935e-06,0.0018113693222403526,0.0003645883989520371],[-2.247138399980031e-05,-0.00032340214238502085,-0.007211322896182537,-3.995955466962187e-06,0.04120190814137459,0.0009920414304360747],[0.0009952796390280128,-0.0007430577534250915,0.0019497129833325744,-2.9405551686068065e-05,-0.0021856087259948254,0.0014984688023105264]]]},"payload_sha256":"5830cb04df6edbce29ef1a86e565a80ef538b7b03dec1ae97a3d90a8a919d179","physical_robot_ready":false,"provenance":{"dataset":{"id":"dataset:678630ea95cd0cd31e0d8414","sample_count":1135,"sha256":"678630ea95cd0cd31e0d84141fc1743432a756d4c8b0e31c7abc85627f8f8135","train_cases":["example-02","example-04"]},"evaluation":null,"training":{"algorithm":"normalized-rbf-ridge","dataset_sha256":"678630ea95cd0cd31e0d84141fc1743432a756d4c8b0e31c7abc85627f8f8135","id":"training:5d36c4d0b6f66aacf5c4090f","movement_head_sha256":"5d36c4d0b6f66aacf5c4090fea81c3ca755b8e2b7a9c5115aea7c71ab3369299","regularization":0.01,"sample_count":1135,"seed":19047}},"runtime":{"action_contract":"six joint targets in radians; symmetric gripper width in metres","dependencies":{"mujoco":"3.15.0","numpy":"2.5.3"},"files_sha256":{"arm_distill.py":"14640f91ec9c91d10a405fa56eb21bda8b55b285854ce2b5859a71d1ffd06ea1","arm_env.py":"e4af9b75364a568284b1e6d96469deacf498dd87a60b067fa5e310fe15c54c00","arm_policy.py":"d6d53d87703fb8608390092309285afb6a2c8fd3103a7abd77668be85469eab3","arm_runner.py":"affb5812b6a395f6cca9cb4e900af2ef28162e7ff1e52a51bd29b789daa98df1","capsule_installer.py":"60bb1652cae830fd03ae6eaa5e10150c957be13e1dc630a5631c1a2692ae37ec","cli.py":"192ec6a0852c7fd5dd3ae912815dae2fc7c9cfb93b97c85d528060e00a975b17","env.py":"929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4","installer.py":"d3a2c5a4913920c52beef89df9954494b89fd0becb08bb4847264dae48d3761c","skill_capsule.py":"ecb7dbd60d3a11f40bf65c4520f024b449c0d63f9ae27c601c9c9fe17b32e6a0","skillspace.py":"34b651609358ae9a58b9a610545ce7b85b83fb3d6c699613414ff8e0ea457b3c"},"kind":"hybrid-distilled-placement-runtime"},"schema_version":1,"scope":"simulation-only","source":{"sha256":"23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15","text":"{\n \"vdia\": \"0\",\n \"skill\": \"place_cup\",\n \"version\": \"0.3.0\",\n \"task\": \"Set a cup on the mark.\",\n \"rail\": \"physical\",\n \"demos\": 1,\n \"format\": \"lerobot/v3\",\n \"codebase_version\": \"v3.0\",\n \"robot_type\": \"dvidia.wrist_room\",\n \"cameras\": [\"ego\", \"room\"],\n \"residual\": {\"claimer\": 0.5, \"contributors\": 0.5, \"book\": \"not-live\"}\n}\n"},"status":"candidate"} diff --git a/benchmarks/sample-count-19047/groups-004.skill-capsule.json b/benchmarks/sample-count-19047/groups-004.skill-capsule.json new file mode 100644 index 0000000..14883b3 --- /dev/null +++ b/benchmarks/sample-count-19047/groups-004.skill-capsule.json @@ -0,0 +1 @@ +{"compatibility":{"actions":[{"name":"joint_targets","semantics":"absolute_joint_position","shape":[6],"units":"rad"},{"name":"gripper_width","semantics":"total_symmetric_jaw_gap","shape":[],"units":"m"}],"adapter_id":"contact-pick-place-box-v0","arm_count":1,"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","control_hz":50.0,"coordinates":"world_metres","gripper":{"gap_limits_m":[0.0,0.08],"independent_commands":1,"jaw_joints":["jaw_left","jaw_right"],"kind":"symmetric_parallel_jaw","maximum_jaw_actuator_force_n":15.0},"independent_commands":7,"joint_limits_rad":[[-2.6,2.6],[-2.2,1.6],[-2.7,2.7],[-3.0,3.0],[-2.8,2.8],[-3.1,3.1]],"joint_order":["j0","j1","j2","j3","j4","j5"],"maximum_joint_actuator_torque_nm":40.0,"object_profile":"upright_axis_aligned_rigid_box_30_to_50_mm","observation_origin":"privileged_native_simulator_state","observations":[{"frame":"joint","name":"joint_position","shape":[6],"units":"rad"},{"frame":"joint","name":"joint_velocity","shape":[6],"units":"rad/s"},{"frame":"joint","name":"joint_actuator_torque","shape":[6],"units":"N*m"},{"frame":"world","name":"end_effector_position","shape":[3],"units":"m"},{"frame":"world","name":"end_effector_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"end_effector_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_position","shape":[3],"units":"m"},{"frame":"world","name":"object_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"object_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_angular_velocity","shape":[3],"units":"rad/s"},{"frame":"world","name":"target_position","shape":[3],"units":"m"},{"frame":"object","name":"object_size","shape":[3],"units":"m"},{"frame":"gripper","name":"gripper_width","shape":[],"units":"m"},{"frame":"gripper","name":"grasp_contacts","order":["left","right"],"shape":[2],"units":"count"},{"frame":"gripper","name":"pad_normal_forces","order":["left","right"],"shape":[2],"units":"N"},{"frame":"contact","name":"effective_sliding_friction","order":["jaw_object","table_object"],"shape":[2],"units":"coefficient"},{"frame":"contact","name":"table_contacts","shape":[],"units":"count"}],"physical_robot_joints":8,"supervisor_revision":"dimension-aware-contact-rates-recovery-round2","target_rate_limits":{"jaw_gap_m_s":0.16,"joint_rad_s":3.0},"tool_orientation":"fixed_identity_rotation","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29],"raised_tool_radial_max_m":0.56}},"format":"dvidia.skill-capsule","grounding":{"adapter_id":"contact-pick-place-box-v0","arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","frame":"world","language_aliases":["Set a cup on the mark."],"object":{"friction":0.8,"kind":"box","mass":0.04,"position":[0.42,-0.04,0.31],"size":[0.04,0.04,0.04]},"procedure":["approach","descend","close","lift","transfer","place","release","retreat","verify"],"provenance":"authored_simulation_adapter","schema_version":1,"table_height":0.29,"target":{"dwell_seconds":0.25,"position":[0.54,0.1,0.31],"position_tolerance":0.025},"task_type":"place_cup","units":"m","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29]}},"movement_head":{"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","centers":[[1.0952481031417847,1.4786365032196045,-0.4066437780857086,-0.0001683255541138351,-0.7023463845252991,-1.0930813550949097,0.11961405724287033,-0.48227033019065857,0.5983936190605164,-1.4714564085006714,-1.0591720342636108,1.0906479358673096],[-2.051717519760132,0.5771871209144592,-1.6053154468536377,-0.00045156216947361827,1.6163346767425537,2.052483558654785,-0.5606028437614441,2.711369037628174,-1.749647617340088,0.5318674445152283,2.027329921722412,-1.4864994287490845],[-0.17980121076107025,-0.6125081777572632,1.4768214225769043,-0.00022166130656842142,-1.4242357015609741,0.18727779388427734,0.08154928684234619,-0.4666137099266052,0.6481788158416748,0.8514948487281799,0.1714441031217575,1.1870871782302856],[-0.5742805600166321,-1.1863031387329102,0.4118760824203491,-2.4051167201832868e-05,0.4373609721660614,0.5729440450668335,-0.029070289805531502,-0.16607998311519623,-1.0730925798416138,0.801588773727417,0.6238506436347961,-0.6678048372268677],[-1.3186688423156738,-0.43477416038513184,-0.557410717010498,-5.2834468078799546e-05,1.091172695159912,1.3122950792312622,-0.3087652325630188,1.361598253250122,-1.514675498008728,0.6532946825027466,1.2172342538833618,-1.2021613121032715],[0.07013949006795883,-0.9108601212501526,0.5327163338661194,0.0004116890486329794,0.07262016087770462,-0.07257338613271713,0.09307553619146347,-0.4692267179489136,0.677079975605011,0.5470996499061584,-0.03703682869672775,-0.2503352165222168],[0.07928389310836792,-0.8236402273178101,0.004014946985989809,3.1684638088336214e-05,0.6736140251159668,-0.0880952849984169,0.09569564461708069,0.13706950843334198,-1.2312722206115723,0.813100278377533,-0.41910919547080994,-0.8592169880867004],[0.9696006178855896,0.8045573830604553,-0.9090734720230103,0.00026796406018547714,0.5232499837875366,-0.9723449945449829,0.12043283134698868,-0.4810422658920288,0.448578417301178,-1.171658992767334,-0.9171620011329651,-0.32057249546051025]],"context_mean":[-0.08539111167192459,-1.2819104194641113,1.4482640027999878,3.151679038637667e-06,-0.16629981994628906,0.08380704373121262,-0.0020384909585118294,0.028692591935396194,-0.06215041130781174,0.06195763871073723,0.1035265326499939,-0.09723600745201111],"context_scale":[0.21892431378364563,0.15593896806240082,0.244650736451149,0.019999999552965164,0.1865261197090149,0.21799060702323914,0.019999999552965164,0.059721753001213074,0.06732939928770065,0.04171416163444519,0.09698643535375595,0.0556398406624794],"dataset_sha256":"4c825e3c4dbbbe01c2bdb3591e4a75726b66c9d75d3d6f90df5238ab3f3e0e55","feature_contract":"q6_object_minus_tcp3_target_minus_tcp3__capped_goal_delta3_upright_orientation_delta3_v1","kernel_width":1.4348705803030808,"kind":"dvidia.numpy-rbf-movement-head","physical_robot_ready":false,"schema_version":1,"scope":"simulation-only","status":"candidate","train_cases":["example-01","example-02","example-04","example-09"],"training":{"algorithm":"normalized-rbf-ridge","regularization":0.01,"sample_count":2281,"seed":19047},"weights":[[[-0.0009391106432303786,0.0092207882553339,-0.017622115090489388,1.4474488125415519e-05,0.0051565696485340595,-0.0016848052619025111],[0.011352160014212132,0.004119087476283312,-0.0027642063796520233,1.2338193755567772e-06,-0.002115732291713357,-0.008547216653823853],[-5.01994181831833e-05,-0.009584128856658936,-0.007207629270851612,-2.3207667254609987e-06,0.01643352396786213,0.0002074205403914675],[-0.0007961012306623161,-0.007885357365012169,-0.003751604352146387,1.0898297659878153e-05,0.006298211868852377,0.0003705110866576433],[-0.00038872446748428047,-0.0008376457262784243,-0.0088749835267663,-1.3197766747907735e-05,0.0463976189494133,0.0019251331686973572],[0.001950140343979001,-0.0016601841198280454,0.003961422946304083,-5.828155917697586e-05,-0.003437258070334792,0.0024396313820034266]],[[0.004906550515443087,0.013038596138358116,-0.01770108751952648,4.7807661758270115e-06,0.003500924212858081,-0.00462203286588192],[0.011549415066838264,-0.012720933184027672,0.01883142627775669,4.8806909944687504e-06,-0.006464136764407158,-0.010420597158372402],[0.00833666417747736,0.02334628440439701,-0.05129079893231392,3.3586175050004385e-06,0.022554244846105576,-0.00458488380536437],[-0.0051430813036859035,-0.012427872978150845,0.023310650140047073,9.999274880101439e-06,-0.004095600452274084,0.008866599760949612],[-0.005597038194537163,-0.012512371875345707,0.023510590195655823,1.0230090992990881e-05,-0.0019750308711081743,0.012232845649123192],[0.008536066859960556,0.028352854773402214,-0.05501212552189827,-6.971612310735509e-05,0.033555347472429276,0.0206657312810421]],[[0.007783011067658663,0.018675794824957848,-0.02409668080508709,-2.2661823095404543e-05,0.001202471787109971,-0.008822926320135593],[0.018511226400732994,-0.0006110751419328153,-0.0008596186526119709,-1.5985851860023104e-06,-0.001839016447775066,-0.014762579463422298],[8.513373177265748e-05,-0.012266301549971104,-0.00829425361007452,7.705575626459904e-06,0.020268945023417473,-0.0002003648114623502],[-0.0008377415360882878,-0.017756497487425804,0.0335213728249073,2.0717356164823286e-05,-0.011040021665394306,0.00038958562072366476],[0.0009308313601650298,-1.8733520334990317e-07,-0.009044964797794819,5.308186518959701e-05,0.04672485217452049,-0.001721257925964892],[0.0021361946128308773,0.0033847049344331026,-0.004877257160842419,-0.00010786673374241218,0.004937516059726477,0.004603039938956499]],[[0.009023161605000496,0.009495910257101059,-0.003493459662422538,4.205630102660507e-05,-0.004993824288249016,-0.010487983003258705],[0.016210023313760757,-0.014505628496408463,0.016641275957226753,3.5351742553757504e-05,-0.001100199413485825,-0.014877421781420708],[1.7866484995465726e-05,-0.00525956554338336,-0.01594589836895466,-2.168381615774706e-05,0.019108334556221962,-0.00044569920282810926],[-0.0007412866107188165,-0.001550445449538529,0.00227760779671371,-2.3871994926594198e-05,0.004411962348967791,-0.0003096733707934618],[-0.001466928282752633,0.007101660128682852,-0.012633493170142174,-0.0001781648606993258,0.03254485875368118,-0.005651645362377167],[0.0011175278341397643,-0.002625752706080675,0.0063176872208714485,-0.00029237219132483006,-0.013238605111837387,0.02951914444565773]],[[0.0075853681191802025,0.02691001258790493,-0.033297453075647354,-7.564104635093827e-06,0.005898066330701113,-0.004738871939480305],[0.013547376729547977,-0.004030718468129635,0.005109154153615236,4.433035883266712e-06,-0.0005552797229029238,-0.010084456764161587],[-0.0009489739895798266,0.017577752470970154,-0.03121141716837883,1.7308924725512043e-05,0.013770348392426968,-0.0022848222870379686],[0.0012215861352160573,0.004241135902702808,-0.006858165841549635,3.4062579743476817e-06,0.0034779126290231943,-0.001271154498681426],[0.001489076530560851,0.0036691066343337297,-0.006709117442369461,-3.3269136565650115e-06,0.007349493447691202,-0.003286907682195306],[-0.002580349799245596,-0.008396352641284466,0.014790250919759274,-4.853043719776906e-05,-0.008277319371700287,0.009118864312767982]],[[-0.0008646526257507503,0.03338460251688957,-0.041017450392246246,1.6818238464111346e-06,0.007462213281542063,0.00021121501049492508],[0.016142435371875763,-0.0018158152233809233,0.0030660664197057486,5.536483149626292e-05,-0.0023865322582423687,-0.01409624982625246],[-0.00034599832724779844,-0.003389472607523203,-0.015827951952815056,-5.2627969125751406e-05,0.017829688265919685,0.0022228346206247807],[-0.0007721903966739774,-0.024411220103502274,0.02696863003075123,-3.5914606996811926e-05,-0.00012530447565950453,0.003490789094939828],[-0.0021312441676855087,-0.0029243824537843466,-0.002823546528816223,-0.00036525988252833486,0.03748006373643875,0.015919720754027367],[-0.001015953836031258,0.00801722053438425,-0.01293187029659748,-0.00047543965047225356,0.017269518226385117,0.031470801681280136]],[[-0.0071506197564303875,-0.006381059996783733,0.014287636615335941,-5.198942017159425e-05,-0.007671384606510401,0.009707457385957241],[0.014479144476354122,0.006560684181749821,-0.008220293559134007,1.4056447071197908e-05,0.0030963937751948833,-0.01262819953262806],[0.00010603765986161307,4.86319768242538e-05,-0.019526682794094086,-3.5844022931996733e-06,0.01768283173441887,-0.0019393160473555326],[-0.0002856598293874413,-0.003653530729934573,0.0033584539778530598,-1.4439872757066041e-05,0.0022177989594638348,-0.0005005822167731822],[0.000465223885839805,0.013937951996922493,-0.01974116452038288,-8.853100007399917e-05,0.02754245325922966,-0.01199234090745449],[0.0005953946965746582,-0.007547849789261818,0.01026980485767126,-0.00011660268501145765,-0.01392177864909172,0.02358447201550007]],[[-0.00018182751955464482,0.025975020602345467,-0.03602733090519905,1.7102709080063505e-06,0.01014616247266531,3.763240601983853e-05],[0.011586765758693218,0.004190283361822367,-0.005828198976814747,1.5672145309508778e-05,0.0025743157602846622,-0.009174836799502373],[0.0003191686701029539,0.001488947425968945,-0.021644311025738716,1.4832527085673064e-05,0.01944088190793991,-0.0014457375509664416],[-0.0017018367070704699,-0.008034344762563705,0.005038321018218994,1.7343595573038328e-06,-0.0014801216311752796,0.0025543957017362118],[0.002938708756119013,0.0061780428513884544,-0.012278808280825615,0.0001233224757015705,0.039677053689956665,-0.013537165708839893],[0.004368926398456097,-0.010595904663205147,0.02182239480316639,-0.00023796614550519735,-0.018986046314239502,0.013091168366372585]]]},"payload_sha256":"0eac4db58e6188988e0b4172cee3ff57733b45f0c6e3ee1aa762b126335f28b3","physical_robot_ready":false,"provenance":{"dataset":{"id":"dataset:4c825e3c4dbbbe01c2bdb359","sample_count":2281,"sha256":"4c825e3c4dbbbe01c2bdb3591e4a75726b66c9d75d3d6f90df5238ab3f3e0e55","train_cases":["example-01","example-02","example-04","example-09"]},"evaluation":null,"training":{"algorithm":"normalized-rbf-ridge","dataset_sha256":"4c825e3c4dbbbe01c2bdb3591e4a75726b66c9d75d3d6f90df5238ab3f3e0e55","id":"training:59459226319eff5bb80a615f","movement_head_sha256":"59459226319eff5bb80a615fabddf1364fc5b5688cf188534fd06c7283f0e3f2","regularization":0.01,"sample_count":2281,"seed":19047}},"runtime":{"action_contract":"six joint targets in radians; symmetric gripper width in metres","dependencies":{"mujoco":"3.15.0","numpy":"2.5.3"},"files_sha256":{"arm_distill.py":"14640f91ec9c91d10a405fa56eb21bda8b55b285854ce2b5859a71d1ffd06ea1","arm_env.py":"e4af9b75364a568284b1e6d96469deacf498dd87a60b067fa5e310fe15c54c00","arm_policy.py":"d6d53d87703fb8608390092309285afb6a2c8fd3103a7abd77668be85469eab3","arm_runner.py":"affb5812b6a395f6cca9cb4e900af2ef28162e7ff1e52a51bd29b789daa98df1","capsule_installer.py":"60bb1652cae830fd03ae6eaa5e10150c957be13e1dc630a5631c1a2692ae37ec","cli.py":"192ec6a0852c7fd5dd3ae912815dae2fc7c9cfb93b97c85d528060e00a975b17","env.py":"929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4","installer.py":"d3a2c5a4913920c52beef89df9954494b89fd0becb08bb4847264dae48d3761c","skill_capsule.py":"ecb7dbd60d3a11f40bf65c4520f024b449c0d63f9ae27c601c9c9fe17b32e6a0","skillspace.py":"34b651609358ae9a58b9a610545ce7b85b83fb3d6c699613414ff8e0ea457b3c"},"kind":"hybrid-distilled-placement-runtime"},"schema_version":1,"scope":"simulation-only","source":{"sha256":"23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15","text":"{\n \"vdia\": \"0\",\n \"skill\": \"place_cup\",\n \"version\": \"0.3.0\",\n \"task\": \"Set a cup on the mark.\",\n \"rail\": \"physical\",\n \"demos\": 1,\n \"format\": \"lerobot/v3\",\n \"codebase_version\": \"v3.0\",\n \"robot_type\": \"dvidia.wrist_room\",\n \"cameras\": [\"ego\", \"room\"],\n \"residual\": {\"claimer\": 0.5, \"contributors\": 0.5, \"book\": \"not-live\"}\n}\n"},"status":"candidate"} diff --git a/benchmarks/sample-count-19047/groups-007.skill-capsule.json b/benchmarks/sample-count-19047/groups-007.skill-capsule.json new file mode 100644 index 0000000..3a4c5ce --- /dev/null +++ b/benchmarks/sample-count-19047/groups-007.skill-capsule.json @@ -0,0 +1 @@ +{"compatibility":{"actions":[{"name":"joint_targets","semantics":"absolute_joint_position","shape":[6],"units":"rad"},{"name":"gripper_width","semantics":"total_symmetric_jaw_gap","shape":[],"units":"m"}],"adapter_id":"contact-pick-place-box-v0","arm_count":1,"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","control_hz":50.0,"coordinates":"world_metres","gripper":{"gap_limits_m":[0.0,0.08],"independent_commands":1,"jaw_joints":["jaw_left","jaw_right"],"kind":"symmetric_parallel_jaw","maximum_jaw_actuator_force_n":15.0},"independent_commands":7,"joint_limits_rad":[[-2.6,2.6],[-2.2,1.6],[-2.7,2.7],[-3.0,3.0],[-2.8,2.8],[-3.1,3.1]],"joint_order":["j0","j1","j2","j3","j4","j5"],"maximum_joint_actuator_torque_nm":40.0,"object_profile":"upright_axis_aligned_rigid_box_30_to_50_mm","observation_origin":"privileged_native_simulator_state","observations":[{"frame":"joint","name":"joint_position","shape":[6],"units":"rad"},{"frame":"joint","name":"joint_velocity","shape":[6],"units":"rad/s"},{"frame":"joint","name":"joint_actuator_torque","shape":[6],"units":"N*m"},{"frame":"world","name":"end_effector_position","shape":[3],"units":"m"},{"frame":"world","name":"end_effector_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"end_effector_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_position","shape":[3],"units":"m"},{"frame":"world","name":"object_quaternion","shape":[4],"units":"wxyz"},{"frame":"world","name":"object_velocity","shape":[3],"units":"m/s"},{"frame":"world","name":"object_angular_velocity","shape":[3],"units":"rad/s"},{"frame":"world","name":"target_position","shape":[3],"units":"m"},{"frame":"object","name":"object_size","shape":[3],"units":"m"},{"frame":"gripper","name":"gripper_width","shape":[],"units":"m"},{"frame":"gripper","name":"grasp_contacts","order":["left","right"],"shape":[2],"units":"count"},{"frame":"gripper","name":"pad_normal_forces","order":["left","right"],"shape":[2],"units":"N"},{"frame":"contact","name":"effective_sliding_friction","order":["jaw_object","table_object"],"shape":[2],"units":"coefficient"},{"frame":"contact","name":"table_contacts","shape":[],"units":"count"}],"physical_robot_joints":8,"supervisor_revision":"dimension-aware-contact-rates-recovery-round2","target_rate_limits":{"jaw_gap_m_s":0.16,"joint_rad_s":3.0},"tool_orientation":"fixed_identity_rotation","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29],"raised_tool_radial_max_m":0.56}},"format":"dvidia.skill-capsule","grounding":{"adapter_id":"contact-pick-place-box-v0","arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","frame":"world","language_aliases":["Set a cup on the mark."],"object":{"friction":0.8,"kind":"box","mass":0.04,"position":[0.42,-0.04,0.31],"size":[0.04,0.04,0.04]},"procedure":["approach","descend","close","lift","transfer","place","release","retreat","verify"],"provenance":"authored_simulation_adapter","schema_version":1,"table_height":0.29,"target":{"dwell_seconds":0.25,"position":[0.54,0.1,0.31],"position_tolerance":0.025},"task_type":"place_cup","units":"m","workspace":{"max":[0.6,0.2,0.55],"min":[0.25,-0.2,0.29]}},"movement_head":{"arm_profile":"dvidia-authored-6dof-parallel-jaw-v0","centers":[[0.6339258551597595,-0.2160557508468628,1.0381486415863037,-0.0002205160999437794,-1.1808849573135376,-0.6299140453338623,0.06620757281780243,-0.4514952600002289,0.5863461494445801,0.8006615042686462,-0.9926416277885437,1.0274115800857544],[0.9565039873123169,0.8453730344772339,-0.9366454482078552,0.00019379732839297503,0.5214436054229736,-0.9578704237937927,0.10281636565923691,-0.46769386529922485,0.426466703414917,-1.1946368217468262,-0.878881573677063,-0.29879316687583923],[-1.018141746520996,-0.7979374527931213,-0.17432986199855804,2.7898731786990538e-05,0.898112416267395,1.0113462209701538,-0.18047775328159332,0.7096797227859497,-1.4221247434616089,0.7255598306655884,0.946341872215271,-1.0680559873580933],[0.058549460023641586,-0.8984605073928833,0.3752375543117523,0.0007045206148177385,0.276520311832428,-0.0648612231016159,0.08345752209424973,-0.4663245379924774,0.8109862208366394,0.36117878556251526,0.07666011899709702,-0.44880515336990356],[-0.5357311964035034,-0.8198392391204834,1.506136417388916,-0.000227519849431701,-1.28715181350708,0.5440400242805481,0.0567193366587162,-0.44774818420410156,0.5684131383895874,0.8620448708534241,0.6363914608955383,1.0213857889175415],[1.0686136484146118,1.4808646440505981,-0.40466147661209106,-0.00016377170686610043,-0.7179440855979919,-1.0660611391067505,0.10113926231861115,-0.46881458163261414,0.5935651659965515,-1.4658749103546143,-0.9891906380653381,1.1152480840682983],[-1.840522289276123,0.3744827210903168,-1.40673828125,-0.00038892365409992635,1.5272470712661743,1.839113473892212,-0.43544524908065796,2.4933784008026123,-1.7333128452301025,0.5932144522666931,1.7100062370300293,-1.44508695602417],[-0.03720121085643768,-0.9610395431518555,0.2946753203868866,-9.8458505817689e-05,0.4074999690055847,0.034918081015348434,0.06313076615333557,-0.12276197969913483,-1.0538870096206665,0.8163340091705322,-0.16425137221813202,-0.6219044923782349]],"context_mean":[-0.08494970947504044,-1.2753459215164185,1.4391602277755737,3.2158650355995633e-06,-0.16374659538269043,0.08336960524320602,-0.0016702174907550216,0.028426559641957283,-0.061182960867881775,0.061943169683218,0.10520140826702118,-0.09742140024900436],"context_scale":[0.23045633733272552,0.15614385902881622,0.24317370355129242,0.019999999552965164,0.1849329173564911,0.22960226237773895,0.019999999552965164,0.0608404204249382,0.06705088168382645,0.041913315653800964,0.10561560094356537,0.055341437458992004],"dataset_sha256":"2a7388f027aeb4e1147885f1b1dff75c606bf115a80cf694926c55a666061112","feature_contract":"q6_object_minus_tcp3_target_minus_tcp3__capped_goal_delta3_upright_orientation_delta3_v1","kernel_width":1.5750644943237233,"kind":"dvidia.numpy-rbf-movement-head","physical_robot_ready":false,"schema_version":1,"scope":"simulation-only","status":"candidate","train_cases":["example-01","example-02","example-04","example-05","example-07","example-08","example-09"],"training":{"algorithm":"normalized-rbf-ridge","regularization":1e-06,"sample_count":4029,"seed":19047},"weights":[[[-0.007631983142346144,0.04103546962141991,-0.05363023653626442,-9.466131450608373e-05,0.012225634418427944,0.0075868889689445496],[0.01886693760752678,0.01448760088533163,-0.019772207364439964,2.9948407245683484e-05,0.0050268955528736115,-0.018776552751660347],[0.00021768784790765494,-0.011223186738789082,-0.008145404048264027,3.909702627424849e-06,0.019290201365947723,-0.00021573457343038172],[-0.006204559467732906,0.06670284271240234,-0.13205677270889282,0.00012565978977363557,0.063412144780159,0.006215392146259546],[-0.0008027649018913507,-0.0008236858993768692,-0.017966527491807938,5.1453811465762556e-05,0.059358254075050354,0.0008156930562108755],[-0.006276881787925959,0.07572483271360397,-0.08300246298313141,-0.0006258393405005336,0.007755032274872065,0.045855894684791565]],[[-0.0006520844181068242,0.028978466987609863,-0.041332513093948364,-1.9930921553168446e-06,0.012198294512927532,0.0006510212551802397],[0.011744796298444271,0.004559977445751429,-0.007384186610579491,3.9881604607217014e-05,0.0028931363485753536,-0.011717344634234905],[6.753015622962266e-05,0.0023198940325528383,-0.0229587871581316,-3.982885118603008e-06,0.020502833649516106,-6.735296483384445e-05],[-0.0039056220557540655,-0.02849564515054226,0.05686420947313309,0.00014114032092038542,-0.03387346863746643,0.003900832263752818],[-8.775795140536502e-05,0.004425622057169676,-0.007721017114818096,-3.053900218219496e-05,0.04285656660795212,8.792683365754783e-05],[-0.0018638992914929986,-0.009236867539584637,0.03352529928088188,-0.0004810421378351748,-0.024574046954512596,0.04186456277966499]],[[0.009112752974033356,0.028520092368125916,-0.034080781042575836,7.669298793189228e-05,0.005413788836449385,-0.00907079502940178],[0.010594332590699196,-0.007881480269134045,0.0075575439259409904,0.00011405825352994725,5.52296987734735e-05,-0.010545344091951847],[0.002483600750565529,0.0002523483126424253,-0.01985505409538746,-7.216049561975524e-05,0.019794274121522903,-0.0024808209855109453],[-0.03955378755927086,0.10494481772184372,-0.16439871490001678,0.0006463938625529408,0.08144070953130722,0.03940321505069733],[0.044712141156196594,-0.061994798481464386,0.10165143758058548,-0.0010417747544124722,-0.00038274083635769784,-0.044623974710702896],[0.029086051508784294,-0.004460724536329508,0.023308224976062775,-0.0010410910472273827,-0.016322871670126915,0.01076523307710886]],[[0.006206058897078037,-0.0005875902716070414,0.0013552620075643063,5.536489697988145e-05,-0.0006973088602535427,-0.006175674963742495],[0.015971576794981956,-0.008920772932469845,0.0209761131554842,6.215594476088881e-05,-0.011990568600594997,-0.015928979963064194],[-0.0017436216585338116,0.010671225376427174,-0.042849261313676834,4.3562929931795225e-05,0.03209527209401131,0.001750842435285449],[0.001627303776331246,-0.0788082480430603,0.08242375403642654,6.066646164981648e-05,0.0017200703732669353,-0.0016104287933558226],[-0.01252227183431387,0.07725413143634796,-0.18641194701194763,0.00038479582872241735,0.14904287457466125,0.012595917098224163],[-0.016616499051451683,0.07370162755250931,-0.16924528777599335,-0.00046377311809919775,0.0951412245631218,0.056524891406297684]],[[0.01845916546881199,0.01211838610470295,-0.014679014682769775,-3.0881947168381885e-05,0.002479329938068986,-0.01839897409081459],[0.021948980167508125,-0.017720019444823265,0.0135036064311862,6.52531161904335e-05,0.004363196901977062,-0.021848393604159355],[5.2387622417882085e-05,-0.01289182435721159,-0.007921300828456879,3.5337297958903946e-06,0.02070559561252594,-5.09408455400262e-05],[-0.005365821532905102,-0.022346002981066704,-0.03704201430082321,6.1030808865325525e-05,0.07106423377990723,0.005362110678106546],[0.0037251010071486235,-0.005820101592689753,0.021756799891591072,-1.0860722795769107e-05,0.022406747564673424,-0.0037116778548806906],[-0.015678834170103073,0.009149977937340736,0.03682507574558258,-0.0006412990624085069,-0.04700959473848343,0.05530446022748947]],[[-0.001603159704245627,0.009394978173077106,-0.016823066398501396,-3.515740536386147e-05,0.007406794931739569,0.0015902745071798563],[0.010327586904168129,0.0037426387425512075,0.0013344984035938978,8.329296542797238e-05,-0.00534614035859704,-0.010242583230137825],[-4.9955218855757266e-05,-0.01018503587692976,-0.0059228925965726376,-2.159230461984407e-06,0.016026291996240616,4.9016292905434966e-05],[0.0017976309172809124,-0.011074493639171124,-0.03480115160346031,-9.884222527034581e-05,0.03877933323383331,-0.0018037823028862476],[6.464094622060657e-05,-0.0004285962786525488,-0.014546002261340618,-2.8170721634523943e-05,0.05417110398411751,-7.344601181102917e-05],[0.011392449028789997,-0.019383620470762253,-0.03341318666934967,-0.0009447257034480572,0.05393056944012642,0.02800358459353447]],[[0.00445644510909915,0.015852438285946846,-0.019454989582300186,3.1154118005360942e-06,0.003524558385834098,-0.0044428822584450245],[0.011533059179782867,-0.01287093386054039,0.017321066930890083,1.897147376439534e-05,-0.004377682693302631,-0.01149533037096262],[0.006213520187884569,0.01852402091026306,-0.05527359992265701,0.0001068751880666241,0.03645725175738335,-0.006204609759151936],[-0.24249592423439026,-0.8172127604484558,1.582891821861267,-0.0013883687788620591,-0.7282412648200989,0.24198953807353973],[0.1484261006116867,0.5335811972618103,-1.0666593313217163,0.0014070492470636964,0.5548117160797119,-0.14821189641952515],[0.005398736335337162,0.016365276649594307,-0.020695019513368607,-0.00023025221889838576,0.004493372049182653,0.034524936228990555]],[[-0.014355484396219254,0.044744569808244705,-0.04422866925597191,-6.124206265667453e-05,-0.0004923359374515712,0.014298963360488415],[0.012599064968526363,0.01854773424565792,-0.03858627378940582,9.534024866297841e-05,0.02008083276450634,-0.01251994725316763],[0.0034198849461972713,-0.018792888149619102,0.020060187205672264,-0.00014309256221167743,-0.001467979047447443,-0.003445724258199334],[-0.025717424228787422,0.15130196511745453,-0.18646809458732605,0.00024409344769082963,0.033566124737262726,0.025650309398770332],[0.02225201576948166,-0.12284719944000244,0.2875998914241791,-0.00114319974090904,-0.12493278831243515,-0.022495198994874954],[0.026788078248500824,-0.15259969234466553,0.31179681420326233,-0.0007039538468234241,-0.1595928817987442,0.012903436087071896]]]},"payload_sha256":"9c31c70ae3dfa1a080e8ec15744950822276ff57f7862e8468d443ed8332f813","physical_robot_ready":false,"provenance":{"dataset":{"id":"dataset:2a7388f027aeb4e1147885f1","sample_count":4029,"sha256":"2a7388f027aeb4e1147885f1b1dff75c606bf115a80cf694926c55a666061112","train_cases":["example-01","example-02","example-04","example-05","example-07","example-08","example-09"]},"evaluation":null,"training":{"algorithm":"normalized-rbf-ridge","dataset_sha256":"2a7388f027aeb4e1147885f1b1dff75c606bf115a80cf694926c55a666061112","id":"training:1398ecbb57a23bbe1005f5f7","movement_head_sha256":"1398ecbb57a23bbe1005f5f78c1490a85fbd1ace8dc09bbd11894837468d7f12","regularization":1e-06,"sample_count":4029,"seed":19047}},"runtime":{"action_contract":"six joint targets in radians; symmetric gripper width in metres","dependencies":{"mujoco":"3.15.0","numpy":"2.5.3"},"files_sha256":{"arm_distill.py":"14640f91ec9c91d10a405fa56eb21bda8b55b285854ce2b5859a71d1ffd06ea1","arm_env.py":"e4af9b75364a568284b1e6d96469deacf498dd87a60b067fa5e310fe15c54c00","arm_policy.py":"d6d53d87703fb8608390092309285afb6a2c8fd3103a7abd77668be85469eab3","arm_runner.py":"affb5812b6a395f6cca9cb4e900af2ef28162e7ff1e52a51bd29b789daa98df1","capsule_installer.py":"60bb1652cae830fd03ae6eaa5e10150c957be13e1dc630a5631c1a2692ae37ec","cli.py":"192ec6a0852c7fd5dd3ae912815dae2fc7c9cfb93b97c85d528060e00a975b17","env.py":"929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4","installer.py":"d3a2c5a4913920c52beef89df9954494b89fd0becb08bb4847264dae48d3761c","skill_capsule.py":"ecb7dbd60d3a11f40bf65c4520f024b449c0d63f9ae27c601c9c9fe17b32e6a0","skillspace.py":"34b651609358ae9a58b9a610545ce7b85b83fb3d6c699613414ff8e0ea457b3c"},"kind":"hybrid-distilled-placement-runtime"},"schema_version":1,"scope":"simulation-only","source":{"sha256":"23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15","text":"{\n \"vdia\": \"0\",\n \"skill\": \"place_cup\",\n \"version\": \"0.3.0\",\n \"task\": \"Set a cup on the mark.\",\n \"rail\": \"physical\",\n \"demos\": 1,\n \"format\": \"lerobot/v3\",\n \"codebase_version\": \"v3.0\",\n \"robot_type\": \"dvidia.wrist_room\",\n \"cameras\": [\"ego\", \"room\"],\n \"residual\": {\"claimer\": 0.5, \"contributors\": 0.5, \"book\": \"not-live\"}\n}\n"},"status":"candidate"} diff --git a/benchmarks/sample-count-19047/models-frozen.json b/benchmarks/sample-count-19047/models-frozen.json new file mode 100644 index 0000000..e923153 --- /dev/null +++ b/benchmarks/sample-count-19047/models-frozen.json @@ -0,0 +1 @@ +{"candidates":[{"capsule_file_sha256":"c998cab9a704fb37f5e60493eafc52c20b285d579065c3e4fbeaf182079c95d3","capsule_path":"candidates/groups-002/candidate.skill-capsule.json","installation_id":"70a8dd0e999c5675a2845b53","movement_test":{"episodes":1,"learned_joint_delta_mse":1.120344178000201e-06,"samples":621,"train_mean_delta_mse":7.834385571673985e-05,"zero_delta_mse":8.172739843340321e-05},"training_action_samples":1135,"training_episode_count":2,"training_group_count":2,"training_pipeline_seconds":0.5916076248977333,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.11391330667979953,"learned_rgb_mse":0.0024406688519574495,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.11391330667979953,"learned_rgb_mse":0.0024406688519574495,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003992404562246332,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003992404562246332,"transitions":11}},{"capsule_file_sha256":"32c62628ed40064de57e3bcb3e32c041b16b268a48dd59307c32f778c3b5ec4d","capsule_path":"candidates/groups-004/candidate.skill-capsule.json","installation_id":"0bd068b6b2ecc0d06deb1d49","movement_test":{"episodes":1,"learned_joint_delta_mse":1.1087842308134674e-06,"samples":621,"train_mean_delta_mse":7.839595075163347e-05,"zero_delta_mse":8.172739843340321e-05},"training_action_samples":2281,"training_episode_count":4,"training_group_count":4,"training_pipeline_seconds":0.8397391671314836,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.11912927941122996,"learned_rgb_mse":0.002360370868993712,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.11912927941122996,"learned_rgb_mse":0.002360370868993712,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003866846205347564,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003866846205347564,"transitions":11}},{"capsule_file_sha256":"712eb4ed16939294b0898d8e6f39500201f0decf31c18122575de593887bf15a","capsule_path":"candidates/groups-007/candidate.skill-capsule.json","installation_id":"205924e092b684a9e4c67ad6","movement_test":{"episodes":1,"learned_joint_delta_mse":1.3251631022866396e-06,"samples":621,"train_mean_delta_mse":7.842909795405035e-05,"zero_delta_mse":8.172739843340321e-05},"training_action_samples":4029,"training_episode_count":7,"training_group_count":7,"training_pipeline_seconds":1.2524780000094324,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.12449758348668184,"learned_rgb_mse":0.0023421455366137525,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.12449758348668184,"learned_rgb_mse":0.0023421455366137525,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003859011472511888,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003859011472511888,"transitions":11}}],"protocol_sha256":"1e6d475aed2b505ec1ed4ef3990c723b7f9171863647bbfeb01610ba9c8c8eec"} diff --git a/benchmarks/sample-count-19047/protocol.json b/benchmarks/sample-count-19047/protocol.json new file mode 100644 index 0000000..b7f6406 --- /dev/null +++ b/benchmarks/sample-count-19047/protocol.json @@ -0,0 +1 @@ +{"benchmark_source_sha256":{"footage_benchmark.py":"a2c08b4836b92ee8c98c56dd5d56a788d7b71e591be63a8181584693312418bd","footage_data.py":"c54828b712afb1c7e89dc764b34dec1fe1ab28517c3ec589ba0977a2146c450d","footage_pipeline.py":"195278556a2b40dc628dbe528ec653c8b705a4ff45fa8d1074bf5341daec4dfd","footage_train.py":"2457a37a10016df6f359433f224c006b8c67f920b671383cbcfd5cfa78fc626b"},"input_files_sha256":{"grounding.json":"c53d3966399eb54ae7975829b43db64d16e2c7f5847d5d0171d04170db0154f6","source/dataset.json":"e0edc2a275d465ad58ebe1649e1df69c6ec37c30028f64f1c21c483c0f95cad9","subsets/groups-002/dataset.json":"039e94668770a58a4ba4415e1404accb7490529533f30c5fdc1fcceef2d076b8","subsets/groups-004/dataset.json":"891f9cd5cf2d2a1300b345b3314942e4c9718a8ddcfea3a48deb2e34d6122602","subsets/groups-007/dataset.json":"fea403d1092d929af7d17788de5a29949a35b44173a05b7abd0664d31e8f6d2d","task.skill.json":"23f27bea18abdf54ffd8a123fc099036ab7ee81fcd44714c894f5d1b12720a15"},"kind":"dvidia.footage-sample-count-benchmark","physical_robot_ready":false,"rules":{"control":"Each candidate uses identical recorded joint targets with jaws forced open.","counts":"Nested whole connected training groups; development and test episodes never change.","freeze":"Freeze this protocol before fitting; freeze all candidates before any scene evaluation.","limits":"No human-video action recovery, demonstration sufficiency threshold, hardware qualification or end-to-end speedup is implied.","qualification":"Existing native exact-scene predicate and installation gate are unchanged.","teacher":"One authored baseline per exact scene, shared by all candidates."},"runtime":{"action_contract":"six joint targets in radians; symmetric gripper width in metres","dependencies":{"mujoco":"3.15.0","numpy":"2.5.3"},"files_sha256":{"arm_distill.py":"14640f91ec9c91d10a405fa56eb21bda8b55b285854ce2b5859a71d1ffd06ea1","arm_env.py":"e4af9b75364a568284b1e6d96469deacf498dd87a60b067fa5e310fe15c54c00","arm_policy.py":"d6d53d87703fb8608390092309285afb6a2c8fd3103a7abd77668be85469eab3","arm_runner.py":"affb5812b6a395f6cca9cb4e900af2ef28162e7ff1e52a51bd29b789daa98df1","capsule_installer.py":"60bb1652cae830fd03ae6eaa5e10150c957be13e1dc630a5631c1a2692ae37ec","cli.py":"192ec6a0852c7fd5dd3ae912815dae2fc7c9cfb93b97c85d528060e00a975b17","env.py":"929c1d8c3c4cafe0054c633ee5f3e51c551dd6b26ae014ab1ea9219b58eca8e4","installer.py":"d3a2c5a4913920c52beef89df9954494b89fd0becb08bb4847264dae48d3761c","skill_capsule.py":"ecb7dbd60d3a11f40bf65c4520f024b449c0d63f9ae27c601c9c9fe17b32e6a0","skillspace.py":"34b651609358ae9a58b9a610545ce7b85b83fb3d6c699613414ff8e0ea457b3c"},"kind":"hybrid-distilled-placement-runtime"},"scenes":[{"group":"nominal","id":"scene-00","layout_sha256":"24b72cdd2c0cb7f15ab37b50d7dad354b5853c5a8d963ec6060637975ea4cb56","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.02,"object_position":[0.512199,0.020606,0.305],"object_size":[0.035,0.03,0.03],"seed":19047,"target_position":[0.345408,-0.134129,0.305]},"scene_sha256":"475e17e2384897fda253446c24cc607833d967bfa50e9139845615e393e5cb79"},{"group":"nominal","id":"scene-01","layout_sha256":"00d006be928d8da35ebbcbc6715095e53122fa99b7fe3571f4861cf324e47e80","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.4,"object_mass":0.04,"object_position":[0.365344,-0.01378,0.305],"object_size":[0.04,0.045,0.03],"seed":19048,"target_position":[0.335159,0.154923,0.305]},"scene_sha256":"7e4e4e2cd26ff45516d92d33c2b0197fd065ba62c7b3b29bbeea7fa527b5d411"},{"group":"nominal","id":"scene-02","layout_sha256":"c1732804a4639163df340b664074c20f9f6e4d66773c1afd73cd303c13f4ca1a","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.4,"object_mass":0.08,"object_position":[0.397733,-0.101964,0.305],"object_size":[0.04,0.035,0.03],"seed":19049,"target_position":[0.442884,0.007667,0.305]},"scene_sha256":"143843fd9c15aec4d7069ad803765e8c7d37aaa063bd1bcc420ada2b725211b5"},{"group":"nominal","id":"scene-03","layout_sha256":"57757e9c51b94a463bcfb40656a6e83a4ab87ada987fa5c34526557db4501964","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":1.5,"object_mass":0.08,"object_position":[0.496833,-0.097782,0.3075],"object_size":[0.04,0.045,0.035],"seed":19050,"target_position":[0.303898,-0.160651,0.3075]},"scene_sha256":"3c66ce9494f44d0cd1880a7e6dfdf51ab53758a026a29bc6076b8ce624edfa2e"},{"group":"nominal","id":"scene-04","layout_sha256":"1cfabe70dd6799df3d859aeffcd86655a10016d302a9aeb87fe103ebfee00c35","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.4,"object_mass":0.06,"object_position":[0.314577,0.048273,0.31],"object_size":[0.05,0.04,0.04],"seed":19051,"target_position":[0.453724,-0.159947,0.31]},"scene_sha256":"9dd2ff2faead021108c63ac86dcb352d93fa31eda3bcd9b1413b9fe350ae4898"},{"group":"nominal","id":"scene-05","layout_sha256":"51f20a7a98fc248021f12f9ea26a54df76dacd84246e3f4b6879a5924a276a2a","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.4,"object_mass":0.04,"object_position":[0.378142,0.019372,0.3125],"object_size":[0.035,0.035,0.045],"seed":19052,"target_position":[0.425134,-0.137639,0.3125]},"scene_sha256":"84d5d3a2053c61a65249fb9f551b17f7665821bbd7ea73e64d8d9a5ef130b746"},{"group":"nominal","id":"scene-06","layout_sha256":"158a6aa35379023ae6db9ac61ee4abf0723e1efd4f329ea756198498c2e849a3","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":1.2,"object_mass":0.04,"object_position":[0.350677,0.161117,0.31],"object_size":[0.03,0.05,0.04],"seed":19053,"target_position":[0.337214,-0.154152,0.31]},"scene_sha256":"ac34ce607dbb0223912f6ef20a290530ed2e418819cc306602e5360d462bc6ee"},{"group":"nominal","id":"scene-07","layout_sha256":"bd154673d7f06407585b2b41ff53840b5aa2b571e4a2678dd9b0f12dcfcaf95f","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":1.5,"object_mass":0.08,"object_position":[0.308186,-0.010576,0.305],"object_size":[0.05,0.03,0.03],"seed":19054,"target_position":[0.317993,-0.15223,0.305]},"scene_sha256":"bc97d7f83474f39d4f43750fdf5058403f948fc2cf614b0f9cfee1e7ff32c5a9"},{"group":"actuator_stress","id":"scene-08","layout_sha256":"5c8d77d75bd8be39bc7d8daeff73c4ba5a3acb029f3b729ae10a61d841470e5a","scene":{"jaw_force_limit":0.3,"joint_torque_limit":5.0,"object_friction":0.8,"object_mass":0.02,"object_position":[0.455864,0.118741,0.315],"object_size":[0.04,0.035,0.05],"seed":19055,"target_position":[0.313503,-0.139928,0.315]},"scene_sha256":"a9e5b6158f1b7bf4cc9afbb4a2a6f67b078f7ae23885fcadc5466d3728a2dab6"},{"group":"actuator_stress","id":"scene-09","layout_sha256":"eeadb42b28bfc60ac577be96706235391a7bdbca3171099c63f8f105b7a7002c","scene":{"jaw_force_limit":0.3,"joint_torque_limit":5.0,"object_friction":1.2,"object_mass":0.04,"object_position":[0.415324,0.137974,0.31],"object_size":[0.035,0.04,0.04],"seed":19056,"target_position":[0.416452,-0.165466,0.31]},"scene_sha256":"66cec8a3c410f790c326ce9a031202135533aaaf3b72864729e8e870c59e3d62"},{"group":"actuator_stress","id":"scene-10","layout_sha256":"3a61f6a0936cba78b37f78e0be744c02a77a26ee9bb2685734f3eaee100f6077","scene":{"jaw_force_limit":0.3,"joint_torque_limit":5.0,"object_friction":1.2,"object_mass":0.04,"object_position":[0.359419,0.158784,0.3075],"object_size":[0.05,0.035,0.035],"seed":19057,"target_position":[0.334051,-0.099017,0.3075]},"scene_sha256":"4f7aff58d9eaeac570a93839acff8588c7488a02213f26478cdaa6df66cc2404"},{"group":"actuator_stress","id":"scene-11","layout_sha256":"18e4edde94d8ebd1ef50aa76764892336fd92c213e56501b42e8808ce3c5b947","scene":{"jaw_force_limit":0.3,"joint_torque_limit":5.0,"object_friction":1.2,"object_mass":0.08,"object_position":[0.507221,0.021997,0.31],"object_size":[0.03,0.035,0.04],"seed":19058,"target_position":[0.413335,0.148658,0.31]},"scene_sha256":"20c2d7585a0f02738d49fd98403d880bff2c93e88d8f8c5a85eeeb021b74c416"}],"schema_version":1,"scope":"simulation-only","seed":19047,"source_dataset_sha256":"e0edc2a275d465ad58ebe1649e1df69c6ec37c30028f64f1c21c483c0f95cad9","source_scene_exclusion":{"bindings":[{"episode_id":"example-00","evidence_sha256":"b22a5889e43f9cfa2b8db9d60b2b4b67810681f0c9a7543b00723e381fa28d2f","layout_sha256":"632834d98d0b186be42f654ec1fe85f401f1eb81305f8ba1435c11360f066cdc","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.34,-0.11,0.31],"object_size":[0.04,0.04,0.04],"seed":8000,"target_position":[0.44,0.12,0.31]},"split":"dev"},{"episode_id":"example-01","evidence_sha256":"24432975726cd2eec5f8d17065cbb7e02d004e7a78abf2a3f2f9a1641e8ac052","layout_sha256":"66bd2f7c18ad4c729c357715982b8cc127a0c0febb0b07941c7dd322e93fb2a9","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.35300000000000004,-0.08,0.31],"object_size":[0.04,0.04,0.04],"seed":8001,"target_position":[0.452,0.09899999999999999,0.31]},"split":"train"},{"episode_id":"example-02","evidence_sha256":"db4ea00255207ce1702b758f6e33923423d04a0391fca3cd26b6363f1e8b7fb1","layout_sha256":"7fd753722722e199bbd5db8c14df66f4d5313674df5f9aebafe2561c3f8c7cac","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.36600000000000005,-0.05,0.31],"object_size":[0.04,0.04,0.04],"seed":8002,"target_position":[0.464,0.07799999999999999,0.31]},"split":"train"},{"episode_id":"example-03","evidence_sha256":"4930ef61016374bfc117142c1a9e0a432231599dfabb411ab265e5d8f62f4685","layout_sha256":"0fe6f0e7a22aebd252407995b4b05f8aa32f35fc74d19502db1da98ab7d8de74","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.379,-0.020000000000000004,0.31],"object_size":[0.04,0.04,0.04],"seed":8003,"target_position":[0.476,0.056999999999999995,0.31]},"split":"dev"},{"episode_id":"example-04","evidence_sha256":"0402402fe205c87e3bf7c23feaf97b21afb1e875425dc752bb4c9c8fa85fb543","layout_sha256":"f2fd31fd202bf37275f84f8f5112b69fcfde8c12e84888e7994f8e91f95601f8","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.392,0.009999999999999995,0.31],"object_size":[0.04,0.04,0.04],"seed":8004,"target_position":[0.488,0.03599999999999999,0.31]},"split":"train"},{"episode_id":"example-05","evidence_sha256":"3795888d487c8f122f344f6c24395b3012b0efaf48f546487ec82b1e8eaeb130","layout_sha256":"77a290098abe061d87a761802cec8da62b46485d523a52e4af1bd34041737f85","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.405,0.039999999999999994,0.31],"object_size":[0.04,0.04,0.04],"seed":8005,"target_position":[0.5,0.014999999999999986,0.31]},"split":"train"},{"episode_id":"example-06","evidence_sha256":"2055437ae141a1095c7e78fe14967b8151a7e0750b39f6eb90fcda3d241234f1","layout_sha256":"fa51312086ad8927786ae3544709c39e92b84322b5e3dfe124dfe21d6f1b13ba","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.34,0.06999999999999999,0.31],"object_size":[0.04,0.04,0.04],"seed":8006,"target_position":[0.44,-0.006000000000000005,0.31]},"split":"test"},{"episode_id":"example-07","evidence_sha256":"15c83e62bddb98d68297ca85f212f0f586437490f114bf1eac570484964f7600","layout_sha256":"15abf37dcef32b55bf6a06a77b0cfd8b462b4165404e68202e0207280504f36e","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.35300000000000004,-0.11,0.31],"object_size":[0.04,0.04,0.04],"seed":8007,"target_position":[0.452,0.12,0.31]},"split":"train"},{"episode_id":"example-08","evidence_sha256":"7fb5f068e0f6ab833b49c36296164ceb0d534ba7fbf656fd411de787642fb2b2","layout_sha256":"c52bb34ab83c357446a7bfbc26b8445b1603715d99114cca08ec2ff37d570e4a","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.36600000000000005,-0.08,0.31],"object_size":[0.04,0.04,0.04],"seed":8008,"target_position":[0.464,0.09899999999999999,0.31]},"split":"train"},{"episode_id":"example-09","evidence_sha256":"06eb182c2deaae1728930de055f641fb5de80bb332b9238915ee9ff10c586c8c","layout_sha256":"11f7c0ae1f8ede87df9e622255511e69548e1e45ba8dc1a13ef5797d26f6c42c","scene":{"jaw_force_limit":15.0,"joint_torque_limit":40.0,"object_friction":0.8,"object_mass":0.04,"object_position":[0.379,-0.05,0.31],"object_size":[0.04,0.04,0.04],"seed":8009,"target_position":[0.476,0.07799999999999999,0.31]},"split":"train"}],"limits":"Hashes and supplier scene declarations check consistency, not independent capture attestation.","status":"excluded_hash_bound_native_declarations"},"subsets":[{"dataset_path":"subsets/groups-002/dataset.json","dev_episode_ids":["example-00","example-03"],"test_episode_ids":["example-06"],"train_episode_ids":["example-02","example-04"],"train_group_ids":["group-0a99eb58f0a04c1c272d5742","group-263dda07ae1e06d3275961fc"],"training_group_count":2},{"dataset_path":"subsets/groups-004/dataset.json","dev_episode_ids":["example-00","example-03"],"test_episode_ids":["example-06"],"train_episode_ids":["example-01","example-02","example-04","example-09"],"train_group_ids":["group-0a99eb58f0a04c1c272d5742","group-263dda07ae1e06d3275961fc","group-2127b070eb84b4c744b61f97","group-c2c47e374effedd5ba7944ca"],"training_group_count":4},{"dataset_path":"subsets/groups-007/dataset.json","dev_episode_ids":["example-00","example-03"],"test_episode_ids":["example-06"],"train_episode_ids":["example-01","example-02","example-04","example-05","example-07","example-08","example-09"],"train_group_ids":["group-0a99eb58f0a04c1c272d5742","group-263dda07ae1e06d3275961fc","group-2127b070eb84b4c744b61f97","group-c2c47e374effedd5ba7944ca","group-fef08eda11494994ea2b90bd","group-86e23892ec31bc5a44ec2e88","group-df7454102682828fa9d9ae2b"],"training_group_count":7}],"training":{"frames_per_clip":12,"latent_dim":8,"movement_centers":8,"regularization_grid":[1e-06,0.0001,0.01,1.0],"resolution":16,"selection":"The same fixed development episodes select regularization for each count."}} diff --git a/benchmarks/sample-count-19047/protocol.sha256 b/benchmarks/sample-count-19047/protocol.sha256 new file mode 100644 index 0000000..01089a3 --- /dev/null +++ b/benchmarks/sample-count-19047/protocol.sha256 @@ -0,0 +1 @@ +1e6d475aed2b505ec1ed4ef3990c723b7f9171863647bbfeb01610ba9c8c8eec diff --git a/benchmarks/sample-count-19047/report.json b/benchmarks/sample-count-19047/report.json new file mode 100644 index 0000000..f46faf9 --- /dev/null +++ b/benchmarks/sample-count-19047/report.json @@ -0,0 +1 @@ +{"analysis_scope":{"composition":"Single composition-order pilot: count is confounded with which training groups are added.","modality":"The movement candidate learns aligned numerical actions; the RGB predictor does not control the arm.","novelty":"Only exact source object/target layout exclusion is checked relative to supplied hash-bound native declarations; no out-of-distribution or human-transfer claim.","scene_independence":"The same distinct benchmark scenes are reused for every count; condition-by-scene runs are not independent scenes.","stratification":"Nominal and actuator-stress outcomes are reported separately. Combined counts and clocks are execution totals, not a pooled performance estimate."},"benchmark_wall_seconds":102.72678875015117,"comparisons":[{"capsule_file_sha256":"c998cab9a704fb37f5e60493eafc52c20b285d579065c3e4fbeaf182079c95d3","capsule_path":"candidates/groups-002/candidate.skill-capsule.json","installation_id":"70a8dd0e999c5675a2845b53","movement_test":{"episodes":1,"learned_joint_delta_mse":1.120344178000201e-06,"samples":621,"train_mean_delta_mse":7.834385571673985e-05,"zero_delta_mse":8.172739843340321e-05},"outcomes":[{"case_id":"scene-00","control_episode_wall_seconds":0.9864663751795888,"control_final_target_error_m":0.2275129830525695,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.24694733940837,"control_success":false,"controls_path":"qualification/groups-002/scene-00/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-00/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-00/run.json","scene_sha256":"475e17e2384897fda253446c24cc607833d967bfa50e9139845615e393e5cb79","status":"completed","student_episode_wall_seconds":0.9328550829086453,"student_final_target_error_m":0.00047787544055272294,"student_reason":"success","student_simulated_seconds":13.398999999998013,"student_simulated_seconds_per_wall_second":14.363431411254025,"student_success":true},{"case_id":"scene-01","control_episode_wall_seconds":1.0290995419491082,"control_final_target_error_m":0.17138213857242646,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.49101934872804,"control_success":false,"controls_path":"qualification/groups-002/scene-01/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-01/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-01/run.json","scene_sha256":"7e4e4e2cd26ff45516d92d33c2b0197fd065ba62c7b3b29bbeea7fa527b5d411","status":"completed","student_episode_wall_seconds":0.9153490841854364,"student_final_target_error_m":0.0010932362332275432,"student_reason":"success","student_simulated_seconds":12.532999999998493,"student_simulated_seconds_per_wall_second":13.692044069887865,"student_success":true},{"case_id":"scene-02","control_episode_wall_seconds":0.9929166659712791,"control_final_target_error_m":0.11856461703898681,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.128409580466926,"control_success":false,"controls_path":"qualification/groups-002/scene-02/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-02/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-02/run.json","scene_sha256":"143843fd9c15aec4d7069ad803765e8c7d37aaa063bd1bcc420ada2b725211b5","status":"completed","student_episode_wall_seconds":0.8250070831272751,"student_final_target_error_m":0.0005344236240047005,"student_reason":"success","student_simulated_seconds":10.165999999999805,"student_simulated_seconds_per_wall_second":12.322318447818079,"student_success":true},{"case_id":"scene-03","control_episode_wall_seconds":1.0176400411874056,"control_final_target_error_m":0.202919750549968,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.6879832469998,"control_success":false,"controls_path":"qualification/groups-002/scene-03/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-03/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-03/run.json","scene_sha256":"3c66ce9494f44d0cd1880a7e6dfdf51ab53758a026a29bc6076b8ce624edfa2e","status":"completed","student_episode_wall_seconds":0.9167484999634326,"student_final_target_error_m":0.00015317354311940795,"student_reason":"success","student_simulated_seconds":12.242999999998654,"student_simulated_seconds_per_wall_second":13.354807780418517,"student_success":true},{"case_id":"scene-04","control_episode_wall_seconds":1.0070810839533806,"control_final_target_error_m":0.25043453186604464,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.873436694231728,"control_success":false,"controls_path":"qualification/groups-002/scene-04/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-04/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-04/run.json","scene_sha256":"9dd2ff2faead021108c63ac86dcb352d93fa31eda3bcd9b1413b9fe350ae4898","status":"completed","student_episode_wall_seconds":1.0329305829945952,"student_final_target_error_m":0.0005042642032832402,"student_reason":"success","student_simulated_seconds":14.029999999997663,"student_simulated_seconds_per_wall_second":13.582713331347916,"student_success":true},{"case_id":"scene-05","control_episode_wall_seconds":1.0062469588592649,"control_final_target_error_m":0.16389234629153743,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.88825282056482,"control_success":false,"controls_path":"qualification/groups-002/scene-05/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-05/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-05/run.json","scene_sha256":"84d5d3a2053c61a65249fb9f551b17f7665821bbd7ea73e64d8d9a5ef130b746","status":"completed","student_episode_wall_seconds":0.8706468748860061,"student_final_target_error_m":0.0009830181538191193,"student_reason":"success","student_simulated_seconds":12.460999999998533,"student_simulated_seconds_per_wall_second":14.312346784257455,"student_success":true},{"case_id":"scene-06","control_episode_wall_seconds":0.9911459169816226,"control_final_target_error_m":0.31555632491116936,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.160797206142114,"control_success":false,"controls_path":"qualification/groups-002/scene-06/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-06/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-06/run.json","scene_sha256":"ac34ce607dbb0223912f6ef20a290530ed2e418819cc306602e5360d462bc6ee","status":"completed","student_episode_wall_seconds":1.2890703750308603,"student_final_target_error_m":0.0009092701355944005,"student_reason":"success","student_simulated_seconds":17.434999999998325,"student_simulated_seconds_per_wall_second":13.525250706022106,"student_success":true},{"case_id":"scene-07","control_episode_wall_seconds":1.0150966250803322,"control_final_target_error_m":0.14199307695541274,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.732302083631243,"control_success":false,"controls_path":"qualification/groups-002/scene-07/controls.json","group":"nominal","qualification_path":"qualification/groups-002/scene-07/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-07/run.json","scene_sha256":"bc97d7f83474f39d4f43750fdf5058403f948fc2cf614b0f9cfee1e7ff32c5a9","status":"completed","student_episode_wall_seconds":0.8479621671140194,"student_final_target_error_m":0.0007645987314948491,"student_reason":"success","student_simulated_seconds":12.130999999998716,"student_simulated_seconds_per_wall_second":14.306062782595285,"student_success":true},{"case_id":"scene-08","control_episode_wall_seconds":1.0475559579208493,"control_final_target_error_m":0.2952563357562951,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.182852967324777,"control_success":false,"controls_path":"qualification/groups-002/scene-08/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-002/scene-08/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-08/run.json","scene_sha256":"a9e5b6158f1b7bf4cc9afbb4a2a6f67b078f7ae23885fcadc5466d3728a2dab6","status":"completed","student_episode_wall_seconds":1.1784365000203252,"student_final_target_error_m":0.0023149570431931555,"student_reason":"success","student_simulated_seconds":16.00599999999658,"student_simulated_seconds_per_wall_second":13.582403464013984,"student_success":true},{"case_id":"scene-09","control_episode_wall_seconds":0.9894085000269115,"control_final_target_error_m":0.3034420930583171,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.19268785290345,"control_success":false,"controls_path":"qualification/groups-002/scene-09/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-002/scene-09/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-09/run.json","scene_sha256":"66cec8a3c410f790c326ce9a031202135533aaaf3b72864729e8e870c59e3d62","status":"completed","student_episode_wall_seconds":1.1757019171491265,"student_final_target_error_m":0.003626524341898727,"student_reason":"success","student_simulated_seconds":16.085999999996677,"student_simulated_seconds_per_wall_second":13.68203943989684,"student_success":true},{"case_id":"scene-10","control_episode_wall_seconds":1.0057518330868334,"control_final_target_error_m":0.25904611838839087,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.897059103291692,"control_success":false,"controls_path":"qualification/groups-002/scene-10/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-002/scene-10/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-002/scene-10/run.json","scene_sha256":"4f7aff58d9eaeac570a93839acff8588c7488a02213f26478cdaa6df66cc2404","status":"completed","student_episode_wall_seconds":1.3028882921207696,"student_final_target_error_m":0.0022274781654255406,"student_reason":"success","student_simulated_seconds":16.362999999997015,"student_simulated_seconds_per_wall_second":12.55901990903781,"student_success":true},{"case_id":"scene-11","control_episode_wall_seconds":1.0130176672246307,"control_final_target_error_m":0.1576629029336082,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.768693066640882,"control_success":false,"controls_path":"qualification/groups-002/scene-11/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-002/scene-11/qualification.json","qualification_status":"failed","qualified":false,"run_path":"qualification/groups-002/scene-11/run.json","scene_sha256":"20c2d7585a0f02738d49fd98403d880bff2c93e88d8f8c5a85eeeb021b74c416","status":"completed","student_episode_wall_seconds":1.1889005419798195,"student_final_target_error_m":0.1576629029336082,"student_reason":"horizon","student_simulated_seconds":17.999999999999016,"student_simulated_seconds_per_wall_second":15.14003851829731,"student_success":false}],"paired_with_teacher":{"both_failure":1,"both_success":11,"errors":0,"student_only":0,"teacher_only":0},"paired_with_teacher_by_group":{"actuator_stress":{"both_failure":1,"both_success":3,"errors":0,"student_only":0,"teacher_only":0},"nominal":{"both_failure":0,"both_success":8,"errors":0,"student_only":0,"teacher_only":0}},"summary":{"control_episode_wall_seconds":12.101427167421207,"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.2173052682812272,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":17.84913440469992,"timed_scenes":12,"total_episode_wall_seconds":12.101427167421207,"total_simulated_seconds":215.99999999998818},"control_successes":0,"errors":0,"groups":{"actuator_stress":{"control_measurements":{"final_target_error_max_m":0.3034420930583171,"final_target_error_mean_m":0.2538518625341528,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":17.752643723924,"timed_scenes":4,"total_episode_wall_seconds":4.055733958259225,"total_simulated_seconds":71.99999999999606},"control_successes":0,"errors":0,"planned_scenes":4,"qualified_scenes":3,"student_measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.041457965621031406,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":13.713577722936819,"timed_scenes":4,"total_episode_wall_seconds":4.845927251270041,"total_simulated_seconds":66.45499999998928},"student_successes":3},"nominal":{"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.19903197115476437,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":17.89777415773361,"timed_scenes":8,"total_episode_wall_seconds":8.045693209161982,"total_simulated_seconds":143.99999999999213},"control_successes":0,"errors":0,"planned_scenes":8,"qualified_scenes":8,"student_measurements":{"final_target_error_max_m":0.0010932362332275432,"final_target_error_mean_m":0.000677482508136998,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":13.681547173736453,"timed_scenes":8,"total_episode_wall_seconds":7.63056975021027,"total_simulated_seconds":104.3979999999882},"student_successes":8}},"planned_scenes":12,"qualified_scenes":11,"student_episode_wall_seconds":12.476497001480311,"student_measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.014270976879101801,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":13.693987982340406,"timed_scenes":12,"total_episode_wall_seconds":12.476497001480311,"total_simulated_seconds":170.8529999999775},"student_successes":11},"training_action_samples":1135,"training_episode_count":2,"training_group_count":2,"training_pipeline_seconds":0.5916076248977333,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.11391330667979953,"learned_rgb_mse":0.0024406688519574495,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.11391330667979953,"learned_rgb_mse":0.0024406688519574495,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003992404562246332,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003992404562246332,"transitions":11}},{"capsule_file_sha256":"32c62628ed40064de57e3bcb3e32c041b16b268a48dd59307c32f778c3b5ec4d","capsule_path":"candidates/groups-004/candidate.skill-capsule.json","installation_id":"0bd068b6b2ecc0d06deb1d49","movement_test":{"episodes":1,"learned_joint_delta_mse":1.1087842308134674e-06,"samples":621,"train_mean_delta_mse":7.839595075163347e-05,"zero_delta_mse":8.172739843340321e-05},"outcomes":[{"case_id":"scene-00","control_episode_wall_seconds":1.0255248330067843,"control_final_target_error_m":0.2275129830525719,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.5519884264762,"control_success":false,"controls_path":"qualification/groups-004/scene-00/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-00/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-00/run.json","scene_sha256":"475e17e2384897fda253446c24cc607833d967bfa50e9139845615e393e5cb79","status":"completed","student_episode_wall_seconds":0.975333167007193,"student_final_target_error_m":0.00029304442976889,"student_reason":"success","student_simulated_seconds":13.360999999998034,"student_simulated_seconds_per_wall_second":13.698908692909752,"student_success":true},{"case_id":"scene-01","control_episode_wall_seconds":0.9994049579836428,"control_final_target_error_m":0.17138213857242646,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.010717133438135,"control_success":false,"controls_path":"qualification/groups-004/scene-01/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-01/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-01/run.json","scene_sha256":"7e4e4e2cd26ff45516d92d33c2b0197fd065ba62c7b3b29bbeea7fa527b5d411","status":"completed","student_episode_wall_seconds":0.9384836670942605,"student_final_target_error_m":0.0005155264457278859,"student_reason":"success","student_simulated_seconds":12.618999999998445,"student_simulated_seconds_per_wall_second":13.4461583535806,"student_success":true},{"case_id":"scene-02","control_episode_wall_seconds":1.0290954171214253,"control_final_target_error_m":0.11856461703898681,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.491089456358115,"control_success":false,"controls_path":"qualification/groups-004/scene-02/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-02/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-02/run.json","scene_sha256":"143843fd9c15aec4d7069ad803765e8c7d37aaa063bd1bcc420ada2b725211b5","status":"completed","student_episode_wall_seconds":0.7152245421893895,"student_final_target_error_m":0.0004457606398904108,"student_reason":"success","student_simulated_seconds":10.092999999999845,"student_simulated_seconds_per_wall_second":14.11165222197765,"student_success":true},{"case_id":"scene-03","control_episode_wall_seconds":0.9979065000079572,"control_final_target_error_m":0.202919750549968,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.037762054717035,"control_success":false,"controls_path":"qualification/groups-004/scene-03/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-03/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-03/run.json","scene_sha256":"3c66ce9494f44d0cd1880a7e6dfdf51ab53758a026a29bc6076b8ce624edfa2e","status":"completed","student_episode_wall_seconds":0.8866178749594837,"student_final_target_error_m":0.00018120280206082543,"student_reason":"success","student_simulated_seconds":12.180999999998688,"student_simulated_seconds_per_wall_second":13.738725942735284,"student_success":true},{"case_id":"scene-04","control_episode_wall_seconds":0.9997394590172917,"control_final_target_error_m":0.25043453186604464,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":18.004690959875063,"control_success":false,"controls_path":"qualification/groups-004/scene-04/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-04/qualification.json","qualification_status":"failed","qualified":false,"run_path":"qualification/groups-004/scene-04/run.json","scene_sha256":"9dd2ff2faead021108c63ac86dcb352d93fa31eda3bcd9b1413b9fe350ae4898","status":"completed","student_episode_wall_seconds":1.2128829590510577,"student_final_target_error_m":0.25043453186604464,"student_reason":"horizon","student_simulated_seconds":17.999999999999016,"student_simulated_seconds_per_wall_second":14.840673509076225,"student_success":false},{"case_id":"scene-05","control_episode_wall_seconds":1.1448569998610765,"control_final_target_error_m":0.16389234629153743,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":15.722487613896966,"control_success":false,"controls_path":"qualification/groups-004/scene-05/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-05/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-05/run.json","scene_sha256":"84d5d3a2053c61a65249fb9f551b17f7665821bbd7ea73e64d8d9a5ef130b746","status":"completed","student_episode_wall_seconds":0.926846917020157,"student_final_target_error_m":0.0005472133552983627,"student_reason":"success","student_simulated_seconds":12.911999999998283,"student_simulated_seconds_per_wall_second":13.931103144315118,"student_success":true},{"case_id":"scene-06","control_episode_wall_seconds":1.0348108750768006,"control_final_target_error_m":0.31555632491116936,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.394482831138696,"control_success":false,"controls_path":"qualification/groups-004/scene-06/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-06/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-06/run.json","scene_sha256":"ac34ce607dbb0223912f6ef20a290530ed2e418819cc306602e5360d462bc6ee","status":"completed","student_episode_wall_seconds":1.262525541940704,"student_final_target_error_m":0.0004225180646386118,"student_reason":"success","student_simulated_seconds":17.555999999998473,"student_simulated_seconds_per_wall_second":13.90546124953012,"student_success":true},{"case_id":"scene-07","control_episode_wall_seconds":1.0128609160892665,"control_final_target_error_m":0.14199307695541274,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.7714429632633,"control_success":false,"controls_path":"qualification/groups-004/scene-07/controls.json","group":"nominal","qualification_path":"qualification/groups-004/scene-07/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-07/run.json","scene_sha256":"bc97d7f83474f39d4f43750fdf5058403f948fc2cf614b0f9cfee1e7ff32c5a9","status":"completed","student_episode_wall_seconds":0.9243724578991532,"student_final_target_error_m":0.0002860877117699497,"student_reason":"success","student_simulated_seconds":13.047999999998208,"student_simulated_seconds_per_wall_second":14.115522253500236,"student_success":true},{"case_id":"scene-08","control_episode_wall_seconds":1.0468113750685006,"control_final_target_error_m":0.2952563357562783,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.195074899545435,"control_success":false,"controls_path":"qualification/groups-004/scene-08/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-004/scene-08/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-08/run.json","scene_sha256":"a9e5b6158f1b7bf4cc9afbb4a2a6f67b078f7ae23885fcadc5466d3728a2dab6","status":"completed","student_episode_wall_seconds":1.1764931669458747,"student_final_target_error_m":0.002352153016650641,"student_reason":"success","student_simulated_seconds":15.883999999996636,"student_simulated_seconds_per_wall_second":13.501140887397428,"student_success":true},{"case_id":"scene-09","control_episode_wall_seconds":1.0072949170134962,"control_final_target_error_m":0.3034420930583171,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.869642441328676,"control_success":false,"controls_path":"qualification/groups-004/scene-09/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-004/scene-09/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-09/run.json","scene_sha256":"66cec8a3c410f790c326ce9a031202135533aaaf3b72864729e8e870c59e3d62","status":"completed","student_episode_wall_seconds":1.1833238748367876,"student_final_target_error_m":0.00305607451861461,"student_reason":"success","student_simulated_seconds":16.053999999996638,"student_simulated_seconds_per_wall_second":13.56686900465937,"student_success":true},{"case_id":"scene-10","control_episode_wall_seconds":1.0370904998853803,"control_final_target_error_m":0.25904611838839087,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.356248082484974,"control_success":false,"controls_path":"qualification/groups-004/scene-10/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-004/scene-10/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-004/scene-10/run.json","scene_sha256":"4f7aff58d9eaeac570a93839acff8588c7488a02213f26478cdaa6df66cc2404","status":"completed","student_episode_wall_seconds":1.2258898338768631,"student_final_target_error_m":0.002396864296420193,"student_reason":"success","student_simulated_seconds":16.403999999997065,"student_simulated_seconds_per_wall_second":13.381300298510181,"student_success":true},{"case_id":"scene-11","control_episode_wall_seconds":1.0209977920167148,"control_final_target_error_m":0.1576629029336082,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.62981285634782,"control_success":false,"controls_path":"qualification/groups-004/scene-11/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-004/scene-11/qualification.json","qualification_status":"failed","qualified":false,"run_path":"qualification/groups-004/scene-11/run.json","scene_sha256":"20c2d7585a0f02738d49fd98403d880bff2c93e88d8f8c5a85eeeb021b74c416","status":"completed","student_episode_wall_seconds":1.1392408749088645,"student_final_target_error_m":0.1576629029336082,"student_reason":"horizon","student_simulated_seconds":17.999999999999016,"student_simulated_seconds_per_wall_second":15.799994888209183,"student_success":false}],"paired_with_teacher":{"both_failure":1,"both_success":10,"errors":0,"student_only":0,"teacher_only":1},"paired_with_teacher_by_group":{"actuator_stress":{"both_failure":1,"both_success":3,"errors":0,"student_only":0,"teacher_only":0},"nominal":{"both_failure":0,"both_success":7,"errors":0,"student_only":0,"teacher_only":1}},"summary":{"control_episode_wall_seconds":12.356394542148337,"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.21730526828122598,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":17.480827377533178,"timed_scenes":12,"total_episode_wall_seconds":12.356394542148337,"total_simulated_seconds":215.99999999998818},"control_successes":0,"errors":0,"groups":{"actuator_stress":{"control_measurements":{"final_target_error_max_m":0.3034420930583171,"final_target_error_mean_m":0.2538518625341486,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":17.50889908770781,"timed_scenes":4,"total_episode_wall_seconds":4.112194583984092,"total_simulated_seconds":71.99999999999606},"control_successes":0,"errors":0,"planned_scenes":4,"qualified_scenes":3,"student_measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.04136699869132342,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":14.040790184824523,"timed_scenes":4,"total_episode_wall_seconds":4.72494775056839,"total_simulated_seconds":66.34199999998935},"student_successes":3},"nominal":{"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.19903197115476468,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":17.466825250567666,"timed_scenes":8,"total_episode_wall_seconds":8.244199958164245,"total_simulated_seconds":143.99999999999213},"control_successes":0,"errors":0,"planned_scenes":8,"qualified_scenes":7,"student_measurements":{"final_target_error_max_m":0.25043453186604464,"final_target_error_mean_m":0.03164073566439995,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":13.997192173671591,"timed_scenes":8,"total_episode_wall_seconds":7.8422871271613985,"total_simulated_seconds":109.769999999989},"student_successes":7}},"planned_scenes":12,"qualified_scenes":10,"student_episode_wall_seconds":12.567234877729788,"student_measurements":{"final_target_error_max_m":0.25043453186604464,"final_target_error_mean_m":0.03488282334004111,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":14.013583872142297,"timed_scenes":12,"total_episode_wall_seconds":12.567234877729788,"total_simulated_seconds":176.11199999997834},"student_successes":10},"training_action_samples":2281,"training_episode_count":4,"training_group_count":4,"training_pipeline_seconds":0.8397391671314836,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.11912927941122996,"learned_rgb_mse":0.002360370868993712,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.11912927941122996,"learned_rgb_mse":0.002360370868993712,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003866846205347564,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003866846205347564,"transitions":11}},{"capsule_file_sha256":"712eb4ed16939294b0898d8e6f39500201f0decf31c18122575de593887bf15a","capsule_path":"candidates/groups-007/candidate.skill-capsule.json","installation_id":"205924e092b684a9e4c67ad6","movement_test":{"episodes":1,"learned_joint_delta_mse":1.3251631022866396e-06,"samples":621,"train_mean_delta_mse":7.842909795405035e-05,"zero_delta_mse":8.172739843340321e-05},"outcomes":[{"case_id":"scene-00","control_episode_wall_seconds":1.0522389591205865,"control_final_target_error_m":0.22751298305258918,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.106380488936274,"control_success":false,"controls_path":"qualification/groups-007/scene-00/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-00/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-00/run.json","scene_sha256":"475e17e2384897fda253446c24cc607833d967bfa50e9139845615e393e5cb79","status":"completed","student_episode_wall_seconds":1.0187591251451522,"student_final_target_error_m":0.00020602498085761276,"student_reason":"success","student_simulated_seconds":13.72799999999783,"student_simulated_seconds_per_wall_second":13.475216723130577,"student_success":true},{"case_id":"scene-01","control_episode_wall_seconds":1.0250907498411834,"control_final_target_error_m":0.17138213857242646,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.55942096130293,"control_success":false,"controls_path":"qualification/groups-007/scene-01/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-01/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-01/run.json","scene_sha256":"7e4e4e2cd26ff45516d92d33c2b0197fd065ba62c7b3b29bbeea7fa527b5d411","status":"completed","student_episode_wall_seconds":0.9042723751626909,"student_final_target_error_m":0.00023687303671160166,"student_reason":"success","student_simulated_seconds":12.563999999998476,"student_simulated_seconds_per_wall_second":13.894043813666254,"student_success":true},{"case_id":"scene-02","control_episode_wall_seconds":1.199300125008449,"control_final_target_error_m":0.11856461703898681,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":15.00875354271493,"control_success":false,"controls_path":"qualification/groups-007/scene-02/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-02/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-02/run.json","scene_sha256":"143843fd9c15aec4d7069ad803765e8c7d37aaa063bd1bcc420ada2b725211b5","status":"completed","student_episode_wall_seconds":0.7511838749051094,"student_final_target_error_m":0.0003804395290101563,"student_reason":"success","student_simulated_seconds":10.19199999999979,"student_simulated_seconds_per_wall_second":13.567916379045354,"student_success":true},{"case_id":"scene-03","control_episode_wall_seconds":1.0244278330355883,"control_final_target_error_m":0.202919750549968,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.5707838263837,"control_success":false,"controls_path":"qualification/groups-007/scene-03/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-03/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-03/run.json","scene_sha256":"3c66ce9494f44d0cd1880a7e6dfdf51ab53758a026a29bc6076b8ce624edfa2e","status":"completed","student_episode_wall_seconds":0.9355659999419004,"student_final_target_error_m":0.0001355685086806787,"student_reason":"success","student_simulated_seconds":12.59199999999846,"student_simulated_seconds_per_wall_second":13.45923216617581,"student_success":true},{"case_id":"scene-04","control_episode_wall_seconds":1.0113932499662042,"control_final_target_error_m":0.25043453186604464,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.79723169064109,"control_success":false,"controls_path":"qualification/groups-007/scene-04/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-04/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-04/run.json","scene_sha256":"9dd2ff2faead021108c63ac86dcb352d93fa31eda3bcd9b1413b9fe350ae4898","status":"completed","student_episode_wall_seconds":1.0899150827899575,"student_final_target_error_m":0.0004827575304041458,"student_reason":"success","student_simulated_seconds":14.237999999997548,"student_simulated_seconds_per_wall_second":13.063403034621016,"student_success":true},{"case_id":"scene-05","control_episode_wall_seconds":1.0344085418619215,"control_final_target_error_m":0.16389234629153743,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.40124841544643,"control_success":false,"controls_path":"qualification/groups-007/scene-05/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-05/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-05/run.json","scene_sha256":"84d5d3a2053c61a65249fb9f551b17f7665821bbd7ea73e64d8d9a5ef130b746","status":"completed","student_episode_wall_seconds":0.9386573750525713,"student_final_target_error_m":0.00028924933930814023,"student_reason":"success","student_simulated_seconds":12.820999999998334,"student_simulated_seconds_per_wall_second":13.658871001019161,"student_success":true},{"case_id":"scene-06","control_episode_wall_seconds":1.0170817920006812,"control_final_target_error_m":0.31555632491116936,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.697691711294503,"control_success":false,"controls_path":"qualification/groups-007/scene-06/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-06/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-06/run.json","scene_sha256":"ac34ce607dbb0223912f6ef20a290530ed2e418819cc306602e5360d462bc6ee","status":"completed","student_episode_wall_seconds":1.266663874965161,"student_final_target_error_m":0.00020186653579116234,"student_reason":"success","student_simulated_seconds":17.526999999998438,"student_simulated_seconds_per_wall_second":13.837135759856189,"student_success":true},{"case_id":"scene-07","control_episode_wall_seconds":1.0234625830780715,"control_final_target_error_m":0.14199307695541274,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.587355217094384,"control_success":false,"controls_path":"qualification/groups-007/scene-07/controls.json","group":"nominal","qualification_path":"qualification/groups-007/scene-07/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-07/run.json","scene_sha256":"bc97d7f83474f39d4f43750fdf5058403f948fc2cf614b0f9cfee1e7ff32c5a9","status":"completed","student_episode_wall_seconds":0.9081952499691397,"student_final_target_error_m":0.00020528229345727388,"student_reason":"success","student_simulated_seconds":12.600999999998455,"student_simulated_seconds_per_wall_second":13.874769770516455,"student_success":true},{"case_id":"scene-08","control_episode_wall_seconds":1.0471471659839153,"control_final_target_error_m":0.2952563357562766,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.189560918198104,"control_success":false,"controls_path":"qualification/groups-007/scene-08/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-007/scene-08/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-08/run.json","scene_sha256":"a9e5b6158f1b7bf4cc9afbb4a2a6f67b078f7ae23885fcadc5466d3728a2dab6","status":"completed","student_episode_wall_seconds":1.1848265409935266,"student_final_target_error_m":0.002168267471331513,"student_reason":"success","student_simulated_seconds":16.255999999996884,"student_simulated_seconds_per_wall_second":13.720151800756884,"student_success":true},{"case_id":"scene-09","control_episode_wall_seconds":1.0092129579279572,"control_final_target_error_m":0.3034420930583171,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.835680624786377,"control_success":false,"controls_path":"qualification/groups-007/scene-09/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-007/scene-09/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-09/run.json","scene_sha256":"66cec8a3c410f790c326ce9a031202135533aaaf3b72864729e8e870c59e3d62","status":"completed","student_episode_wall_seconds":1.1926772079896182,"student_final_target_error_m":0.0028262699461039345,"student_reason":"success","student_simulated_seconds":16.150999999996756,"student_simulated_seconds_per_wall_second":13.541803173400917,"student_success":true},{"case_id":"scene-10","control_episode_wall_seconds":1.0655337499920279,"control_final_target_error_m":0.25904611838839087,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":16.892942152356685,"control_success":false,"controls_path":"qualification/groups-007/scene-10/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-007/scene-10/qualification.json","qualification_status":"validated_simulation_scene","qualified":true,"run_path":"qualification/groups-007/scene-10/run.json","scene_sha256":"4f7aff58d9eaeac570a93839acff8588c7488a02213f26478cdaa6df66cc2404","status":"completed","student_episode_wall_seconds":1.2062284171115607,"student_final_target_error_m":0.00241545688701494,"student_reason":"success","student_simulated_seconds":16.346999999996996,"student_simulated_seconds_per_wall_second":13.552159581136038,"student_success":true},{"case_id":"scene-11","control_episode_wall_seconds":1.0099192920606583,"control_final_target_error_m":0.1576629029336082,"control_reason":"horizon","control_simulated_seconds":17.999999999999016,"control_simulated_seconds_per_wall_second":17.823206410159248,"control_success":false,"controls_path":"qualification/groups-007/scene-11/controls.json","group":"actuator_stress","qualification_path":"qualification/groups-007/scene-11/qualification.json","qualification_status":"failed","qualified":false,"run_path":"qualification/groups-007/scene-11/run.json","scene_sha256":"20c2d7585a0f02738d49fd98403d880bff2c93e88d8f8c5a85eeeb021b74c416","status":"completed","student_episode_wall_seconds":1.1374621251598,"student_final_target_error_m":0.1576629029336082,"student_reason":"horizon","student_simulated_seconds":17.999999999999016,"student_simulated_seconds_per_wall_second":15.824702732383487,"student_success":false}],"paired_with_teacher":{"both_failure":1,"both_success":11,"errors":0,"student_only":0,"teacher_only":0},"paired_with_teacher_by_group":{"actuator_stress":{"both_failure":1,"both_success":3,"errors":0,"student_only":0,"teacher_only":0},"nominal":{"both_failure":0,"both_success":8,"errors":0,"student_only":0,"teacher_only":0}},"summary":{"control_episode_wall_seconds":12.519216999877244,"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.2173052682812273,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":17.2534751975388,"timed_scenes":12,"total_episode_wall_seconds":12.519216999877244,"total_simulated_seconds":215.99999999998818},"control_successes":0,"errors":0,"groups":{"actuator_stress":{"control_measurements":{"final_target_error_max_m":0.3034420930583171,"final_target_error_mean_m":0.2538518625341482,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":17.42576372840128,"timed_scenes":4,"total_episode_wall_seconds":4.131813165964559,"total_simulated_seconds":71.99999999999606},"control_successes":0,"errors":0,"planned_scenes":4,"qualified_scenes":3,"student_measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.04126822430951465,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":14.139218994576035,"timed_scenes":4,"total_episode_wall_seconds":4.7211942912545055,"total_simulated_seconds":66.75399999998965},"student_successes":3},"nominal":{"control_measurements":{"final_target_error_max_m":0.31555632491116936,"final_target_error_mean_m":0.19903197115476684,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":17.16860221010925,"timed_scenes":8,"total_episode_wall_seconds":8.387403833912686,"total_simulated_seconds":143.99999999999213},"control_successes":0,"errors":0,"planned_scenes":8,"qualified_scenes":8,"student_measurements":{"final_target_error_max_m":0.0004827575304041458,"final_target_error_mean_m":0.00026725771927759643,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":13.600422844242726,"timed_scenes":8,"total_episode_wall_seconds":7.8132129579316825,"total_simulated_seconds":106.26299999998733},"student_successes":8}},"planned_scenes":12,"qualified_scenes":11,"student_episode_wall_seconds":12.534407249186188,"student_measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.013934246582689947,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":13.803365134096016,"timed_scenes":12,"total_episode_wall_seconds":12.534407249186188,"total_simulated_seconds":173.01699999997697},"student_successes":11},"training_action_samples":4029,"training_episode_count":7,"training_group_count":7,"training_pipeline_seconds":1.2524780000094324,"training_pipeline_timing_scope":"Intake, frame decoding and learning; final receipt hashes, run JSON, ZIP creation and integrity verification excluded.","visual_test":{"episodes":1,"learned_latent_mse":0.12449758348668184,"learned_rgb_mse":0.0023421455366137525,"per_episode":[{"episode_id":"example-06","group_id":"group-ba2b7429bec52ca533fd7951","learned_latent_mse":0.12449758348668184,"learned_rgb_mse":0.0023421455366137525,"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003859011472511888,"transitions":11}],"persistence_rgb_mse":0.0035549714230000973,"train_mean_rgb_mse":0.003859011472511888,"transitions":11}}],"declared_source_exact_layout_exclusion_established":true,"kind":"dvidia.footage-sample-count-result","limitations":["Training group counts and numerical action sample counts are different evidence units.","The same frozen source dev/test episodes and benchmark scenes are shared across counts; they are not independent repeated studies.","Scene exclusion is relative to hash-bound supplier declarations when supplied; otherwise source-scene disjointness is unestablished.","Actuator-stress failures can be shared physical/controller limits, not a demonstrated need for more labels.","Privileged native simulator state and authored phases, gripper/contact logic and recovery remain in use.","No raw human-video action bridge, minimum demonstration count, physical robot skill or guaranteed performance is established.","Aggregate results never authorize an untested scene; installed execution still needs its exact current-runtime qualification."],"models_frozen_sha256":"1ca16dbd8d0aa26f36139056b98e7a808800a7edf5b259fc1ffc0f93fff51cc0","physical_robot_ready":false,"protocol_sha256":"1e6d475aed2b505ec1ed4ef3990c723b7f9171863647bbfeb01610ba9c8c8eec","schema_version":1,"scope":"simulation-only","source_scene_exclusion":"excluded_hash_bound_native_declarations","teacher":{"episode_wall_seconds":10.324916874989867,"errors":0,"groups":{"actuator_stress":{"errors":0,"measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.04089109596152566,"measured_error_scenes":4,"measured_scenes":4,"simulated_seconds_per_wall_second":16.188699866338872,"timed_scenes":4,"total_episode_wall_seconds":3.929654668085277,"total_simulated_seconds":63.61599999999005},"planned_scenes":4,"successes":3},"nominal":{"errors":0,"measurements":{"final_target_error_max_m":0.0005302789172681855,"final_target_error_mean_m":0.0003172641866316759,"measured_error_scenes":8,"measured_scenes":8,"simulated_seconds_per_wall_second":15.701154503341211,"timed_scenes":8,"total_episode_wall_seconds":6.39526220690459,"total_simulated_seconds":100.41299999998786},"planned_scenes":8,"successes":8}},"measurements":{"final_target_error_max_m":0.1576629029336082,"final_target_error_mean_m":0.013841874778263005,"measured_error_scenes":12,"measured_scenes":12,"simulated_seconds_per_wall_second":15.88671385793979,"timed_scenes":12,"total_episode_wall_seconds":10.324916874989867,"total_simulated_seconds":164.0289999999779},"outcomes":[{"case_id":"scene-00","episode_wall_seconds":0.8514194579329342,"group":"nominal","reason":"success","run_path":"teacher/scene-00/run.json","scene_sha256":"475e17e2384897fda253446c24cc607833d967bfa50e9139845615e393e5cb79","status":"completed","success":true,"teacher_episode_wall_seconds":0.8514194579329342,"teacher_final_target_error_m":0.00036169360706507633,"teacher_simulated_seconds":13.29699999999807,"teacher_simulated_seconds_per_wall_second":15.61744904477561},{"case_id":"scene-01","episode_wall_seconds":0.742814667057246,"group":"nominal","reason":"success","run_path":"teacher/scene-01/run.json","scene_sha256":"7e4e4e2cd26ff45516d92d33c2b0197fd065ba62c7b3b29bbeea7fa527b5d411","status":"completed","success":true,"teacher_episode_wall_seconds":0.742814667057246,"teacher_final_target_error_m":0.0002093485590012056,"teacher_simulated_seconds":11.836999999998879,"teacher_simulated_seconds_per_wall_second":15.935334242783126},{"case_id":"scene-02","episode_wall_seconds":0.634299207944423,"group":"nominal","reason":"success","run_path":"teacher/scene-02/run.json","scene_sha256":"143843fd9c15aec4d7069ad803765e8c7d37aaa063bd1bcc420ada2b725211b5","status":"completed","success":true,"teacher_episode_wall_seconds":0.634299207944423,"teacher_final_target_error_m":0.0003909199501342075,"teacher_simulated_seconds":10.130999999999824,"teacher_simulated_seconds_per_wall_second":15.97195751328685},{"case_id":"scene-03","episode_wall_seconds":0.7504690408241004,"group":"nominal","reason":"success","run_path":"teacher/scene-03/run.json","scene_sha256":"3c66ce9494f44d0cd1880a7e6dfdf51ab53758a026a29bc6076b8ce624edfa2e","status":"completed","success":true,"teacher_episode_wall_seconds":0.7504690408241004,"teacher_final_target_error_m":0.0005302789172681855,"teacher_simulated_seconds":11.767999999998917,"teacher_simulated_seconds_per_wall_second":15.6808600486388},{"case_id":"scene-04","episode_wall_seconds":0.8684928750153631,"group":"nominal","reason":"success","run_path":"teacher/scene-04/run.json","scene_sha256":"9dd2ff2faead021108c63ac86dcb352d93fa31eda3bcd9b1413b9fe350ae4898","status":"completed","success":true,"teacher_episode_wall_seconds":0.8684928750153631,"teacher_final_target_error_m":0.00035671316413181386,"teacher_simulated_seconds":13.56599999999792,"teacher_simulated_seconds_per_wall_second":15.620162686720885},{"case_id":"scene-05","episode_wall_seconds":0.7877573750447482,"group":"nominal","reason":"success","run_path":"teacher/scene-05/run.json","scene_sha256":"84d5d3a2053c61a65249fb9f551b17f7665821bbd7ea73e64d8d9a5ef130b746","status":"completed","success":true,"teacher_episode_wall_seconds":0.7877573750447482,"teacher_final_target_error_m":0.00021578091315435074,"teacher_simulated_seconds":12.3399999999986,"teacher_simulated_seconds_per_wall_second":15.664721640083195},{"case_id":"scene-06","episode_wall_seconds":0.9992966251447797,"group":"nominal","reason":"success","run_path":"teacher/scene-06/run.json","scene_sha256":"ac34ce607dbb0223912f6ef20a290530ed2e418819cc306602e5360d462bc6ee","status":"completed","success":true,"teacher_episode_wall_seconds":0.9992966251447797,"teacher_final_target_error_m":0.0002655248873198246,"teacher_simulated_seconds":15.956999999996595,"teacher_simulated_seconds_per_wall_second":15.968231652623384},{"case_id":"scene-07","episode_wall_seconds":0.7607129579409957,"group":"nominal","reason":"success","run_path":"teacher/scene-07/run.json","scene_sha256":"bc97d7f83474f39d4f43750fdf5058403f948fc2cf614b0f9cfee1e7ff32c5a9","status":"completed","success":true,"teacher_episode_wall_seconds":0.7607129579409957,"teacher_final_target_error_m":0.00020785349497874333,"teacher_simulated_seconds":11.516999999999056,"teacher_simulated_seconds_per_wall_second":15.139744735217677},{"case_id":"scene-08","episode_wall_seconds":0.9803391250316054,"group":"actuator_stress","reason":"success","run_path":"teacher/scene-08/run.json","scene_sha256":"a9e5b6158f1b7bf4cc9afbb4a2a6f67b078f7ae23885fcadc5466d3728a2dab6","status":"completed","success":true,"teacher_episode_wall_seconds":0.9803391250316054,"teacher_final_target_error_m":0.001955660537489348,"teacher_simulated_seconds":15.237999999996994,"teacher_simulated_seconds_per_wall_second":15.543600791721675},{"case_id":"scene-09","episode_wall_seconds":1.005271667148918,"group":"actuator_stress","reason":"success","run_path":"teacher/scene-09/run.json","scene_sha256":"66cec8a3c410f790c326ce9a031202135533aaaf3b72864729e8e870c59e3d62","status":"completed","success":true,"teacher_episode_wall_seconds":1.005271667148918,"teacher_final_target_error_m":0.002028526319252969,"teacher_simulated_seconds":15.410999999996898,"teacher_simulated_seconds_per_wall_second":15.330184370663218},{"case_id":"scene-10","episode_wall_seconds":0.9717558338306844,"group":"actuator_stress","reason":"success","run_path":"teacher/scene-10/run.json","scene_sha256":"4f7aff58d9eaeac570a93839acff8588c7488a02213f26478cdaa6df66cc2404","status":"completed","success":true,"teacher_episode_wall_seconds":0.9717558338306844,"teacher_final_target_error_m":0.0019172940557521127,"teacher_simulated_seconds":14.966999999997144,"teacher_simulated_seconds_per_wall_second":15.40201713119321},{"case_id":"scene-11","episode_wall_seconds":0.9722880420740694,"group":"actuator_stress","reason":"horizon","run_path":"teacher/scene-11/run.json","scene_sha256":"20c2d7585a0f02738d49fd98403d880bff2c93e88d8f8c5a85eeeb021b74c416","status":"completed","success":false,"teacher_episode_wall_seconds":0.9722880420740694,"teacher_final_target_error_m":0.1576629029336082,"teacher_simulated_seconds":17.999999999999016,"teacher_simulated_seconds_per_wall_second":18.513032374235213}],"planned_scenes":12,"successes":11},"timing_scope":"Benchmark function wall time includes arm dependency imports, source validation, scene planning, input copies, frozen plans, fits, installation, native evaluations and artifact output before the final report. Process startup, top-level module imports and final report writing are excluded. Pipeline and native episode clocks retain their own narrower scopes."} diff --git a/docs/benchmarks.md b/docs/benchmarks.md new file mode 100644 index 0000000..870acb9 --- /dev/null +++ b/docs/benchmarks.md @@ -0,0 +1,140 @@ +# Frozen sample-count and simulation qualification benchmark + +`dvidia-training-benchmark` compares nested sets of **whole connected training +groups** from an actual prepared dataset. It keeps the original development and +test episodes fixed, trains visual predictors and aligned-label movement heads, +builds and installs candidate capsules, and qualifies each candidate in a new +seeded suite of bounded native simulation scenes. + +This is a simulation experiment. It does not recover actions from human video, +establish a sufficient demonstration count, qualify a physical arm, or broaden +the installed capsule's existing exact-scene execution gate. + +## Run locally + +Install the optional arm dependencies first: + +```sh +.venv/bin/python -m pip install ".[arm]" +``` + +Use a prepared `dataset.json` containing real encoded media and validated action +sidecars. The default comparison uses 2, 4, and 7 training groups when available; +smaller datasets use the eligible counts. Every episode needs aligned movement +supervision, and the existing fitter requires at least 32 distinct nonzero +training action examples. Related clips remain together. + +```sh +.venv/bin/dvidia-training-benchmark path/to/native/prepared/dataset.json \ + --task-source path/to/native/source/task.skill.json \ + --source-evidence path/to/native/source/evidence \ + --counts 2 4 7 --nominal 8 --stress 4 --seed 19047 \ + --output runs/sample-count-19047 +``` + +From a source checkout, `.venv/bin/python -m dvidia_training.footage_benchmark` +exposes the same command. Use a fresh output +directory. A benchmark cannot overwrite or resume a frozen experiment. To make +a small smoke run, use one count with `--nominal 1 --stress 0`; a smoke run is not +general scene evidence. + +The default task source is the bundled authored rigid-box placement envelope. +Supply `--task-source` and, if needed, `--grounding` explicitly for your supported +task. The benchmark does not infer a task adapter from videos. + +## What is frozen + +Before any fitting or physics evaluation, `protocol.json` and `protocol.sha256` +record source dataset and training code hashes, the exact runtime, nested group +and episode identities, unchanged development/test episodes, full scene inputs +and hashes, model settings, development-only regularization selection, and the +existing success/control rules. Copies of the prepared inputs are retained. + +All models are fitted and installed before any benchmark scene is evaluated. +`models-frozen.json` then binds the exact candidate bytes. Source, protocol, +input, runtime, and model checks repeat before subsequent fitting/evaluation. +Results cannot silently redefine the plan. New tuning requires a new experiment +and fresh evaluation scenes; choosing another output directory with the same +seed merely repeats known cases. + +One authored teacher baseline runs per exact scene. Each installed candidate is +evaluated through the existing `capsule_worker(qualify=True)` path, including its +own recorded-joint-target/open-jaw control. The original `run.json`, +`controls.json`, `qualification.json`, and replay remain under each candidate's +scene directory. Raw student completion, control completion, valid +qualification, and execution errors are reported separately. A student that +completes while its control also completes does not earn qualification. + +## Source-scene exclusion + +Prepared datasets retain evidence hashes but do not always contain complete +source scene declarations. When `--source-evidence` is supplied, every action +sidecar must bind a corresponding native telemetry evidence JSON. Its declared +zero-jitter scene is validated, and evaluation layouts exclude object/target +layouts from **all** source training, development, and test episodes. Changing +only a seed or material cannot make a source layout new. + +These checks establish consistency relative to hash-bound supplier declarations, +not independent attestation of simulation or capture history. Without the +evidence directory, the report explicitly marks source-scene disjointness as +unestablished. Do not present that run as independent scene generalization. + +## Interpret the result + +`report.json` gives training-group counts, training-episode and numerical-action +sample counts, fixed held-out prediction errors, per-scene teacher/student/control +outcomes, nominal/stress qualification totals, paired teacher comparisons, and +scoped clocks. Every recorded teacher, student, and control also retains final +target error, simulated time, wall time, and throughput. Aggregate throughput is +total simulated time divided by total episode wall time, rather than an average +of individual speed ratios. Error means/maxima count only available finite +numeric target distances. `measured_error_scenes` is separate from `timed_scenes`; +rejected, invalid, and execution-error cases remain in planned-scene counts. The same test +recordings and evaluation scenes are shared across +counts; they are not repeated independent studies. A single nesting order +confounds count with which groups are added. This is a bounded composition-order +pilot, not a universal learning curve or performance guarantee. Nominal and +actuator-stress outcomes and paired teacher comparisons are stratified. Combined +clocks/counts are execution accounting, not a pooled performance estimate. Exact +source layout exclusion establishes neither out-of-distribution robustness nor +human-to-robot transfer. + +Stress cases use reduced actuator limits. A shared teacher/student grasp failure +may reflect that constraint rather than too few training labels. Privileged +simulator observations and authored phases, grasp/contact logic and recovery +remain in use. Camera-based control, human-video action recovery, hardware +transfer, and broader embodiments remain unmeasured. + +The existing published one-scene receipt recorded about 0.82 seconds for its +candidate and 0.91 seconds for its open-jaw control on one host. A default +three-count, twelve-scene run therefore adds 72 candidate/control episodes and +twelve teacher episodes. Allow a few minutes on a comparable machine, plus +installation and output I/O; this is an estimate, not a requirement or guarantee. +Full native traces, controls, copied media, and replays can require hundreds of +MiB. The report measures the actual run. Dependencies and data must be installed +before offline execution. + +## Published pilot and reproduction + +The [8 October frozen result](../benchmarks/sample-count-19047/README.md) records +the completed 2/4/7 comparison, including every failed case. Repeating seed 19047 +reproduces known layouts; use a new, recorded seed for a subsequent evaluation +after changing a learner. + +To reproduce the published source intake, download the fixed +[native Skillspace archive](https://huggingface.co/datasets/Dvidia/dvidia-training-examples/resolve/d351a32923d54e3166e8ff780fd827c80174a903/archives/native-skillspace.zip?download=true) +before disconnecting. Its SHA256 is +`5091233dbc21ea84022e57101a5e2036be44bb6b226adf2610265f12263349e9`. +Extract it into `examples/native-source`, then prepare it with: + +```sh +.venv/bin/dvidia-train prepare examples/native-source --output runs/native-prepared +``` + +Use `runs/native-prepared/dataset/dataset.json`, +`examples/native-source/task.skill.json`, and +`examples/native-source/evidence` in the benchmark command above. Keep the +default preparation split seed; changing the source split creates a different +experiment. Training and qualification need no network after dependencies and +these inputs are installed. The source archive contains simulator recordings +and aligned numeric labels, rather than human shoe demonstrations. diff --git a/publishing/huggingface/releases/2026-10-08.json b/publishing/huggingface/releases/2026-10-08.json new file mode 100644 index 0000000..1d41949 --- /dev/null +++ b/publishing/huggingface/releases/2026-10-08.json @@ -0,0 +1,44 @@ +{ + "schema_version": 1, + "kind": "dvidia.public-huggingface-pilot-verification", + "date": "2026-10-08", + "scope": "Public authored synthetic v0.1.0 evidence; no human recordings or hardware qualification.", + "repositories": [ + { + "repo_id": "Dvidia/dvidia-training-examples", + "repo_type": "dataset", + "commit": "04598efff57cf47125855881ae18a53d17cc640c", + "verified_files": 86, + "inventory_sha256": "f5566c087984e9a6f1783e6f3f70f17bc098acca03ece1c12812a032616e7012" + }, + { + "repo_id": "Dvidia/dvidia-training-pilot", + "repo_type": "model", + "commit": "d249e81596d774fbdd2abf2e1438977ccfad69a3", + "verified_files": 30, + "inventory_sha256": "0bb45f23db29c744915097a4ad0c67c6c8401efb4d5ef3abec694d2db6090e2d" + }, + { + "repo_id": "Dvidia/dvidia-training", + "repo_type": "space", + "commit": "61e9d5f0041bca1422188f06e7a7115f61935b44", + "verified_files": 8, + "inventory_sha256": "bed21dc57a57cdeead882acceed77cf0e5b383b636ee5391bd0437cf7dde6af9" + } + ], + "verification": "Every planned file downloaded anonymously at its fixed commit and matched by byte count and SHA256. The native ZIP used HTTP byte ranges due to a stalled whole-file transfer.", + "dataset_card_update": { + "repo_id": "Dvidia/dvidia-training-examples", + "commit": "d351a32923d54e3166e8ff780fd827c80174a903", + "path": "README.md", + "prior_sha256": "40be5f617f01da1d55835a131d600b26fe32e5bf26ec304f2aaef576ac252ab4", + "sha256": "92590f355fb7fc2aa65edc65328e4d3f39c7a74a8b6804409a3f07ae33c67f2d", + "scope": "Archive viewer metadata and explanatory documentation only; dataset files unchanged." + }, + "consumer_reproduction": { + "source": "Public visual-skillspace.zip", + "offline": true, + "test_rgb_mse": 0.00030387928412713475, + "physical_robot_ready": false + } +} diff --git a/pyproject.toml b/pyproject.toml index 5dec71a..29ac577 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ Issues = "https://github.com/Dvidia-Inference/dvidia-training/issues" dvidia-train = "dvidia_training.footage_pipeline:main" dvidia-training-studio = "dvidia_training.training_studio:main" dvidia-training-doctor = "dvidia_training.doctor:main" +dvidia-training-benchmark = "dvidia_training.footage_benchmark:main" dvidia-skill-capsule = "dvidia_training.arm_entrypoints:capsule_main" [tool.setuptools.packages.find] diff --git a/src/dvidia_training/footage_benchmark.py b/src/dvidia_training/footage_benchmark.py new file mode 100644 index 0000000..f24e6e1 --- /dev/null +++ b/src/dvidia_training/footage_benchmark.py @@ -0,0 +1,498 @@ +"""Frozen sample-count comparisons and exact-scene native qualification. + +This experiment learns from existing aligned numerical labels, not actions +recovered from human video. It never broadens an installed capsule's scene gate. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +from hashlib import sha256 +import json +import math +from pathlib import Path +import time + +from .footage_pipeline import _canonical, _fresh, _json, _prepare, _prepared_copy + +DEFAULT_COUNTS = (2, 4, 7) +SOURCE_FILES = ('footage_benchmark.py', 'footage_pipeline.py', 'footage_data.py', 'footage_train.py') +EVIDENCE_FILE_MAX_BYTES = 8 * 1024 * 1024 +EVIDENCE_TOTAL_MAX_BYTES = 128 * 1024 * 1024 + + +def _digest(value): + return sha256(_canonical(value)).hexdigest() + + +def _write(path, value): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(_canonical(value) + b'\n') + + +def _integer(value, low, high, label): + if type(value) is not int or not low <= value <= high: + raise ValueError(f'{label} must be an integer in [{low}, {high}].') + return value + + +def normalize_counts(counts, available): + """Counts measure whole training groups, not frames or sidecar samples.""" + _integer(available, 1, 64, 'Available training groups') + if counts is None: + return tuple(value for value in DEFAULT_COUNTS if value <= available) or (available,) + if not isinstance(counts, (list, tuple)) or not counts: + raise ValueError('Supply a nonempty list of training group counts.') + if len(counts) > 8: + raise ValueError('Compare at most eight training group counts in one benchmark.') + for value in counts: + _integer(value, 1, available, 'Training group count') + if len(set(counts)) != len(counts): + raise ValueError('Training group counts must be distinct.') + return tuple(sorted(counts)) + + +def nested_subsets(dataset, counts=None, seed=17): + """Select nested connected groups; retain all development/test episodes.""" + from .footage_data import _groups + _integer(seed, 0, 999999, 'Subset seed') + episodes = dataset.get('episodes') + if type(episodes) is not list or not episodes: + raise ValueError('A prepared dataset with episodes is required.') + checked = deepcopy(episodes) + groups = _groups(checked) + assignments = {} + for original, recalculated in zip(episodes, checked): + split = original.get('split') + if split not in ('train', 'dev', 'test'): + raise ValueError('Each source episode needs a train, dev or test split.') + group = recalculated['group_id'] + if group in assignments and assignments[group] != split: + raise ValueError('Connected source groups cross training/development/test splits.') + assignments[group] = split + if original.get('group_id') != group: + raise ValueError('Declared source groups differ from connected recording lineage.') + fixed = {name: [row['id'] for row in episodes if row['split'] == name] + for name in ('train', 'dev', 'test')} + if not all(fixed.values()): + raise ValueError('Keep nonempty training, development and test groups.') + if dataset.get('splits') is not None and dataset['splits'] != fixed: + raise ValueError('Prepared split metadata differs from episode assignments.') + training = [group['id'] for group in groups if assignments[group['id']] == 'train'] + ordered = sorted(training, key=lambda group: _digest([seed, group])) + result = [] + for count in normalize_counts(counts, len(training)): + selected = ordered[:count] + result.append({'training_group_count': count, 'train_group_ids': selected, + 'train_episode_ids': [row['id'] for row in episodes + if row['split'] == 'train' and row['group_id'] in selected], + 'dev_episode_ids': fixed['dev'], 'test_episode_ids': fixed['test']}) + return result + + +def _layout_digest(scene): + # A new seed or material value cannot turn the same layout into new coverage. + return _digest({key: scene[key] for key in ('object_position', 'target_position')}) + + +def make_scenes(seed, nominal=8, stress=4, source_scenes=()): + """Create distinct bounded box layouts without observing any outcome.""" + import numpy as np + from .arm_runner import validate_scene + from .skill_capsule import scene_sha256 + _integer(seed, 0, 999900, 'Scene seed') + _integer(nominal, 0, 24, 'Nominal scene count') + _integer(stress, 0, 24, 'Stress scene count') + if not 1 <= nominal + stress <= 24: + raise ValueError('Use one to 24 total evaluation scenes.') + excluded = {_layout_digest(validate_scene(scene)) for scene in source_scenes} + rng = np.random.default_rng(seed) + cases = [] + for index in range(nominal + stress): + for attempt in range(10000): + xy = np.round(rng.uniform([.27, -.17, .29, -.17], [.53, .17, .54, .17]), 6) + if (np.linalg.norm(xy[:2]) > .54 or np.linalg.norm(xy[2:]) > .54 + or np.linalg.norm(xy[:2] - xy[2:]) < .10): + continue + size = rng.choice([.03, .035, .04, .045, .05], 3).tolist() + z = .29 + size[2] / 2 + stressed = index >= nominal + scene = validate_scene({ + 'object_position': [*xy[:2].tolist(), z], 'target_position': [*xy[2:].tolist(), z], + 'object_size': size, 'object_mass': float(rng.choice([.02, .04, .06, .08])), + 'object_friction': float(rng.choice([.4, .8, 1.2, 1.5])), + 'jaw_force_limit': .3 if stressed else 15., + 'joint_torque_limit': 5. if stressed else 40., 'seed': seed + index}) + layout = _layout_digest(scene) + if layout in excluded: + continue + excluded.add(layout) + cases.append({'id': f'scene-{index:02}', 'group': 'actuator_stress' if stressed else 'nominal', + 'scene': scene, 'scene_sha256': scene_sha256(scene), 'layout_sha256': layout}) + break + else: + raise ValueError('Could not create a distinct supported evaluation layout.') + return cases + + +def freeze_protocol(output, protocol): + """An exclusive plan is written before any fitting or physics evaluation.""" + output = Path(output) + output.mkdir(parents=True, exist_ok=True) + raw = _canonical(protocol) + b'\n' + with (output / 'protocol.json').open('xb') as handle: + handle.write(raw) + with (output / 'protocol.sha256').open('x') as handle: + handle.write(sha256(raw).hexdigest() + '\n') + return sha256(raw).hexdigest() + + +def verify_protocol(output, protocol): + output = Path(output) + raw = (output / 'protocol.json').read_bytes() + expected = _canonical(protocol) + b'\n' + if raw != expected or (output / 'protocol.sha256').read_text().strip() != sha256(raw).hexdigest(): + raise ValueError('Frozen benchmark protocol changed. Choose a fresh experiment.') + + +def _source_hashes(): + return {name: sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() for name in SOURCE_FILES} + + +def _source_scenes(dataset, sidecars, directory): + if directory is None: + return [], {'status': 'unestablished', 'reason': 'No hash-bound native scene evidence was supplied.'} + from .arm_runner import validate_scene + from .footage_train import _json as parse_evidence + directory = Path(directory) + if not directory.is_dir() or any(path.is_symlink() for path in (directory, *directory.parents)): + raise ValueError('Source evidence must be a local directory without symbolic links.') + paths = list(directory.glob('*.json')) + if len(paths) > 1200: + raise ValueError('Source evidence directory exceeds 1200 JSON files.') + index, total = {}, 0 + for path in paths: + if path.is_symlink() or not path.is_file() or path.stat().st_size > EVIDENCE_FILE_MAX_BYTES: + raise ValueError('Native evidence must use regular JSON files of at most 8 MiB.') + raw = path.read_bytes() + if len(raw) > EVIDENCE_FILE_MAX_BYTES: + raise ValueError('Native evidence grew beyond its bounded size while being read.') + total += len(raw) + if total > EVIDENCE_TOTAL_MAX_BYTES: + raise ValueError('Native evidence exceeds the 128 MiB benchmark limit.') + index[sha256(raw).hexdigest()] = raw + scenes, bindings = [], [] + for row in dataset['episodes']: + provenance = sidecars[row['id']]['provenance'] + if provenance['source_kind'] != 'native-simulation-telemetry': + raise ValueError('Native scene exclusion requires native telemetry evidence for every episode.') + digest = provenance['validation']['evidence_sha256'] + if digest not in index: + raise ValueError(f"Missing hash-bound native evidence for {row['id']}.") + record = parse_evidence(index[digest], maximum=EVIDENCE_FILE_MAX_BYTES) + if (record.get('kind') != 'dvidia.native-telemetry-evidence' or record.get('schema_version') != 1 + or record.get('scope') != 'simulation-only'): + raise ValueError('Expected simulation-only native telemetry evidence.') + config = record.get('final_info', {}).get('config', {}) + if config.get('scene_jitter') != 0 or config.get('table_height') != .29: + raise ValueError('Scene exclusion supports declared zero-jitter scenes on the existing table.') + try: + scene = validate_scene({key: config[key] for key in ( + 'object_position', 'target_position', 'object_mass', 'object_friction', + 'jaw_force_limit', 'joint_torque_limit')} | { + 'object_size': [2 * value for value in config['object_half_size']], 'seed': record['seed']}) + except (KeyError, TypeError) as exc: + raise ValueError('Native evidence lacks a complete supported scene declaration.') from exc + scenes.append(scene) + bindings.append({'episode_id': row['id'], 'split': row['split'], 'evidence_sha256': digest, + 'layout_sha256': _layout_digest(scene), 'scene': scene}) + return scenes, {'status': 'excluded_hash_bound_native_declarations', 'bindings': bindings, + 'limits': 'Hashes and supplier scene declarations check consistency, not independent capture attestation.'} + + +def _materialize_subset(snapshot, spec, output): + dataset = _json(snapshot) + chosen = set(spec['train_episode_ids'] + spec['dev_episode_ids'] + spec['test_episode_ids']) + dataset['episodes'] = [row for row in dataset['episodes'] if row['id'] in chosen] + dataset['groups'] = [row for row in dataset['groups'] if any(identifier in chosen for identifier in row['episode_ids'])] + dataset['splits'] = {name: [row['id'] for row in dataset['episodes'] if row['split'] == name] + for name in ('train', 'dev', 'test')} + dataset['actual_media_episodes'] = len(dataset['episodes']) + dataset['independent_groups'] = len(dataset['groups']) + dataset['benchmark_parent_dataset_sha256'] = sha256(snapshot.read_bytes()).hexdigest() + temporary = snapshot.parent / f"subset-{spec['training_group_count']:03}.json" + with temporary.open('xb') as handle: + handle.write(_canonical(dataset) + b'\n') + try: + _prepared_copy(temporary, output) + finally: + temporary.unlink() + + +def _assert_inputs(output, protocol): + from .capsule_installer import capsule_runtime + verify_protocol(output, protocol) + if capsule_runtime() != protocol['runtime'] or _source_hashes() != protocol['benchmark_source_sha256']: + raise ValueError('Frozen benchmark runtime or training source changed.') + for path, expected in protocol['input_files_sha256'].items(): + if sha256((output / path).read_bytes()).hexdigest() != expected: + raise ValueError('Frozen benchmark input changed.') + + +def _episode_metrics(episode, prefix): + wall = episode['episode_wall_seconds'] + simulated = episode['simulated_seconds'] + return {f'{prefix}_final_target_error_m': episode['final_distance'], + f'{prefix}_simulated_seconds': simulated, + f'{prefix}_episode_wall_seconds': wall, + f'{prefix}_simulated_seconds_per_wall_second': simulated / wall if wall else None} + + +def _measurement_summary(rows, prefix): + def finite(value): + return type(value) in (int, float) and math.isfinite(value) and value >= 0 + measured = [row for row in rows if finite(row.get(f'{prefix}_episode_wall_seconds')) + and finite(row.get(f'{prefix}_simulated_seconds'))] + wall = sum(row[f'{prefix}_episode_wall_seconds'] for row in measured) + simulated = sum(row[f'{prefix}_simulated_seconds'] for row in measured) + errors = [row[f'{prefix}_final_target_error_m'] for row in rows + if finite(row.get(f'{prefix}_final_target_error_m'))] + return {'measured_scenes': len(measured), 'timed_scenes': len(measured), + 'measured_error_scenes': len(errors), 'total_simulated_seconds': simulated, + 'total_episode_wall_seconds': wall, + 'simulated_seconds_per_wall_second': simulated / wall if wall else None, + 'final_target_error_mean_m': sum(errors) / len(errors) if errors else None, + 'final_target_error_max_m': max(errors) if errors else None} + + +def _aggregate(rows): + groups = {} + for group in sorted({row['group'] for row in rows}): + values = [row for row in rows if row['group'] == group] + groups[group] = {'planned_scenes': len(values), + 'student_successes': sum(row.get('student_success') is True for row in values), + 'qualified_scenes': sum(row.get('qualified') is True for row in values), + 'control_successes': sum(row.get('control_success') is True for row in values), + 'errors': sum(row.get('status') == 'error' for row in values), + 'student_measurements': _measurement_summary(values, 'student'), + 'control_measurements': _measurement_summary(values, 'control')} + student_measurements = _measurement_summary(rows, 'student') + control_measurements = _measurement_summary(rows, 'control') + return {'planned_scenes': len(rows), 'student_successes': sum(row.get('student_success') is True for row in rows), + 'qualified_scenes': sum(row.get('qualified') is True for row in rows), + 'control_successes': sum(row.get('control_success') is True for row in rows), + 'errors': sum(row.get('status') == 'error' for row in rows), 'groups': groups, + 'student_episode_wall_seconds': student_measurements['total_episode_wall_seconds'], + 'control_episode_wall_seconds': control_measurements['total_episode_wall_seconds'], + 'student_measurements': student_measurements, + 'control_measurements': control_measurements} + + +def _paired_outcomes(teachers, rows): + paired = {'both_success': 0, 'teacher_only': 0, 'student_only': 0, 'both_failure': 0, 'errors': 0} + for teacher, row in zip(teachers, rows): + if row['status'] == 'error' or teacher['status'] == 'error': + paired['errors'] += 1 + else: + key = ('both_success' if teacher['success'] else 'student_only') if row['student_success'] else ( + 'teacher_only' if teacher['success'] else 'both_failure') + paired[key] += 1 + return paired + + +def benchmark(dataset_path, output, *, counts=None, seed=19043, nominal=8, stress=4, + task_source=None, grounding=None, source_evidence=None): + """Fit every frozen count before evaluating any of its scene outcomes.""" + started = time.perf_counter() + from .arm_entrypoints import require_arm + require_arm() + from .arm_runner import authored_grounding, run_skill, write_run + from .capsule_installer import capsule_runtime, capsule_worker, install_capsule + from .footage_pipeline import inspect_run, run + from .footage_train import _load_dataset, RIDGE_GRID + from .skillspace import load_skillspace + _integer(seed, 0, 999900, 'Benchmark seed') + dataset_path = Path(dataset_path) + dataset, original_dataset_hash, _, sidecars = _load_dataset(dataset_path) + if set(sidecars) != {row['id'] for row in dataset['episodes']}: + raise ValueError('Every benchmark episode needs validated aligned movement supervision.') + subsets = nested_subsets(dataset, counts, seed) + original_scenes, exclusion = _source_scenes(dataset, sidecars, source_evidence) + cases = make_scenes(seed, nominal, stress, original_scenes) + task_source = Path(task_source) if task_source is not None else Path(__file__).with_name('place_cup.skill.json') + task = _json(task_source, limit=128 * 1024) + raw = task_source.read_bytes() + resolved_grounding = _json(Path(grounding)) if grounding is not None else ( + None if 'simulation_grounding' in task else authored_grounding(raw)) + load_skillspace(raw, resolved_grounding) + output = _fresh(output) + _prepare(dataset_path, output / 'source', offline=True, seed=seed) + if sha256(dataset_path.read_bytes()).hexdigest() != original_dataset_hash: + raise ValueError('The prepared source changed while the benchmark snapshot was being made.') + snapshot = output / 'source' / 'dataset.json' + (output / 'task.skill.json').write_bytes(raw) + input_hashes = {'source/dataset.json': sha256(snapshot.read_bytes()).hexdigest(), + 'task.skill.json': sha256(raw).hexdigest()} + if resolved_grounding is not None: + _write(output / 'grounding.json', resolved_grounding) + input_hashes['grounding.json'] = sha256((output / 'grounding.json').read_bytes()).hexdigest() + for spec in subsets: + relative = f"subsets/groups-{spec['training_group_count']:03}" + destination = output / relative + destination.parent.mkdir(parents=True, exist_ok=True) + _materialize_subset(snapshot, spec, destination) + spec['dataset_path'] = f'{relative}/dataset.json' + input_hashes[spec['dataset_path']] = sha256((output / spec['dataset_path']).read_bytes()).hexdigest() + protocol = {'schema_version': 1, 'kind': 'dvidia.footage-sample-count-benchmark', + 'scope': 'simulation-only', 'physical_robot_ready': False, + 'source_dataset_sha256': original_dataset_hash, + 'runtime': capsule_runtime(), 'benchmark_source_sha256': _source_hashes(), + 'input_files_sha256': input_hashes, 'seed': seed, 'subsets': subsets, + 'scenes': cases, 'source_scene_exclusion': exclusion, + 'training': {'frames_per_clip': 12, 'resolution': 16, 'latent_dim': 8, + 'movement_centers': 8, 'regularization_grid': list(RIDGE_GRID), + 'selection': 'The same fixed development episodes select regularization for each count.'}, + 'rules': {'counts': 'Nested whole connected training groups; development and test episodes never change.', + 'freeze': 'Freeze this protocol before fitting; freeze all candidates before any scene evaluation.', + 'teacher': 'One authored baseline per exact scene, shared by all candidates.', + 'control': 'Each candidate uses identical recorded joint targets with jaws forced open.', + 'qualification': 'Existing native exact-scene predicate and installation gate are unchanged.', + 'limits': 'No human-video action recovery, demonstration sufficiency threshold, hardware qualification or end-to-end speedup is implied.'}} + protocol_hash = freeze_protocol(output, protocol) + candidates = [] + for spec in subsets: + _assert_inputs(output, protocol) + directory = f"candidates/groups-{spec['training_group_count']:03}" + result = run(output / spec['dataset_path'], output / directory, seed=seed, + task_source=output / 'task.skill.json', + grounding=output / 'grounding.json' if resolved_grounding is not None else None) + inspect_run(output / directory) + if (result['movement']['train_episode_ids'] != spec['train_episode_ids'] + or [row['episode_id'] for row in result['movement']['alignment_records'] if row['split'] == 'dev'] != spec['dev_episode_ids'] + or [row['episode_id'] for row in result['movement']['alignment_records'] if row['split'] == 'test'] != spec['test_episode_ids']): + raise ValueError('Training changed the frozen source-group assignments.') + capsule_path = output / directory / 'candidate.skill-capsule.json' + installation = install_capsule(capsule_path.read_bytes(), output / 'installs') + candidates.append({'training_group_count': spec['training_group_count'], + 'training_episode_count': len(spec['train_episode_ids']), + 'training_action_samples': result['movement']['training']['sample_count'], + 'training_pipeline_seconds': result['elapsed_seconds'], + 'training_pipeline_timing_scope': result['timing_scope'], + 'visual_test': result['visual']['test'], 'movement_test': result['movement']['test'], + 'installation_id': installation['installation_id'], 'capsule_path': f'{directory}/candidate.skill-capsule.json', + 'capsule_file_sha256': sha256(capsule_path.read_bytes()).hexdigest()}) + models = {'protocol_sha256': protocol_hash, 'candidates': candidates} + with (output / 'models-frozen.json').open('xb') as handle: + handle.write(_canonical(models) + b'\n') + models_hash = sha256((output / 'models-frozen.json').read_bytes()).hexdigest() + def verify(): + _assert_inputs(output, protocol) + if sha256((output / 'models-frozen.json').read_bytes()).hexdigest() != models_hash: + raise ValueError('Frozen candidate set changed.') + for candidate in candidates: + if sha256((output / candidate['capsule_path']).read_bytes()).hexdigest() != candidate['capsule_file_sha256']: + raise ValueError('Frozen candidate weights changed.') + teachers = [] + for case in cases: + verify() + relative = f"teacher/{case['id']}" + record = {'case_id': case['id'], 'group': case['group'], 'scene_sha256': case['scene_sha256'], + 'status': 'completed', 'run_path': f'{relative}/run.json'} + try: + payload = run_skill(raw, resolved_grounding, case['scene'], offline=True) + write_run(output / relative, payload, source=raw, grounding=resolved_grounding) + episode = payload['episodes'][0] + record.update(success=episode['success'], reason=episode['reason'], + episode_wall_seconds=episode['episode_wall_seconds'], **_episode_metrics(episode, 'teacher')) + except (ValueError, RuntimeError) as exc: + record.update(status='error', success=False, error=str(exc)) + _write(output / relative / 'benchmark-error.json', record) + teachers.append(record) + comparisons = [] + for candidate in candidates: + rows = [] + for case in cases: + verify() + relative = f"qualification/groups-{candidate['training_group_count']:03}/{case['id']}" + record = {'case_id': case['id'], 'group': case['group'], 'scene_sha256': case['scene_sha256'], + 'status': 'completed', 'qualified': False, 'run_path': f'{relative}/run.json', + 'controls_path': f'{relative}/controls.json', 'qualification_path': f'{relative}/qualification.json'} + try: + payload = capsule_worker(output / 'installs', candidate['installation_id'], case['scene'], + output / relative, qualify=True) + controls = _json(output / record['controls_path']) + student, control = payload['episodes'][0], controls['episodes'][0] + receipt = payload['local_qualification'] + record.update(student_success=student['success'], student_reason=student['reason'], + control_success=control['success'], control_reason=control['reason'], + qualified=receipt.get('status') == 'validated_simulation_scene', + qualification_status=receipt.get('status'), + **_episode_metrics(student, 'student'), **_episode_metrics(control, 'control')) + except (ValueError, RuntimeError) as exc: + record.update(status='error', error=str(exc)) + _write(output / relative / 'benchmark-error.json', record) + rows.append(record) + paired_groups = {group: _paired_outcomes([row for row in teachers if row['group'] == group], + [row for row in rows if row['group'] == group]) + for group in sorted({row['group'] for row in rows})} + comparisons.append({**candidate, 'outcomes': rows, 'summary': _aggregate(rows), + 'paired_with_teacher': _paired_outcomes(teachers, rows), + 'paired_with_teacher_by_group': paired_groups}) + verify() + report = {'schema_version': 1, 'kind': 'dvidia.footage-sample-count-result', + 'scope': 'simulation-only', 'physical_robot_ready': False, + 'protocol_sha256': protocol_hash, 'models_frozen_sha256': models_hash, + 'source_scene_exclusion': exclusion['status'], + 'declared_source_exact_layout_exclusion_established': exclusion['status'] == 'excluded_hash_bound_native_declarations', + 'analysis_scope': {'composition': 'Single composition-order pilot: count is confounded with which training groups are added.', + 'scene_independence': 'The same distinct benchmark scenes are reused for every count; condition-by-scene runs are not independent scenes.', + 'stratification': 'Nominal and actuator-stress outcomes are reported separately. Combined counts and clocks are execution totals, not a pooled performance estimate.', + 'novelty': 'Only exact source object/target layout exclusion is checked relative to supplied hash-bound native declarations; no out-of-distribution or human-transfer claim.', + 'modality': 'The movement candidate learns aligned numerical actions; the RGB predictor does not control the arm.'}, + 'teacher': {'outcomes': teachers, 'planned_scenes': len(cases), + 'successes': sum(row['success'] for row in teachers), + 'errors': sum(row['status'] == 'error' for row in teachers), + 'episode_wall_seconds': sum(row.get('episode_wall_seconds', 0.) for row in teachers), + 'measurements': _measurement_summary(teachers, 'teacher'), + 'groups': {group: {'planned_scenes': len(values), + 'successes': sum(row['success'] for row in values), + 'errors': sum(row['status'] == 'error' for row in values), + 'measurements': _measurement_summary(values, 'teacher')} + for group in sorted({row['group'] for row in teachers}) + for values in [[row for row in teachers if row['group'] == group]]}}, + 'comparisons': comparisons, 'benchmark_wall_seconds': time.perf_counter() - started, + 'timing_scope': 'Benchmark function wall time includes arm dependency imports, source validation, scene planning, input copies, frozen plans, fits, installation, native evaluations and artifact output before the final report. Process startup, top-level module imports and final report writing are excluded. Pipeline and native episode clocks retain their own narrower scopes.', + 'limitations': ['Training group counts and numerical action sample counts are different evidence units.', + 'The same frozen source dev/test episodes and benchmark scenes are shared across counts; they are not independent repeated studies.', + 'Scene exclusion is relative to hash-bound supplier declarations when supplied; otherwise source-scene disjointness is unestablished.', + 'Actuator-stress failures can be shared physical/controller limits, not a demonstrated need for more labels.', + 'Privileged native simulator state and authored phases, gripper/contact logic and recovery remain in use.', + 'No raw human-video action bridge, minimum demonstration count, physical robot skill or guaranteed performance is established.', + 'Aggregate results never authorize an untested scene; installed execution still needs its exact current-runtime qualification.']} + _write(output / 'report.json', report) + return report + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('dataset', type=Path, help='Actual prepared dataset.json with aligned action sidecars.') + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--counts', type=int, nargs='+', help='Nested whole training-group counts; default eligible 2, 4, 7.') + parser.add_argument('--seed', type=int, default=19043) + parser.add_argument('--nominal', type=int, default=8) + parser.add_argument('--stress', type=int, default=4) + parser.add_argument('--task-source', type=Path) + parser.add_argument('--grounding', type=Path) + parser.add_argument('--source-evidence', type=Path, help='Hash-bound native evidence JSON directory for source-layout exclusion.') + args = parser.parse_args() + report = benchmark(args.dataset, args.output, counts=args.counts, seed=args.seed, + nominal=args.nominal, stress=args.stress, task_source=args.task_source, + grounding=args.grounding, source_evidence=args.source_evidence) + print(json.dumps({'output': str(args.output), 'scope': report['scope'], + 'source_scene_exclusion': report['source_scene_exclusion'], + 'comparisons': [{'training_group_count': row['training_group_count'], **row['summary']} + for row in report['comparisons']]})) + + +if __name__ == '__main__': + main() diff --git a/tests/test_footage_benchmark.py b/tests/test_footage_benchmark.py new file mode 100644 index 0000000..1e85c97 --- /dev/null +++ b/tests/test_footage_benchmark.py @@ -0,0 +1,358 @@ +"""Independent group selection, frozen experiment plans and unseen scene bounds.""" +from copy import deepcopy +from hashlib import sha256 +import importlib.util +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from dvidia_training import footage_benchmark as benchmark + + +def receipt_group_id(episode_ids): + """The prepared receipt names a connected group by its sorted episode IDs.""" + encoded = json.dumps(sorted(episode_ids), separators=(',', ':')).encode() + return 'group-' + sha256(encoded).hexdigest()[:24] + + +def grouped_dataset(): + """Metadata fixture: several recordings have more than one video episode.""" + episodes, groups = [], [] + for split, sizes in (('train', [2, 1, 3, 1, 2, 1, 2]), + ('dev', [2, 1]), ('test', [2, 1])): + for index, size in enumerate(sizes): + group_id = f'{split}-recording-{index}' + ids = [] + for clip in range(size): + identifier = f'{group_id}-clip-{clip}' + ids.append(identifier) + episodes.append({ + 'id': identifier, 'group_id': group_id, 'split': split, + 'media_sha256': sha256(identifier.encode()).hexdigest(), + 'source_recording_id': group_id, + 'session_id': f'{group_id}-session', + 'shoe_pair_id': f'{group_id}-pair', + }) + group_id = receipt_group_id(ids) + for episode in episodes[-size:]: + episode['group_id'] = group_id + groups.append({'id': group_id, 'episode_ids': ids, 'split': split}) + return {'episodes': episodes, 'groups': groups, + 'splits': {split: [row['id'] for row in episodes if row['split'] == split] + for split in ('train', 'dev', 'test')}} + + +class CountTests(unittest.TestCase): + def test_counts_are_sorted_and_defaults_fit_available_training_groups(self): + self.assertEqual(benchmark.normalize_counts([7, 2, 4], 7), (2, 4, 7)) + self.assertEqual(benchmark.normalize_counts([1], 7), (1,)) + self.assertEqual(benchmark.normalize_counts(None, 7), (2, 4, 7)) + self.assertEqual(benchmark.normalize_counts(None, 3), (2,)) + self.assertEqual(benchmark.normalize_counts(None, 1), (1,)) + + def test_invalid_counts_cannot_silently_change_the_experiment(self): + for counts in ([], [2, 2], [True], [False, 2], [1.5], ['2'], + [0], [-1], [8]): + with self.subTest(counts=counts), self.assertRaises(ValueError): + benchmark.normalize_counts(counts, 7) + for available in (0, True, 65): + with self.subTest(available=available), self.assertRaises(ValueError): + benchmark.normalize_counts(None, available) + with self.assertRaises(ValueError): + benchmark.normalize_counts(list(range(1, 10)), 9) + + +class GroupSelectionTests(unittest.TestCase): + def test_nested_subsets_keep_recordings_whole_and_evaluation_fixed(self): + dataset = grouped_dataset() + original = deepcopy(dataset) + subsets = benchmark.nested_subsets(dataset, [2, 4, 7], seed=391) + self.assertEqual(dataset, original) + self.assertEqual(subsets, benchmark.nested_subsets(dataset, [2, 4, 7], seed=391)) + self.assertEqual([row['training_group_count'] for row in subsets], [2, 4, 7]) + dev = {e['id'] for e in dataset['episodes'] if e['split'] == 'dev'} + test = {e['id'] for e in dataset['episodes'] if e['split'] == 'test'} + prior = set() + for row in subsets: + selected = set(row['train_group_ids']) + self.assertEqual(len(selected), row['training_group_count']) + self.assertTrue(prior <= selected) + expected = {e['id'] for e in dataset['episodes'] + if e['split'] == 'train' and e['group_id'] in selected} + self.assertEqual(set(row['train_episode_ids']), expected) + self.assertEqual(set(row['dev_episode_ids']), dev) + self.assertEqual(set(row['test_episode_ids']), test) + self.assertTrue(expected.isdisjoint(dev | test)) + self.assertTrue(dev.isdisjoint(test)) + prior = selected + self.assertEqual(set(subsets[-1]['train_episode_ids']), + {e['id'] for e in dataset['episodes'] if e['split'] == 'train'}) + + def test_cross_split_source_identity_is_rejected_before_subsetting(self): + for key in ('media_sha256', 'source_recording_id', 'session_id', 'shoe_pair_id'): + dataset = grouped_dataset() + heldout = next(e for e in dataset['episodes'] if e['split'] == 'test') + affected = [e for e in dataset['episodes'] + if e['group_id'] in (dataset['episodes'][0]['group_id'], heldout['group_id'])] + dataset['episodes'][0][key] = heldout[key] + merged = receipt_group_id([e['id'] for e in affected]) + for episode in affected: + episode['group_id'] = merged + with self.subTest(identity=key), self.assertRaisesRegex(ValueError, 'cross.*splits'): + benchmark.nested_subsets(dataset, [2], seed=391) + + def test_declared_groups_cannot_split_connected_recordings(self): + dataset = grouped_dataset() + dataset['episodes'][0]['group_id'] = 'pretend-independent-recording' + with self.assertRaisesRegex(ValueError, 'connected recording lineage'): + benchmark.nested_subsets(dataset, [2], seed=391) + + def test_absent_development_or_test_cannot_become_a_training_curve(self): + for missing in ('dev', 'test'): + dataset = grouped_dataset() + dataset['episodes'] = [e for e in dataset['episodes'] if e['split'] != missing] + dataset['groups'] = [g for g in dataset['groups'] if g['split'] != missing] + with self.subTest(split=missing), self.assertRaises(ValueError): + benchmark.nested_subsets(dataset, [2], seed=391) + + def test_split_receipt_cannot_disagree_with_episode_assignments(self): + dataset = grouped_dataset() + dataset['splits']['test'] = dataset['splits']['dev'] + with self.assertRaisesRegex(ValueError, 'split metadata'): + benchmark.nested_subsets(dataset, [2], seed=391) + + +class QualificationSummaryTests(unittest.TestCase): + def test_control_success_and_errors_remain_visible_without_qualification(self): + rows = [ + {'group': 'nominal', 'status': 'completed', 'student_success': True, + 'control_success': False, 'qualified': True, + 'student_episode_wall_seconds': 0.2, 'student_simulated_seconds': 10., + 'student_final_target_error_m': 0.002, + 'control_episode_wall_seconds': 0.3, 'control_simulated_seconds': 18., + 'control_final_target_error_m': 0.12}, + {'group': 'nominal', 'status': 'completed', 'student_success': True, + 'control_success': True, 'qualified': False}, + {'group': 'actuator_stress', 'status': 'error', 'qualified': False}, + {'group': 'actuator_stress', 'status': 'completed', 'student_success': False, + 'control_success': False, 'qualified': False}, + ] + summary = benchmark._aggregate(rows) + self.assertEqual(summary['planned_scenes'], 4) + self.assertEqual(summary['student_successes'], 2) + self.assertEqual(summary['qualified_scenes'], 1) + self.assertEqual(summary['control_successes'], 1) + self.assertEqual(summary['errors'], 1) + self.assertEqual(summary['groups']['nominal']['planned_scenes'], 2) + self.assertEqual(summary['groups']['nominal']['qualified_scenes'], 1) + self.assertEqual(summary['groups']['actuator_stress']['planned_scenes'], 2) + self.assertEqual(summary['groups']['actuator_stress']['qualified_scenes'], 0) + self.assertEqual(summary['groups']['actuator_stress']['errors'], 1) + self.assertAlmostEqual(summary['student_episode_wall_seconds'], 0.2) + self.assertAlmostEqual(summary['control_episode_wall_seconds'], 0.3) + self.assertEqual(summary['student_measurements']['measured_scenes'], 1) + self.assertEqual(summary['groups']['actuator_stress']['student_measurements']['measured_scenes'], 0) + + def test_throughput_uses_total_simulated_time_over_total_wall_time(self): + rows = [ + {'student_episode_wall_seconds': 2., 'student_simulated_seconds': 10., + 'student_final_target_error_m': 0.01, + 'control_episode_wall_seconds': 1., 'control_simulated_seconds': 100., + 'control_final_target_error_m': 0.05}, + {'student_episode_wall_seconds': 8., 'student_simulated_seconds': 8., + 'student_final_target_error_m': 0.03}, + {'status': 'error'}, + ] + summary = benchmark._measurement_summary(rows, 'student') + self.assertEqual(summary['measured_scenes'], 2) + self.assertEqual(summary['total_simulated_seconds'], 18.) + self.assertEqual(summary['total_episode_wall_seconds'], 10.) + self.assertAlmostEqual(summary['simulated_seconds_per_wall_second'], 1.8) + self.assertNotAlmostEqual(summary['simulated_seconds_per_wall_second'], (5. + 1.) / 2) + self.assertAlmostEqual(summary['final_target_error_mean_m'], 0.02) + self.assertAlmostEqual(summary['final_target_error_max_m'], 0.03) + self.assertEqual(benchmark._measurement_summary(rows, 'control')['measured_scenes'], 1) + unmeasured = benchmark._measurement_summary([{'status': 'error'}], 'student') + self.assertEqual(unmeasured['measured_scenes'], 0) + self.assertIsNone(unmeasured['simulated_seconds_per_wall_second']) + self.assertIsNone(unmeasured['final_target_error_mean_m']) + + def test_missing_or_nonfinite_target_errors_do_not_discard_timed_failures(self): + distances = [0.01, None, True, False, float('nan'), float('inf'), 0.03, 0] + rows = [{ + 'group': 'nominal', 'status': 'error' if index in {1, 2, 3, 4, 5} else 'completed', + 'qualified': False, 'student_episode_wall_seconds': float(index + 1), + 'student_simulated_seconds': float(10 * (index + 1)), + 'student_final_target_error_m': distance, + } for index, distance in enumerate(distances)] + measurements = benchmark._measurement_summary(rows, 'student') + self.assertEqual(measurements['timed_scenes'], 8) + self.assertEqual(measurements['measured_scenes'], 8) + self.assertEqual(measurements['measured_error_scenes'], 3) + self.assertEqual(measurements['total_episode_wall_seconds'], 36.) + self.assertEqual(measurements['total_simulated_seconds'], 360.) + self.assertEqual(measurements['simulated_seconds_per_wall_second'], 10.) + self.assertAlmostEqual(measurements['final_target_error_mean_m'], 0.04 / 3) + self.assertEqual(measurements['final_target_error_max_m'], 0.03) + summary = benchmark._aggregate(rows) + self.assertEqual(summary['planned_scenes'], 8) + self.assertEqual(summary['errors'], 5) + self.assertEqual(summary['groups']['nominal']['planned_scenes'], 8) + self.assertEqual(summary['groups']['nominal']['errors'], 5) + self.assertEqual(summary['student_measurements'], measurements) + + +class FrozenProtocolTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.output = Path(temporary.name) / 'new-experiment' + self.protocol = {'schema_version': 1, 'seed': 391, + 'candidate_sha256': 'a' * 64, + 'scenes': [{'id': 'unseen-scene', 'mass': 0.04}]} + + def test_frozen_plan_verifies_and_refuses_replacement(self): + benchmark.freeze_protocol(self.output, self.protocol) + original = {name: (self.output / name).read_bytes() + for name in ('protocol.json', 'protocol.sha256')} + self.assertEqual(json.loads(original['protocol.json']), self.protocol) + benchmark.verify_protocol(self.output, self.protocol) + with self.assertRaises((ValueError, FileExistsError)): + benchmark.freeze_protocol(self.output, {**self.protocol, 'seed': 999}) + self.assertEqual({name: (self.output / name).read_bytes() for name in original}, original) + + def test_changed_plan_json_hash_or_expected_protocol_is_rejected(self): + benchmark.freeze_protocol(self.output, self.protocol) + expected = deepcopy(self.protocol) + expected['scenes'][0]['mass'] = 0.08 + with self.assertRaises(ValueError): + benchmark.verify_protocol(self.output, expected) + for name in ('protocol.json', 'protocol.sha256'): + path = self.output / name + original = path.read_bytes() + path.write_bytes(original + b'changed') + with self.subTest(file=name), self.assertRaises(ValueError): + benchmark.verify_protocol(self.output, self.protocol) + path.write_bytes(original) + benchmark.verify_protocol(self.output, self.protocol) + + def test_matching_replacement_json_and_hash_do_not_replace_the_expected_plan(self): + benchmark.freeze_protocol(self.output, self.protocol) + replacement = {**self.protocol, 'seed': 999} + raw = json.dumps(replacement, sort_keys=True, separators=(',', ':')).encode() + b'\n' + (self.output / 'protocol.json').write_bytes(raw) + (self.output / 'protocol.sha256').write_text(sha256(raw).hexdigest() + '\n') + with self.assertRaises(ValueError): + benchmark.verify_protocol(self.output, self.protocol) + + +@unittest.skipUnless(importlib.util.find_spec('mujoco') is not None, + 'Scene generation needs the optional native arm engine.') +class SceneProtocolTests(unittest.TestCase): + def test_source_evidence_total_limit_uses_retained_bytes_after_small_stat(self): + payloads = [json.dumps({'recording': index, 'padding': 'x' * 48}).encode() + for index in range(2)] + file_limit = max(map(len, payloads)) + 1 + total_limit = sum(map(len, payloads)) - 1 + self.assertLess(file_limit, total_limit) + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary).resolve() + evidence_paths = {directory / f'recording-{index}.json' + for index in range(len(payloads))} + for path, raw in zip(sorted(evidence_paths), payloads): + path.write_bytes(raw) + original_stat = Path.stat + + def stale_small_stat(path, *args, **kwargs): + result = original_stat(path, *args, **kwargs) + if path in evidence_paths: + fields = list(result) + fields[6] = 1 # Preserve real file mode, but simulate pre-read growth. + return os.stat_result(fields) + return result + + with patch.object(Path, 'stat', stale_small_stat), \ + patch.object(benchmark, 'EVIDENCE_FILE_MAX_BYTES', file_limit), \ + patch.object(benchmark, 'EVIDENCE_TOTAL_MAX_BYTES', total_limit): + self.assertLess(sum(path.stat().st_size for path in evidence_paths), total_limit) + with self.assertRaisesRegex(ValueError, 'Native evidence exceeds'): + benchmark._source_scenes({'episodes': []}, {}, directory) + + def test_scene_protocol_is_deterministic_varied_and_within_supported_bounds(self): + from dvidia_training.arm_runner import validate_scene + rows = benchmark.make_scenes(804213, nominal=8, stress=4) + self.assertEqual(rows, benchmark.make_scenes(804213, nominal=8, stress=4)) + self.assertEqual(len(rows), 12) + self.assertEqual(len({r['id'] for r in rows}), 12) + self.assertEqual(len({r['scene_sha256'] for r in rows}), 12) + self.assertEqual(len({r['layout_sha256'] for r in rows}), 12) + self.assertEqual(sum(r['group'] == 'nominal' for r in rows), 8) + self.assertEqual(sum(r['group'] == 'actuator_stress' for r in rows), 4) + for row in rows: + scene = validate_scene(row['scene']) + self.assertEqual(scene, row['scene']) + for key in ('scene_sha256', 'layout_sha256'): + self.assertRegex(row[key], r'^[a-f0-9]{64}$') + for field in ('object_position', 'target_position', 'object_size', + 'object_mass', 'object_friction'): + values = {json.dumps(r['scene'][field]) for r in rows} + self.assertGreater(len(values), 1, field) + self.assertNotEqual(rows, benchmark.make_scenes(804214, nominal=8, stress=4)) + + def test_source_layout_is_excluded_even_when_materials_or_seed_change(self): + original = benchmark.make_scenes(804213, nominal=8, stress=4) + source = deepcopy(original[0]['scene']) + source['seed'] += 10000 + source['object_mass'] = 0.08 if source['object_mass'] != 0.08 else 0.02 + fresh = benchmark.make_scenes(804213, nominal=8, stress=4, source_scenes=[source]) + self.assertNotIn(original[0]['layout_sha256'], {r['layout_sha256'] for r in fresh}) + self.assertEqual(len(fresh), 12) + self.assertEqual(len({r['layout_sha256'] for r in fresh}), 12) + + def test_source_scene_uses_hashed_bytes_when_the_file_changes_before_parsing(self): + from dvidia_training import footage_train + expected = {'object_position': [.42, -.04, .31], 'target_position': [.54, .10, .31], + 'object_size': [.04, .04, .04], 'object_mass': .04, 'object_friction': .8, + 'jaw_force_limit': 15., 'joint_torque_limit': 40., 'seed': 17} + evidence = {'kind': 'dvidia.native-telemetry-evidence', 'schema_version': 1, + 'scope': 'simulation-only', 'seed': 17, + 'final_info': {'config': { + 'object_position': expected['object_position'], + 'target_position': expected['target_position'], + 'object_half_size': [.02, .02, .02], 'object_mass': .04, + 'object_friction': .8, 'jaw_force_limit': 15., + 'joint_torque_limit': 40., 'scene_jitter': 0., 'table_height': .29}}} + original = json.dumps(evidence, sort_keys=True).encode() + digest = sha256(original).hexdigest() + replacement = deepcopy(evidence) + replacement['final_info']['config']['object_mass'] = .08 + replacement['final_info']['config']['object_position'] = [.35, .08, .31] + changed = json.dumps(replacement, sort_keys=True).encode() + dataset = {'episodes': [{'id': 'original-source', 'split': 'train'}]} + sidecars = {'original-source': {'provenance': { + 'source_kind': 'native-simulation-telemetry', + 'validation': {'evidence_sha256': digest}}}} + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary).resolve() + path = directory / 'original-source.json' + path.write_bytes(original) + parser = footage_train._json + + def mutate_then_parse(raw, **kwargs): + path.write_bytes(changed) + return parser(raw, **kwargs) + + with patch.object(footage_train, '_json', side_effect=mutate_then_parse) as parsing: + scenes, receipt = benchmark._source_scenes(dataset, sidecars, directory) + self.assertEqual(parsing.call_count, 1) + self.assertEqual(path.read_bytes(), changed) + self.assertEqual(scenes, [expected]) + self.assertEqual(receipt['bindings'][0]['evidence_sha256'], digest) + self.assertEqual(receipt['bindings'][0]['scene'], expected) + + +if __name__ == '__main__': + unittest.main()