From 031311f5fc804c3f2eed6067f8546fa6b56de3eb Mon Sep 17 00:00:00 2001 From: Tarzelf Date: Tue, 6 Oct 2026 09:04:30 -0400 Subject: [PATCH] Document evidence-based Skillspace graduation and Argus pilot --- README.md | 4 + docs/topics.json | 26 ++-- reports/From Skillspace to robot skill.md | 138 ++++++++++++++++++ topics/evaluation-provenance/README.md | 1 + .../experiments/0002-skill-graduation.md | 66 +++++++++ topics/evaluation-provenance/references.bib | 56 +++++++ topics/evaluation-provenance/topic.json | 26 ++-- 7 files changed, 297 insertions(+), 20 deletions(-) create mode 100644 reports/From Skillspace to robot skill.md create mode 100644 topics/evaluation-provenance/experiments/0002-skill-graduation.md diff --git a/README.md b/README.md index 131d20a..1de501b 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,10 @@ Small, reproducible studies of what makes a physical demonstration useful: visib All four covers are **AI-generated editorial illustrations**, not footage, measurements or experimental evidence. [Image provenance](docs/assets/PROMPTS.md). +## Latest study + +[From Skillspace to robot skill](reports/From%20Skillspace%20to%20robot%20skill.md) reviews Argus and proposes a measurable path from accepted videos to robot-tested releases. Includes a current implementation gap analysis, DOF compatibility requirements and an unrun [annotation pilot](topics/evaluation-provenance/experiments/0002-skill-graduation.md). + ## A useful first contribution Choose one numbered question in a topic README. Open a [research question](https://github.com/Dvidia-Inference/dvidia-research/issues/new?template=research-question.yml), identify the uncertainty and link the primary source. A source correction, clearer annotation definition or tiny reproducible fixture is a useful contribution before any model run. diff --git a/docs/topics.json b/docs/topics.json index e9224a6..a5b7ff6 100644 --- a/docs/topics.json +++ b/docs/topics.json @@ -197,13 +197,6 @@ "type": "paper", "year": 2021 }, - { - "id": "rlds2026schema", - "title": "RLDS dataset schema (repository accessed 2026)", - "url": "https://github.com/google-research/rlds", - "type": "code", - "year": 2026 - }, { "id": "lerobot2026v3", "title": "LeRobotDataset v3.0 (documentation accessed 2026)", @@ -219,9 +212,16 @@ "year": 2021 }, { - "id": "lerobot2026license", - "title": "LeRobot source license and third-party notices (accessed 2026)", - "url": "https://github.com/huggingface/lerobot/blob/main/LICENSE", + "id": "li2026argus", + "title": "Argus: An Open-Source Annotator for Robotics Data", + "url": "https://pantheon.inc/research/argus", + "type": "essay", + "year": 2026 + }, + { + "id": "pantheon2026argus", + "title": "Argus source at reviewed revision (accessed 2026)", + "url": "https://github.com/Pantheon-Industries-Inc/argus/tree/6c99686a3d93027c517f02f37f24b5b68532e12a", "type": "code", "year": 2026 } @@ -244,6 +244,12 @@ "text": "How much does recording-level leakage change the selected evaluation metric?", "state": "open", "issueUrl": null + }, + { + "id": "ep-04", + "text": "Do grounded temporal annotations reduce false completion labels, and what evidence should gate a robot-tested skill release?", + "state": "open", + "issueUrl": "https://github.com/Dvidia-Inference/dvidia-research/issues/5" } ], "milestones": [ diff --git a/reports/From Skillspace to robot skill.md b/reports/From Skillspace to robot skill.md new file mode 100644 index 0000000..ea81d8d --- /dev/null +++ b/reports/From Skillspace to robot skill.md @@ -0,0 +1,138 @@ +# From Skillspace to robot skill + +**Source review and proposal · 6 October 2026 · Experiments not run** + +A Skillspace can become the home for a skill's learning history. More uploaded videos do not, by themselves, establish a better policy. We need evidence that new demonstrations add useful coverage, survive quality checks, and improve held-out performance on a specified robot. + +This review follows [Varun Nair's pipeline post](https://x.com/_varunnair/status/2107188112980054270), its linked Argus article, selected upstream source files, and an audit of the DVIDIA website. The design below is our proposal, not an implemented training service. No user footage was sent to any model, no training was run, and no robot was operated for this review. + +## What the linked work contributes + +Pantheon's **Argus** combines temporal visual annotation with deterministic data checks. The authors report medium-or-higher issues in 27% of 3,546 audited episodes. That is an author-reported finding on their sample, not an estimate for our corpus. Their comparison uses Astra as a reference; agreement with it and higher annotation density do not independently establish correctness or better policy learning. The article's moving-camera and misleading-instruction examples are especially relevant to neck-mounted POV capture. [li2026argus] + +The inspected repository revision is `6c99686a3d93027c517f02f37f24b5b68532e12a`. It separates preparation, checks, labeling, review and display. Runs preserve settings and outputs; its documentation acknowledges nondeterministic labels. `label/frames.py` checks exact frame PTS, respects rotation/mirroring, limits decoding threads and releases full-resolution images after preparing smaller views. This is a useful reference for a server-side adapter, not a mobile recording dependency. [pantheon2026argus] + +Upstream code is Apache-2.0 with third-party notices. Optional hand-pose dependencies and outputs carry additional restrictions, including non-commercial terms described by upstream. Do not assume the repository license covers checkpoints, source footage or generated annotations. Keep the initial integration limited to ingestion, checks and labels; review optional dependencies separately. [argus2026notices] + +**Recommendation:** adopt the evidence discipline first. Evaluate a narrow Argus-compatible adapter before importing its full dashboard, hand-pose stack or model choices. + +## What DVIDIA actually has + +Implementation audit snapshot: 6 October 2026. These are findings from internal source review, not claims about uninspected private records. This public report summarizes behavior without reproducing private source or account data. + +| Area | Present foundation | Missing connection | +| --- | --- | --- | +| Account Skillspaces | Owned spaces, private originals, file hashes and consent records | Personal spaces are currently named collections; their generic saved brief is not a versioned task specification | +| Curation | Separate tooling supports temporal segments, before/after states, uncertainty, revisioned briefs and review decisions | New account uploads do not automatically become curated training episodes | +| Dataset releases | Rights-aware metadata releases and provenance | A metadata export is not a synchronized observation/action training set | +| Coverage | Counts distinguish several evidence states | Coverage is explicitly unknown; no target distribution or measured learning curve | +| Readiness | An explanatory four-stage interface | No automatic training or graduation process | +| Robot delivery | Exact-profile package validation and a limited adapter path | No demonstrated transfer from these Skillspaces, independent graduation test or arbitrary-robot support | + +The valuable foundation is traceability. Connect those existing pieces through stable IDs and immutable versions rather than introducing another disconnected upload workflow. + +## The proposed learning loop + +```mermaid +flowchart LR + A[Record or upload] --> B[Checks and suggested annotations] + B --> C[Accepted task episodes] + C --> D[Versioned dataset] + D --> E[Training run for a robot profile] + E --> F[Held-out evaluation] + F --> G[Versioned robot-tested skill] + F --> H[Missing situations and failure examples] + H --> A +``` + +An uploaded original remains immutable. Annotation revisions, consent, dataset membership and model artifacts reference it; they never overwrite it. An episode with a useful recovery can be valuable even when the attempt failed. Preserve outcomes and mark segments eligible for imitation, recovery learning or evaluation separately. + +Proposed records: + +- **Task version:** observable goal, start/end conditions, required objects, success rubric, failure categories and target capture situations. +- **Episode:** original hash, consent reference, task version, source/session grouping, camera setup, exact timebase and segment boundaries. Missing calibration or robot actions remain explicitly missing. +- **Annotation run:** input/frame digests, model/provider revision, prompt and recipe hashes, cost, evidence intervals, confidence and abstentions. Keep proposed and reviewed judgments distinct. +- **Dataset version:** accepted episode revisions, rights decision, duplicate groups, exclusions, frozen splits and intended training purpose. +- **Training run:** dataset digest, base checkpoint, code/environment, seed, optimization budget, action schema and output checkpoint digest. +- **Evaluation and skill release:** robot profile, test-suite version, trials, strict successes, interventions, failure slices, uncertainty, reviewer and release signature. + +An annotation outcome and a policy success rate belong to different records. A generated pose or inferred action must never silently become measured robot supervision. + +## What “more videos” should mean + +Volume is useful only relative to a task and learning recipe. Repeated near-identical clips may add little; corrupted labels or train/test leakage can make an apparent gain misleading. Open X-Embodiment provides evidence that combined robot datasets can produce positive transfer in studied settings, not that every additional phone video improves every robot. [oxe2025] + +For the initial task, define a small coverage matrix: object variation, starting arrangement, camera setup, demonstrator/session, successful completion, and recoverable failure. Derive it from the task and deployment environment. Do not grade a person's skill, location or filming aesthetics as a substitute for usable evidence. + +Show separate counts for **uploaded**, **accepted**, **unique sessions**, **outcomes visible**, and **robot-aligned**. A coverage percentage is meaningful only after its denominator is versioned. Never label bytes stored, videos tagged or a checklist percentage “training complete.” + +Choose the next requested recording from an uncovered situation or measured model failure. Candidate priorities can consider the missing condition, annotation uncertainty, duplicate burden and collection cost. Keep those scores provisional until an ablation shows they select more useful training data. + +## The bridge from phone video to robot action + +POV video can support task descriptions, temporal segmentation, visual representations, outcome recognition and collection planning. Direct supervised control usually needs an action-learning bridge: paired robot demonstrations, measured interaction hardware, validated retargeting, or a separately evaluated method for learning actions from observation. + +UMI is relevant because it combines handheld grippers with a designed trajectory/control interface and latency handling. Its reported transfer relies on that system; it does not demonstrate that ordinary neck-mounted phone footage alone supplies equivalent action supervision. [chi2024umi] + +Our lowest-risk first control experiment is a simple object-placement task on one fixed robot/camera setup. Keep a robot-demonstration baseline. Add human-video-derived signals as a separate controlled change; otherwise improvement cannot be attributed to the phone footage. + +## DOFs and compatibility + +Here **DOF** means degrees of freedom. It describes independent motion capability; actuators and a controller execute a policy. Matching a numeric DOF count is insufficient to select a compatible skill. + +RT-X's published interface, for example, distinguishes translation, rotation and gripper movement, with explicit action semantics. These are not interchangeable with an arbitrary vector of joint angles. MoveIt likewise depends on a robot model, joint limits and collision geometry. [oxe2026interface] [moveit2026model] + +A proposed `robot_profile` should specify robot/model revision; ordered joints and units; control mode and rate; actuator limits; gripper type/opening; camera streams, intrinsics/extrinsics and calibration hashes; coordinate frames; observation/action schema; and runtime requirements. Execution should reject unknown or mismatched profiles. + +DVIDIA's inspected SO-101 profile names **five arm joints plus a gripper channel**. Six output channels do not imply unrestricted six-DOF Cartesian motion. A policy tested on this profile must not receive a generic “all robots” badge. New hardware needs an adapter and its own evaluation record. + +## Graduation should require evidence + +| Stage shown to people | Evidence required to enter | Main next action | +| --- | --- | --- | +| Collecting | Task and rights defined | Record the missing example | +| Dataset ready | Frozen accepted episodes, quality checks, splits and intended use | Review the dataset version | +| Training | Real run ID and resource budget | View run status | +| Testing | Checkpoint and fixed evaluation protocol | See results and failures | +| Robot-tested | Passed task-specific criteria on a named physical setup | View compatible robots and install requirements | + +A trained checkpoint may remain in testing, fail, or be retired. Simulation success and physical success must remain separately labeled. Test criteria must be agreed before results are inspected. A release should state its operating limits; a badge never replaces them. + +Keep **Everyday / Skilled / Specialist / Artisan** as task-character descriptors. They are separate from training maturity, compatibility and reward eligibility. Artisan is not automatically more robot-ready. + +For composed skills, store preconditions, postconditions, required resources and failure/recovery transitions. A video showing A before B is evidence of sequence, not proof that A is a prerequisite. Test the complete chain: isolated component success is not end-to-end reliability. + +## A simple mobile presentation + +Retain three views: **Overview · Videos · Results**. The overview has a short stage rail, one useful gap and one primary action. Example copy, explicitly a design example rather than live counts: + +> Collecting · We need examples with different starting positions. +> +> Record an example + +Videos have a thumbnail and one status: Checking, Needs detail, Accepted or Excluded. Open a video to see its timeline and optional corrections. Results shows the dataset version, real evaluation evidence and a robot compatibility chip when available. Put technical details behind a disclosure, not in the upload path. + +Keep upload progress and background annotation progress separate. Show actual bytes for upload; use named analysis stages when remaining time is unknown. Dense analysis runs asynchronously on the server, with cached frame/recipe results, bounded queues and retry-safe jobs. A model timeout must not lose a saved original or block normal account navigation. + +## Next work, in order + +1. Link personal spaces to a stable task specification and existing curation IDs. +2. Add a versioned annotation adapter and deterministic media checks with explicit permissions and cost caps. +3. Define one coverage matrix and frozen episode-level split; route users toward a missing example. +4. Execute the [small annotation pilot](../topics/evaluation-provenance/experiments/0002-skill-graduation.md). +5. Separately validate the observation-to-action bridge and a robot training/evaluation loop. +6. Only then activate a graduation badge backed by a signed, versioned evaluation receipt. + +The first deliverable is evidence that our accepted episodes and annotations are reliable—not an automatic promise of robot learning. Commercial training permissions and contributor revenue terms remain separate from storage, public visibility and these research protocols. + +## Sources + +- [li2026argus] Eric Li, [Argus: An Open-Source Annotator for Robotics Data](https://pantheon.inc/research/argus), 1 October 2026. Author-reported results; not independently reproduced here. +- [pantheon2026argus] Pantheon, [Argus at inspected revision](https://github.com/Pantheon-Industries-Inc/argus/tree/6c99686a3d93027c517f02f37f24b5b68532e12a), accessed 6 October 2026. [Exact frame decoding](https://github.com/Pantheon-Industries-Inc/argus/blob/6c99686a3d93027c517f02f37f24b5b68532e12a/label/frames.py). +- [argus2026notices] Pantheon, [third-party notices](https://github.com/Pantheon-Industries-Inc/argus/blob/6c99686a3d93027c517f02f37f24b5b68532e12a/THIRD_PARTY_NOTICES.txt), accessed 6 October 2026. +- [chi2024umi] Cheng Chi et al., [Universal Manipulation Interface](https://arxiv.org/abs/2402.10329v3), 2024. +- [oxe2025] Open X-Embodiment Collaboration, [Open X-Embodiment: Robotic Learning Datasets and RT-X Models](https://arxiv.org/abs/2310.08864v9), revision 2025; initially submitted 2023. +- [oxe2026interface] Google DeepMind, [RT-X observation/action interface](https://github.com/google-deepmind/open_x_embodiment/blob/main/README.md), living documentation accessed 6 October 2026. +- [moveit2026model] MoveIt, [URDF and SRDF](https://moveit.picknik.ai/main/doc/examples/urdf_srdf/urdf_srdf_tutorial.html), living documentation accessed 6 October 2026. + +Bibliographic entries: [evaluation and provenance bibliography](../topics/evaluation-provenance/references.bib). This report's open documentation license does not license user videos or downstream training data. diff --git a/topics/evaluation-provenance/README.md b/topics/evaluation-provenance/README.md index 61ddb08..f8f191f 100644 --- a/topics/evaluation-provenance/README.md +++ b/topics/evaluation-provenance/README.md @@ -27,6 +27,7 @@ RLDS describes episode boundaries and optional action fields [rlds2026schema]. L 1. **ep-01** — Can independent environments reproduce identical inventories, grouped splits and supported episode readback? 2. **ep-02** — Which corruptions and duplicated derivatives evade a small provenance validator? 3. **ep-03** — How much does recording-level leakage change the selected evaluation metric? +4. **ep-04** — Do grounded temporal annotations reduce false completion labels, and what evidence should gate a robot-tested skill release? See the [Argus review and graduation proposal](../../reports/From%20Skillspace%20to%20robot%20skill.md) and [proposed pilot](experiments/0002-skill-graduation.md). ## First small experiment diff --git a/topics/evaluation-provenance/experiments/0002-skill-graduation.md b/topics/evaluation-provenance/experiments/0002-skill-graduation.md new file mode 100644 index 0000000..e45d896 --- /dev/null +++ b/topics/evaluation-provenance/experiments/0002-skill-graduation.md @@ -0,0 +1,66 @@ +# Pilot 0002 — Annotation quality before skill graduation + +**Status: proposed, not run.** Linked question: **ep-04** ([issue #5](https://github.com/Dvidia-Inference/dvidia-research/issues/5)). No paid inference, recording transfer, hardware purchase or robot operation is authorized by this document. Resource ceilings below are proposed stop limits, not a spending commitment. + +## Hypothesis and scope + +For one narrowly defined object-placement task, timestamp-grounded annotations plus deterministic media checks reduce false task-completion labels compared with a fixed visual-annotation baseline (condition C versus B below). Existing coarse tagging is included only as product-capability and cost context. This measures annotation utility only. It does not establish policy improvement or robot transfer. + +A result that contradicts the hypothesis is no improvement in false-completion errors on the held-out set, worse abstention-adjusted coverage, or higher cost without useful error reduction. + +## Data and split + +Use up to 40 original or permission-cleared episodes, at most 30 minutes of footage and 2 GB. No private account footage enters a third-party service without the applicable analysis permission. Source licenses and training eligibility are recorded separately. + +Use one observable task: move an object into a specified tray and release it. Include success, incomplete attempts, successful recovery, camera motion and occlusion. Two reviewers independently mark the visible completion state and event intervals, then adjudicate disagreements. They must not see model output first. Keep unobservable outcomes unknown. + +Freeze 20 development and 20 held-out episodes, grouped by capture session and demonstrator. No original/derivative or near-duplicate group crosses splits. If grouping cannot produce these sizes, report the actual split and reduce scope before opening the test set. Record original hashes, display rotation/mirroring, PTS/timebase and camera mount. + +## Conditions + +A. Existing coarse tag/routing output, retaining its actual capabilities. If it has no completion or interval field, report those metrics as unavailable; do not turn a category into a success label. + +B. A fixed baseline visual annotation recipe supplying task/setup context and ordered sampled frames. + +C. The same visual model and budget with timestamp-exact frames, explicit uncertainty and observation-grounded task verification. Keep deterministic check results separate from model judgments. + +Use the same footage, source frame selection budget and pinned model settings where possible; record unavoidable differences. The Argus revision reviewed in the report is a reference implementation, not a result. Evaluate any cheaper model in a separate comparison using the frozen inputs and independent labels. + +## Metrics and reporting + +- Strict completion precision: adjudicated true successes among episodes called successful with an observable reference outcome. Report numerator, denominator and Wilson 95% interval; abstentions are not true negatives. Separately count unsupported success claims on reference-unknown episodes. +- False completion count, especially for occluded outcomes and success later undone. +- Success recall: correctly predicted successes divided by all adjudicated visible successes, including model abstentions in the denominator. Report numerator, denominator and Wilson 95% interval. Publish the full success/failure/unknown confusion matrix; reference-unknown episodes do not enter visible-success precision or recall denominators and are reported separately. +- Coverage: non-abstained episodes divided by eligible episodes, with abstention reason counts. +- Event matching: one-to-one same-type matching at temporal IoU >=0.5; report precision, recall, F1 and boundary absolute error for matched events. Unobserved events stay unknown. +- Deterministic faults: seed at least one truncated file, wrong episode boundary and incorrect display transform in separate synthetic fixtures. Report each detection and any false alarms on valid fixtures. +- Cost and time: actual billed or explicitly estimated inference cost, per-footage-minute processing time, retries, peak memory and cache reuse. Separate local preprocessing from model latency. + +Predeclare failure slices and show per-episode errors. Advance C only if it reduces false-completion errors relative to B without lowering success recall or coverage or increasing unsupported success claims on the held-out set; otherwise report a tradeoff or an inconclusive result. A suggested engineering target is zero undetected seeded timestamp/boundary faults and >=90% completion precision, but the small pilot cannot certify a production model; intervals, coverage and critical errors govern the next decision. Review the protocol before running, not after seeing results. + +## Resources and stop conditions + +Proposed cap: 40 source episodes, 30 footage minutes, 2 GB, four operator-hours, CPU preprocessing concurrency <=2, and a separately approved inference cap <=US$20. Cache keys include source hash, timestamps, display transform, sampling recipe and model/prompt version. Retries consume the same cap. Stop on missing permission, unresolved dependency terms, frame mismatch, leakage, budget exhaustion or a model receiving media outside the allowlist. + +Do not run upstream quickstarts that download unrelated datasets. No robot-control output or “Robot-tested” badge is produced by this pilot. + +## Follow-up needed to test “more data improves the robot” + +A separate preregistered experiment needs an actual compatible robot, validated calibration and synchronized action demonstrations. Freeze nested smaller/larger training sets by session and keep held-out sessions unchanged. Compare one pinned policy recipe at equal optimizer-step/compute budgets across several seeds; use the same randomized test conditions and a predeclared strict success rubric. Report successes/trials, interventions, confidence intervals, failure slices and all failed runs. + +To isolate the benefit of ordinary human video, compare the same robot baseline with and without the proposed human-video-derived training signal. Simply adding robot demonstrations would test a different hypothesis. Video count, annotation count and a successful export are not outcome metrics. + +Before physical trials, agree robot profile, allowed workspace, operating limits, supervision, stop mechanism and a bounded trial/compute budget. A resource estimate and power/precision analysis determine trial counts; this document does not pretend 40 annotation episodes are sufficient for hardware graduation. + +## Reproducibility and publication + +Publish eligible manifest metadata, split IDs, source/model/code revisions, recipe and frame digests, metrics and error analysis. Keep participant contacts, private consent documents, credentials and unlicensed media out of this repository. Record checkpoint lineage only after a real training run. + +- [ ] Protocol and rights reviewed +- [ ] Fixture faults and split validated +- [ ] Independent reference labels completed +- [ ] Annotation pilot executed +- [ ] Results and limitations reviewed +- [ ] Separate control-learning experiment preregistered + +**Results: not run.** [Design and source review](../../../reports/From%20Skillspace%20to%20robot%20skill.md). diff --git a/topics/evaluation-provenance/references.bib b/topics/evaluation-provenance/references.bib index 44ae289..281e016 100644 --- a/topics/evaluation-provenance/references.bib +++ b/topics/evaluation-provenance/references.bib @@ -37,3 +37,59 @@ @misc{lerobot2026license url = {https://github.com/huggingface/lerobot/blob/main/LICENSE}, note = {Living or undated source; year records access, not initial publication. Accessed 2026-10-06.} } + +@misc{li2026argus, + author = {Li, Eric}, + title = {{Argus: An Open-Source Annotator for Robotics Data}}, + year = {2026}, + url = {https://pantheon.inc/research/argus}, + note = {Published 2026-10-01; accessed 2026-10-06. Author-reported results, not reproduced here.} +} + +@misc{pantheon2026argus, + author = {{Pantheon Industries}}, + title = {{Argus source at reviewed revision}}, + year = {2026}, + url = {https://github.com/Pantheon-Industries-Inc/argus/tree/6c99686a3d93027c517f02f37f24b5b68532e12a}, + note = {Revision 6c99686a3d93027c517f02f37f24b5b68532e12a. Accessed 2026-10-06.} +} + +@misc{argus2026notices, + author = {{Pantheon Industries}}, + title = {{Argus third-party notices}}, + year = {2026}, + url = {https://github.com/Pantheon-Industries-Inc/argus/blob/6c99686a3d93027c517f02f37f24b5b68532e12a/THIRD_PARTY_NOTICES.txt}, + note = {Accessed 2026-10-06. Source code and optional artifacts have separate terms.} +} + +@misc{chi2024umi, + author = {Chi, Cheng and Xu, Zhenjia and Pan, Chuer and Cousineau, Eric and Burchfiel, Benjamin and Feng, Siyuan and Tedrake, Russ and Song, Shuran}, + title = {{Universal Manipulation Interface: In-The-Wild Robot Teaching Without In-The-Wild Robots}}, + year = {2024}, + url = {https://arxiv.org/abs/2402.10329v3}, + note = {Accessed 2026-10-06. Reviewed abstract and project summary; no reproduction.} +} + +@misc{oxe2025, + author = {{Open X-Embodiment Collaboration}}, + title = {{Open X-Embodiment: Robotic Learning Datasets and RT-X Models}}, + year = {2025}, + url = {https://arxiv.org/abs/2310.08864v9}, + note = {Initially submitted 2023; revision v9 in 2025. Accessed 2026-10-06.} +} + +@misc{oxe2026interface, + author = {{Google DeepMind}}, + title = {{RT-X observation and action interface}}, + year = {2026}, + url = {https://github.com/google-deepmind/open_x_embodiment/blob/main/README.md}, + note = {Living documentation; year records access, not publication. Accessed 2026-10-06.} +} + +@misc{moveit2026model, + author = {{MoveIt contributors}}, + title = {{URDF and SRDF}}, + year = {2026}, + url = {https://moveit.picknik.ai/main/doc/examples/urdf_srdf/urdf_srdf_tutorial.html}, + note = {Living documentation; year records access, not publication. Accessed 2026-10-06.} +} diff --git a/topics/evaluation-provenance/topic.json b/topics/evaluation-provenance/topic.json index 641311f..1218ebb 100644 --- a/topics/evaluation-provenance/topic.json +++ b/topics/evaluation-provenance/topic.json @@ -15,13 +15,6 @@ "type": "paper", "year": 2021 }, - { - "id": "rlds2026schema", - "title": "RLDS dataset schema (repository accessed 2026)", - "url": "https://github.com/google-research/rlds", - "type": "code", - "year": 2026 - }, { "id": "lerobot2026v3", "title": "LeRobotDataset v3.0 (documentation accessed 2026)", @@ -37,9 +30,16 @@ "year": 2021 }, { - "id": "lerobot2026license", - "title": "LeRobot source license and third-party notices (accessed 2026)", - "url": "https://github.com/huggingface/lerobot/blob/main/LICENSE", + "id": "li2026argus", + "title": "Argus: An Open-Source Annotator for Robotics Data", + "url": "https://pantheon.inc/research/argus", + "type": "essay", + "year": 2026 + }, + { + "id": "pantheon2026argus", + "title": "Argus source at reviewed revision (accessed 2026)", + "url": "https://github.com/Pantheon-Industries-Inc/argus/tree/6c99686a3d93027c517f02f37f24b5b68532e12a", "type": "code", "year": 2026 } @@ -62,6 +62,12 @@ "text": "How much does recording-level leakage change the selected evaluation metric?", "state": "open", "issueUrl": null + }, + { + "id": "ep-04", + "text": "Do grounded temporal annotations reduce false completion labels, and what evidence should gate a robot-tested skill release?", + "state": "open", + "issueUrl": "https://github.com/Dvidia-Inference/dvidia-research/issues/5" } ], "milestones": [