From 3f30847a7285a180f93b8b0beac0def1a7ceb912 Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 31 Oct 2026 23:47:11 -0400 Subject: [PATCH 1/2] Freeze bounded incremental PiPER hold and fresh four-control study --- docs/designs/piper-incremental-hold-v03.md | 115 ++++ docs/research/agent-workflows-20261031.md | 45 ++ .../piper_incremental_benchmark.py | 501 ++++++++++++++++++ src/dvidia_training/piper_incremental_hold.py | 188 +++++++ tests/test_piper_incremental_benchmark.py | 322 +++++++++++ tests/test_piper_incremental_hold.py | 403 ++++++++++++++ 6 files changed, 1574 insertions(+) create mode 100644 docs/designs/piper-incremental-hold-v03.md create mode 100644 docs/research/agent-workflows-20261031.md create mode 100644 src/dvidia_training/piper_incremental_benchmark.py create mode 100644 src/dvidia_training/piper_incremental_hold.py create mode 100644 tests/test_piper_incremental_benchmark.py create mode 100644 tests/test_piper_incremental_hold.py diff --git a/docs/designs/piper-incremental-hold-v03.md b/docs/designs/piper-incremental-hold-v03.md new file mode 100644 index 0000000..52c8d55 --- /dev/null +++ b/docs/designs/piper-incremental-hold-v03.md @@ -0,0 +1,115 @@ +# Bounded incremental pose correction after a test lift + +Status: reviewed before source freeze, 31 October 2026 local date. +This specification is reviewed before native trials. No success is presumed. + +## Problem and decision + +The frozen v0.2 Cartesian controller produced a nearly constant actuator target +while TCP displacement crossed its 4 mm limit. `pose_action` returns an absolute +joint target: measured joint position plus a damped inverse-kinematic error step. +Subtracting its admission value nearly cancels measured drift against that step. +The saved-record explanation is testable; it is not proof of a working remedy. + +Preserve the three v0.2 controls unchanged and add exactly one authored candidate. +The candidate's first admitted command equals the preceding issued target. At +each subsequent 20 ms boundary, obtain the current pose-error increment as +`pose_action(admission_pose) - current_measured_joint_position`. Add this increment +to the accumulated correction around the preceding issued admission target. +The update gain is exactly 1.0 per control boundary, chosen before any native run. +Limit each accepted correction change to 0.002 rad per joint. Refuse if the +unlimited cumulative proposal exceeds ±0.03 rad; do not clip or integrate beyond +that total bound. Update internal correction only after all checks pass. This +terminal refusal supplies the anti-windup rule. Do not tune gains on the old or +new benchmark. A failure remains a failure. + +## Invariants + +- Original evidence-gated acquisition, supported retry and terminal refusal. +- Same fixed admission pose; initial bias bound 0.08 rad; target margin 0.02 rad. +- Candidate-only measured-joint eligibility: at admission, every subsequent + update and final observation, refuse measured joints outside that same 0.02 rad + native target margin before calling the pose solver. Its absolute-target + clipping would otherwise create a nonzero increment at zero pose error. + Record `measured_joint_margin_exhausted` as a terminal bound refusal, without + integrating correction. The three original variants remain unchanged. +- Same jaw target after admission, native gains, force caps and 1 ms target slew. +- Same 4 mm / 0.05 rad terminal pose error, original relative-grip checks, and + synthetic 10 N per-finger checking ceiling. These are not physical safety limits. +- Invalid or noncausal measurements refuse without issuing a command. Final + observations check evidence but do not solve or issue a hypothetical next action. +- No hardware control, new camera model, learning, transport or placement. +- Keep candidate identity, update law, increments, proposed and applied cumulative + corrections, bound refusals and final controller status explicit in records. + +## Fixed comparison + +Eight fresh finite development scenes: masses 0.0475 / 0.0525 kg; friction +0.78 / 0.52; starts (0.304, -0.024, 0.082) and (0.308, -0.035, 0.082) metres. +Seeds 311200–311207. Four variants: measured_joint, issued_joint, cartesian, +incremental. Thirty-two attempts, each 20 simulated seconds, once each. +Four balanced orders repeated twice: [0,1,3,2], [1,2,0,3], [2,3,1,0], [3,0,2,1]. +Each variant occupies every run position twice. No trial resume, outcome-based +retry, candidate selection, or post-result parameter changes. + +Primary native task endpoint: correctly qualified 100 ms admission prefix, +continuous native grip retention AND TCP stationarity for the next five seconds. +Use every 1 ms native sample and exactly the v0.2 definitions: bilateral contacts, +forces strictly >0.01 N and <=10 N, unsupported, at least 6 mm probe rise, at most +3 mm TCP-local drift, TCP displacement <=4 mm and orientation <=0.05 rad. Same +distance epsilon 1e-12. Require full valid warning-free recording, no unexpected +contact anywhere, and no post-admission force-ceiling violation. Missing admission +or incomplete follow-up fails. All attempts remain in the denominator. + +Report separately: grip-only diagnostic; controller retained at 5 s; conjunction +of native task success and controller qualification; false initial admissions; +false maintained claims against grip and against pose; final qualification; +failure timing/reasons; active correction magnitude/rate; wall and CPU timings. +A terminal fallback may retain an object: its outcome cannot be attributed to +continued active Cartesian regulation. No general success or speedup claim follows +from these deterministic cells. Previously published learned placement is unchanged. + +## Verification and publication gate + +1. Fresh-context reviewer receives this document and referenced source only. +2. Pure tests establish correct sign, accumulation, zero-error invariance, + bumpless admission, rate/total bounds, no update on refusal, frozen jaw, invalid + measurements, terminal status and final non-command observation behavior. +3. Freeze source commit, source map, protocol, runtime and asset hashes before + the 32 native trials. Root alone runs the native matrix once. +4. Reconstruct requested-to-applied controls; verify exact common acquisition + prefixes across four variants up through earliest treatment observation, + excluding the first treatment command. Preserve unknown/incomplete records. +5. Independently recompute native grip/pose and correction bounds from saved + numeric arrays and command records, with no runtime/scorer import or physics. +6. Publish all failures and exact source/evidence only after tests, review and + actual browser inspection. Adoption into the stable local product is a + separate decision; this comparison cannot establish hardware readiness. + +## Workflow sources and boundary + +The approach takes inspiration from EGM's context-independent design review and +Harness2's propose/probe/compose and retained lessons. We implement an original, +bounded robotics experiment using our existing sandbox and native evaluator. +We do not install their plugins or adopt unrestricted execution. Model weights +are not being trained by either workflow. This study does not run Harness2 or +claim its benchmark results for robotics. The final evaluation stays separate +from proposal development; inference about a next change is explicitly post-hoc. + +- https://github.com/google/elephant-goldfish-model/tree/403d8881d62022490637d740657b33f01e321ea8 +- https://github.com/google-research/harness2/tree/1cfe254e8d25b15f37048c3730134686abe60335 +- Existing source: src/dvidia_training/piper_retention.py and piper_task.py. +- Existing method: benchmarks/piper-retention-v02/README.md. +- Frozen baseline: bc106d6f66367f8896787a2303aa0a124c69ea56. + +## Pre-execution review disposition + +Fresh review found that absolute-target clipping could manufacture a nonzero +update at zero pose error outside the joint margin. The candidate now refuses +that measurement before admission, updates and final inspection. A pure test +reproduces the counterexample. Reciprocal review found setup/export exceptions +and nonfinite JSON could truncate the scheduled matrix; the runner now retains +invalid attempts and explicit invalid-number markers. A mocked all-invalid +32-attempt lifecycle checks retention and refusal to resume. Both reviewers +cleared their blockers before any native trial. The three old controllers are +byte-for-byte unchanged. diff --git a/docs/research/agent-workflows-20261031.md b/docs/research/agent-workflows-20261031.md new file mode 100644 index 0000000..efcc240 --- /dev/null +++ b/docs/research/agent-workflows-20261031.md @@ -0,0 +1,45 @@ +# What EGM and Harness² contribute to DVIDIA + +Reviewed 31 October 2026. These are agent engineering workflows, not robotics +policies, physics engines or video-to-actuator models. + +EGM separates a context-rich design author from reviewers who receive only the +artifact and referenced source. This is useful for finding assumptions shared +by the original implementation conversation. Review supplements execution tests. +Source: [pinned README](https://github.com/google/elephant-goldfish-model/blob/403d8881d62022490637d740657b33f01e321ea8/README.md). + +Harness² improves the surrounding agent harness with fixed model weights. Its +propose–probe–compose loop and cross-task lessons suggest a disciplined way to +compare changes and preserve both successes and failures. Its reported office, +coding and professional-task results do not demonstrate robotics improvements. +Sources: [pinned README](https://github.com/google-research/harness2/blob/1cfe254e8d25b15f37048c3730134686abe60335/README.md), +[experiment guide](https://github.com/google-research/harness2/blob/1cfe254e8d25b15f37048c3730134686abe60335/docs/EXPERIMENTS.md). + +## Applied here + +One bounded incremental grip-hold candidate is proposed from the prior recorded +failure. A reviewer receives the short design and relevant source without the +implementation discussion. The review caught a concrete joint-margin clipping +counterexample to zero-error invariance before any simulation was run. The +candidate refuses that state; all three old controllers remain unchanged. + +The fixed comparison contains eight new development scenes and four controls. +Source, assets, runtime, limits, order and outcomes are frozen before execution. +An independent numeric reader reconstructs physical outcomes from saved records. +No failure is retried to improve a score. The resulting lesson is retained even +if the candidate fails. Native stationary grip and active controller success +are separate; fallback retention cannot masquerade as active regulation. + +This is an original adaptation of workflow ideas, not an installation or run of +either framework. We do not adopt unrestricted execution settings. Runtime +limits remain enforced in code; videos and external text are data, not control +instructions. No upstream code or prompt text is copied. Both repositories have +Apache-2.0 licenses; this note links their pinned sources for attribution. + +## Remaining bridge to the product + +The skill pipeline still needs causal camera observations, demonstrations with +issued actuator commands, learned action prediction, and held-out execution +evaluation. Better engineering review can help us build that bridge; it does +not itself supply a learned skill or demonstrate arbitrary video-to-any-arm +transfer. This bounded hold supports a verified pause before later movement. diff --git a/src/dvidia_training/piper_incremental_benchmark.py b/src/dvidia_training/piper_incremental_benchmark.py new file mode 100644 index 0000000..3c8ecfb --- /dev/null +++ b/src/dvidia_training/piper_incremental_benchmark.py @@ -0,0 +1,501 @@ +"""One frozen incremental PiPER hold candidate against three unchanged controls. + +Every scheduled attempt remains in the denominator. The primary task endpoint +requires native grip AND TCP stationarity; grip alone and controller qualification +remain separately inspectable. No physics runs at import time. +""" +from __future__ import annotations + +import argparse +from copy import deepcopy +from dataclasses import asdict, replace +from datetime import datetime, timezone +import gzip +from hashlib import sha256 +import json +import math +from pathlib import Path +import subprocess +import sys +import time + +import numpy as np + +from . import piper_retention_benchmark as legacy +from . import piper_probe_benchmark as prior + +SCHEMA = 'dvidia.piper-incremental-hold-study/v1' +INSPECTION_SCHEMA = 'dvidia.piper-incremental-hold-inspection/v1' +VARIANTS = (*legacy.VARIANTS, 'incremental') +ORDERS = ((0, 1, 3, 2), (1, 2, 0, 3), (2, 3, 1, 0), (3, 0, 2, 1)) +PLANNED_ATTEMPTS = 32 +HOLD_S = legacy.HOLD_S +FORCE_CEILING_N = legacy.FORCE_CEILING_N +TCP_POSITION_CEILING_M = legacy.TCP_POSITION_CEILING_M +TCP_ORIENTATION_CEILING_RAD = legacy.TCP_ORIENTATION_CEILING_RAD +BOUND_REFUSALS = legacy.BOUND_REFUSALS | {'measured_joint_margin_exhausted'} +MAX_FILE = legacy.MAX_FILE +SCOPE = ('One authored bounded incremental pose-hold candidate and three frozen v0.2 controls. ' + 'Eight fresh finite synthetic development scenes with privileged simulator state; ' + 'no learning, full placement, physical sensing or hardware qualification.') +encoded, digest, write, read = prior.encoded, prior.digest, prior.write, prior.read +roster, _timing = legacy.roster, legacy._timing +_index, probe_series = legacy._index, legacy.probe_series +_read_native, _controls, _compact_policy = legacy._read_native, legacy._controls, legacy._compact_policy + + +def case_definitions(): + """The preregistered full 2 x 2 x 2 factorial with fixed Williams orders.""" + from .piper_task import PiperTaskConfig + base, cases = PiperTaskConfig(), [] + for mass in (.0475, .0525): + for friction in (.78, .52): + for layout, start in enumerate(((.304, -.024), (.308, -.035))): + index = len(cases) + config = replace(base, mass_kg=mass, friction=friction, object_position_m=(*start, .082)) + cases.append({'id': f'm{round(mass*10000):04}-f{round(friction*100):02}-p{layout}', + 'seed': 311200+index, 'config': asdict(config), + 'variant_order': [VARIANTS[i] for i in ORDERS[index % len(ORDERS)]]}) + return cases + + +def _source_commit(): + return subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD'], + cwd=Path(__file__).resolve().parents[2], text=True).strip() + + +def make_protocol(assets): + from .piper_incremental_hold import incremental_hold_contract + protocol = legacy.make_protocol(assets) + grip = deepcopy(protocol['primary_endpoint']) + stationary = deepcopy(protocol['secondary_endpoint']) + if 'definition' in stationary: + stationary['definition'] = stationary['definition'].replace('all three controllers', 'all four controllers') + protocol.update(schema=SCHEMA, scope=SCOPE, cases=case_definitions(), variants=list(VARIANTS), + planned_attempts=PLANNED_ATTEMPTS, source_commit=_source_commit(), + candidate_controller_contract=incremental_hold_contract(), + baseline_source_commit='bc106d6f66367f8896787a2303aa0a124c69ea56', + grip_only_endpoint=grip) + protocol['primary_endpoint'] = { + **grip, 'name': 'grip_retention_and_tcp_stationarity', + 'position_ceiling_m': TCP_POSITION_CEILING_M, + 'orientation_ceiling_rad': TCP_ORIENTATION_CEILING_RAD, + 'definition': grip['definition']+' Additionally every native TCP pose from admission through five seconds must stay within 4 mm and 0.05 rad of its admission pose.', + 'predicate': grip['predicate']+' TCP displacement <=0.004 m and orientation error <=0.05 rad with epsilon 1e-12, identically for all four variants.', + } + protocol['secondary_endpoint'] = { + 'grip_only': grip, 'stationarity_details': stationary, + 'active_controller_conjunction': 'Primary native task success AND controller qualified at the five-second endpoint. Refusal is terminal, so endpoint qualification requires uninterrupted controller admission.', + 'maintained_claims': 'At every post-admission boundary claimed qualified, check the complete native history from admission through that boundary. Count grip and pose contradictions separately; false initial admission also invalidates subsequent grip claims.', + } + protocol['order'] = ('Four Williams orders repeated twice: [0,1,3,2], [1,2,0,3], [2,3,1,0], [3,0,2,1]. ' + 'Each variant occupies every position twice. One attempt per scene/variant; no resume, outcome-based retry, candidate selection or post-result tuning.') + return protocol + + +def _frozen_protocol(output): + path = Path(output)/'protocol.json' + protocol = read(path) + if sha256(path.read_bytes()).hexdigest() != read(Path(output)/'frozen.json')['protocol_sha256']: + raise ValueError('Frozen protocol digest changed.') + return protocol + + +def _failed_evaluation(reason): + primary = {'required_s': HOLD_S, 'complete_5s': False, 'native_pause_success': False, + 'success': False, 'controller_retained_at_5s': False, + 'combined_qualified_pause': False, 'reason': reason, + 'first_tcp_pose_failure_s': None} + return {'status': 'audit_refused', 'reason': reason, 'primary': primary, + 'grip_only': deepcopy(primary), 'first_admission': None, + 'final': {'controller_qualified': False, 'native_qualified': False, + 'both_qualified': False, 'evidence_admissible': False}, + 'stationarity': {'complete_5s': False, 'tcp_pose_within_bounds': False, + 'grip_retention_and_stationarity': False}, + 'claims': {'maintained_checks': 0, 'false_maintained_grip': 0, 'false_maintained_pose': 0}, + 'physical_robot_ready': False} + + +def _json_record(result): + """Retain invalid values explicitly without admitting nonfinite JSON numbers.""" + invalid = [] + + def plain(value, path): + if isinstance(value, (float, np.floating)) and not math.isfinite(value): + label = 'nan' if math.isnan(value) else 'positive_infinity' if value > 0 else 'negative_infinity' + invalid.append({'path': path, 'value': label}) + return {'invalid_nonfinite_number': label} + if isinstance(value, dict): + return {key: plain(item, path+'/'+str(key)) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [plain(item, path+'/'+str(index)) for index, item in enumerate(value)] + if isinstance(value, np.ndarray): + return plain(value.tolist(), path) + if isinstance(value, np.generic): + return value.item() + return value + + recorded = plain(result, '') + if invalid: + recorded['valid'] = False + recorded['json_invalid_numbers'] = invalid + if recorded.get('error') is None: + recorded['error'] = {'type': 'NonfiniteEvidence', + 'message': 'Nonfinite JSON observations retained as explicit invalid-number markers.'} + return recorded + + +def _unavailable_native(arrays=None): + """A truthful zero-row/partial sidecar when native telemetry cannot export.""" + from .probe_evidence import FIELDS + if arrays is None: + arrays = {name: np.empty((0,) if width == 1 else (0, width), dtype=np.float64) + for name, width in FIELDS} + metadata = {'schema': 'dvidia.piper-feedback-native/v1', 'native_timestep_s': .001, + 'rows': len(arrays['t_seconds']), 'fields': [{'name': name, 'width': width} for name, width in FIELDS], + 'side_order': ['left', 'right'], 'physical_robot_ready': False, + 'recording_status': 'unavailable_or_incomplete_after_exception'} + return arrays, metadata + + +def _claims_and_pose_failure(result, arrays, evaluation): + claims = {'maintained_checks': 0, 'false_maintained_grip': 0, 'false_maintained_pose': 0, + 'scope': 'Claimed qualified boundaries after admission, compared with every native row since admission; grip includes correct initial prefix. Counts are diagnostic, not independent trials.'} + admission = evaluation['first_admission'] + if admission is None: + return claims, None + times = arrays['t_seconds'] + start = _index(admission['time_s'], times) + position = np.linalg.norm(arrays['tcp_position_m'][start:]-arrays['tcp_position_m'][start], axis=1) + angles = 2*np.arccos(np.clip(np.abs(arrays['tcp_quaternion_wxyz'][start:] @ arrays['tcp_quaternion_wxyz'][start]), 0., 1.)) + pose = (position <= TCP_POSITION_CEILING_M+1e-12) & (angles <= TCP_ORIENTATION_CEILING_RAD+1e-12) + policy = result['policy'] + probe = next(row for row in policy['probes'] if row['probe_index'] == admission['probe_index']) + condition = probe_series(arrays, probe)[0] + grip = np.logical_and.accumulate(condition[start:]) & admission['native_prefix_qualified'] + pose_continuous = np.logical_and.accumulate(pose) + for check in policy['qualification_checks']: + if (check['probe_index'] == admission['probe_index'] and check['qualified'] is True + and check['t_seconds'] > admission['time_s']+1e-8): + index = _index(check['t_seconds'], times)-start + claims['maintained_checks'] += 1 + claims['false_maintained_grip'] += int(not grip[index]) + claims['false_maintained_pose'] += int(not pose_continuous[index]) + in_window = times[start:] <= admission['time_s']+HOLD_S+1e-8 + failures = np.flatnonzero(~pose & in_window) + return claims, float(times[start+int(failures[0])]) if failures.size else None + + +def evaluate(result, arrays, metadata, protocol): + """Reuse the frozen native evaluator; promote its stationary-grip endpoint.""" + try: + evaluation = legacy.evaluate(result, arrays, metadata, protocol) + evaluation['grip_only'] = deepcopy(evaluation['primary']) + primary, stationarity = evaluation['primary'], evaluation['stationarity'] + primary['success'] = bool(stationarity['grip_retention_and_stationarity']) + primary['combined_qualified_pause'] = bool(primary['success'] and primary['controller_retained_at_5s']) + if evaluation['grip_only']['success'] and not primary['success']: + primary['reason'] = 'tcp_stationarity_lost' + claims, failure = _claims_and_pose_failure(result, arrays, evaluation) + evaluation['claims'] = claims + primary['first_tcp_pose_failure_s'] = failure + records = ((result.get('policy') or {}).get('retention') or {}).get('hold_records', []) + increments, rates = [], [] + for row in records: + if row.get('pose_error_increment_rad') is not None: + increments.append(np.asarray(row['pose_error_increment_rad'], dtype=float)) + if row.get('command_issued') and row.get('correction_update_accepted'): + rates.append((np.asarray(row['accumulated_correction_rad'])-np.asarray(row['correction_before_update_rad']))/.02) + evaluation['corrections'].update( + maximum_abs_pose_error_increment_rad=float(np.max(np.abs(increments))) if increments else None, + maximum_abs_accepted_correction_rate_rad_s=float(np.max(np.abs(rates))) if rates else None) + return evaluation + except (ValueError, TypeError, KeyError, IndexError) as exc: + return _failed_evaluation(type(exc).__name__+': '+str(exc)[:300]) + + +def aggregates(rows): + result = {} + for variant in VARIANTS: + selected = [row for row in rows if row['variant'] == variant] + admissions = [row['evaluation']['first_admission'] for row in selected if row['evaluation']['first_admission'] is not None] + result[variant] = { + 'attempts': len(selected), 'valid': sum(row['valid'] for row in selected), + 'admissions': len(admissions), + 'false_initial_admissions': sum(not row['native_prefix_qualified'] for row in admissions), + 'native_5s_pause_success': sum(row['evaluation']['primary']['native_pause_success'] for row in selected), + 'native_5s_stationary_success': sum(row['evaluation']['primary']['success'] for row in selected), + 'controller_retained_at_5s': sum(row['evaluation']['primary']['controller_retained_at_5s'] for row in selected), + 'combined_qualified_5s_pause': sum(row['evaluation']['primary']['combined_qualified_pause'] for row in selected), + 'grip_retention_and_stationarity': sum(row['evaluation']['stationarity']['grip_retention_and_stationarity'] for row in selected), + 'false_maintained_grip': sum(row['evaluation']['claims']['false_maintained_grip'] for row in selected), + 'false_maintained_pose': sum(row['evaluation']['claims']['false_maintained_pose'] for row in selected), + 'final_both_qualified': sum(row['evaluation']['final']['both_qualified'] for row in selected), + } + return result + + +def prepare(output, assets): + output = Path(output) + if output.exists() or output.is_symlink(): + raise ValueError('Choose a fresh output directory; no evidence is overwritten.') + protocol = make_protocol(assets) + output.mkdir(parents=True) + write(output/'protocol.json', protocol) + write(output/'frozen.json', {'created_at_utc': datetime.now(timezone.utc).isoformat(), + 'protocol_sha256': sha256((output/'protocol.json').read_bytes()).hexdigest()}) + sources = output/'sources'; sources.mkdir() + for name, expected in protocol['source_sha256'].items(): + raw = Path(__file__).with_name(name).read_bytes() + if sha256(raw).hexdigest() != expected: + raise ValueError('Source changed during preparation.') + (sources/name).write_bytes(raw) + return protocol + +def _attempt(assets, case, variant): + from .piper_task import PiperTaskConfig + from .piper_feedback_env import FeedbackPiperTaskEnv + from .piper_retention import RetentionPolicy + from .piper_incremental_hold import IncrementalHoldPolicy + from .reference_sim import model_digest + env = None + result = {'schema': SCHEMA, 'id': case['id']+'-'+variant, 'scene_id': case['id'], 'variant': variant, + 'seed': case['seed'], 'config': case['config'], 'valid': False, 'physical_robot_ready': False, + 'trajectory': [], 'actions': [], 'error': None, 'metrics': {}, 'policy': None} + timers = {name: [] for name in ('policy_wall', 'policy_cpu', 'step_wall', 'step_cpu')} + policy = None + began = time.perf_counter() + try: + env = FeedbackPiperTaskEnv(assets, PiperTaskConfig(**case['config'])) + start = time.perf_counter() + obs, info = env.reset(case['seed']) + result['setup_reset_wall_s'] = time.perf_counter()-start + result.update(initial_object_position_m=env.initial_object_position.tolist(), + compiled_model_sha256=model_digest(env.model), + native_force_limits=env.model.actuator_forcerange.tolist()) + policy = IncrementalHoldPolicy(env) if variant == 'incremental' else RetentionPolicy(env, variant=variant) + result['trajectory'].append(obs) + for _ in range(round(env.config.timeout_s/env.config.control_dt)): + wall, cpu = time.perf_counter(), time.process_time() + action = policy(obs) + timers['policy_wall'].append(time.perf_counter()-wall) + timers['policy_cpu'].append(time.process_time()-cpu) + record = {'t_seconds': obs['t_seconds'], **action} + result['actions'].append(record) + obs.update(command=record, issued_joint_targets_rad=list(action['joint_targets_rad']), + issued_gripper_position_m=action['gripper_position_m']) + wall, cpu = time.perf_counter(), time.process_time() + obs, info = env.step(action) + timers['step_wall'].append(time.perf_counter()-wall) + timers['step_cpu'].append(time.process_time()-cpu) + if obs is not None: + result['trajectory'].append(obs) + if env.done: + break + if obs is not None: + policy.observe_final(obs) + result.update(valid=info['valid'], metrics=info, policy=policy.summary()) + except Exception as exc: + result['error'] = {'type': type(exc).__name__, 'message': str(exc)[:300]} + if env is not None and env.ready: + try: + result['metrics'] = env.info() + except Exception as info_exc: + result['metrics_error'] = {'type': type(info_exc).__name__, 'message': str(info_exc)[:300]} + if policy is not None: + try: + result['policy'] = policy.summary() + except Exception as summary_exc: + result['policy_summary_error'] = {'type': type(summary_exc).__name__, + 'message': str(summary_exc)[:300]} + arrays, metadata = _unavailable_native() + if env is not None: + try: + arrays = env.native_arrays() + metadata = env.native_metadata() + except Exception as native_exc: + result['valid'] = False + result['native_recording_error'] = {'type': type(native_exc).__name__, 'message': str(native_exc)[:300]} + if result['error'] is None: + result['error'] = deepcopy(result['native_recording_error']) + arrays, metadata = _unavailable_native(arrays) + result['timing'] = {'attempt_wall_s': time.perf_counter()-began, + 'summaries': {name: _timing(values) for name, values in timers.items()}, 'samples': timers} + return result, arrays, metadata + +def run(output, assets): + output = Path(output) + protocol = _frozen_protocol(output) + if encoded(protocol) != encoded(make_protocol(assets)): + raise ValueError('Frozen protocol/source/runtime/assets changed; create a new study.') + write(output/'started.json', {'protocol_sha256': sha256((output/'protocol.json').read_bytes()).hexdigest(), + 'started_at_utc': datetime.now(timezone.utc).isoformat()}) + rows = [] + for case, variant in roster(protocol): + if prior.sources() != protocol['source_sha256']: + raise ValueError('Source changed during the fixed study.') + folder = output/(case['id']+'-'+variant); folder.mkdir() + write(folder/'started.json', {'scene_id': case['id'], 'variant': variant, + 'protocol_sha256': sha256((output/'protocol.json').read_bytes()).hexdigest()}) + result, arrays, metadata = _attempt(assets, case, variant) + result = _json_record(result) + (folder/'episode.json.gz').write_bytes(gzip.compress(encoded(result), mtime=0)) + with (folder/'native.npz').open('xb') as stream: + np.savez_compressed(stream, **arrays) + write(folder/'native.json', metadata) + try: + evaluation = evaluate(result, arrays, metadata, protocol) + except Exception as exc: + evaluation = _failed_evaluation(type(exc).__name__+': '+str(exc)[:300]) + write(folder/'evaluation.json', evaluation) + files = {path.name: {'bytes': path.stat().st_size, 'sha256': sha256(path.read_bytes()).hexdigest()} + for path in sorted(folder.iterdir())} + row = {'id': result['id'], 'scene_id': case['id'], 'variant': variant, + 'mass_kg': case['config']['mass_kg'], 'friction': case['config']['friction'], + 'valid': result['valid'], 'error': result['error'], 'metrics': result['metrics'], + 'policy': _compact_policy(result.get('policy')), 'timing': result['timing']['summaries'], + 'evaluation': evaluation, 'files': files} + rows.append(row) + write(folder/'summary.json', row) + print(json.dumps({'attempt': result['id'], 'valid': result['valid'], 'primary': evaluation['primary']['success']}), flush=True) + if prior.sources() != protocol['source_sha256']: + raise ValueError('Source changed during an attempt; all evidence remains retained.') + if prior.runtime() != protocol['runtime']: + raise ValueError('Runtime changed during the study.') + report = {'schema': SCHEMA, 'scope': SCOPE, 'planned_attempts': PLANNED_ATTEMPTS, 'attempted': len(rows), + 'valid_attempts': sum(row['valid'] for row in rows), 'rows': rows, 'aggregates': aggregates(rows), + 'physical_robot_ready': False, 'source_sha256': protocol['source_sha256'], + 'previous_learned_heldout_result': '5/12 unchanged. No learning or full-placement evaluation in this study.'} + write(output/'report.json', report) + return report + +def _treatment_boundary(policy): + events = (policy or {}).get('events', []) + admissions = [event for event in events if event['type'] == 'evidence_qualified'] + admission = min((event['t_seconds'] for event in admissions), default=None) + candidates = [] + for event in events: + if event['type'] == 'evidence_qualified': + reason = 'admission' + elif (event['type'] == 'hold_invalidated' and event.get('reason') in BOUND_REFUSALS + and (admission is None or event['t_seconds'] <= admission+1e-8)): + reason = 'pre_admission_bound_refusal' + else: + continue + candidates.append({'time_s': event['t_seconds'], 'reason': reason, 'detail': event.get('reason')}) + return min(candidates, key=lambda row: row['time_s']) if candidates else None + +def _common_cutoff(rows): + boundaries = [] + for row in rows: + boundary = _treatment_boundary(row.get('policy')) + if boundary is not None: + boundaries.append({'variant': row['variant'], **boundary}) + boundaries.sort(key=lambda row: row['variant']) + timestamp = min((row['time_s'] for row in boundaries), default=None) + reasons = sorted({row['reason'] for row in boundaries if abs(row['time_s']-timestamp) <= 1e-8}) if timestamp is not None else [] + return {'time_s': timestamp, 'reason': '+'.join(reasons) if reasons else 'full_run_no_treatment', + 'treatment_boundaries': boundaries} + +def _prefix(episode, arrays, *, cutoff=None): + return legacy._prefix(episode, arrays, cutoff=_common_cutoff([episode]) if cutoff is None else cutoff) + + +def inspect(output): + from .piper_grip_benchmark import decode_attempt + from .probe_evidence import _arrays + output = Path(output) + protocol_path = output/'protocol.json' + if protocol_path.is_symlink() or not protocol_path.is_file() or protocol_path.stat().st_size > 16*1024*1024: + raise ValueError('Expected a bounded plain frozen protocol.') + with protocol_path.open('rb') as stream: + raw_protocol = stream.read(16*1024*1024+1) + if (len(raw_protocol) > 16*1024*1024 + or sha256(raw_protocol).hexdigest() != read(output/'frozen.json')['protocol_sha256']): + raise ValueError('Frozen protocol digest changed.') + protocol, report = json.loads(raw_protocol), read(output/'report.json') + if prior.sources() != protocol['source_sha256'] or report['source_sha256'] != protocol['source_sha256']: + raise ValueError('Inspect with the frozen implementation.') + if protocol['schema'] != SCHEMA or protocol['variants'] != list(VARIANTS) or encoded(protocol['cases']) != encoded(case_definitions()): + raise ValueError('Frozen candidate roster differs from the authored protocol.') + planned = roster(protocol) + ids = [case['id']+'-'+variant for case, variant in planned] + if ([row['id'] for row in report['rows']] != ids or len(ids) != PLANNED_ATTEMPTS + or report['attempted'] != PLANNED_ATTEMPTS or protocol['planned_attempts'] != PLANNED_ATTEMPTS): + raise ValueError('Attempt roster/denominator differs from frozen protocol.') + checked, prefixes = [], {} + cutoffs = {case['id']: _common_cutoff([row for row in report['rows'] if row['scene_id'] == case['id']]) + for case in protocol['cases']} + for row, (case, variant) in zip(report['rows'], planned): + folder = output/row['id'] + if set(row['files']) != {'started.json', 'episode.json.gz', 'native.npz', 'native.json', 'evaluation.json'}: + raise ValueError('Unexpected attempt artifact roster.') + for name, expected in row['files'].items(): + path = folder/name + if path.is_symlink() or not path.is_file() or path.stat().st_size > MAX_FILE: + raise ValueError('Expected bounded plain evidence file.') + raw = path.read_bytes() + if {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} != expected: + raise ValueError('Frozen artifact digest changed.') + episode = json.loads(decode_attempt((folder/'episode.json.gz').read_bytes())) + if (episode['id'] != row['id'] or episode['variant'] != variant or episode['seed'] != case['seed'] + or episode['config'] != case['config'] or episode['metrics'] != row['metrics'] or episode['valid'] != row['valid'] + or encoded(_compact_policy(episode.get('policy'))) != encoded(row['policy'])): + raise ValueError('Episode identity/config/metrics disagree.') + arrays, metadata = _read_native(folder/'native.npz'), read(folder/'native.json') + reevaluated = evaluate(episode, arrays, metadata, protocol) + if encoded(reevaluated) != encoded(row['evaluation']) or encoded(read(folder/'evaluation.json')) != encoded(reevaluated): + raise ValueError('Recomputed endpoint/receipt differs.') + native_validated, reconstruction, reason = False, False, None + try: + _arrays(arrays, metadata) + native_validated = True + if episode['valid'] and episode['error'] is None: + _controls(episode, arrays) + reconstruction = True + prefixes.setdefault(case['id'], []).append((variant, _prefix(episode, arrays, cutoff=cutoffs[case['id']]))) + except (ValueError, TypeError, KeyError) as exc: + reason = str(exc) + if episode['valid'] and episode['error'] is None: + raise ValueError('Valid attempt failed native readback: '+reason) from exc + checked.append({'id': row['id'], 'native_rows': len(arrays.get('t_seconds', [])), + 'native_validated': native_validated, 'controls_recomputed': reconstruction, + 'retained_refusal': not reconstruction, 'reason': reason, 'evaluation_recomputed': True}) + prefix_checks = [] + for case in protocol['cases']: + group = prefixes.get(case['id'], []) + if len(group) == 4: + base = group[0][1] + if any(encoded(value) != encoded(base) for _, value in group[1:]): + raise ValueError('Acquisition prefixes differ before any hold treatment.') + prefix_checks.append({'scene_id': case['id'], 'verified': True, **base, + 'treatment_boundaries': cutoffs[case['id']]['treatment_boundaries']}) + else: + prefix_checks.append({'scene_id': case['id'], 'verified': False, 'reason': 'invalid_attempt_prevents_complete_four_variant_comparison'}) + if encoded(aggregates(report['rows'])) != encoded(report['aggregates']): + raise ValueError('Aggregate counts differ from retained rows.') + return {'schema': INSPECTION_SCHEMA, 'attempts': len(checked), + 'native_rows': sum(row['native_rows'] for row in checked), 'rows': checked, + 'acquisition_prefixes': prefix_checks, 'aggregates': report['aggregates'], 'physical_robot_ready': False, + 'scope': 'Read-only recorded consistency and causal native predicate reconstruction; hashes do not authenticate external data or hardware.'} + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest='command', required=True) + for name in ('prepare', 'run'): + command = sub.add_parser(name) + command.add_argument('--output', type=Path, required=True) + command.add_argument('--assets', type=Path, required=True) + command = sub.add_parser('inspect'); command.add_argument('--output', type=Path, required=True) + args = parser.parse_args(argv) + try: + result = inspect(args.output) if args.command == 'inspect' else globals()[args.command](args.output, args.assets) + print(json.dumps(result if args.command == 'inspect' else {'schema': SCHEMA, 'planned_attempts': PLANNED_ATTEMPTS, + 'physical_robot_ready': False}, indent=2)) + return 0 + except (ValueError, OSError, KeyError, ImportError) as exc: + print('Incremental hold study refused: '+str(exc), file=sys.stderr) + return 2 + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/src/dvidia_training/piper_incremental_hold.py b/src/dvidia_training/piper_incremental_hold.py new file mode 100644 index 0000000..d8539a1 --- /dev/null +++ b/src/dvidia_training/piper_incremental_hold.py @@ -0,0 +1,188 @@ +"""One fixed incremental pose hold, preserving the frozen v0.2 controls. + +The existing pose solver returns measured joints plus an error step. This +candidate accumulates that step around the last target issued before admission. +All observations and limits have the same synthetic simulator scope as v0.2. +""" +from __future__ import annotations + +from copy import deepcopy + +from . import piper_probe as probe +from . import piper_retention as retention + + +SCHEMA = 'dvidia.piper-incremental-hold/v3' +VARIANT = 'incremental' +UPDATE_GAIN = 1.0 +UPDATE_LAW = ('increment = pose_action(frozen_admission_pose) - current_measured_joints; ' + 'proposal = previous_correction + 1.0 * increment; ' + 'refuse unlimited proposal outside +/-0.03 rad; ' + 'slew accepted correction by at most 0.002 rad per joint per 20 ms; ' + 'target = admission_issued_target + accepted_correction') + + +def incremental_hold_contract(): + """The single preregistered candidate; no runtime gain selection.""" + base = retention.retention_contract() + return { + 'schema': SCHEMA, 'variants': [VARIANT], 'candidate': VARIANT, + 'physical_robot_ready': False, 'observation_origin': probe.ORIGIN, + 'control_dt_s': probe.DT, 'update_gain_per_boundary': UPDATE_GAIN, + 'update_law': UPDATE_LAW, 'limits': deepcopy(base['limits']), + 'acquisition': base['acquisition'], + 'admission': 'Freeze the measured TCP pose and preceding issued target. Check the unchanged 0.08 rad initial bias and 0.02 rad joint margin. Repeat the preceding issued target exactly; initialize cumulative correction to zero.', + 'measured_joint_eligibility': 'At admission, subsequent updates and final observations, refuse measured joints outside the native 0.02 rad target margin before solving. This prevents absolute-target clipping from manufacturing a nonzero increment at zero pose error.', + 'anti_windup': 'Refuse the unlimited cumulative proposal before slew limiting. Commit correction only after all checks pass. Refusal is terminal and leaves accumulated correction unchanged.', + 'rate_scope': base['rate_scope'], 'invalid_measurements': base['invalid_measurements'], + 'refusal': 'Original grip, force, pose, initial-bias, joint-margin and timeout refusals remain terminal. A valid terminal observation latches measured joints and the frozen jaw; its fallback is not active incremental regulation.', + 'final_observation': 'Check force, original eligibility and pose error without solving, integrating or issuing another command.', + 'sampling': base['sampling'], 'qualifications': base['qualifications'], + } + + +class IncrementalHoldPolicy(retention.RetentionPolicy): + """Add one candidate without changing any original retention variant.""" + + def __init__(self, environment): + # Reuse validated native limits and unchanged evidence acquisition. The + # hold-specific methods below never dispatch v0.2 Cartesian regulation. + super().__init__(environment, variant='issued_joint') + self.retention_variant = VARIANT + + def _event(self, kind, observation, reason): + super()._event(kind, observation, reason) + self.events[-1].update( + variant=VARIANT, acquisition_variant='evidence', candidate=VARIANT, + update_law=UPDATE_LAW, update_gain_per_boundary=UPDATE_GAIN, + accumulated_correction_rad=list(self.correction), + proposed_cumulative_correction_rad=deepcopy(self.calculation.get('desired_correction_rad')), + pose_error_increment_rad=deepcopy(self.calculation.get('pose_error_increment_rad')), + refused_bounds=self.calculation.get('refused_bounds'), + active_regulation=self.admitted, final_observation=self.final_inspection) + + def _admit(self, observation, kind): + if self.last_requested is None: + return self._refuse_bound(observation, 'missing_prior_issued_target') + issued = probe._vector(self.last_requested['joint_targets_rad'], 'prior issued targets', 6) + jaw = self._jaw_to_freeze(observation) + self.hold_reference = { + 't_seconds': observation['t_seconds'], + 'position_m': list(observation['tcp_position_m']), + 'quaternion_wxyz': list(observation['tcp_quaternion_wxyz']), + 'issued_joint_targets_rad': issued, + 'measured_joint_position_rad': list(observation['joint_position_rad']), + 'frozen_jaw_target_m': jaw, 'raw_pose_joint_targets_rad': None, + 'fixed_bias_rad': None, + } + if self._range_failure(observation['joint_position_rad'], margin=retention.JOINT_MARGIN_RAD): + return self._refuse_bound(observation, 'measured_joint_margin_exhausted') + try: + raw = self.env.pose_action(self.hold_reference['position_m'], + self.hold_reference['quaternion_wxyz'], jaw) + raw = probe._vector(raw['joint_targets_rad'], 'admission pose action', 6) + except (ValueError, TypeError, KeyError, ArithmeticError): + return self._refuse_bound(observation, 'pose_solver_refused') + bias = [target-current for target, current in zip(issued, raw)] + self.hold_reference.update(raw_pose_joint_targets_rad=raw, fixed_bias_rad=bias) + self.calculation = { + 'raw_pose_joint_targets_rad': raw, 'desired_correction_rad': [0.]*6, + 'correction_before_update_rad': list(self.correction), + 'correction_rate_limited_axes': [], 'refused_bounds': None, + 'correction_update_accepted': False, + } + if max(map(abs, bias)) > retention.INITIAL_BIAS_CEILING_RAD+1e-12: + return self._refuse_bound(observation, 'initial_bias_exhausted') + if self._range_failure(issued, margin=retention.JOINT_MARGIN_RAD): + return self._refuse_bound(observation, 'native_joint_margin_exhausted') + # Call the original acquisition admission directly, then preserve the + # prior issued vector exactly (no roundoff through raw + bias). + probe.ProbeStopPolicy._admit(self, observation, kind) + self.stop_targets = list(issued) + self.diagnostic['decision'] = 'bounded_incremental_hold' + self.events[-1]['retention_hold_reference'] = deepcopy(self.hold_reference) + self.events[-1]['diagnostic']['decision'] = self.diagnostic['decision'] + return self._hold() + + def _hold_check(self, observation): + action = probe.ProbeStopPolicy._hold_check(self, observation) + if not self.admitted: + return action + position_error, orientation_error = self._pose_errors(observation) + self.calculation = { + 'raw_pose_joint_targets_rad': None, 'desired_correction_rad': None, + 'pose_error_increment_rad': None, + 'correction_before_update_rad': list(self.correction), + 'correction_rate_limited_axes': [], 'refused_bounds': None, + 'correction_update_accepted': False, + } + if position_error > retention.POSITION_ERROR_CEILING_M+1e-12: + return self._refuse_bound(observation, 'tcp_position_error_exhausted') + if orientation_error > retention.ORIENTATION_ERROR_CEILING_RAD+1e-12: + return self._refuse_bound(observation, 'tcp_orientation_error_exhausted') + if self._range_failure(observation['joint_position_rad'], margin=retention.JOINT_MARGIN_RAD): + return self._refuse_bound(observation, 'measured_joint_margin_exhausted') + if self.final_inspection: + self.diagnostic['decision'] = 'check_incremental_hold_without_update' + return action + reference = self.hold_reference + try: + raw = self.env.pose_action(reference['position_m'], reference['quaternion_wxyz'], + self.frozen_jaw_target) + raw = probe._vector(raw['joint_targets_rad'], 'incremental pose action', 6) + except (ValueError, TypeError, KeyError, ArithmeticError): + return self._refuse_bound(observation, 'pose_solver_refused') + increment = [target-current for target, current in zip(raw, observation['joint_position_rad'])] + proposed = [prior+UPDATE_GAIN*step for prior, step in zip(self.correction, increment)] + self.calculation.update(raw_pose_joint_targets_rad=raw, + pose_error_increment_rad=increment, + desired_correction_rad=proposed) + # Check the unlimited accumulation first: slew must not conceal an + # exhausted total bound. Failed proposals never modify the integrator. + if max(map(abs, proposed)) > retention.CORRECTION_CEILING_RAD+1e-12: + return self._refuse_bound(observation, 'joint_correction_exhausted') + step = retention.CORRECTION_RATE_RAD_S*probe.DT + limited = [index for index, (wanted, prior) in enumerate(zip(proposed, self.correction)) + if abs(wanted-prior) > step+1e-12] + correction = [prior+max(-step, min(step, wanted-prior)) + for wanted, prior in zip(proposed, self.correction)] + targets = [baseline+value for baseline, value in zip(reference['issued_joint_targets_rad'], correction)] + self.calculation['correction_rate_limited_axes'] = limited + if self._range_failure(targets, margin=retention.JOINT_MARGIN_RAD): + return self._refuse_bound(observation, 'native_joint_margin_exhausted') + self.correction, self.stop_targets = correction, targets + self.calculation['correction_update_accepted'] = True + self.diagnostic['decision'] = 'bounded_incremental_pose_regulation' + return self._hold() + + def _record_hold(self, observation, action, *, final=False): + count = len(self.hold_records) + super()._record_hold(observation, action, final=final) + if len(self.hold_records) == count: + return + record = self.hold_records[-1] + record.update( + candidate=VARIANT, update_law=UPDATE_LAW, update_gain_per_boundary=UPDATE_GAIN, + measured_joint_position_rad=list(observation['joint_position_rad']), + pose_error_increment_rad=deepcopy(self.calculation.get('pose_error_increment_rad')), + correction_before_update_rad=list(self.calculation.get('correction_before_update_rad', self.correction)), + proposed_cumulative_correction_rad=deepcopy(self.calculation.get('desired_correction_rad')), + accumulated_correction_rad=list(self.correction), + applied_cumulative_correction_rad=list(self.correction) if self.admitted and not final else None, + correction_rad=list(self.correction) if self.admitted else None, + correction_update_accepted=self.calculation.get('correction_update_accepted', False), + active_regulation=self.admitted) + if final: + # Preserve the last issued target separately from this observation, + # which deliberately has no new requested command. + record['preceding_issued_joint_targets_rad'] = record['joint_targets_rad'] + record['joint_targets_rad'] = None + + def summary(self): + result = super().summary() + result.update(schema=SCHEMA, candidate=VARIANT, contract=incremental_hold_contract()) + result['retention'].update( + candidate=VARIANT, update_law=UPDATE_LAW, update_gain_per_boundary=UPDATE_GAIN, + accumulated_correction_rad=list(self.correction), active_regulation=self.admitted, + final_controller_status=self.status) + return result diff --git a/tests/test_piper_incremental_benchmark.py b/tests/test_piper_incremental_benchmark.py new file mode 100644 index 0000000..44084a0 --- /dev/null +++ b/tests/test_piper_incremental_benchmark.py @@ -0,0 +1,322 @@ +"""Pure fixed-roster, endpoint and evidence-lifecycle tests; no physics.""" +from collections import Counter +from copy import deepcopy +from hashlib import sha256 +from pathlib import Path +from tempfile import TemporaryDirectory +import io +import gzip +import json +import unittest +from unittest.mock import patch + +import numpy as np + +from dvidia_training import piper_incremental_benchmark as study +from tests.test_piper_retention_benchmark import fixture, controls_fixture + + +def endpoint(result, arrays, metadata, protocol): + with patch.object(study.prior, '_evaluate', return_value={'verification': {'admissible': False}}): + return study.evaluate(result, arrays, metadata, protocol) + + +def seal(folder, protocol): + path = Path(folder)/'protocol.json' + study.write(path, protocol) + study.write(Path(folder)/'frozen.json', {'protocol_sha256': sha256(path.read_bytes()).hexdigest()}) + + +class IncrementalBenchmarkTests(unittest.TestCase): + def test_exact_fresh_factorial_and_williams_order(self): + cases = study.case_definitions() + self.assertEqual(len(cases), 8) + self.assertEqual([row['seed'] for row in cases], list(range(311200, 311208))) + self.assertEqual({row['config']['mass_kg'] for row in cases}, {.0475, .0525}) + self.assertEqual({row['config']['friction'] for row in cases}, {.78, .52}) + self.assertEqual({tuple(row['config']['object_position_m']) for row in cases}, + {(.304, -.024, .082), (.308, -.035, .082)}) + self.assertEqual(len({row['id'] for row in cases}), 8) + self.assertEqual(len(study.roster({'cases': cases})), 32) + self.assertEqual(len({(row['id'], variant) for row, variant in study.roster({'cases': cases})}), 32) + self.assertEqual(study.ORDERS, ((0, 1, 3, 2), (1, 2, 0, 3), (2, 3, 1, 0), (3, 0, 2, 1))) + for variant in study.VARIANTS: + self.assertEqual([sum(row['variant_order'][i] == variant for row in cases) for i in range(4)], + [2, 2, 2, 2]) + transitions = Counter((a, b) for row in cases + for a, b in zip(row['variant_order'], row['variant_order'][1:])) + self.assertEqual(len(transitions), 12) + self.assertEqual(set(transitions.values()), {2}) + + def test_protocol_promotes_stationarity_and_keeps_both_contracts(self): + previous = {'primary_endpoint': {'definition': 'native grip', 'predicate': 'all samples'}, + 'secondary_endpoint': {'name': 'grip_retention_and_tcp_stationarity'}, + 'controller_contract': {'baseline': True}} + with patch.object(study.legacy, 'make_protocol', return_value=deepcopy(previous)), \ + patch.object(study, '_source_commit', return_value='a'*40): + protocol = study.make_protocol('unused-assets') + self.assertEqual(protocol['schema'], study.SCHEMA) + self.assertEqual(protocol['planned_attempts'], 32) + self.assertEqual(protocol['variants'], list(study.VARIANTS)) + self.assertEqual(protocol['controller_contract'], previous['controller_contract']) + self.assertEqual(protocol['candidate_controller_contract']['candidate'], 'incremental') + self.assertEqual(protocol['primary_endpoint']['name'], 'grip_retention_and_tcp_stationarity') + self.assertEqual(protocol['grip_only_endpoint'], previous['primary_endpoint']) + self.assertEqual(protocol['source_commit'], 'a'*40) + + def test_stationary_grip_and_controller_conjunction_succeed(self): + result, arrays, metadata, protocol = fixture() + evaluation = study.evaluate(result, arrays, metadata, protocol) + self.assertTrue(evaluation['primary']['success']) + self.assertTrue(evaluation['primary']['native_pause_success']) + self.assertTrue(evaluation['primary']['combined_qualified_pause']) + self.assertTrue(evaluation['grip_only']['success']) + self.assertTrue(evaluation['receipt']['verification']['admissible']) + self.assertIsNone(evaluation['primary']['first_tcp_pose_failure_s']) + + def test_one_native_pose_excursion_fails_primary_for_every_variant(self): + for variant in study.VARIANTS: + with self.subTest(variant=variant): + result, arrays, metadata, protocol = fixture() + result['variant'] = variant + arrays['tcp_position_m'][3499, 0] += .004001 + arrays['object_position_m'][3499, 0] += .004001 + evaluation = endpoint(result, arrays, metadata, protocol) + self.assertFalse(evaluation['primary']['success']) + self.assertFalse(evaluation['primary']['combined_qualified_pause']) + self.assertTrue(evaluation['primary']['native_pause_success']) + self.assertTrue(evaluation['grip_only']['success']) + self.assertEqual(evaluation['primary']['reason'], 'tcp_stationarity_lost') + self.assertEqual(evaluation['primary']['first_tcp_pose_failure_s'], 3.5) + self.assertEqual(evaluation['claims']['false_maintained_pose'], 2) + self.assertEqual(evaluation['claims']['false_maintained_grip'], 0) + self.assertEqual(evaluation['stationarity']['terminal_tcp_position_delta_m'], [0., 0., 0.]) + + def test_orientation_sign_and_exact_bound_remain_common(self): + result, arrays, metadata, protocol = fixture() + angle = .05 + arrays['tcp_quaternion_wxyz'][3499] = [np.cos(angle/2), 0., 0., np.sin(angle/2)] + self.assertTrue(endpoint(result, arrays, metadata, protocol)['primary']['success']) + angle += .000001 + arrays['tcp_quaternion_wxyz'][3499] = [np.cos(angle/2), 0., 0., np.sin(angle/2)] + first = endpoint(result, arrays, metadata, protocol) + self.assertFalse(first['primary']['success']) + arrays['tcp_quaternion_wxyz'][3499] *= -1 + second = endpoint(result, arrays, metadata, protocol) + self.assertEqual(first['stationarity'], second['stationarity']) + + def test_five_second_pose_endpoint_and_full_run_force_veto(self): + for time_s, expected in ((7., False), (7.001, True)): + result, arrays, metadata, protocol = fixture() + index = round(time_s/.001)-1 + arrays['tcp_position_m'][index, 0] += .004001 + arrays['object_position_m'][index, 0] += .004001 + self.assertEqual(endpoint(result, arrays, metadata, protocol)['primary']['success'], expected) + result, arrays, metadata, protocol = fixture() + arrays['pad_normal_forces_n'][15000, 0] = 10.0001 + evaluation = endpoint(result, arrays, metadata, protocol) + self.assertFalse(evaluation['primary']['success']) + self.assertEqual(evaluation['primary']['reason'], 'force_ceiling_exceeded') + + def test_terminal_fallback_cannot_become_active_control_success(self): + result, arrays, metadata, protocol = fixture() + result['policy']['admitted'] = False + for claim in result['policy']['qualification_checks'][1:]: + claim['qualified'] = False + evaluation = endpoint(result, arrays, metadata, protocol) + self.assertTrue(evaluation['primary']['success']) + self.assertTrue(evaluation['grip_only']['success']) + self.assertFalse(evaluation['primary']['controller_retained_at_5s']) + self.assertFalse(evaluation['primary']['combined_qualified_pause']) + + def test_false_maintained_claims_include_between_boundary_loss(self): + result, arrays, metadata, protocol = fixture() + arrays['pad_normal_forces_n'][2004, 0] = 0. + evaluation = endpoint(result, arrays, metadata, protocol) + self.assertFalse(evaluation['primary']['success']) + self.assertEqual(evaluation['claims']['false_maintained_grip'], 2) + self.assertEqual(evaluation['claims']['false_maintained_pose'], 0) + + def test_no_admission_incomplete_or_invalid_attempts_stay_failed(self): + for failure in ('no_admission', 'short_followup', 'incomplete', 'warning', 'malformed'): + with self.subTest(failure=failure): + result, arrays, metadata, protocol = fixture(15.02 if failure == 'short_followup' else 2.) + if failure == 'no_admission': + result['policy']['events'] = [] + result['policy']['admitted'] = False + elif failure == 'incomplete': result['actions'].pop() + elif failure == 'warning': result['metrics']['warnings'] = ['native warning'] + elif failure == 'malformed': arrays['native_tick'][100] += 1 + evaluation = endpoint(result, arrays, metadata, protocol) + self.assertFalse(evaluation['primary']['success']) + self.assertFalse(evaluation['primary']['combined_qualified_pause']) + self.assertFalse(evaluation['grip_only']['success']) + + def test_aggregates_keep_grip_and_stationary_denominators_distinct(self): + result, arrays, metadata, protocol = fixture() + arrays['tcp_position_m'][3499, 0] += .004001 + arrays['object_position_m'][3499, 0] += .004001 + row = {'variant': 'incremental', 'valid': True, + 'evaluation': endpoint(result, arrays, metadata, protocol)} + failed = {'variant': 'incremental', 'valid': False, + 'evaluation': study._failed_evaluation('retained failure')} + totals = study.aggregates([row, failed])['incremental'] + self.assertEqual(totals['attempts'], 2) + self.assertEqual(totals['native_5s_pause_success'], 1) + self.assertEqual(totals['native_5s_stationary_success'], 0) + self.assertEqual(totals['combined_qualified_5s_pause'], 0) + + def test_four_way_prefix_includes_measured_margin_refusal(self): + result, arrays = controls_fixture() + rows = [{**deepcopy(result), 'variant': variant} for variant in study.VARIANTS] + candidate = rows[-1] + candidate['policy']['events'] = [{'type': 'hold_invalidated', 't_seconds': 1.98, + 'reason': 'measured_joint_margin_exhausted', 'probe_index': 0}] + cutoff = study._common_cutoff(rows) + self.assertEqual(cutoff['time_s'], 1.98) + self.assertEqual(cutoff['reason'], 'pre_admission_bound_refusal') + self.assertEqual(len(cutoff['treatment_boundaries']), 4) + candidate['actions'][99]['joint_targets_rad'][0] += .01 + prefixes = [study._prefix(row, arrays, cutoff=cutoff) for row in rows] + self.assertTrue(all(prefix == prefixes[0] for prefix in prefixes)) + self.assertEqual(prefixes[0]['native_rows'], 1980) + self.assertEqual(prefixes[0]['commands'], 99) + candidate['actions'][98]['joint_targets_rad'][0] += .01 + self.assertNotEqual(study._prefix(candidate, arrays, cutoff=cutoff), prefixes[0]) + self.assertEqual(study._prefix(candidate, arrays)['cutoff_time_s'], 1.98) + + def test_control_readback_still_checks_native_slew(self): + result, arrays = controls_fixture() + study._controls(result, arrays) + arrays['arm_actuator_controls_rad'][54, 0] += 1e-6 + with self.assertRaisesRegex(ValueError, 'slew law'): + study._controls(result, arrays) + + def test_prepare_does_not_replace_existing_evidence(self): + with TemporaryDirectory() as folder, patch.object(study, 'make_protocol') as make: + with self.assertRaises(ValueError): study.prepare(folder, 'unused-assets') + make.assert_not_called() + + def test_started_study_cannot_resume(self): + protocol = {'schema': study.SCHEMA, 'cases': []} + with TemporaryDirectory() as folder, patch.object(study, 'make_protocol', return_value=protocol), \ + patch.object(study, '_attempt') as attempt: + seal(folder, protocol) + study.write(Path(folder)/'started.json', {'original': 'retained'}) + with self.assertRaises(FileExistsError): study.run(folder, 'unused-assets') + attempt.assert_not_called() + self.assertEqual(study.read(Path(folder)/'started.json'), {'original': 'retained'}) + + def test_changed_protocol_or_freeze_rejected_before_native(self): + for change_seal in (False, True): + protocol = {'schema': study.SCHEMA, 'planned_attempts': 32, 'cases': []} + with TemporaryDirectory() as folder, patch.object(study, 'make_protocol', return_value=protocol), \ + patch.object(study, '_attempt') as attempt: + seal(folder, {**protocol, 'planned_attempts': 31}) + if change_seal: + path = Path(folder)/'protocol.json' + path.write_bytes(path.read_bytes().replace(b'31', b'32')) + with self.assertRaises(ValueError): study.run(folder, 'unused-assets') + attempt.assert_not_called() + self.assertFalse((Path(folder)/'started.json').exists()) + + def test_inspect_checks_freeze_before_report_and_source_access(self): + with TemporaryDirectory() as folder, patch.object(study.prior, 'sources') as sources: + seal(folder, {'planned_attempts': 32}) + path = Path(folder)/'protocol.json' + path.write_bytes(path.read_bytes().replace(b'32', b'31')) + with self.assertRaisesRegex(ValueError, 'Frozen protocol digest changed'): + study.inspect(folder) + sources.assert_not_called() + + def test_all_32_invalid_attempts_are_written_and_read_back_without_resume(self): + _, arrays, metadata, _ = fixture() + empty = {name: value[:0].copy() for name, value in arrays.items()} + metadata['rows'] = 0 + protocol = {'schema': study.SCHEMA, 'planned_attempts': 32, 'cases': study.case_definitions(), + 'variants': list(study.VARIANTS), 'source_sha256': {}, 'runtime': {}, + 'source_commit': 'a'*40, 'force_ceiling_n': 10., 'asset_lock_sha256': 'b'*64} + + def refused_attempt(assets, case, variant): + return ({'schema': study.SCHEMA, 'id': case['id']+'-'+variant, 'scene_id': case['id'], + 'variant': variant, 'seed': case['seed'], 'config': case['config'], + 'valid': False, 'error': {'type': 'RuntimeError', 'message': 'retained mock failure'}, + 'metrics': {'valid': False, 'warnings': [], 'simulation_time': 0.}, + 'policy': None, 'actions': [], + 'trajectory': [{'tcp_position_m': [float('nan'), float('inf'), -float('inf')]}], + 'timing': {'summaries': {}}}, + empty, metadata) + + with TemporaryDirectory() as folder, patch.object(study, 'make_protocol', return_value=protocol), \ + patch.object(study.prior, 'sources', return_value={}), \ + patch.object(study.prior, 'runtime', return_value={}), \ + patch.object(study, '_attempt', side_effect=refused_attempt) as attempt, \ + patch('sys.stdout', new_callable=io.StringIO): + seal(folder, protocol) + report = study.run(folder, 'unused-assets') + self.assertEqual(attempt.call_count, 32) + self.assertEqual(report['attempted'], 32) + self.assertEqual(report['valid_attempts'], 0) + for aggregate in report['aggregates'].values(): + self.assertEqual(aggregate['attempts'], 8) + self.assertEqual(aggregate['native_5s_stationary_success'], 0) + # A later tooling commit does not change source-byte identity. + with patch.object(study, '_source_commit', side_effect=AssertionError('inspect must not read HEAD')): + inspection = study.inspect(folder) + self.assertEqual(inspection['attempts'], 32) + self.assertEqual(len(inspection['acquisition_prefixes']), 8) + self.assertTrue(all(row['retained_refusal'] for row in inspection['rows'])) + self.assertTrue(all(not row['verified'] for row in inspection['acquisition_prefixes'])) + first = Path(folder)/report['rows'][0]['id']/'episode.json.gz' + recorded = json.loads(gzip.decompress(first.read_bytes())) + self.assertEqual(recorded['trajectory'][0]['tcp_position_m'][0], + {'invalid_nonfinite_number': 'nan'}) + self.assertEqual(len(recorded['json_invalid_numbers']), 3) + self.assertFalse(recorded['valid']) + with self.assertRaises(FileExistsError): study.run(folder, 'unused-assets') + self.assertEqual(attempt.call_count, 32) + + def test_environment_construction_failure_retains_empty_unknown_native_evidence(self): + with patch('dvidia_training.piper_feedback_env.FeedbackPiperTaskEnv', + side_effect=RuntimeError('construction failed')): + result, arrays, metadata = study._attempt('unused-assets', study.case_definitions()[0], 'incremental') + self.assertFalse(result['valid']) + self.assertEqual(result['error']['message'], 'construction failed') + self.assertEqual(len(arrays), 22) + self.assertTrue(all(len(values) == 0 for values in arrays.values())) + self.assertEqual(metadata['rows'], 0) + self.assertIn('unavailable', metadata['recording_status']) + self.assertFalse(study.evaluate(result, arrays, metadata, {'force_ceiling_n': 10.})['primary']['success']) + + def test_native_export_failure_retains_available_arrays_and_refuses(self): + _, arrays, _, _ = fixture() + arrays['joint_position_rad'][0, 0] = float('nan') + with patch('dvidia_training.piper_feedback_env.FeedbackPiperTaskEnv') as constructor: + env = constructor.return_value + env.ready = False + env.reset.side_effect = RuntimeError('reset failed') + env.native_arrays.return_value = arrays + env.native_metadata.side_effect = RuntimeError('metadata failed') + result, saved, metadata = study._attempt('unused-assets', study.case_definitions()[0], 'incremental') + self.assertFalse(result['valid']) + self.assertEqual(result['error']['message'], 'reset failed') + self.assertEqual(result['native_recording_error']['message'], 'metadata failed') + self.assertEqual(metadata['rows'], 20000) + self.assertTrue(np.isnan(saved['joint_position_rad'][0, 0])) + self.assertIs(saved, arrays) + + def test_native_array_export_exception_retains_zero_rows(self): + with patch('dvidia_training.piper_feedback_env.FeedbackPiperTaskEnv') as constructor: + env = constructor.return_value + env.ready = False + env.reset.side_effect = RuntimeError('reset failed') + env.native_arrays.side_effect = RuntimeError('array export failed') + result, arrays, metadata = study._attempt('unused-assets', study.case_definitions()[0], 'incremental') + self.assertFalse(result['valid']) + self.assertEqual(result['native_recording_error']['message'], 'array export failed') + self.assertEqual(len(arrays), 22) + self.assertEqual(metadata['rows'], 0) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_piper_incremental_hold.py b/tests/test_piper_incremental_hold.py new file mode 100644 index 0000000..c6ce771 --- /dev/null +++ b/tests/test_piper_incremental_hold.py @@ -0,0 +1,403 @@ +"""Pure causal controller tests using a simple Cartesian joint kinematic model.""" +from copy import deepcopy +import math +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from dvidia_training import piper_incremental_hold as incremental +from dvidia_training import piper_retention as retention +from dvidia_training import piper_task + + +class KinematicEnvironment: + """One metre per radian for translation axes; no simulator is constructed.""" + + def __init__(self): + self.config = SimpleNamespace(control_dt=.02, timeout_s=20., + tool_orientation_wxyz=[1., 0., 0., 0.], + goal_position_tolerance_m=.025) + self.model = SimpleNamespace(actuator_ctrlrange=[[-2., 2.]]*6+[[0., .035]]) + self.observation, self.raw_override = None, None + self.requests = [] + + def pose_action(self, position, quaternion, jaw): + self.requests.append((list(position), list(quaternion), jaw)) + if isinstance(self.raw_override, Exception): + raise self.raw_override + if self.raw_override is not None: + return {'joint_targets_rad': deepcopy(self.raw_override), 'gripper_position_m': .012} + targets = list(self.observation['joint_position_rad']) + # Exact inverse of x(q + dq) = x(q) + dq for the first three joints. + for axis in range(3): + targets[axis] += position[axis]-self.observation['tcp_position_m'][axis] + desired_roll = 2*math.atan2(quaternion[1], quaternion[0]) + measured_quat = self.observation['tcp_quaternion_wxyz'] + targets[3] += desired_roll-2*math.atan2(measured_quat[1], measured_quat[0]) + # Reproduce the real pose solver's absolute target margin. Outside it, + # even zero pose error can produce raw target minus measured q != 0. + targets = [max(low+.02, min(high-.02, target)) + for target, (low, high) in zip(targets, self.model.actuator_ctrlrange[:6])] + return {'joint_targets_rad': targets, 'gripper_position_m': .012} + + +class Teacher: + def __init__(self, env): + self.env, self.phase, self.issued_phase, self.ticks = env, 'close', 'close', 35 + self.held_offset, self.grasp_position, self.calls = None, [.3, 0., .082], [] + + def __call__(self, observation): + if self.phase not in ('approach', 'descend', 'close'): + raise AssertionError('Full transport must not be dispatched') + self.calls.append(self.phase) + self.issued_phase = self.phase + self.phase, self.ticks = 'lift', 0 + return {'joint_targets_rad': [.05]*6, 'gripper_position_m': .012} + + +def observation(t, *, rise=None, displacement=0., joint_drift=0., force=.4, + both=True, supported=False, slip=0., roll=0.): + rise = max(0., min(.01, .02*(t-.02))) if rise is None else rise + return { + 't_seconds': t, 'joint_position_rad': [.02+joint_drift]+[.02]*5, + 'gripper_joint_position_m': [.02, -.02], + 'tcp_position_m': [.3+displacement, 0., .082+rise], + 'tcp_quaternion_wxyz': [math.cos(roll/2), math.sin(roll/2), 0., 0.], + 'object_position_m': [.3+displacement+slip, 0., .08+rise], + 'target_position_m': [.35, .08, .08], + 'grasp_contact_counts': {'left': int(both), 'right': int(both)}, + 'pad_normal_forces_n': {'left': force, 'right': force}, + 'table_object_contact_count': int(supported), 'unexpected_contact_count': 0, + 'native_actuator_controls': [0.]*6+[.027], + } + + +def make(variant='incremental'): + env = KinematicEnvironment() + with patch.object(piper_task, 'PiperTaskPolicy', Teacher): + value = (incremental.IncrementalHoldPolicy(env) if variant == 'incremental' + else retention.RetentionPolicy(env, variant)) + return value, env + + +def call(value, env, obs): + env.observation = obs + return value(obs) + + +def admit(value, env): + for index in range(40): + action = call(value, env, observation(index*.02)) + if value.admitted: + return action + raise AssertionError('Kinematic fixture did not admit') + + +def next_observation(value, **kwargs): + return observation(value.last_observed_s+.02, + rise=value.hold_reference['position_m'][2]-.082, **kwargs) + + +class IncrementalHoldTests(unittest.TestCase): + def assertVectorAlmostEqual(self, actual, expected): + self.assertEqual(len(actual), len(expected)) + for a, b in zip(actual, expected): + self.assertAlmostEqual(a, b, places=12) + + def test_contract_fixes_one_candidate_gain_and_original_limits(self): + contract = incremental.incremental_hold_contract() + self.assertEqual(contract['variants'], ['incremental']) + self.assertEqual(contract['update_gain_per_boundary'], 1.) + self.assertEqual(contract['limits'], retention.retention_contract()['limits']) + self.assertEqual(contract['control_dt_s'], .02) + self.assertFalse(contract['physical_robot_ready']) + self.assertEqual(retention.VARIANTS, ('measured_joint', 'issued_joint', 'cartesian')) + + def test_common_acquisition_prefix_and_exact_bumpless_admission(self): + policies = [make(variant) for variant in (*retention.VARIANTS, 'incremental')] + previous = None + for tick in range(40): + actions = [call(value, env, observation(tick*.02)) for value, env in policies] + if any(value.admitted for value, _ in policies): + self.assertTrue(all(value.admitted for value, _ in policies)) + self.assertEqual(actions[-1], previous[-1]) + self.assertEqual(actions[-1], actions[1]) + break + self.assertTrue(all(action == actions[0] for action in actions)) + previous = actions + else: + self.fail('All variants must admit') + value = policies[-1][0] + self.assertEqual(value.correction, [0.]*6) + self.assertEqual(value.teacher.calls, ['close']) + self.assertEqual(value.hold_records[-1]['variant'], 'incremental') + self.assertFalse(value.hold_records[-1]['correction_update_accepted']) + self.assertEqual(value.events[-1]['diagnostic']['decision'], 'bounded_incremental_hold') + + def test_kinematic_error_produces_opposing_signed_target_correction(self): + for displacement in (-.001, .001): + with self.subTest(displacement=displacement): + value, env = make() + first = admit(value, env) + action = call(value, env, next_observation(value, displacement=displacement, + joint_drift=displacement)) + correction = action['joint_targets_rad'][0]-first['joint_targets_rad'][0] + self.assertLess(displacement*correction, 0.) + self.assertAlmostEqual(correction, -displacement) + # The kinematic predicted displacement after that joint change + # reaches the frozen reference in this idealized model. + self.assertAlmostEqual(displacement+correction, 0.) + + def test_repeated_drift_accumulates_even_when_absolute_pose_target_cancels(self): + value, env = make() + first = admit(value, env) + raw_at_admission = value.hold_reference['raw_pose_joint_targets_rad'] + for index in range(1, 5): + action = call(value, env, next_observation(value, displacement=.001, + joint_drift=.001)) + record = value.hold_records[-1] + self.assertVectorAlmostEqual(record['raw_pose_joint_targets_rad'], raw_at_admission) + self.assertAlmostEqual(record['pose_error_increment_rad'][0], -.001) + self.assertAlmostEqual(value.correction[0], -.001*index) + self.assertAlmostEqual(action['joint_targets_rad'][0], first['joint_targets_rad'][0]-.001*index) + + def test_zero_error_keeps_existing_accumulation_despite_new_measured_joints(self): + value, env = make() + admit(value, env) + before = call(value, env, next_observation(value, displacement=.001)) + accumulated = list(value.correction) + for drift in (.005, -.008, .013): + action = call(value, env, next_observation(value, joint_drift=drift)) + self.assertEqual(action, before) + self.assertEqual(value.correction, accumulated) + self.assertEqual(value.hold_records[-1]['pose_error_increment_rad'], [0.]*6) + + def test_opposite_error_unwinds_accumulation_and_pose_and_jaw_stay_frozen(self): + value, env = make() + first = admit(value, env) + reference = deepcopy(value.hold_reference) + call(value, env, next_observation(value, displacement=.001)) + action = call(value, env, next_observation(value, displacement=-.001)) + self.assertVectorAlmostEqual(value.correction, [0.]*6) + self.assertVectorAlmostEqual(action['joint_targets_rad'], first['joint_targets_rad']) + self.assertEqual(action['gripper_position_m'], first['gripper_position_m']) + self.assertEqual(value.hold_reference, reference) + self.assertEqual(env.requests[-1], (reference['position_m'], reference['quaternion_wxyz'], .012)) + + def test_slew_is_per_joint_and_unlimited_proposal_is_explicit(self): + value, env = make() + first = admit(value, env) + env.raw_override = [.03, .01, .021, .019, .02, .02] + action = call(value, env, next_observation(value)) + self.assertVectorAlmostEqual(value.correction, [.002, -.002, .001, -.001, 0., 0.]) + record = value.hold_records[-1] + self.assertVectorAlmostEqual(record['proposed_cumulative_correction_rad'], [.01, -.01, .001, -.001, 0., 0.]) + self.assertVectorAlmostEqual(record['applied_cumulative_correction_rad'], value.correction) + self.assertEqual(record['correction_rate_limited_axes'], [0, 1]) + self.assertTrue(record['correction_update_accepted']) + self.assertVectorAlmostEqual(action['joint_targets_rad'], + [a+b for a, b in zip(first['joint_targets_rad'], value.correction)]) + + def test_total_bound_refuses_unlimited_proposal_before_slew_and_preserves_state(self): + for direction in (-1., 1.): + with self.subTest(direction=direction): + value, env = make() + admit(value, env) + env.raw_override = [.02+direction*.002]+[.02]*5 + for _ in range(15): + call(value, env, next_observation(value)) + self.assertTrue(value.admitted) + previous = list(value.correction) + self.assertAlmostEqual(previous[0], direction*.03) + action = call(value, env, next_observation(value)) + self.assertEqual(value.abort_reason, 'joint_correction_exhausted') + self.assertEqual(value.correction, previous) + self.assertEqual(action['joint_targets_rad'], [.02]*6) + record = value.hold_records[-1] + self.assertAlmostEqual(record['proposed_cumulative_correction_rad'][0], direction*.032) + self.assertEqual(record['accumulated_correction_rad'], previous) + self.assertIsNone(record['applied_cumulative_correction_rad']) + self.assertFalse(record['correction_update_accepted']) + self.assertEqual(value.events[-1]['refused_bounds'], 'joint_correction_exhausted') + calls = len(env.requests) + again = call(value, env, next_observation(value)) + self.assertEqual(again, action) + self.assertEqual(len(env.requests), calls) + self.assertFalse(value.admitted) + self.assertIsNone(value.hold_records[-1]['pose_error_increment_rad']) + value, env = make() + admit(value, env) + env.raw_override = [.051]+[.02]*5 + call(value, env, next_observation(value)) + self.assertEqual(value.abort_reason, 'joint_correction_exhausted') + self.assertEqual(value.correction, [0.]*6) + + def test_original_grip_force_pose_and_margin_refusals_never_integrate(self): + cases = [({'both': False}, 'bilateral_contact_or_force_loss'), + ({'supported': True}, 'support_returned'), + ({'slip': .0031}, 'relative_slip'), + ({'force': 10.0001}, 'declared_force_ceiling_exceeded'), + ({'displacement': .0041}, 'tcp_position_error_exhausted'), + ({'roll': .051}, 'tcp_orientation_error_exhausted')] + for kwargs, reason in cases: + with self.subTest(reason=reason): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + previous, requests = list(value.correction), len(env.requests) + action = call(value, env, next_observation(value, **kwargs)) + self.assertEqual(value.abort_reason, reason) + self.assertEqual(value.correction, previous) + self.assertEqual(len(env.requests), requests) + self.assertEqual(action['gripper_position_m'], .012) + self.assertFalse(value.admitted) + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + previous = list(value.correction) + # Joint 0 already sits on this margin. The next negative correction + # must refuse, even though the cumulative correction remains bounded. + low = value.stop_targets[0]-retention.JOINT_MARGIN_RAD + value.joint_limits[0] = [low, 2.] + call(value, env, next_observation(value, displacement=.001)) + self.assertEqual(value.abort_reason, 'native_joint_margin_exhausted') + self.assertEqual(value.correction, previous) + + def test_admission_bias_and_margin_are_still_checked(self): + for bound in ('bias', 'margin'): + value, env = make() + for tick in range(40): + if value.qualification_since is not None and tick*.02-value.qualification_since >= .1-1e-9: + if bound == 'bias': + env.raw_override = [-.2]*6 + else: + value.joint_limits[0] = [0., 2.] + value.last_requested['joint_targets_rad'][0] = .01 + call(value, env, observation(tick*.02)) + if value.aborted: + break + self.assertEqual(value.abort_reason, 'initial_bias_exhausted' if bound == 'bias' + else 'native_joint_margin_exhausted') + self.assertFalse(value.ever_admitted) + self.assertEqual(value.correction, [0.]*6) + + def test_clipped_zero_error_solver_is_refused_at_measured_margin_on_admission(self): + value, env = make() + for tick in range(40): + obs = observation(tick*.02) + admitting = (value.qualification_since is not None + and tick*.02-value.qualification_since >= .1-1e-9) + if admitting: + low = value.joint_limits[0][0] + obs['joint_position_rad'][0] = low+.01 + value.last_requested['joint_targets_rad'][0] = low+.02 + env.observation = obs + raw = env.pose_action(obs['tcp_position_m'], obs['tcp_quaternion_wxyz'], .012) + self.assertAlmostEqual(raw['joint_targets_rad'][0]-obs['joint_position_rad'][0], .01) + calls = len(env.requests) + call(value, env, obs) + if value.aborted: + self.assertTrue(admitting) + self.assertEqual(len(env.requests), calls) + break + self.assertEqual(value.abort_reason, 'measured_joint_margin_exhausted') + self.assertFalse(value.ever_admitted) + self.assertEqual(value.correction, [0.]*6) + self.assertEqual(value.hold_records[-1]['refused_bounds'], 'measured_joint_margin_exhausted') + + def test_measured_margin_checks_update_and_final_without_solving_or_integrating(self): + for final in (False, True): + with self.subTest(final=final): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + previous, solves, commands = list(value.correction), len(env.requests), len(value.requested_controls) + obs = next_observation(value) + obs['joint_position_rad'][0] = value.joint_limits[0][0]+.01 + env.raw_override = AssertionError('Measured margin must be checked before IK') + if final: + value.observe_final(obs) + else: + call(value, env, obs) + self.assertEqual(value.abort_reason, 'measured_joint_margin_exhausted') + self.assertEqual(value.correction, previous) + self.assertEqual(len(env.requests), solves) + self.assertEqual(len(value.requested_controls), commands+int(not final)) + self.assertFalse(value.hold_records[-1]['correction_update_accepted']) + + def test_malformed_solver_refuses_without_integrating(self): + for bad in ([float('nan')]*6, [0.]*5, 'bad', ValueError('solver')): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + previous = list(value.correction) + env.raw_override = bad + call(value, env, next_observation(value)) + self.assertEqual(value.abort_reason, 'pose_solver_refused') + self.assertEqual(value.correction, previous) + self.assertFalse(value.admitted) + + def test_invalid_or_noncausal_input_refuses_without_command_or_integration(self): + for invalid in ({'joint_position_rad': [float('nan')]*6}, + {'t_seconds': .01}, {'tcp_quaternion_wxyz': [1., 1., 0., 0.]}, + {'pad_normal_forces_n': {'left': -.1, 'right': .4}}): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + before = (len(value.requested_controls), len(env.requests), list(value.correction)) + obs = next_observation(value) + obs.update(invalid) + with self.assertRaises(ValueError): + call(value, env, obs) + self.assertEqual((len(value.requested_controls), len(env.requests), value.correction), before) + self.assertEqual(value.status, 'invalid_measurement_refused') + self.assertFalse(value.input_refusals[-1]['command_issued']) + with self.assertRaises(ValueError): + call(value, env, next_observation(value)) + + def test_final_observation_checks_evidence_without_solve_command_or_integration(self): + for kwargs, reason in (({}, None), ({'displacement': .0041}, 'tcp_position_error_exhausted'), + ({'force': 10.001}, 'declared_force_ceiling_exceeded'), + ({'both': False}, 'bilateral_contact_or_force_loss'), + ({'roll': .051}, 'tcp_orientation_error_exhausted')): + with self.subTest(reason=reason): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + before = (len(value.requested_controls), len(env.requests), list(value.correction)) + env.raw_override = AssertionError('Final observation must never solve') + result = value.observe_final(next_observation(value, **kwargs)) + self.assertEqual((len(value.requested_controls), len(env.requests), value.correction), before) + self.assertEqual(result['abort_reason'], reason) + record = result['retention']['hold_records'][-1] + self.assertTrue(record['final_observation']) + self.assertFalse(record['command_issued']) + self.assertFalse(record['correction_update_accepted']) + self.assertIsNone(record['pose_error_increment_rad']) + self.assertIsNone(record['joint_targets_rad']) + self.assertIsNone(record['applied_cumulative_correction_rad']) + self.assertEqual(result['retention']['final_controller_status'], value.status) + + def test_summary_is_detached_and_every_event_has_truthful_candidate_identity(self): + value, env = make() + admit(value, env) + call(value, env, next_observation(value, displacement=.001)) + summary = value.summary() + self.assertEqual(summary['schema'], incremental.SCHEMA) + self.assertEqual(summary['variant'], 'incremental') + self.assertEqual(summary['contract']['variants'], ['incremental']) + self.assertEqual(summary['acquisition_variant'], 'evidence') + for event in summary['events']: + self.assertEqual(event['variant'], 'incremental') + self.assertEqual(event['retention_variant'], 'incremental') + self.assertEqual(event['acquisition_variant'], 'evidence') + summary['retention']['hold_reference']['position_m'][0] = 99. + summary['retention']['hold_records'][-1]['accumulated_correction_rad'][0] = 99. + summary['retention']['accumulated_correction_rad'][0] = 99. + self.assertNotEqual(value.hold_reference['position_m'][0], 99.) + self.assertNotEqual(value.correction[0], 99.) + self.assertNotEqual(value.hold_records[-1]['accumulated_correction_rad'][0], 99.) + + +if __name__ == '__main__': + unittest.main() From 569e8d3aec52f63eea6c87f109dc4e87d6a7af0e Mon Sep 17 00:00:00 2001 From: DVIDIA Date: Sat, 31 Oct 2026 23:57:09 -0400 Subject: [PATCH 2/2] Publish reproducible incremental-hold comparison and independent audit tools --- .../piper-incremental-hold-v03/README.md | 93 ++ .../independent_audit.py | 1003 +++++++++++++++++ .../package_evidence.py | 428 +++++++ .../render_report.py | 70 ++ .../test_package_evidence.py | 177 +++ .../publish_piper_incremental_hold_v03.py | 184 +++ ...test_publish_piper_incremental_hold_v03.py | 143 +++ tests/test_incremental_independent_audit.py | 307 +++++ 8 files changed, 2405 insertions(+) create mode 100644 benchmarks/piper-incremental-hold-v03/README.md create mode 100644 benchmarks/piper-incremental-hold-v03/independent_audit.py create mode 100644 benchmarks/piper-incremental-hold-v03/package_evidence.py create mode 100644 benchmarks/piper-incremental-hold-v03/render_report.py create mode 100644 benchmarks/piper-incremental-hold-v03/test_package_evidence.py create mode 100644 publishing/huggingface/publish_piper_incremental_hold_v03.py create mode 100644 publishing/huggingface/test_publish_piper_incremental_hold_v03.py create mode 100644 tests/test_incremental_independent_audit.py diff --git a/benchmarks/piper-incremental-hold-v03/README.md b/benchmarks/piper-incremental-hold-v03/README.md new file mode 100644 index 0000000..41a9fae --- /dev/null +++ b/benchmarks/piper-incremental-hold-v03/README.md @@ -0,0 +1,93 @@ +# DVIDIA · PiPER incremental hold v0.3 + +One bounded pose-error accumulator is compared with the unchanged measured-joint, +issued-joint and Cartesian controls across eight fresh synthetic scenes. The +primary outcome requires native grip **and** TCP stationarity for five seconds. +Active controller qualification is a separate conjunction, so a fallback that +retains an object cannot count as active regulation. + +This is an authored CPU simulation experiment. It does not train a policy or +evaluate videos, camera perception, complete placement or physical hardware. + +## Run locally + +Python 3.12, NumPy and MuJoCo are used. No GPU, API key or language model is needed. +The first dependency and pinned-asset installation needs network access; +subsequent simulation runs locally. This source preview is separate from the +stable local application. Minimum memory/CPU requirements have not been qualified. + +```sh +git clone --branch piper-incremental-hold-v0.3-preview https://github.com/Dvidia-Inference/dvidia-training.git +cd dvidia-training +python3.12 -m venv .venv +. .venv/bin/activate +python -m pip install '.[arm]' +dvidia-robot-sim fetch piper --output runs/robot-profile-upstream/assets +``` + +Use your absolute asset path and a new output directory: + +```sh +python -m dvidia_training.piper_incremental_benchmark prepare \ + --assets /absolute/path/to/robot-profile-upstream/assets \ + --output runs/piper-incremental-hold-v03 +python -m dvidia_training.piper_incremental_benchmark run \ + --assets /absolute/path/to/robot-profile-upstream/assets \ + --output runs/piper-incremental-hold-v03 +python -m dvidia_training.piper_incremental_benchmark inspect \ + --output runs/piper-incremental-hold-v03 > runs/piper-incremental-hold-v03/inspection.json +python benchmarks/piper-incremental-hold-v03/render_report.py \ + runs/piper-incremental-hold-v03/report.json +python benchmarks/piper-incremental-hold-v03/independent_audit.py \ + --study runs/piper-incremental-hold-v03 --repo . \ + --output runs/piper-incremental-hold-v03-audit/audit.json +python benchmarks/piper-incremental-hold-v03/package_evidence.py \ + --study runs/piper-incremental-hold-v03 \ + --audit runs/piper-incremental-hold-v03-audit \ + --output runs/piper-incremental-hold-v03-release +``` + +The run refuses changed source/runtime/assets and refuses overwriting evidence. +Each scheduled scene/controller pair runs once. No automatic outcome-based retry +or tuning occurs. The independent audit reads saved arrays with NumPy and the +standard library, without importing the controller, scorer or simulator. + +The package contains all run and audit files with a deterministic inventory and +standalone archive verifier. Original DVIDIA code and synthetic records are MIT; +upstream assets and dependencies retain their licenses. No third-party arm +meshes, private footage, credentials or pretrained model weights are bundled. + +## Why this experiment + +The prior Cartesian update nearly cancelled measured joint drift against its +pose-error step. This candidate accumulates the current error step around the +admission-issued target, with fixed gain, rate, total-correction and pose bounds. +Review caught a zero-error clipping edge case; measured joints outside the target +margin now cause refusal before solving. Jaw requests and native gains are unchanged. + +See the [frozen design](../../docs/designs/piper-incremental-hold-v03.md) and +[EGM/Harness² workflow research](../../docs/research/agent-workflows-20261031.md). +These frameworks inspired the review and comparison process; this is not a run +of either framework and does not claim their benchmark gains for robotics. + +## Frozen result + +Source freeze: `3f30847a7285a180f93b8b0beac0def1a7ceb912`. All 32 attempts +completed valid recordings: 640,000 native 1 ms rows and 32,000 commands. +Each variant admitted four scenes with zero false initial admissions. + +| Controller | Native grip 5 s | Native stationary grip 5 s | Active stationary grip 5 s | +| --- | ---: | ---: | ---: | +| Measured joint | 0/8 | 0/8 | 0/8 | +| Issued joint | 4/8 | 0/8 | 0/8 | +| Cartesian | 4/8 | 0/8 | 0/8 | +| Incremental | 1/8 | 1/8 | 0/8 | + +The candidate terminally refused every admitted hold after 0.32 s, when its +unlimited correction proposal exceeded 0.03 rad. Its one native stationary-grip +result includes fallback; it is not a successful active controller. Four +lower-friction scenes lacked admission in every variant. No controller is promoted. + +[Research note](https://research.dvidia.org/papers/piper-incremental-hold-v03/) · +[GitHub evidence](https://github.com/Dvidia-Inference/dvidia-training/releases/tag/piper-incremental-hold-v0.3-preview) · +[Hugging Face](https://huggingface.co/datasets/Dvidia/piper-incremental-hold-v03) diff --git a/benchmarks/piper-incremental-hold-v03/independent_audit.py b/benchmarks/piper-incremental-hold-v03/independent_audit.py new file mode 100644 index 0000000..4c6ecb9 --- /dev/null +++ b/benchmarks/piper-incremental-hold-v03/independent_audit.py @@ -0,0 +1,1003 @@ +"""Independent readback of fixed32 incremental PiPER evidence. + +No simulator, controller, benchmark scorer, or robot runtime imports. This +checks the consistency of saved numerical evidence and control requests; it +does not authenticate sensors, rerun trials, or establish physical safety. +""" +from __future__ import annotations + +if not __debug__: + raise RuntimeError("Independent incremental-hold audit refuses optimized Python.") + +import argparse +from collections import Counter +import gzip +from hashlib import sha256 +import json +import math +from pathlib import Path +import zipfile + +import numpy as np + +MAX_FILE = 32 * 1024 * 1024 +VARIANTS = ("measured_joint", "issued_joint", "cartesian", "incremental") +SCENES = tuple(f"scene-{index}" for index in range(8)) +FIELDS = { + "t_seconds": 1, "native_tick": 1, + "object_position_m": 3, "object_quaternion_wxyz": 4, + "object_linear_velocity_m_s": 3, "object_angular_velocity_rad_s": 3, + "tcp_position_m": 3, "tcp_quaternion_wxyz": 4, + "object_minus_tcp_local_m": 3, "grasp_contact_counts": 2, + "pad_normal_forces_n": 2, "table_object_contact_count": 1, + "unexpected_contact_count": 1, "gripper_joint_position_m": 2, + "joint_position_rad": 6, "arm_actuator_controls_rad": 6, + "requested_jaw_target_m": 1, "normal_rate_limited_jaw_target_m": 1, + "applied_jaw_target_m": 1, "pulse_active": 1, + "pulse_minimum_opening_m": 1, "pulse_closing_tail": 1, +} +ADMISSIONS = ("evidence_qualified", "timer_admitted") +BOUND_REFUSALS = frozenset({ + "initial_bias_exhausted", "native_joint_margin_exhausted", + "measured_joint_margin_exhausted", "tcp_position_error_exhausted", + "tcp_orientation_error_exhausted", "joint_correction_exhausted", + "pose_solver_refused", "missing_prior_issued_target", +}) +CORRECTION_LIMIT_RAD = .03 +CORRECTION_STEP_RAD = .002 +INITIAL_BIAS_LIMIT_RAD = .08 +CONTROL_DT_S = .02 +CONTROL_SUBSTEP_S = .001 +TCP_POSITION_LIMIT_M = .004 +TCP_ORIENTATION_LIMIT_RAD = .05 + + +def canonical(value): + return json.dumps(value, sort_keys=True, separators=(",", ":"), allow_nan=False).encode() + + +def unique(pairs): + result = {} + for key, value in pairs: + if key in result: + raise ValueError("Duplicate JSON field") + result[key] = value + return result + + +def bad(token): + raise ValueError("Nonfinite JSON " + token) + + +def identity(path): + size, digest = 0, sha256() + assert not path.is_symlink() and path.is_file(), f"Missing or unsafe file: {path}" + with path.open("rb") as stream: + while chunk := stream.read(65536): + size += len(chunk) + assert size <= MAX_FILE, f"File exceeds audit size limit: {path}" + digest.update(chunk) + return {"bytes": size, "sha256": digest.hexdigest()} + + +def read(path): + identity(path) + result = json.loads(path.read_bytes(), object_pairs_hook=unique, parse_constant=bad) + canonical(result) + return result + + +def episode(path): + identity(path) + with gzip.open(path, "rb") as stream: + raw = stream.read(MAX_FILE + 1) + assert len(raw) <= MAX_FILE + result = json.loads(raw, object_pairs_hook=unique, parse_constant=bad) + canonical(result) + return result + + +def row_at(seconds, times): + index = round(seconds / CONTROL_SUBSTEP_S) - 1 + assert 0 <= index < len(times) and abs(times[index] - seconds) < 1e-8 + return index + + +def _unit_quaternions(quaternions): + norms = np.linalg.norm(quaternions, axis=1) + assert np.max(np.abs(norms - 1.)) < 1e-8 + + +def rotation_local(arrays): + q = arrays["tcp_quaternion_wxyz"] + _unit_quaternions(q) + w, x, y, z = q.T + rotation = np.empty((len(q), 3, 3), dtype=np.float64) + rotation[:, 0, 0] = 1 - 2 * (y*y + z*z) + rotation[:, 0, 1] = 2 * (x*y - w*z) + rotation[:, 0, 2] = 2 * (x*z + w*y) + rotation[:, 1, 0] = 2 * (x*y + w*z) + rotation[:, 1, 1] = 1 - 2 * (x*x + z*z) + rotation[:, 1, 2] = 2 * (y*z - w*x) + rotation[:, 2, 0] = 2 * (x*z - w*y) + rotation[:, 2, 1] = 2 * (y*z + w*x) + rotation[:, 2, 2] = 1 - 2 * (x*x + y*y) + vector = arrays["object_position_m"] - arrays["tcp_position_m"] + local = np.einsum("nji,nj->ni", rotation, vector) + assert np.max(np.abs(local - arrays["object_minus_tcp_local_m"])) < 1e-9 + return arrays["object_minus_tcp_local_m"] + + +def native_predicate(arrays, probe): + """Recompute v0.2's acquisition qualification predicate at each 1 ms row.""" + times = arrays["t_seconds"] + origin = row_at(probe["start_time_s"], times) + assert np.linalg.norm(arrays["object_position_m"][origin] - probe["object_origin_m"]) < 1e-9 + assert np.linalg.norm(arrays["object_minus_tcp_local_m"][origin] - probe["local_offset_origin_m"]) < 1e-9 + rise = arrays["object_position_m"][:, 2] - probe["object_origin_m"][2] + drift = np.linalg.norm(arrays["object_minus_tcp_local_m"] - probe["local_offset_origin_m"], axis=1) + components = { + "bilateral_contact": np.all(arrays["grasp_contact_counts"] > 0, axis=1), + "bilateral_force": np.all(arrays["pad_normal_forces_n"] > .01, axis=1), + "force_ceiling": np.all(arrays["pad_normal_forces_n"] <= 10., axis=1), + "unsupported": arrays["table_object_contact_count"] == 0, + "no_unexpected_contact": arrays["unexpected_contact_count"] == 0, + "minimum_rise": rise + 1e-12 >= .006, + "maximum_local_drift": drift <= .003 + 1e-12, + } + condition = np.arange(len(times)) > origin + for component in components.values(): + condition &= component + streak = np.empty(len(times), dtype=np.int64) + previous = 0 + for index, good in enumerate(condition): + previous = previous + 1 if good else 0 + streak[index] = previous + return condition, streak, streak >= 100, components, rise, drift + + +def reasons_at(index, components): + return [name for name, values in components.items() if not values[index]] + + +def stationarity(arrays, start, end, complete, grip_success): + """Recompute 5 s TCP pose stationarity from the admission observation.""" + positions = arrays["tcp_position_m"][start:end + 1] + quaternions = arrays["tcp_quaternion_wxyz"][start:end + 1] + _unit_quaternions(quaternions) + displacement = np.linalg.norm(positions - positions[0], axis=1) + dots = np.abs(quaternions @ quaternions[0]) + angles = 2 * np.arccos(np.clip(dots, 0., 1.)) + max_position, max_angle = float(np.max(displacement)), float(np.max(angles)) + within = (max_position <= TCP_POSITION_LIMIT_M + 1e-12 + and max_angle <= TCP_ORIENTATION_LIMIT_RAD + 1e-12) + return { + "maximum_tcp_position_excursion_m": max_position, + "maximum_tcp_orientation_excursion_rad": max_angle, + "terminal_tcp_position_delta_m": (positions[-1] - positions[0]).tolist(), + "terminal_tcp_orientation_delta_rad": float(angles[-1]), + "tcp_pose_within_bounds": bool(complete and within), + "grip_retention_and_stationarity": bool(complete and grip_success and within), + } + + +def treatment_boundary(policy): + """Earliest admission or pre-admission bound refusal for a scene/variant.""" + events = policy.get("events") if type(policy) is dict else None + if type(events) is not list: + raise ValueError("Policy event tape is missing.") + admissions = [event for event in events + if type(event) is dict and event.get("type") in ADMISSIONS] + if len(admissions) > 1: + raise ValueError("Policy admits more than once.") + admission_time = admissions[0]["t_seconds"] if admissions else None + candidates = [] + if admissions: + candidates.append({"time_s": admission_time, "reason": "admission", + "detail": admissions[0].get("reason", admissions[0].get("type"))}) + for event in events: + if (type(event) is dict and event.get("type") == "hold_invalidated" + and event.get("reason") in BOUND_REFUSALS): + timestamp = event.get("t_seconds") + if type(timestamp) not in (int, float) or not math.isfinite(timestamp): + raise ValueError("Bound refusal has malformed timestamp.") + if admission_time is None or timestamp <= admission_time + 1e-8: + candidates.append({"time_s": float(timestamp), "reason": "pre_admission_bound_refusal", + "detail": event["reason"]}) + if not candidates: + return None + return min(candidates, key=lambda row: row["time_s"]) + + +def common_cutoff(boundaries): + """One common scene cutoff across all four scheduled variants.""" + if set(boundaries) != set(VARIANTS): + raise ValueError("Common cutoff requires all four variant records.") + candidates = [{"variant": variant, **row} for variant, row in boundaries.items() if row is not None] + candidates.sort(key=lambda row: row["variant"]) + timestamp = min((row["time_s"] for row in candidates), default=None) + reasons = sorted({row["reason"] for row in candidates + if timestamp is not None and abs(row["time_s"] - timestamp) <= 1e-8}) + return {"time_s": timestamp, "reason": "+".join(reasons) if reasons else "full_run_no_treatment", + "treatment_boundaries": candidates} + + +def prefix_identity(arrays, actions, cutoff): + """Digest native rows through cutoff, excluding command at cutoff.""" + if len(arrays["t_seconds"]) != 20000 or len(actions) != 1000: + return {"verified": False, "reason": "incomplete_native_or_command_prefix"} + if cutoff["time_s"] is None: + native_rows, command_rows = 20000, list(actions) + else: + native_rows = row_at(cutoff["time_s"], arrays["t_seconds"]) + 1 + command_rows = [action for action in actions + if action["t_seconds"] < cutoff["time_s"] - 1e-8] + prefix_hash = sha256() + for name in sorted(arrays): + value = np.ascontiguousarray(arrays[name][:native_rows]) + prefix_hash.update(json.dumps({"name": name, "shape": list(value.shape), "dtype": str(value.dtype)}, + sort_keys=True, indent=2, allow_nan=False).encode() + b"\n") + prefix_hash.update(value.tobytes()) + return {"verified": True, "cutoff_time_s": cutoff["time_s"], "cutoff_reason": cutoff["reason"], + "native_rows": native_rows, "native_sha256": prefix_hash.hexdigest(), + "commands": len(command_rows), + "commands_sha256": sha256(json.dumps(command_rows, sort_keys=True, indent=2, + allow_nan=False).encode() + b"\n").hexdigest()} + + +def paired_prefix(scene, attempts): + """Preserve incomplete attempts as unknown; compare all complete four-way prefixes.""" + if set(attempts) != set(VARIANTS): + raise ValueError("Scene must retain all four scheduled variants.") + if not all(attempts[variant].get("valid") is True for variant in VARIANTS): + entries = [{"variant": variant, "verified": False, + "reason": attempts[variant].get("reason", "common_quartet_incomplete") + if attempts[variant].get("valid") is not True else "common_quartet_incomplete", + "native_rows_available": attempts[variant].get("native_rows_available")} + for variant in VARIANTS] + return {"scene": scene, "verified": False, + "reason": "invalid_attempt_prevents_complete_quartet", "variants": entries, + "treatment_boundaries": []} + cutoff = common_cutoff({variant: attempts[variant]["boundary"] for variant in VARIANTS}) + prefixes = {variant: prefix_identity(attempts[variant]["arrays"], attempts[variant]["actions"], cutoff) + for variant in VARIANTS} + if any(record.get("verified") is not True for record in prefixes.values()): + return {"scene": scene, "verified": False, "reason": "incomplete_common_prefix_evidence", + "variants": prefixes, "treatment_boundaries": cutoff["treatment_boundaries"]} + identities = {(record["cutoff_time_s"], record["cutoff_reason"], record["native_rows"], + record["native_sha256"], record["commands"], record["commands_sha256"]) + for record in prefixes.values()} + if len(identities) != 1: + raise ValueError("Common acquisition prefixes differ: " + scene) + expected = next(iter(prefixes.values())) + return {"scene": scene, "verified": True, **expected, + "treatment_boundaries": cutoff["treatment_boundaries"], + "variants": [{"variant": variant, **prefixes[variant]} for variant in VARIANTS]} + + +def _vec(value, label): + result = np.asarray(value, dtype=np.float64) + assert result.shape == (6,) and np.isfinite(result).all(), label + return result + + +def inside_joint_margin(joints, limits, margin=.02): + joints, limits = _vec(joints, "measured joints"), np.asarray(limits, dtype=np.float64) + assert limits.shape == (6, 2) and np.isfinite(limits).all() + return bool(np.all(joints >= limits[:, 0] + margin - 1e-12) + and np.all(joints <= limits[:, 1] - margin + 1e-12)) + + +def check_reported_stationarity(actual, reported, complete): + assert reported["complete_5s"] is bool(complete) + for field in ("maximum_tcp_position_excursion_m", "maximum_tcp_orientation_excursion_rad"): + assert abs(reported[field] - actual[field]) <= 1e-10, field + assert np.allclose(reported["terminal_tcp_position_delta_m"], actual["terminal_tcp_position_delta_m"], + atol=1e-10, rtol=0.) + assert abs(reported["terminal_tcp_orientation_delta_rad"] - actual["terminal_tcp_orientation_delta_rad"]) <= 1e-10 + for field in ("tcp_pose_within_bounds", "grip_retention_and_stationarity"): + assert reported[field] is actual[field] + + +def correction_audit(policy, actions, arrays, boundary, variant): + """Reconstruct candidate law, refusal behavior, margin, and final no-command row.""" + retention = policy.get("retention") or {} + records = retention.get("hold_records") + reference = retention.get("hold_reference") + assert policy.get("variant") == variant + if type(records) is not list: + raise ValueError("Hold diagnostics are missing.") + if variant != "incremental": + # Legacy controls have their own audit in the historical study; for this + # fixed comparison we still check bumpless admission, jaw freeze, and final seal. + return baseline_hold_audit(policy, actions, arrays, boundary, variant) + + assert policy.get("candidate") == "incremental" + law = retention.get("update_law") + assert type(law) is str and law + assert retention.get("update_gain_per_boundary") == 1. + limits = np.asarray(retention.get("native_joint_control_limits_rad"), dtype=np.float64) + assert limits.shape == (6, 2) and np.isfinite(limits).all() + if not records: + assert reference is None + assert boundary is None or boundary.get("reason") == "pre_admission_bound_refusal" + return {"hold_record_count": 0, "bound_refusal_counts": {}, + "rate_limited_axis_occurrences": 0, + "maximum_abs_pose_error_increment_rad": None, + "maximum_abs_active_correction_rad": None, + "maximum_abs_proposed_correction_rad": None, + "first_admitted_target_continuous": False, + "final_observation_no_ik_or_command": False} + refused_counts = Counter() + rate_limited_axes = 0 + max_increment = max_active = max_proposal = 0. + previous = np.zeros(6, dtype=np.float64) + rate_limited_axes = sum(len(row.get("correction_rate_limited_axes") or []) for row in records) + accepted_rates = [] + inactive_latched_target = None + command_by_time = {float(row["t_seconds"]): row for row in actions} + assert len(command_by_time) == len(actions) + admission = boundary is not None and boundary["reason"] == "admission" + + if reference is not None: + assert reference.get("frozen_jaw_target_m") == policy["frozen_jaw_target_m"] + issued = _vec(reference.get("issued_joint_targets_rad"), "admission issued target") + measured = _vec(reference.get("measured_joint_position_rad"), "admission measured joints") + first_time = float(reference["t_seconds"]) + if boundary is None or boundary.get("detail") != "measured_joint_margin_exhausted": + assert inside_joint_margin(measured, limits) + if admission or boundary is None or boundary.get("detail") != "native_joint_margin_exhausted": + assert inside_joint_margin(issued, limits) + if admission: + first = records[0] + assert first["command_issued"] is True and abs(first["t_seconds"] - first_time) <= 1e-8 + assert _vec(first["joint_targets_rad"], "first admitted target").tolist() == issued.tolist() + first_action = command_by_time[first_time] + assert first_action["joint_targets_rad"] == first["joint_targets_rad"] + assert first_action["gripper_position_m"] == reference["frozen_jaw_target_m"] + assert abs(first_time - boundary["time_s"]) <= 1e-8 + previous_actions = [row for row in actions if row["t_seconds"] < first_time - 1e-8] + if previous_actions: + assert previous_actions[-1]["joint_targets_rad"] == reference["issued_joint_targets_rad"] + + for record in records: + time = float(record["t_seconds"]) + assert record.get("candidate") == "incremental" + assert record.get("update_law") == law + assert record.get("update_gain_per_boundary") == 1. + accumulated = _vec(record.get("accumulated_correction_rad"), "stored correction") + assert np.max(np.abs(accumulated)) <= CORRECTION_LIMIT_RAD + 1e-12 + before = _vec(record.get("correction_before_update_rad"), "prior correction") + assert np.allclose(before, previous, atol=1e-12, rtol=0.) + refused = record.get("refused_bounds") + if refused is not None: + refused_counts[refused] += 1 + assert any(event.get("type") == "hold_invalidated" and event.get("reason") == refused + and abs(event.get("t_seconds", -1.) - time) <= 1e-8 + for event in policy["events"]) + + final = record.get("final_observation") is True + command_issued = record.get("command_issued") is True + assert not final or not command_issued + measured_q = _vec(record.get("measured_joint_position_rad"), "record measured joints") + native_index = row_at(time, arrays["t_seconds"]) + assert np.allclose(measured_q, arrays["joint_position_rad"][native_index], atol=1e-9, rtol=0.) + margin_ok = inside_joint_margin(measured_q, limits) + raw_value = record.get("raw_pose_joint_targets_rad") + increment_value = record.get("pose_error_increment_rad") + proposal_value = record.get("proposed_cumulative_correction_rad") + accepted = record.get("correction_update_accepted") is True + + if not margin_ok: + assert refused == "measured_joint_margin_exhausted" + assert raw_value is None and increment_value is None and proposal_value is None + assert accepted is False and np.allclose(accumulated, before, atol=1e-12, rtol=0.) + elif refused == "measured_joint_margin_exhausted": + raise AssertionError("Measured-joint margin refusal has an in-margin observation") + + if final: + assert raw_value is None and increment_value is None and proposal_value is None + assert accepted is False and record.get("applied_cumulative_correction_rad") is None + assert record.get("joint_targets_rad") is None + assert record.get("preceding_issued_joint_targets_rad") is not None + assert record["preceding_issued_joint_targets_rad"] == actions[-1]["joint_targets_rad"] + assert record.get("active_regulation") is (policy.get("admitted") is True) + assert record.get("correction_rate_limited_axes") == [] + continue + + if record.get("active_regulation") is True and margin_ok and refused is None: + # Admission has a pose solve to establish the bias but intentionally + # does not integrate. Later control boundaries must log the full law. + if time == (reference or {}).get("t_seconds"): + assert raw_value is not None + raw = _vec(raw_value, "admission raw pose target") + bias = _vec(reference.get("fixed_bias_rad"), "initial bias") + assert np.allclose(issued - raw, bias, atol=1e-12, rtol=0.) + assert np.max(np.abs(bias)) <= INITIAL_BIAS_LIMIT_RAD + 1e-12 + assert increment_value is None and accepted is False + assert np.allclose(accumulated, 0., atol=1e-12, rtol=0.) + previous = accumulated + else: + assert raw_value is not None and increment_value is not None and proposal_value is not None + raw = _vec(raw_value, "raw pose target") + increment = _vec(increment_value, "pose error increment") + expected_increment = raw - measured_q + assert np.allclose(increment, expected_increment, atol=1e-12, rtol=0.) + max_increment = max(max_increment, float(np.max(np.abs(increment)))) + proposal = _vec(proposal_value, "cumulative proposal") + expected_proposal = before + increment + assert np.allclose(proposal, expected_proposal, atol=1e-12, rtol=0.) + max_proposal = max(max_proposal, float(np.max(np.abs(proposal)))) + assert np.max(np.abs(proposal)) <= CORRECTION_LIMIT_RAD + 1e-12 + applied = before + np.clip(increment, -CORRECTION_STEP_RAD, CORRECTION_STEP_RAD) + axes = [i for i, (step, old) in enumerate(zip(increment, before)) + if abs(step) > CORRECTION_STEP_RAD + 1e-12] + assert record.get("correction_rate_limited_axes") == axes + assert accepted is True + assert np.allclose(accumulated, applied, atol=1e-12, rtol=0.) + assert np.allclose(record.get("applied_cumulative_correction_rad"), applied, + atol=1e-12, rtol=0.) + target = _vec(record.get("joint_targets_rad"), "incremental joint target") + expected_target = issued + applied + assert np.allclose(target, expected_target, atol=1e-12, rtol=0.) + max_active = max(max_active, float(np.max(np.abs(applied)))) + assert inside_joint_margin(target, limits) + accepted_rates.append(float(np.max(np.abs(applied - before)) / CONTROL_DT_S)) + previous = accumulated + elif record.get("active_regulation") is True: + # The row that refuses measured-joint margin is checked before IK. + assert not margin_ok and refused == "measured_joint_margin_exhausted" + assert raw_value is None and increment_value is None and proposal_value is None + assert accepted is False and np.allclose(accumulated, before, atol=1e-12, rtol=0.) + assert record.get("applied_cumulative_correction_rad") is None + previous = accumulated + elif refused == "joint_correction_exhausted": + assert raw_value is not None and increment_value is not None and proposal_value is not None + raw = _vec(raw_value, "refused raw pose target") + increment = _vec(increment_value, "refused pose error increment") + assert np.allclose(increment, raw - measured_q, atol=1e-12, rtol=0.) + proposal = _vec(proposal_value, "refused proposal") + assert np.allclose(proposal, before + increment, atol=1e-12, rtol=0.) + assert np.max(np.abs(proposal)) > CORRECTION_LIMIT_RAD + 1e-12 + assert accepted is False and np.allclose(accumulated, before, atol=1e-12, rtol=0.) + assert record.get("applied_cumulative_correction_rad") is None + previous = accumulated + elif refused == "native_joint_margin_exhausted": + assert raw_value is not None and increment_value is not None and proposal_value is not None + raw = _vec(raw_value, "joint-margin refusal raw target") + increment = _vec(increment_value, "joint-margin refusal increment") + proposal = _vec(proposal_value, "joint-margin refusal proposal") + assert np.allclose(increment, raw - measured_q, atol=1e-12, rtol=0.) + assert np.allclose(proposal, before + increment, atol=1e-12, rtol=0.) + limited = before + np.clip(increment, -CORRECTION_STEP_RAD, CORRECTION_STEP_RAD) + assert not inside_joint_margin(issued + limited, limits) + assert accepted is False and np.allclose(accumulated, before, atol=1e-12, rtol=0.) + assert record.get("applied_cumulative_correction_rad") is None + previous = accumulated + elif refused == "initial_bias_exhausted": + assert reference is not None and reference.get("fixed_bias_rad") is not None + bias = _vec(reference["fixed_bias_rad"], "refused initial bias") + assert np.max(np.abs(bias)) > INITIAL_BIAS_LIMIT_RAD + 1e-12 + assert accepted is False and np.allclose(accumulated, before, atol=1e-12, rtol=0.) + previous = accumulated + else: + # Terminal grip/pose/solver refusal may have no fresh IK result and + # must never integrate a stale proposal. + assert accepted is False + assert np.allclose(accumulated, before, atol=1e-12, rtol=0.) + assert record.get("applied_cumulative_correction_rad") is None + previous = accumulated + + action = command_by_time.get(time) + assert action is not None + assert action["joint_targets_rad"] == record["joint_targets_rad"] + assert action["gripper_position_m"] == record["frozen_jaw_target_m"] + if record.get("active_regulation") is False: + # Terminal fallback samples measured joints once, then holds that + # latched target. The arm may move slightly after the latch, so + # comparing every later request with each new measurement would + # incorrectly reject a valid terminal hold. + target = _vec(record.get("joint_targets_rad"), "terminal hold target") + if inactive_latched_target is None: + assert np.allclose(target, measured_q, atol=1e-12, rtol=0.) + inactive_latched_target = target.copy() + else: + assert np.allclose(target, inactive_latched_target, atol=1e-12, rtol=0.) + if reference is not None: + assert record["frozen_jaw_target_m"] == reference["frozen_jaw_target_m"] + + # Every requested hold record must map one-to-one onto an issued action. + command_records = [row for row in records if row.get("command_issued") is True] + assert len({float(row["t_seconds"]) for row in command_records}) == len(command_records) + return { + "hold_record_count": len(records), "bound_refusal_counts": dict(refused_counts), + "rate_limited_axis_occurrences": rate_limited_axes, + "maximum_abs_pose_error_increment_rad": max_increment or None, + "maximum_abs_active_correction_rad": max_active or None, + "maximum_abs_proposed_correction_rad": max_proposal or None, + "maximum_abs_accepted_correction_rate_rad_s": max(accepted_rates) if accepted_rates else None, + "first_admitted_target_continuous": bool(admission and reference is not None), + "final_observation_no_ik_or_command": bool(records[-1].get("final_observation") + and records[-1].get("raw_pose_joint_targets_rad") is None + and records[-1].get("command_issued") is False), + } + + +def baseline_hold_audit(policy, actions, arrays, boundary, variant): + """Check inherited holds' admission continuity, frozen jaw, and final seal.""" + retention = policy.get("retention") or {} + records, reference = retention.get("hold_records"), retention.get("hold_reference") + if type(records) is not list: + raise ValueError("Hold diagnostics are missing.") + assert policy.get("variant") == variant + if reference is not None: + issued = _vec(reference["issued_joint_targets_rad"], "admission issued target") + assert reference["frozen_jaw_target_m"] == policy["frozen_jaw_target_m"] + if boundary is not None and boundary["reason"] == "admission": + first = records[0] + assert first["command_issued"] is True + assert abs(first["t_seconds"] - boundary["time_s"]) <= 1e-8 + expected_first = (_vec(reference["measured_joint_position_rad"], "admission measured joints") + if variant == "measured_joint" else issued) + assert np.array_equal(_vec(first["joint_targets_rad"], "admission target"), expected_first) + action = next(row for row in actions if abs(row["t_seconds"] - boundary["time_s"]) <= 1e-8) + assert first["joint_targets_rad"] == action["joint_targets_rad"] + prior = [row for row in actions if row["t_seconds"] < boundary["time_s"] - 1e-8] + if prior: + assert prior[-1]["joint_targets_rad"] == reference["issued_joint_targets_rad"] + for record in records: + if record.get("final_observation"): + assert record.get("command_issued") is False + assert record.get("raw_pose_joint_targets_rad") is None + assert record.get("joint_targets_rad") == actions[-1]["joint_targets_rad"] + continue + if record.get("command_issued"): + action = next(row for row in actions if abs(row["t_seconds"] - record["t_seconds"]) <= 1e-8) + assert action["joint_targets_rad"] == record["joint_targets_rad"] + assert action["gripper_position_m"] == record["frozen_jaw_target_m"] + return {"hold_record_count": len(records), "first_admitted_target_continuous": bool(reference), + "final_observation_no_ik_or_command": bool(records[-1].get("final_observation") + and not records[-1].get("command_issued") + and records[-1].get("raw_pose_joint_targets_rad") is None)} + + +def native_success(arrays, policy, admission): + probe = next(row for row in policy["probes"] if row["probe_index"] == admission["probe_index"]) + condition, streak, qualified, components, rise, drift = native_predicate(arrays, probe) + times = arrays["t_seconds"] + start = row_at(admission["t_seconds"], times) + target = admission["t_seconds"] + 5. + # A NumPy scalar comparison yields np.bool_; reports deserialize as bool, + # so normalize before exact endpoint comparisons and JSON output. + complete = bool(target <= times[-1] + 1e-8) + end = row_at(target, times) if complete else len(times) - 1 + admission_ok = bool(qualified[start]) + grip = bool(admission_ok and complete and np.all(condition[start:end + 1]) + and np.all(arrays["pad_normal_forces_n"][start:] <= 10.)) + pose = stationarity(arrays, start, end, complete, grip) + losses = np.flatnonzero(~condition[start:end + 1]) + loss_index = start + int(losses[0]) if len(losses) else None + return { + "admission_ok": admission_ok, "streak_ms": int(streak[start]), + "grip_only_5s": grip, "grip_and_stationary_5s": pose["grip_retention_and_stationarity"], + "complete_5s": complete, "start": start, "end": end, "condition": condition, + "components": components, "stationarity": pose, + "first_grip_loss_s": float(times[loss_index]) if loss_index is not None else None, + "first_grip_loss_reasons": reasons_at(loss_index, components) if loss_index is not None else [], + "first_tcp_pose_failure_s": next((float(times[start + i]) for i, ok in enumerate( + (np.linalg.norm(arrays["tcp_position_m"][start:end + 1] - arrays["tcp_position_m"][start], axis=1) + <= TCP_POSITION_LIMIT_M + 1e-12) + & (2 * np.arccos(np.clip(np.abs(arrays["tcp_quaternion_wxyz"][start:end + 1] + @ arrays["tcp_quaternion_wxyz"][start]), 0., 1.)) + <= TCP_ORIENTATION_LIMIT_RAD + 1e-12)) if not ok), None), + } + + +def false_maintained_claims(arrays, policy, admission, admission_probe): + """Count explicit true controller claims contradicted by native grip or TCP pose.""" + checks = policy.get("qualification_checks") + if type(checks) is not list: + raise ValueError("Qualification checks are missing.") + condition, _, qualified, components, _, _ = native_predicate(arrays, admission_probe) + start = row_at(admission["t_seconds"], arrays["t_seconds"]) + pose_pos = np.linalg.norm(arrays["tcp_position_m"] - arrays["tcp_position_m"][start], axis=1) + q = arrays["tcp_quaternion_wxyz"] + pose_ang = 2 * np.arccos(np.clip(np.abs(q @ q[start]), 0., 1.)) + grip_continuous = np.logical_and.accumulate(condition[start:]) & bool(qualified[start]) + pose_continuous = np.logical_and.accumulate( + (pose_pos[start:] <= TCP_POSITION_LIMIT_M + 1e-12) + & (pose_ang[start:] <= TCP_ORIENTATION_LIMIT_RAD + 1e-12)) + claims = [row for row in checks if row.get("qualified") is True + and row["t_seconds"] > admission["t_seconds"] + 1e-8] + false_grip, false_pose = [], [] + for claim in claims: + index = row_at(claim["t_seconds"], arrays["t_seconds"]) + offset = index - start + if not grip_continuous[offset]: + false_grip.append({"t_seconds": claim["t_seconds"], + "reasons": reasons_at(index, components), + "native_prefix_qualified": bool(qualified[index])}) + if not pose_continuous[offset]: + false_pose.append({"t_seconds": claim["t_seconds"], + "position_excursion_m": float(pose_pos[index]), + "orientation_excursion_rad": float(pose_ang[index])}) + return {"maintained_checks": len(claims), "false_maintained_grip": false_grip, + "false_maintained_pose": false_pose} + + +def reconstruct_controls(arrays, actions, requested, trajectory, initial_controls, case): + """Replay the fixed requested-to-applied 1 ms per-actuator slew.""" + assert len(actions) == len(requested) == 1000 and len(trajectory) == 1001 + control = np.asarray(initial_controls, dtype=np.float64) + assert control.shape == (7,) and np.isfinite(control).all() + max_error = 0. + rates = np.array([case["config"]["joint_target_rate_rad_s"] * .001] * 6 + + [case["config"]["jaw_target_rate_m_s"] * .001], dtype=np.float64) + for index, (action, request) in enumerate(zip(actions, requested)): + assert abs(action["t_seconds"] - index * CONTROL_DT_S) < 1e-8 + assert request["index"] == index + assert all(action[key] == request[key] for key in ("t_seconds", "joint_targets_rad", "gripper_position_m")) + assert trajectory[index]["command"] == action + assert trajectory[index]["issued_joint_targets_rad"] == action["joint_targets_rad"] + assert trajectory[index]["issued_gripper_position_m"] == action["gripper_position_m"] + target = np.asarray(action["joint_targets_rad"] + [action["gripper_position_m"]], dtype=np.float64) + assert target.shape == (7,) and np.isfinite(target).all() + for tick in range(index * 20, (index + 1) * 20): + control += np.clip(target - control, -rates, rates) + actual = np.r_[arrays["arm_actuator_controls_rad"][tick], arrays["applied_jaw_target_m"][tick]] + error = float(np.max(np.abs(control - actual))) + max_error = max(max_error, error) + assert error < 1e-12 + assert arrays["requested_jaw_target_m"][tick] == action["gripper_position_m"] + for field in ("object_position_m", "joint_position_rad", "tcp_position_m"): + assert np.allclose(trajectory[index + 1][field], arrays[field][(index + 1) * 20 - 1], + atol=1e-9, rtol=0.) + assert all(trajectory[-1].get(key) is None + for key in ("command", "issued_joint_targets_rad", "issued_gripper_position_m")) + return {"requested_commands": len(actions), "native_substeps_reconstructed": 20000, + "maximum_control_reconstruction_error": max_error} + + +def validate_native(arrays, metadata, *, expected_rows=20000): + assert set(arrays) == set(FIELDS) + assert metadata.get("schema") == "dvidia.piper-feedback-native/v1" + assert metadata.get("native_timestep_s") == CONTROL_SUBSTEP_S + assert metadata.get("physical_robot_ready") is False + assert metadata.get("side_order") == ["left", "right"] + assert metadata.get("fields") == [{"name": name, "width": width} for name, width in FIELDS.items()] + times = arrays["t_seconds"] + assert times.ndim == 1 and 0 < len(times) <= 30000 + assert metadata.get("rows") == len(times) + assert expected_rows is None or len(times) == expected_rows + for name, width in FIELDS.items(): + expected = (len(times),) if width == 1 else (len(times), width) + assert arrays[name].dtype == np.dtype("float64") and arrays[name].shape == expected + assert np.isfinite(arrays[name]).all() + ticks = np.arange(1, len(times) + 1, dtype=np.float64) + assert np.array_equal(arrays["native_tick"], ticks) + assert np.max(np.abs(times - ticks * CONTROL_SUBSTEP_S)) <= 1e-8 + for name in ("grasp_contact_counts", "table_object_contact_count", "unexpected_contact_count"): + values = arrays[name] + assert np.all((values >= 0) & (values <= 10000) & (values == np.floor(values))) + assert np.all((arrays["pad_normal_forces_n"] >= 0) & (arrays["pad_normal_forces_n"] <= 1e6)) + assert not np.any(arrays["pulse_active"]) and not np.any(arrays["pulse_closing_tail"]) + rotation_local(arrays) + for name in ("tcp_quaternion_wxyz", "object_quaternion_wxyz"): + _unit_quaternions(arrays[name]) + return len(times) + + +def check_schedule(protocol, report): + assert protocol["planned_attempts"] == report["planned_attempts"] == report["attempted"] == 32 + assert protocol["variants"] == list(VARIANTS) + cases = protocol["cases"] + assert len(cases) == 8 and len({case["id"] for case in cases}) == 8 + assert [case["seed"] for case in cases] == list(range(311200, 311208)) + planned = [(case, variant) for case in cases for variant in case["variant_order"]] + assert len(planned) == 32 and all(set(case["variant_order"]) == set(VARIANTS) for case, _ in planned) + expected_orders = (("measured_joint", "issued_joint", "incremental", "cartesian"), + ("issued_joint", "cartesian", "measured_joint", "incremental"), + ("cartesian", "incremental", "issued_joint", "measured_joint"), + ("incremental", "measured_joint", "cartesian", "issued_joint")) + assert tuple(tuple(case["variant_order"]) for case in cases) == expected_orders * 2 + for variant in VARIANTS: + assert [sum(case["variant_order"][position] == variant for case in cases) + for position in range(4)] == [2, 2, 2, 2] + assert [case["id"] + "-" + variant for case, variant in planned] == [row["id"] for row in report["rows"]] + return cases, planned + + +def inventory(root): + result = {} + for path in sorted(root.rglob("*")): + if path.is_symlink(): + raise ValueError("Symlink in frozen study: " + str(path)) + if path.is_file(): + result[path.relative_to(root).as_posix()] = identity(path) + return result + + +def read_npz(path): + with zipfile.ZipFile(path) as bundle: + entries = bundle.infolist() + assert (len(entries) == 22 and len({entry.filename for entry in entries}) == 22 + and sum(entry.file_size for entry in entries) <= MAX_FILE + and all(entry.file_size <= 2 * 1024 * 1024 and "/" not in entry.filename + and entry.filename.endswith(".npy") for entry in entries)) + with np.load(path, allow_pickle=False) as bundle: + return {name: bundle[name] for name in bundle.files} + + +def compact_policy(policy): + if policy is None: + return None + result = {key: value for key, value in policy.items() + if key not in ("requested_controls", "qualification_checks")} + if "retention" in result: + result["retention"] = {key: value for key, value in result["retention"].items() + if key != "hold_records"} + return result + + +def correction_summary(policy, variant): + records = ((policy or {}).get("retention") or {}).get("hold_records", []) + flags, refusals = {}, Counter() + active, desired, increments, accepted_rates, axis_count = [], [], [], [], 0 + for record in records: + for key, value in record.items(): + if type(value) is bool and any(token in key for token in ("saturat", "limit", "refus")): + flags[key] = flags.get(key, 0) + int(value) + if record.get("refused_bounds") is not None: + refusals[record["refused_bounds"]] += 1 + axis_count += len(record.get("correction_rate_limited_axes") or []) + for key, output in (("correction_rad", active), ("desired_correction_rad", desired)): + if record.get(key) is not None: + output.append(_vec(record[key], key)) + increment = record.get("pose_error_increment_rad") + if increment is not None: + increments.append(_vec(increment, "pose_error_increment_rad")) + if record.get("command_issued") and record.get("correction_update_accepted"): + before = _vec(record["correction_before_update_rad"], "correction_before_update_rad") + after = _vec(record["accumulated_correction_rad"], "accumulated_correction_rad") + accepted_rates.append(float(np.max(np.abs(after - before)) / CONTROL_DT_S)) + max_abs = lambda values: float(np.max(np.abs(values))) if values else None + retention = (policy or {}).get("retention") or {} + result = { + "hold_record_count": len(records), "flag_counts": flags, + "bound_refusal_counts": dict(refusals), "rate_limited_axis_occurrences": axis_count, + "maximum_abs_active_correction_rad": max_abs(active), + "maximum_abs_desired_correction_rad": max_abs(desired), + "maximum_abs_pose_error_increment_rad": None, + "maximum_abs_accepted_correction_rate_rad_s": None, + "input_refusals": retention.get("input_refusals", []), + "scope": "Includes recorded final diagnostic checks; only command_issued rows represent issued controls.", + } + if variant == "incremental": + result.update(maximum_abs_pose_error_increment_rad=max_abs(increments), + maximum_abs_accepted_correction_rate_rad_s=max(accepted_rates) if accepted_rates else None) + return result + + +def compare_endpoint(summary, item, arrays): + evaluation, policy = summary["evaluation"], item.get("policy") + admissions = [event for event in (policy or {}).get("events", []) + if event.get("type") == "evidence_qualified"] + assert len(admissions) <= 1 + if not admissions: + assert evaluation.get("first_admission") is None + assert evaluation["primary"]["success"] is False + assert evaluation["primary"]["native_pause_success"] is False + assert evaluation["primary"]["combined_qualified_pause"] is False + assert evaluation["grip_only"]["success"] is False + assert evaluation["stationarity"]["grip_retention_and_stationarity"] is False + assert evaluation["claims"]["maintained_checks"] == 0 + return {"admitted": False, "false_initial": False, "grip_only_5s": False, + "grip_stationary_5s": False, "controller_retained_5s": False, + "combined_5s": False, "final_both_qualified": bool(evaluation["final"]["both_qualified"]), + "false_maintained_grip": 0, "false_maintained_pose": 0, "maintained_checks": 0, + "scope": "No admission; retained in the denominator as failure."} + event = admissions[0] + native = native_success(arrays, policy, event) + first = evaluation["first_admission"] + assert abs(first["time_s"] - event["t_seconds"]) <= 1e-8 + assert first["probe_index"] == event["probe_index"] + assert first["native_prefix_qualified"] is native["admission_ok"] + assert abs(first["native_qualifying_window_s"] - native["streak_ms"] * .001) <= 1e-12 + primary, grip_only = evaluation["primary"], evaluation["grip_only"] + assert primary["complete_5s"] is native["complete_5s"] + assert primary["native_pause_success"] is native["grip_only_5s"] + assert grip_only["success"] is native["grip_only_5s"] + assert primary["success"] is native["grip_and_stationary_5s"] + target_time = event["t_seconds"] + 5. + checks = [row for row in policy["qualification_checks"] + if row.get("probe_index") == event["probe_index"] + and abs(row["t_seconds"] - target_time) <= 1e-8] + controller = bool(native["complete_5s"] and len(checks) == 1 and checks[0]["qualified"] is True) + assert primary["controller_retained_at_5s"] is controller + assert primary["combined_qualified_pause"] is (native["grip_and_stationary_5s"] and controller) + assert primary["first_tcp_pose_failure_s"] == native["first_tcp_pose_failure_s"] + check_reported_stationarity(native["stationarity"], evaluation["stationarity"], native["complete_5s"]) + probe = next(row for row in policy["probes"] if row["probe_index"] == event["probe_index"]) + false = false_maintained_claims(arrays, policy, + {"t_seconds": event["t_seconds"], "probe_index": event["probe_index"]}, probe) + claims = evaluation["claims"] + assert claims["maintained_checks"] == false["maintained_checks"] + assert claims["false_maintained_grip"] == len(false["false_maintained_grip"]) + assert claims["false_maintained_pose"] == len(false["false_maintained_pose"]) + return {"admitted": True, "false_initial": not native["admission_ok"], + "grip_only_5s": native["grip_only_5s"], + "grip_stationary_5s": native["grip_and_stationary_5s"], + "controller_retained_5s": controller, + "combined_5s": bool(native["grip_and_stationary_5s"] and controller), + "final_both_qualified": bool(evaluation["final"]["both_qualified"]), + "false_maintained_grip": claims["false_maintained_grip"], + "false_maintained_pose": claims["false_maintained_pose"], + "maintained_checks": claims["maintained_checks"], + "native": native, "false_claim_details": false, + "scope": "Independent native predicate recomputation from saved rows; controller claims checked separately."} + + +def _report_aggregates(report, variant): + selected = [row for row in report["rows"] if row["variant"] == variant] + return { + "attempts": len(selected), "valid": sum(row["valid"] for row in selected), + "admissions": sum(row["evaluation"]["first_admission"] is not None for row in selected), + "false_initial_admissions": sum(bool(row["evaluation"]["first_admission"] + and not row["evaluation"]["first_admission"]["native_prefix_qualified"]) + for row in selected), + "native_5s_pause_success": sum(row["evaluation"]["primary"]["native_pause_success"] for row in selected), + "native_5s_stationary_success": sum(row["evaluation"]["primary"]["success"] for row in selected), + "controller_retained_at_5s": sum(row["evaluation"]["primary"]["controller_retained_at_5s"] for row in selected), + "combined_qualified_5s_pause": sum(row["evaluation"]["primary"]["combined_qualified_pause"] for row in selected), + "grip_retention_and_stationarity": sum(row["evaluation"]["stationarity"]["grip_retention_and_stationarity"] for row in selected), + "false_maintained_grip": sum(row["evaluation"]["claims"]["false_maintained_grip"] for row in selected), + "false_maintained_pose": sum(row["evaluation"]["claims"]["false_maintained_pose"] for row in selected), + "final_both_qualified": sum(row["evaluation"]["final"]["both_qualified"] for row in selected), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--study", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + study, repo, output = args.study, args.repo, args.output + if output.exists() or output.is_symlink(): + raise ValueError("Audit output must be new") + raw_before = inventory(study) + protocol, report, seal = read(study / "protocol.json"), read(study / "report.json"), read(study / "frozen.json") + assert seal["protocol_sha256"] == raw_before["protocol.json"]["sha256"] + assert protocol["schema"] == report["schema"] == "dvidia.piper-incremental-hold-study/v1" + cases, planned = check_schedule(protocol, report) + source_before = {} + for name, expected in protocol["source_sha256"].items(): + current = identity(repo / "src" / "dvidia_training" / name)["sha256"] + frozen = identity(study / "sources" / name)["sha256"] + assert current == frozen == expected + source_before[name] = current + assert source_before == report["source_sha256"] + + totals = {variant: Counter(attempts=0, valid=0, admissions=0, false_initial=0, + grip_only_5s=0, grip_stationary_5s=0, controller_retained_5s=0, + combined_5s=0, final_both_qualified=0, + false_maintained_grip=0, false_maintained_pose=0, + maintained_checks=0, native_rows=0, commands=0) + for variant in VARIANTS} + rows, attempts_by_scene, paired = [], {case["id"]: {} for case in cases}, {} + + def retain_attempt(scene, variant, value): + attempts_by_scene[scene][variant] = value + if len(attempts_by_scene[scene]) == 4: + paired[scene] = paired_prefix(scene, attempts_by_scene[scene]) + attempts_by_scene[scene] = {} + + for summary, (case, variant) in zip(report["rows"], planned): + folder = study / summary["id"] + for name, expected in summary["files"].items(): + assert "/" not in name and name not in (".", "..") + assert identity(folder / name) == expected + item = episode(folder / "episode.json.gz") + assert item["schema"] == protocol["schema"] + assert item["id"] == summary["id"] and item["scene_id"] == case["id"] + assert item["config"] == case["config"] and item["variant"] == variant and item["seed"] == case["seed"] + assert item["valid"] is summary["valid"] and item["error"] == summary["error"] + assert item["metrics"] == summary["metrics"] + assert canonical(compact_policy(item.get("policy"))) == canonical(summary["policy"]) + evaluation = read(folder / "evaluation.json") + assert canonical(evaluation) == canonical(summary["evaluation"]) + arrays, metadata = read_npz(folder / "native.npz"), read(folder / "native.json") + counter = totals[variant] + counter["attempts"] += 1 + counter["native_rows"] += len(arrays.get("t_seconds", [])) + if not item["valid"] or item["error"] is not None: + assert evaluation["primary"]["success"] is False and evaluation["grip_only"]["success"] is False + retain_attempt(case["id"], variant, {"valid": False, + "reason": "invalid_attempt_retained_as_unknown", + "native_rows_available": len(arrays.get("t_seconds", []))}) + rows.append({"id": item["id"], "scene_id": case["id"], "variant": variant, "valid": False, + "admitted": False, "grip_only_5s": False, "grip_stationary_5s": False, + "controller_retained_5s": False, "combined_5s": False, + "scope": "Invalid/incomplete; retained in denominator as unknown failure."}) + continue + + validate_native(arrays, metadata) + assert item["metrics"].get("valid") is True and item["metrics"].get("warnings") == [] + unexpected = int(np.count_nonzero(arrays["unexpected_contact_count"])) + assert unexpected == item["metrics"]["unexpected_contact_native_ticks"] == 0 + assert len(item["actions"]) == 1000 and len(item["trajectory"]) == 1001 + controls = reconstruct_controls(arrays, item["actions"], item["policy"]["requested_controls"], + item["trajectory"], item["trajectory"][0]["native_actuator_controls"], case) + counter.update(valid=1, commands=controls["requested_commands"]) + boundary = treatment_boundary(item["policy"]) + endpoint = compare_endpoint(summary, item, arrays) + correction = correction_audit(item["policy"], item["actions"], arrays, boundary, variant) + computed = correction_summary(item["policy"], variant) + assert canonical(computed) == canonical(evaluation["corrections"]) + if variant == "incremental": + assert correction["final_observation_no_ik_or_command"] + counter.update( + admissions=int(endpoint["admitted"]), false_initial=int(endpoint["false_initial"]), + grip_only_5s=int(endpoint["grip_only_5s"]), grip_stationary_5s=int(endpoint["grip_stationary_5s"]), + controller_retained_5s=int(endpoint["controller_retained_5s"]), + combined_5s=int(endpoint["combined_5s"]), + final_both_qualified=int(endpoint["final_both_qualified"]), + false_maintained_grip=int(endpoint["false_maintained_grip"]), + false_maintained_pose=int(endpoint["false_maintained_pose"]), + maintained_checks=int(endpoint["maintained_checks"])) + retain_attempt(case["id"], variant, {"valid": True, "boundary": boundary, + "arrays": arrays, "actions": item["actions"]}) + rows.append({"id": item["id"], "scene_id": case["id"], "variant": variant, "valid": True, + **{key: value for key, value in endpoint.items() + if key not in ("native", "false_claim_details")}, + "corrections": correction, "control_reconstruction": controls}) + + assert len(report["rows"]) == 32 and all(counter["attempts"] == 8 for counter in totals.values()) + paired_rows = [paired[case["id"]] for case in cases] + inspection = read(study / "inspection.json") + assert inspection["schema"] == "dvidia.piper-incremental-hold-inspection/v1" + assert inspection["attempts"] == 32 + assert inspection["native_rows"] == sum(counter["native_rows"] for counter in totals.values()) + inspected = {row["id"]: row for row in inspection["rows"]} + assert set(inspected) == {row["id"] for row in report["rows"]} + for row in report["rows"]: + check = inspected[row["id"]] + assert check["evaluation_recomputed"] is True + if row["valid"]: + assert check["native_validated"] is True and check["controls_recomputed"] is True + else: + assert check["retained_refusal"] is True or check["native_validated"] is False + inspected_prefixes = {row["scene_id"]: row for row in inspection["acquisition_prefixes"]} + assert set(inspected_prefixes) == {case["id"] for case in cases} + for scene, prefix in paired.items(): + check = inspected_prefixes[scene] + assert check["verified"] is prefix["verified"] + if prefix["verified"]: + for field in ("cutoff_time_s", "cutoff_reason", "native_rows", "native_sha256", "commands", "commands_sha256"): + assert check[field] == prefix[field] + assert check["treatment_boundaries"] == prefix["treatment_boundaries"] + else: + assert check["reason"] == "invalid_attempt_prevents_complete_four_variant_comparison" + assert canonical(inspection["aggregates"]) == canonical(report["aggregates"]) + for variant in VARIANTS: + assert canonical(report["aggregates"][variant]) == canonical(_report_aggregates(report, variant)) + + source_after = {name: identity(repo / "src" / "dvidia_training" / name)["sha256"] + for name in source_before} + assert source_after == source_before and raw_before == inventory(study) + output.parent.mkdir(parents=True, exist_ok=True) + value = {"schema": "dvidia.incremental-hold-independent-audit/v1", "passed": True, + "protocol_sha256": raw_before["protocol.json"]["sha256"], + "source_sha256": source_before, "frozen_raw_files": raw_before, + "rows": rows, "totals": {variant: dict(counter) for variant, counter in totals.items()}, + "paired_acquisition_prefixes": paired_rows, "source_and_raw_inputs_unchanged": True, + "scope": "Independent saved-record predicate, controller-law and control-slew audit. No simulator/runtime/scorer imports or trial reruns; hashes do not authenticate external data or hardware."} + with output.open("x", encoding="utf-8") as stream: + json.dump(value, stream, sort_keys=True, indent=2, allow_nan=False) + stream.write("\n") + print(json.dumps({"passed": True, "attempts": 32, "aggregates": report["aggregates"], + "raw_files": len(raw_before)})) + + +if __name__ == "__main__": + main() diff --git a/benchmarks/piper-incremental-hold-v03/package_evidence.py b/benchmarks/piper-incremental-hold-v03/package_evidence.py new file mode 100644 index 0000000..50fa9f1 --- /dev/null +++ b/benchmarks/piper-incremental-hold-v03/package_evidence.py @@ -0,0 +1,428 @@ +"""Package the reviewed retention run and independent audit, offline. + +The entire selected run and audit trees are archived in deterministic order. +Packaging never imports frozen runtime code, runs physics, reads credentials or +publishes. It refuses incomplete or failed audit evidence. +""" +from __future__ import annotations + +import argparse +import gzip +from hashlib import sha256 +import io +import json +import os +from pathlib import Path +import re +import stat +import sys +import zipfile + +SCHEMA = 'dvidia.piper-incremental-hold-study/v1' +VARIANTS = {'measured_joint', 'issued_joint', 'cartesian', 'incremental'} +ARCHIVE_NAME = 'piper-incremental-hold-v03.zip' +HF_NAMES = {'README.md', 'LICENSE', ARCHIVE_NAME, 'report.json', 'protocol.json', 'inspection.json'} +MAX_FILE = 64 * 1024 * 1024 +MAX_ARCHIVE = 192 * 1024 * 1024 +MAX_TOTAL = 480 * 1024 * 1024 +MAX_FILES = 2048 +MAX_JSON = 16 * 1024 * 1024 +PRIVATE = (b'/'+b'Users/', b'/private/'+b'tmp/', b'/opt/'+b'homebrew/', b'/var/'+b'folders/') +SECRETS = tuple(re.compile(value) for value in + (rb'hf_[A-Za-z0-9]{20,}', rb'gh[pousr]_[A-Za-z0-9]{20,}', + rb'apikey_[A-Za-z0-9_]{20,}', rb'AKIA[A-Z0-9]{16}')) + + +def encoded(value): + return json.dumps(value, sort_keys=True, indent=2, allow_nan=False).encode()+b'\n' + + +def pairs(items): + result = {} + for key, value in items: + if key in result: + raise ValueError('Duplicate JSON fields.') + result[key] = value + return result + + +def nonfinite(_value): + raise ValueError('Nonfinite JSON number.') + + +def load_json(path, limit=MAX_JSON): + raw = read(path, limit) + value = json.loads(raw, object_pairs_hook=pairs, parse_constant=nonfinite) + return value, raw + + +def read(path, limit=MAX_FILE): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > limit: + raise ValueError('Missing, linked or oversized evidence file.') + with path.open('rb') as stream: + raw = stream.read(limit+1) + if len(raw) > limit: + raise ValueError('Evidence grew beyond its byte limit.') + return raw + + +def safe_path(name): + if (type(name) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]{0,399}', name) + or any(part in ('', '.', '..') for part in name.split('/'))): + raise ValueError('Unsafe archive path.') + return name + + +def scan_stream(stream, limit=MAX_FILE): + size, tail = 0, b'' + while chunk := stream.read(65536): + size += len(chunk) + if size > limit: + raise ValueError('Decoded evidence exceeds its byte limit.') + block = tail+chunk + if any(value in block for value in PRIVATE) or any(pattern.search(block) for pattern in SECRETS): + raise ValueError('Private path or credential-shaped bytes in evidence.') + tail = block[-256:] + return size + + +def parse_json_bytes(raw): + value = json.loads(raw, object_pairs_hook=pairs, parse_constant=nonfinite) + scan_stream(io.BytesIO(json.dumps(value, ensure_ascii=False, allow_nan=False).encode())) + return value + + +def scan_file(path): + raw = read(path) + scan_stream(io.BytesIO(raw)) + if path.suffix == '.json': + parse_json_bytes(raw) + elif path.suffix == '.jsonl': + for line in raw.splitlines(): + if line.strip(): + parse_json_bytes(line) + elif path.suffix == '.gz': + total = 0 + with gzip.open(path, 'rb') as stream: + chunks = [] + while chunk := stream.read(65536): + total += len(chunk) + if total > MAX_FILE: + raise ValueError('Gzip content exceeds its byte limit.') + scan_stream(io.BytesIO(chunk)) + chunks.append(chunk) + decoded = b''.join(chunks) + if decoded.lstrip().startswith((b'{', b'[')): + parse_json_bytes(decoded) + elif path.suffix in ('.npz', '.zip'): + with zipfile.ZipFile(path) as bundle: + entries = bundle.infolist() + names = [item.filename for item in entries] + if (not entries or len(entries) > MAX_FILES or len(names) != len(set(names)) + or sum(item.file_size for item in entries) > MAX_TOTAL): + raise ValueError('Nested archive exceeds its member/byte bound.') + for item in entries: + safe_path(item.filename) + if (item.file_size > MAX_FILE or item.flag_bits & 1 + or stat.S_IFMT(item.external_attr >> 16) not in (0, stat.S_IFREG) + or item.compress_type not in (zipfile.ZIP_STORED, zipfile.ZIP_DEFLATED)): + raise ValueError('Unsafe nested archive member.') + with bundle.open(item) as stream: + scan_stream(stream) + + +def identity(path, limit=MAX_FILE): + raw = read(path, limit) + return {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} + + +def tree(root): + root = Path(root) + if root.is_symlink() or not root.is_dir(): + raise ValueError('Evidence root must be a regular directory.') + result = {} + for path in sorted(root.rglob('*')): + if path.is_symlink() or not (path.is_file() or path.is_dir()): + raise ValueError('Evidence tree contains a link or special file.') + if path.is_dir(): + continue + name = safe_path(path.relative_to(root).as_posix()) + if name in result: + raise ValueError('Duplicate evidence path.') + scan_file(path) + result[name] = identity(path) + if len(result) > MAX_FILES or sum(item['bytes'] for item in result.values()) > MAX_TOTAL: + raise ValueError('Evidence tree exceeds its count/size bound.') + if not result: + raise ValueError('Evidence tree is empty.') + return result + + +def validate_inputs(study, audit): + study, audit = Path(study), Path(audit) + if 'piper-incremental-hold-v03' not in study.name: + raise ValueError('Study directory must be the new piper-incremental-hold-v03 run.') + study_files, audit_files = tree(study), tree(audit) + for name in ('report.json', 'protocol.json', 'inspection.json'): + if name not in study_files: + raise ValueError('Run is missing '+name+'.') + if 'audit.json' not in audit_files: + raise ValueError('Independent audit is missing audit.json.') + report, report_raw = load_json(study/'report.json') + protocol, protocol_raw = load_json(study/'protocol.json') + inspection, inspection_raw = load_json(study/'inspection.json') + audit_doc, audit_raw = load_json(audit/'audit.json') + for name, raw, identities in (('report.json', report_raw, study_files), + ('protocol.json', protocol_raw, study_files), + ('inspection.json', inspection_raw, study_files), + ('audit.json', audit_raw, audit_files)): + if {'bytes': len(raw), 'sha256': sha256(raw).hexdigest()} != identities[name]: + raise ValueError('Parsed metadata differs from its pinned evidence identity.') + rows = report.get('rows', report.get('attempts')) if type(report) is dict else None + if (type(report) is not dict or report.get('schema') != SCHEMA or type(rows) is not list or len(rows) != 32 + or report.get('planned_attempts') != 32 or report.get('attempted') != 32 + or type(protocol) is not dict or type(inspection) is not dict or type(audit_doc) is not dict): + raise ValueError('Run report, protocol or audit schema is incomplete.') + inspection_rows = inspection.get('rows') + if (inspection.get('schema') != 'dvidia.piper-incremental-hold-inspection/v1' + or inspection.get('attempts') != 32 or type(inspection_rows) is not list + or len(inspection_rows) != 32 + or [(row.get('id'), row.get('evaluation_recomputed')) for row in inspection_rows + if type(row) is dict] != [(row.get('id'), True) for row in rows]): + raise ValueError('Inspection does not cover all 32 attempts with recomputed evaluations.') + if (audit_doc.get('schema') != 'dvidia.incremental-hold-independent-audit/v1' + or type(audit_doc.get('passed')) is not bool or audit_doc['passed'] is not True + or audit_doc.get('source_and_raw_inputs_unchanged') is not True + or inspection.get('passed') is False or inspection.get('status') in ('FAIL', 'failed', 'refused')): + raise ValueError('The independent audit or run inspection did not pass.') + roster = {} + scene_sets = {} + pairs_seen = set() + for row in rows: + variant = row.get('variant') if type(row) is dict else None + scene = row.get('scene_id') if type(row) is dict else None + if variant not in VARIANTS or type(scene) is not str: + raise ValueError('Attempt records must identify a scene and one of the four variants.') + pair = (variant, scene) + if pair in pairs_seen: + raise ValueError('Duplicate scene/variant attempt in report.') + pairs_seen.add(pair) + roster.setdefault(variant, set()).add(scene) + if (set(roster) != VARIANTS or any(len(scenes) != 8 for scenes in roster.values()) + or len({frozenset(scenes) for scenes in roster.values()}) != 1): + raise ValueError('Expected eight scenes for each of the four variants.') + for field in ('protocol_sha256',): + expected = sha256(protocol_raw).hexdigest() + for doc in (audit_doc, report, inspection): + if field in doc and doc[field] != expected: + raise ValueError('Protocol digest differs across run and audit records.') + if audit_doc.get(field) != expected: + raise ValueError('Independent audit is not bound to this frozen protocol.') + source_hashes = protocol.get('source_sha256') + if (type(source_hashes) is not dict or report.get('source_sha256') != source_hashes + or audit_doc.get('source_sha256') != source_hashes + or audit_doc.get('frozen_raw_files') != study_files): + raise ValueError('Independent audit source or frozen-input inventory differs from this run.') + audit_rows = audit_doc.get('rows') + if (type(audit_rows) is not list or len(audit_rows) != 32 + or [(row.get('id'), row.get('valid')) for row in audit_rows] + != [(row.get('id'), row.get('valid')) for row in rows]): + raise ValueError('Independent audit does not cover the exact 32 ordered run attempts.') + return {'study': study, 'audit': audit, 'study_files': study_files, + 'audit_files': audit_files, 'report': report, 'report_raw': report_raw, + 'protocol_raw': protocol_raw, 'inspection_raw': inspection_raw, + 'audit_raw': audit_raw, 'report_doc': report, 'protocol': protocol, + 'inspection': inspection, 'audit_doc': audit_doc} + + +VERIFY_SOURCE = r'''"""Verify/reconstruct a DVIDIA PiPER retention archive; no pickle or imports.""" +import argparse, hashlib, json, re, stat, zipfile +from pathlib import Path +MAX_FILE=64*1024*1024 +MAX_TOTAL=480*1024*1024 +def unique(pairs): + out={} + for k,v in pairs: + if k in out: raise ValueError('duplicate JSON field') + out[k]=v + return out +def safe(name): + if type(name) is not str or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9._/-]{0,399}',name) or any(p in ('','.','..') for p in name.split('/')): raise ValueError('unsafe archive path') + return name +def verify(path,output=None): + with zipfile.ZipFile(path) as z: + es=z.infolist(); names=[e.filename for e in es] + if not 1<=len(es)<=2050 or len(names)!=len(set(names)) or sum(e.file_size for e in es)>MAX_TOTAL: raise ValueError('archive bounds/roster invalid') + for e in es: + safe(e.filename) + if e.file_size>MAX_FILE or e.flag_bits&1 or stat.S_IFMT(e.external_attr>>16) not in (0,stat.S_IFREG) or e.compress_type not in (zipfile.ZIP_STORED,zipfile.ZIP_DEFLATED): raise ValueError('unsafe archive member') + if 'inventory.json' not in names or z.getinfo('inventory.json').file_size>4*1024*1024: raise ValueError('missing/large inventory') + inv=json.loads(z.read('inventory.json'),object_pairs_hook=unique,parse_constant=lambda _:(_ for _ in ()).throw(ValueError('nonfinite'))) + if inv.get('schema')!='dvidia.piper-incremental-hold-inventory/v1' or type(inv.get('files')) is not list: raise ValueError('unknown inventory schema') + records={} + for r in inv['files']: + if type(r) is not dict or set(r)!={'path','bytes','sha256'}: raise ValueError('malformed inventory record') + n=safe(r['path']) + if n in records or n=='inventory.json' or type(r['bytes']) is not int or not 0<=r['bytes']<=MAX_FILE or type(r['sha256']) is not str or not re.fullmatch('[0-9a-f]{64}',r['sha256']): raise ValueError('invalid inventory identity') + records[n]=r + if set(names)!=set(records)|{'inventory.json'}: raise ValueError('inventory roster differs') + for name,r in records.items(): + h=hashlib.sha256(); size=0 + with z.open(name) as f: + while c:=f.read(65536): + size+=len(c) + if size>r['bytes']: raise ValueError('member exceeds expected bytes') + h.update(c) + if size!=r['bytes'] or h.hexdigest()!=r['sha256']: raise ValueError('member digest differs') + if output is not None: + root=Path(output) + if root.exists() or root.is_symlink(): raise ValueError('output must be new') + root.mkdir(parents=True); base=root.resolve() + for n in names: + dest=root/n; dest.parent.mkdir(parents=True,exist_ok=True) + if not dest.parent.resolve().is_relative_to(base): raise ValueError('output escape') + with z.open(n) as src,dest.open('xb') as dst: + while c:=src.read(65536): dst.write(c) + return {'verified':True,'files':len(records),'logical_bytes':sum(r['bytes'] for r in records.values()),'reconstructed':output is not None} +def main(): + p=argparse.ArgumentParser(description=__doc__); p.add_argument('archive'); p.add_argument('--output'); a=p.parse_args(); print(json.dumps(verify(a.archive,a.output),sort_keys=True)) +if __name__=='__main__': main() +''' + + +def archive(path, files, generated, *, expected_identities): + if set(expected_identities) != set(files): + raise ValueError('Archive input roster differs from pinned identities.') + expected = {name: dict(value) for name, value in expected_identities.items()} + inventory = [] + for name, source in sorted(files.items()): + safe_path(name) + if identity(source) != expected[name]: + raise ValueError('Archive input differs from its pinned evidence identity.') + inventory.append({'path': name, **expected[name]}) + for name, raw in sorted(generated.items()): + safe_path(name) + scan_stream(io.BytesIO(raw)) + inventory.append({'path': name, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest()}) + inventory.sort(key=lambda item: item['path']) + if len(inventory) > MAX_FILES or sum(item['bytes'] for item in inventory) > MAX_TOTAL-1024*1024: + raise ValueError('Expanded archive exceeds its limits.') + content = {**files, **generated, 'inventory.json': encoded({'schema':'dvidia.piper-incremental-hold-inventory/v1', + 'files':inventory,'scope':'All run/audit artifacts and included tools; inventory excludes itself.'})} + with zipfile.ZipFile(path, 'x', compression=zipfile.ZIP_DEFLATED, compresslevel=9, allowZip64=True) as z: + for name, source in sorted(content.items()): + info=zipfile.ZipInfo(name, date_time=(1980,1,1,0,0,0)); info.create_system=3 + info.external_attr=(stat.S_IFREG|0o644)<<16 + info.compress_type=zipfile.ZIP_STORED if isinstance(source,Path) and source.suffix in ('.gz','.npz') else zipfile.ZIP_DEFLATED + with z.open(info,'w',force_zip64=True) as dest: + if isinstance(source,Path): + if source.is_symlink() or not source.is_file(): + raise ValueError('Archive input is no longer a plain file.') + copied, copied_hash = 0, sha256() + with source.open('rb') as stream: + while chunk:=stream.read(65536): + copied += len(chunk) + if copied > expected[name]['bytes']: + raise ValueError('Copied archive input exceeds its pinned size.') + copied_hash.update(chunk) + dest.write(chunk) + if {'bytes': copied, 'sha256': copied_hash.hexdigest()} != expected[name]: + raise ValueError('Copied archive input differs from its pinned evidence identity.') + else: + dest.write(source) + return inventory + + +def archive_digest(path): + h=sha256(); size=0 + with path.open('rb') as stream: + while chunk:=stream.read(65536): + size+=len(chunk) + if size>MAX_ARCHIVE: raise ValueError('Compressed archive exceeds its limit.') + h.update(chunk) + return {'bytes':size,'sha256':h.hexdigest()} + + +def prepare(study, audit, output, repo_root): + output=Path(output) + if output.exists() or output.is_symlink(): raise ValueError('Package output must be new.') + frozen=validate_inputs(study,audit) + initial=(frozen['study_files'],frozen['audit_files']) + files={**{'study/'+name:frozen['study']/name for name in frozen['study_files']}, + **{'audit/'+name:frozen['audit']/name for name in frozen['audit_files']}} + expected_identities = {**{'study/'+name: value for name, value in frozen['study_files'].items()}, + **{'audit/'+name: value for name, value in frozen['audit_files'].items()}} + tool_dir=Path(__file__).parent + files.update({ + 'tools/package_evidence.py':Path(__file__), + 'tools/render_report.py':tool_dir/'render_report.py', + 'tools/test_package_evidence.py':tool_dir/'test_package_evidence.py', + 'tools/independent_audit.py':tool_dir/'independent_audit.py', + 'tools/verify_archive.py':None, + 'licenses/agilex_piper-LICENSE.txt':Path(repo_root)/'third_party/robot-models/agilex_piper-LICENSE.txt', + 'licenses/mujoco-LICENSE.txt':Path(repo_root)/'third_party/mujoco/LICENSE.txt'}) + generated_tools={name:raw for name,raw in files.items() if raw is None} + files={name:path for name,path in files.items() if path is not None} + readme=b'''# DVIDIA | PiPER incremental hold v0.3\n\nA bounded native CPU study compares measured-joint, issued-joint, Cartesian and incremental hold strategies on eight fresh synthetic scenes (32 attempts). The primary outcome requires native grip AND TCP stationarity for five seconds after admission. Active controller qualification is reported separately. Every scheduled attempt and failure is retained. See report.json and the protocol for the frozen results and definitions.\n\nThis is not a learned policy, full transport or placement task, physical-sensor validation, hardware qualification or real-object reliability estimate. The simulator uses authored privileged state.\n\nThe ZIP preserves the complete new run tree under study/ and its independent audit under audit/. It includes deterministic inventory hashes and tools/verify_archive.py. To verify and optionally reconstruct without importing project code, extract the archive and run:\n\n python tools/verify_archive.py piper-incremental-hold-v03.zip --output evidence\n\nResearch article: https://research.dvidia.org/papers/piper-incremental-hold-v03/\nSource release: https://github.com/Dvidia-Inference/dvidia-training/releases/tag/piper-incremental-hold-v0.3-preview\nEvidence dataset: https://huggingface.co/datasets/Dvidia/piper-incremental-hold-v03\n\nDVIDIA source and authored synthetic records are MIT. Upstream simulator and robot asset notices are included. No physical robot footage, credentials or hardware data are bundled.\n''' + generated={**generated_tools,'README.md':readme,'LICENSE':read(Path(repo_root)/'LICENSE'), + 'tools/verify_archive.py':VERIFY_SOURCE.encode()} + # Top-level project README/renderer/tool paths are included in the archive; scanner validates them too. + for name, source in files.items(): + scan_file(source) + if name not in expected_identities: + expected_identities[name] = identity(source) + output.mkdir(parents=True); stage=output/'huggingface'; stage.mkdir() + archive_path=stage/ARCHIVE_NAME + inventory=archive(archive_path,files,generated,expected_identities=expected_identities) + verifier={'__name__':'retention_archive_verifier'} + exec(compile(VERIFY_SOURCE,'verify_archive.py','exec'),verifier) + verified=verifier['verify'](archive_path) + if (tree(frozen['study'])!=initial[0] or tree(frozen['audit'])!=initial[1]): + raise ValueError('Study or audit changed during packaging.') + archive_id=archive_digest(archive_path) + table = ['\n## Frozen results\n', '| Hold | Admissions | False initial | Native 5 s grip | Grip + TCP stationarity | Active stationary hold | Final controller + native |', + '|---|---:|---:|---:|---:|---:|---:|'] + for variant in ('measured_joint', 'issued_joint', 'cartesian', 'incremental'): + result = frozen['report']['aggregates'][variant] + table.append('| '+variant+' | '+str(result['admissions'])+'/8 | '+str(result['false_initial_admissions'])+' | '+str(result['native_5s_pause_success'])+'/8 | '+str(result['grip_retention_and_stationarity'])+'/8 | '+str(result['combined_qualified_5s_pause'])+'/8 | '+str(result['final_both_qualified'])+'/8 |') + table.append('\nFresh finite development cells, not a population accuracy estimate. Every scheduled attempt is retained, with no native reruns or post-result tuning. No controller is automatically promoted.\n') + card=b'''---\nlicense: mit\npretty_name: DVIDIA PiPER incremental hold v0.3\ntask_categories:\n- robotics\ntags:\n- robotics\n- simulation\n- piper\n- mujoco\n- research\n---\n\n'''+readme+'\n'.join(table).encode()+b'\nArchive SHA-256: `'+archive_id['sha256'].encode()+b'`\n' + payload={'README.md':card,'LICENSE':generated['LICENSE'], + 'report.json':frozen['report_raw'],'protocol.json':frozen['protocol_raw'], + 'inspection.json':frozen['inspection_raw'],ARCHIVE_NAME:archive_path.read_bytes()} + for name, raw in payload.items(): + scan_stream(io.BytesIO(raw),MAX_ARCHIVE if name==ARCHIVE_NAME else MAX_FILE) + if name == ARCHIVE_NAME: + if (stage/name).read_bytes() != raw: + raise ValueError('Staged archive bytes changed while preparing dataset payload.') + else: + with (stage/name).open('xb') as stream: stream.write(raw) + if {p.name for p in stage.iterdir()}!=HF_NAMES: raise ValueError('Dataset payload must contain exactly six files.') + inventory_path=output/'inventory.json'; inventory_path.write_bytes(encoded({'schema':'dvidia.piper-incremental-hold-package/v1','files':[{'path':p.name,**identity(p,MAX_ARCHIVE if p.name==ARCHIVE_NAME else MAX_FILE)} for p in sorted(stage.iterdir())]})) + receipt={'schema':'dvidia.piper-incremental-hold-package-receipt/v1','remote_operations':False, + 'target_dataset':'Dvidia/piper-incremental-hold-v03','source_tag':'piper-incremental-hold-v0.3-preview', + 'attempts':32,'variants':sorted(VARIANTS),'study_files':len(frozen['study_files']), + 'audit_files':len(frozen['audit_files']),'archive':{'filename':ARCHIVE_NAME,**archive_id}, + 'verified':verified,'frozen_inputs_unchanged':True, + 'pinned_archive_inputs_verified':{'files':len(expected_identities),'at_index':True,'at_stream_copy':True}, + 'inventory_sha256':sha256(inventory_path.read_bytes()).hexdigest()} + (output/'package-receipt.json').write_bytes(encoded(receipt)) + return receipt + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument('--study',type=Path,required=True); parser.add_argument('--audit',type=Path,required=True) + parser.add_argument('--repo',type=Path,default=Path(__file__).resolve().parents[2]) + parser.add_argument('--output',type=Path,required=True) + args=parser.parse_args() + try: + result=prepare(args.study,args.audit,args.output,args.repo) + print(json.dumps({'archive':result['archive'],'verified':result['verified']},indent=2)) + return 0 + except (ValueError,OSError,KeyError,TypeError,zipfile.BadZipFile) as exc: + print('Evidence package refused: '+str(exc),file=sys.stderr); return 2 + + +if __name__=='__main__': raise SystemExit(main()) diff --git a/benchmarks/piper-incremental-hold-v03/render_report.py b/benchmarks/piper-incremental-hold-v03/render_report.py new file mode 100644 index 0000000..219ad1e --- /dev/null +++ b/benchmarks/piper-incremental-hold-v03/render_report.py @@ -0,0 +1,70 @@ +"""Render all fixed incremental-hold attempts without simulator imports.""" +import argparse +import html +import json +from pathlib import Path + +VARIANTS = ('measured_joint', 'issued_joint', 'cartesian', 'incremental') + + +def unique(pairs): + out = {} + for key, value in pairs: + if key in out: + raise ValueError('Duplicate JSON field') + out[key] = value + return out + + +def load(path): + path = Path(path) + if path.is_symlink() or not path.is_file() or path.stat().st_size > 16*1024*1024: + raise ValueError('Expected bounded report file') + def invalid(_): + raise ValueError('Nonfinite JSON') + doc = json.loads(path.read_bytes(), object_pairs_hook=unique, parse_constant=invalid) + rows = doc.get('rows', []) + scenes = {r['scene_id'] for r in rows} + if (doc.get('schema') != 'dvidia.piper-incremental-hold-study/v1' + or doc.get('attempted') != 32 or doc.get('planned_attempts') != 32 + or len(rows) != 32 or len(scenes) != 8 + or {(r['scene_id'], r['variant']) for r in rows} != {(s,v) for s in scenes for v in VARIANTS}): + raise ValueError('Expected complete eight-scene, four-controller study') + return doc + + +def render(doc): + esc = lambda value: html.escape(str(value), quote=True) + truth = lambda value: 'yes' if value is True else 'no' if value is False else 'unknown' + summary, details = [], [] + for variant in VARIANTS: + group = [r for r in doc['rows'] if r['variant'] == variant] + evaluations = [r['evaluation'] for r in group] + count = lambda field: sum(e['primary'].get(field) is True for e in evaluations) + summary.append(''+''.join(f'{esc(v)}' for v in ( + variant.replace('_',' '), f'{sum(e.get("first_admission") is not None for e in evaluations)}/8', + f'{count("native_pause_success")}/8', f'{count("success")}/8', + f'{count("combined_qualified_pause")}/8'))+'') + body = [] + for r in group: + e, p = r['evaluation'], r['evaluation']['primary'] + excursion = e.get('stationarity', {}).get('maximum_tcp_position_excursion_m') + cells = (r['scene_id'], truth(r['valid']), truth(p.get('native_pause_success')), + truth(p.get('success')), truth(p.get('combined_qualified_pause')), + f'{excursion*1000:.3f} mm' if type(excursion) in (int,float) else 'unavailable', + p.get('reason','unknown')) + body.append(''+''.join(f'{esc(v)}' for v in cells)+'') + details.append(f'
{esc(variant.replace("_"," "))} · all eight attempts
{"".join(body)}
SceneValidGripStationary gripActive stationary gripTCP excursionReason
') + return '''A grip that stays in place · DVIDIA research

A grip that stays in place.

A frozen comparison of four hold controllers on eight new simulated scenes. Holding the object and keeping the tool in place are measured together.

32 scheduled attempts. Every failure stays in the denominator. Authored controls using simulator state; no model training or hardware qualification.

Five-second outcomes

'''+''.join(summary)+'''
ControllerAdmittedNative gripNative stationary gripActive stationary grip

The primary endpoint requires continuous grip plus TCP motion within 4 mm and 0.05 rad. “Active stationary grip” additionally requires the controller to remain qualified. A post-refusal fallback can retain an object; that does not establish successful active regulation.

Every attempt

'''+''.join(details)+'''

What changed

The new incremental candidate accumulates the current pose-error step around the target issued before admission. It retains the jaw request, limits each correction update to 0.002 rad and refuses a proposed total above 0.03 rad. Candidate measurements outside the native joint margin are refused before solving. The three previous controllers remain unchanged.

These are nearby deterministic development cells, not independent random trials or a real-world reliability estimate. The study does not test transport, placement, videos, cameras or cross-arm transfer. Timing in the JSON is shared-host offline computation, not sensor latency.

''' + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('report', type=Path) + parser.add_argument('--output', type=Path) + args = parser.parse_args() + output = args.output or args.report.with_name('report.html') + with output.open('x') as stream: + stream.write(render(load(args.report))) diff --git a/benchmarks/piper-incremental-hold-v03/test_package_evidence.py b/benchmarks/piper-incremental-hold-v03/test_package_evidence.py new file mode 100644 index 0000000..2a2402f --- /dev/null +++ b/benchmarks/piper-incremental-hold-v03/test_package_evidence.py @@ -0,0 +1,177 @@ +from hashlib import sha256 +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +import zipfile + +import package_evidence as package + + +def write_json(path, value): + path.write_bytes(package.encoded(value)) + + +class PackageEvidenceTests(unittest.TestCase): + def setUp(self): + self.temporary = tempfile.TemporaryDirectory() + self.addCleanup(self.temporary.cleanup) + self.root = Path(self.temporary.name) + self.study = self.root / 'piper-incremental-hold-v03-run' + self.audit = self.root / 'piper-incremental-hold-v03-audit' + self.study.mkdir(); self.audit.mkdir() + rows = [{'id': f'{scene}-{variant}', 'scene_id': f'scene-{scene}', 'variant': variant} + for scene in range(8) for variant in sorted(package.VARIANTS)] + write_json(self.study / 'report.json', {'schema': package.SCHEMA, 'planned_attempts': 32, + 'attempted': 32, 'rows': rows, 'source_sha256': {}, + 'aggregates': {v: {k: 0 for k in ('admissions', 'false_initial_admissions', + 'native_5s_pause_success', 'grip_retention_and_stationarity', 'combined_qualified_5s_pause', 'final_both_qualified')} + for v in package.VARIANTS}}) + write_json(self.study / 'protocol.json', {'schema': 'synthetic-protocol', 'source_sha256': {}}) + write_json(self.study / 'inspection.json', { + 'schema': 'dvidia.piper-incremental-hold-inspection/v1', 'attempts': 32, + 'rows': [{'id': row['id'], 'evaluation_recomputed': True} for row in rows]}) + (self.study / 'attempt-0.bin').write_bytes(b'synthetic attempt evidence') + (self.audit / 'independent.txt').write_text('synthetic independent audit fixture') + study_inventory = package.tree(self.study) + write_json(self.audit / 'audit.json', { + 'schema': 'dvidia.incremental-hold-independent-audit/v1', 'passed': True, + 'source_and_raw_inputs_unchanged': True, + 'protocol_sha256': package.identity(self.study / 'protocol.json')['sha256'], + 'source_sha256': {}, 'frozen_raw_files': study_inventory, 'rows': rows}) + + def test_prepares_repeatable_complete_archive_with_verified_reconstruction(self): + outputs = [self.root / 'package-a', self.root / 'package-b'] + receipts = [package.prepare(self.study, self.audit, output, Path(__file__).resolve().parents[2]) + for output in outputs] + archive_a = outputs[0] / 'huggingface' / package.ARCHIVE_NAME + archive_b = outputs[1] / 'huggingface' / package.ARCHIVE_NAME + self.assertEqual(archive_a.read_bytes(), archive_b.read_bytes()) + self.assertEqual(receipts[0]['verified']['verified'], True) + self.assertEqual(receipts[0]['attempts'], 32) + self.assertEqual({path.name for path in (outputs[0] / 'huggingface').iterdir()}, package.HF_NAMES) + with zipfile.ZipFile(archive_a) as archive: + self.assertIn('study/report.json', archive.namelist()) + self.assertIn('audit/audit.json', archive.namelist()) + self.assertIn('tools/verify_archive.py', archive.namelist()) + self.assertEqual(json.loads(archive.read('inventory.json'))['schema'], + 'dvidia.piper-incremental-hold-inventory/v1') + verifier = {'__name__': 'retention_archive_verifier'} + exec(compile(package.VERIFY_SOURCE, 'verify_archive.py', 'exec'), verifier) + destination = self.root / 'reconstructed' + proof = verifier['verify'](archive_a, destination) + self.assertTrue(proof['verified']) + self.assertEqual((destination / 'study/report.json').read_bytes(), + (self.study / 'report.json').read_bytes()) + + def test_requires_passed_audit_and_same_eight_scene_roster(self): + audit = json.loads((self.audit / 'audit.json').read_bytes()) + audit['passed'] = False + write_json(self.audit / 'audit.json', audit) + with self.assertRaisesRegex(ValueError, 'audit'): + package.validate_inputs(self.study, self.audit) + audit['passed'] = True + write_json(self.audit / 'audit.json', audit) + report = json.loads((self.study / 'report.json').read_bytes()) + report['rows'][-1]['scene_id'] = 'different-scene' + write_json(self.study / 'report.json', report) + with self.assertRaisesRegex(ValueError, 'eight scenes'): + package.validate_inputs(self.study, self.audit) + + def test_refuses_duplicate_nonfinite_and_private_evidence(self): + (self.study / 'bad.json').write_bytes(b'{"x":1,"x":2}') + with self.assertRaises(ValueError): + package.tree(self.study) + (self.study / 'bad.json').unlink() + (self.study / 'bad.json').write_bytes(b'{"x":NaN}') + with self.assertRaises(ValueError): + package.tree(self.study) + (self.study / 'bad.json').unlink() + (self.study / 'attempt-0.bin').write_bytes(b'/'+b'Users/private/person/video.mp4') + with self.assertRaisesRegex(ValueError, 'Private path'): + package.tree(self.study) + + def test_verifier_rejects_modified_archive_and_refuses_overwrite(self): + receipt = package.prepare(self.study, self.audit, self.root / 'package', + Path(__file__).resolve().parents[2]) + archive_path = self.root / 'package/huggingface' / package.ARCHIVE_NAME + verifier = {'__name__': 'retention_archive_verifier'} + exec(compile(package.VERIFY_SOURCE, 'verify_archive.py', 'exec'), verifier) + output = self.root / 'reconstructed' + verifier['verify'](archive_path, output) + with self.assertRaisesRegex(ValueError, 'new'): + verifier['verify'](archive_path, output) + altered = self.root / 'altered.zip' + with zipfile.ZipFile(archive_path) as source, zipfile.ZipFile(altered, 'w') as dest: + for name in source.namelist(): + raw = source.read(name) + if name == 'study/report.json': + raw += b' ' + dest.writestr(name, raw) + with self.assertRaisesRegex(ValueError, 'expected bytes|digest'): + verifier['verify'](altered) + self.assertEqual(receipt['archive']['sha256'], sha256(archive_path.read_bytes()).hexdigest()) + + def test_prepare_rejects_transient_same_size_mutation_before_archive_indexing(self): + target = self.study/'attempt-0.bin' + original = target.read_bytes() + real_archive = package.archive + def mutate_then_archive(*args, **kwargs): + target.write_bytes(b'X'*len(original)) + try: + return real_archive(*args, **kwargs) + finally: + # Restoration would defeat a final-tree-only check. + target.write_bytes(original) + with patch.object(package, 'archive', side_effect=mutate_then_archive): + with self.assertRaisesRegex(ValueError, 'pinned evidence identity'): + package.prepare(self.study,self.audit,self.root/'mutated-index', + Path(__file__).resolve().parents[2]) + self.assertEqual(target.read_bytes(), original) + + def test_archive_stream_copy_rejects_mutation_after_indexing(self): + target = self.study/'attempt-0.bin' + original, real_zip = target.read_bytes(), zipfile.ZipFile + expected = {'study/attempt-0.bin': package.identity(target)} + def mutate_before_copy(*args, **kwargs): + target.write_bytes(b'Y'*len(original)) + return real_zip(*args, **kwargs) + try: + with patch.object(package.zipfile, 'ZipFile', side_effect=mutate_before_copy): + with self.assertRaisesRegex(ValueError, 'Copied archive input differs'): + package.archive(self.root/'mutated-copy.zip',{'study/attempt-0.bin':target},{}, + expected_identities=expected) + finally: + target.write_bytes(original) + + def test_parsed_metadata_is_bound_to_the_tree_snapshot(self): + real_load = package.load_json + def changed_read(path, *args, **kwargs): + value, raw = real_load(path, *args, **kwargs) + return (value, raw+b' ') if Path(path).name=='report.json' else (value,raw) + with patch.object(package, 'load_json', side_effect=changed_read): + with self.assertRaisesRegex(ValueError, 'Parsed metadata differs'): + package.validate_inputs(self.study,self.audit) + + def test_archive_identity_has_its_own_larger_byte_bound(self): + large = self.root/package.ARCHIVE_NAME + with large.open('wb') as stream: + stream.truncate(package.MAX_FILE+1) + with self.assertRaisesRegex(ValueError, 'oversized'): + package.identity(large) + self.assertEqual(package.identity(large,package.MAX_ARCHIVE),package.archive_digest(large)) + real_identity, archive_calls = package.identity, [] + def checked_identity(path, limit=package.MAX_FILE): + if Path(path).name==package.ARCHIVE_NAME: + self.assertEqual(limit,package.MAX_ARCHIVE) + archive_calls.append(path) + return real_identity(path,limit) + with patch.object(package, 'identity', side_effect=checked_identity): + package.prepare(self.study,self.audit,self.root/'archive-limit', + Path(__file__).resolve().parents[2]) + self.assertEqual(len(archive_calls),1) + + +if __name__ == '__main__': + unittest.main() diff --git a/publishing/huggingface/publish_piper_incremental_hold_v03.py b/publishing/huggingface/publish_piper_incremental_hold_v03.py new file mode 100644 index 0000000..3b53944 --- /dev/null +++ b/publishing/huggingface/publish_piper_incremental_hold_v03.py @@ -0,0 +1,184 @@ +"""Stage or publish the reviewed PiPER incremental hold v0.3 evidence dataset. + +This publisher is intentionally inert until its explicit ``publish`` command. +It freezes exactly six reviewed files and writes one commit only to an empty, +public dataset head. It does not modify any Space. +""" +from __future__ import annotations + +import argparse +from hashlib import sha256 +import json +from pathlib import Path +import re +import sys + +from publish import PublicationError, client +from publish_task_environments_v02 import digest, encoded, plain, public_client, download, scan + + +REPO = 'Dvidia/piper-incremental-hold-v03' +SCHEMA = 'dvidia.piper-incremental-hold-publication/v1' +ARCHIVE = 'piper-incremental-hold-v03.zip' +NAMES = {'README.md', 'LICENSE', ARCHIVE, 'report.json', 'protocol.json', 'inspection.json'} +PLAN_LIMIT = 4 * 1024 * 1024 +LIMITS = {name: (192 * 1024 * 1024 if name == ARCHIVE else + 16 * 1024 * 1024 if name == 'report.json' else 4 * 1024 * 1024) + for name in NAMES} +OID = re.compile(r'(?:[a-f0-9]{40}|[a-f0-9]{64})\Z') + + +def read(path, limit=None): + path = plain(path) + limit = LIMITS.get(path.name, PLAN_LIMIT) if limit is None else limit + if not path.is_file() or path.stat().st_size > limit: + raise PublicationError('Missing or oversized reviewed publication file.') + with path.open('rb') as stream: + raw = stream.read(limit + 1) + if len(raw) > limit: + raise PublicationError('Publication file changed beyond its byte limit.') + return raw + + +def files(root): + root = plain(root) + if not root.is_dir(): + raise PublicationError('Expected a plain publication directory.') + paths = sorted(root.iterdir()) + if {path.name for path in paths} != NAMES: + raise PublicationError('Dataset must contain exactly the reviewed six files.') + inventory = [] + for path in paths: + raw = read(path) + scan(raw) + inventory.append({'path': path.name, **digest(raw), + 'representation': 'lfs' if path.name == ARCHIVE else 'git'}) + return inventory + + +def stage(root, plan): + root, plan = plain(root), plain(plan) + value = {'schema': SCHEMA, 'repo_id': REPO, 'repo_type': 'dataset', + 'private': False, 'root': str(root), 'files': files(root)} + with plan.open('xb') as stream: + stream.write(encoded(value)) + return {'plan_sha256': sha256(read(plan, PLAN_LIMIT)).hexdigest(), + 'files': len(value['files']), 'bytes': sum(row['bytes'] for row in value['files'])} + + +def _pairs(items): + result = {} + for key, value in items: + if key in result: + raise PublicationError('Duplicate keys in reviewed publication plan.') + result[key] = value + return result + + +def _constant(_value): + raise PublicationError('Nonfinite values in reviewed publication plan.') + + +def checked(plan, expected_digest): + raw = read(plan, PLAN_LIMIT) + if sha256(raw).hexdigest() != expected_digest: + raise PublicationError('Reviewed plan digest changed.') + try: + value = json.loads(raw, object_pairs_hook=_pairs, parse_constant=_constant) + except (ValueError, TypeError, UnicodeError) as exc: + raise PublicationError('Malformed reviewed publication plan.') from exc + if (type(value) is not dict + or set(value) != {'schema', 'repo_id', 'repo_type', 'private', 'root', 'files'} + or value['schema'] != SCHEMA or value['repo_id'] != REPO + or value['repo_type'] != 'dataset' or value['private'] is not False + or type(value['root']) is not str or type(value['files']) is not list + or len(value['files']) != len(NAMES)): + raise PublicationError('Reviewed dataset identity changed.') + for row in value['files']: + if (type(row) is not dict or set(row) != {'path', 'bytes', 'sha256', 'representation'} + or type(row['path']) is not str or row['path'] not in NAMES + or type(row['bytes']) is not int or row['bytes'] < 0 + or type(row['sha256']) is not str or not re.fullmatch(r'[a-f0-9]{64}', row['sha256']) + or row['representation'] != ('lfs' if row['path'] == ARCHIVE else 'git')): + raise PublicationError('Reviewed dataset inventory is malformed.') + if ({row['path'] for row in value['files']} != NAMES + or files(value['root']) != value['files']): + raise PublicationError('Reviewed dataset identity or bytes changed.') + return value + + +def _remote(info): + if (getattr(info, 'private', None) is not False + or not OID.fullmatch(getattr(info, 'sha', '') or '') + or not getattr(info, 'siblings', None)): + raise PublicationError('Expected an initialized public dataset with an exact commit.') + + +def publish(plan, expected_digest, cache): + value = checked(plan, expected_digest) + payloads = [] + for row in value['files']: + raw = read(Path(value['root']) / row['path']) + if digest(raw) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Payload changed while freezing reviewed publication bytes.') + payloads.append((row['path'], raw)) + # No payload file is reopened after credential loading or remote access. + api, token = client(cache) + from huggingface_hub import CommitOperationAdd + from huggingface_hub.errors import RepositoryNotFoundError + try: + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + except RepositoryNotFoundError: + api.create_repo(REPO, repo_type='dataset', private=False, exist_ok=False) + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True) + _remote(info) + if len(info.siblings) != 1 or {item.rfilename for item in info.siblings} != {'.gitattributes'}: + raise PublicationError('Dataset is not empty. Verify the existing publication; do not overwrite it.') + operations = [CommitOperationAdd(path_in_repo=name, path_or_fileobj=raw) for name, raw in payloads] + commit = api.create_commit(REPO, repo_type='dataset', operations=operations, + parent_commit=info.sha, + commit_message='Publish PiPER incremental hold v0.3 simulator evidence') + del token, api + return {'repo_id': REPO, 'commit': commit.oid, 'files_added': len(operations)} + + +def verify(plan, expected_digest, cache): + value = checked(plan, expected_digest) + api = public_client() + info = api.repo_info(REPO, repo_type='dataset', files_metadata=True, token=False) + _remote(info) + if (len(info.siblings) != len(NAMES) + 1 + or {item.rfilename for item in info.siblings} != NAMES | {'.gitattributes'}): + raise PublicationError('Public dataset roster differs from the reviewed files.') + for row in value['files']: + raw = download(REPO, row['path'], 'dataset', info.sha, cache) + if digest(raw) != {key: row[key] for key in ('bytes', 'sha256')}: + raise PublicationError('Anonymous public bytes differ from the reviewed artifacts.') + return {'repo_id': REPO, 'commit': info.sha, 'anonymous_files_verified': len(value['files'])} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest='command', required=True) + command = commands.add_parser('stage') + command.add_argument('root'); command.add_argument('--plan', required=True) + for name in ('publish', 'verify'): + command = commands.add_parser(name) + command.add_argument('--plan', required=True) + command.add_argument('--sha256', required=True) + command.add_argument('--cache', required=True) + args = parser.parse_args() + try: + result = (stage(args.root, args.plan) if args.command == 'stage' else + globals()[args.command](args.plan, args.sha256, Path(args.cache))) + print(json.dumps(result, indent=2)) + return 0 + except PublicationError as exc: + print(str(exc), file=sys.stderr) + except Exception: + print('Publication operation failed; no credential or remote response details displayed.', file=sys.stderr) + return 2 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/publishing/huggingface/test_publish_piper_incremental_hold_v03.py b/publishing/huggingface/test_publish_piper_incremental_hold_v03.py new file mode 100644 index 0000000..803ecdc --- /dev/null +++ b/publishing/huggingface/test_publish_piper_incremental_hold_v03.py @@ -0,0 +1,143 @@ +from copy import deepcopy +from hashlib import sha256 +import json +from pathlib import Path +import sys +from types import SimpleNamespace +import tempfile +import unittest +from unittest.mock import Mock, patch + +import publish_piper_incremental_hold_v03 as publisher + + +class RetentionPublicationTests(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.dataset = self.root / 'dataset'; self.dataset.mkdir() + for name in publisher.NAMES: + (self.dataset / name).write_bytes(b'synthetic retention fixture') + self.plan = self.root / 'plan.json' + self.receipt = publisher.stage(self.dataset, self.plan) + class Operation: + def __init__(self, path_in_repo, path_or_fileobj): + self.path_in_repo, self.path_or_fileobj = path_in_repo, path_or_fileobj + class NotFound(Exception): + pass + sdk = SimpleNamespace(CommitOperationAdd=Operation) + errors = SimpleNamespace(RepositoryNotFoundError=NotFound) + patcher = patch.dict(sys.modules, {'huggingface_hub': sdk, + 'huggingface_hub.errors': errors}) + patcher.start(); self.addCleanup(patcher.stop) + + def info(self, names=('.gitattributes',), private=False, revision='a' * 40): + return SimpleNamespace(private=private, sha=revision, + siblings=[SimpleNamespace(rfilename=name) for name in names]) + + def test_staging_checking_and_public_readback_do_not_load_credentials(self): + api = Mock(); api.repo_info.return_value = self.info(sorted(publisher.NAMES | {'.gitattributes'})) + with patch.object(publisher, 'client', side_effect=AssertionError('Credential accessed')), \ + patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', side_effect=lambda repo, name, kind, revision, cache: + (self.dataset / name).read_bytes()) as download: + value = publisher.checked(self.plan, self.receipt['plan_sha256']) + proof = publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(value['repo_id'], 'Dvidia/piper-incremental-hold-v03') + self.assertEqual(proof['anonymous_files_verified'], 6) + self.assertEqual(download.call_count, 6) + api.repo_info.assert_called_once_with(publisher.REPO, repo_type='dataset', + files_metadata=True, token=False) + self.assertTrue(all(call.args[3] == 'a' * 40 for call in download.call_args_list)) + with self.assertRaises(FileExistsError): + publisher.stage(self.dataset, self.plan) + + def test_mutated_payload_and_unreviewed_roster_fail_before_credentials(self): + with patch.object(publisher, 'client') as client: + (self.dataset / 'README.md').write_bytes(b'changed') + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + (self.dataset / 'README.md').write_bytes(b'synthetic retention fixture') + (self.dataset / 'unreviewed.json').write_text('{}') + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + client.assert_not_called() + + def test_resealed_destination_duplicate_and_nonfinite_plans_are_rejected(self): + original = json.loads(self.plan.read_bytes()) + for field, replacement in [('repo_id', 'Dvidia/robot-simulation-lab'), + ('repo_type', 'space'), ('private', True)]: + value = deepcopy(original); value[field] = replacement + raw = publisher.encoded(value); self.plan.write_bytes(raw) + with self.subTest(field=field), self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, sha256(raw).hexdigest()) + for raw in (b'{"schema":1,"schema":2}', b'{"private":NaN}'): + self.plan.write_bytes(raw) + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, sha256(raw).hexdigest()) + value = deepcopy(original); value['files'][1] = value['files'][0] + raw = publisher.encoded(value); self.plan.write_bytes(raw) + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, sha256(raw).hexdigest()) + + def test_existing_private_or_unpinned_dataset_cannot_be_overwritten(self): + for info in (self.info(['README.md']), self.info(private=True), self.info(revision='main')): + api = Mock(); api.repo_info.return_value = info + with self.subTest(info=info), patch.object(publisher, 'client', return_value=(api, 'fixture')): + with self.assertRaises(publisher.PublicationError): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + api.create_commit.assert_not_called() + api.create_repo.assert_not_called() + + def test_payload_freeze_and_upload_bytes_are_immutable_across_credential_boundary(self): + original = publisher.read + reads = 0 + def changed(path, *args, **kwargs): + nonlocal reads + if Path(path).name == 'README.md': + reads += 1 + if reads == 2: + return b'changed after review' + return original(path, *args, **kwargs) + with patch.object(publisher, 'read', side_effect=changed), patch.object(publisher, 'client') as client: + with self.assertRaisesRegex(publisher.PublicationError, 'freezing'): + publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + client.assert_not_called() + api = Mock(); api.repo_info.return_value = self.info() + api.create_commit.return_value = SimpleNamespace(oid='b' * 40) + def credential_boundary(_cache): + (self.dataset / 'README.md').write_bytes(b'late mutation not uploaded') + return api, 'fixture' + with patch.object(publisher, 'client', side_effect=credential_boundary): + result = publisher.publish(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + self.assertEqual(result['files_added'], 6) + commit = api.create_commit.call_args + self.assertEqual(commit.kwargs['parent_commit'], 'a' * 40) + uploaded = {operation.path_in_repo: operation.path_or_fileobj + for operation in commit.kwargs['operations']} + self.assertEqual(uploaded['README.md'], b'synthetic retention fixture') + + def test_oversized_file_and_changed_anonymous_bytes_are_rejected(self): + path = self.dataset / publisher.ARCHIVE + with path.open('wb') as stream: + stream.truncate(publisher.LIMITS[publisher.ARCHIVE] + 1) + with self.assertRaisesRegex(publisher.PublicationError, 'oversized'): + publisher.read(path) + (self.dataset / publisher.ARCHIVE).write_bytes(b'synthetic retention fixture') + self.plan.unlink(); self.receipt = publisher.stage(self.dataset, self.plan) + api = Mock(); api.repo_info.return_value = self.info(sorted(publisher.NAMES | {'.gitattributes'})) + with patch.object(publisher, 'client', side_effect=AssertionError('Credential accessed')), \ + patch.object(publisher, 'public_client', return_value=api), \ + patch.object(publisher, 'download', return_value=b'changed bytes'): + with self.assertRaisesRegex(publisher.PublicationError, 'Anonymous public bytes'): + publisher.verify(self.plan, self.receipt['plan_sha256'], self.root / 'cache') + target = self.root / 'unreviewed'; target.write_text('fixture') + (self.dataset / 'README.md').unlink() + (self.dataset / 'README.md').symlink_to(target) + with self.assertRaises(publisher.PublicationError): + publisher.checked(self.plan, self.receipt['plan_sha256']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/test_incremental_independent_audit.py b/tests/test_incremental_independent_audit.py new file mode 100644 index 0000000..926d370 --- /dev/null +++ b/tests/test_incremental_independent_audit.py @@ -0,0 +1,307 @@ +"""Independent fixed32 record-law tests; no simulator or scorer imports.""" +import sys +from pathlib import Path +import unittest + +import numpy as np + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] + / "benchmarks" / "piper-incremental-hold-v03")) +import independent_audit as audit + + +def correction_fixture(): + n = 1040 + joints = np.full((n, 6), .1, dtype=np.float64) + joints[1019:, 0] = .101 + arrays = {"t_seconds": np.arange(1, n + 1, dtype=np.float64) * .001, + "joint_position_rad": joints} + issued, jaw = [.1] * 6, .012 + actions = [{"t_seconds": 1., "joint_targets_rad": issued.copy(), "gripper_position_m": jaw}, + {"t_seconds": 1.02, "joint_targets_rad": [.099, .1, .1, .1, .1, .1], + "gripper_position_m": jaw}] + reference = {"t_seconds": 1., "position_m": [.3, 0., .1], "quaternion_wxyz": [1., 0., 0., 0.], + "issued_joint_targets_rad": issued.copy(), "measured_joint_position_rad": issued.copy(), + "frozen_jaw_target_m": jaw, "raw_pose_joint_targets_rad": issued.copy(), + "fixed_bias_rad": [0.] * 6} + + def record(time, measured, target, before, proposal, accumulated, *, raw, increment, + accepted, final=False, applied=None): + return {"t_seconds": time, "final_observation": final, "command_issued": not final, + "status": "admitted_hold", "admitted": True, "variant": "incremental", + "candidate": "incremental", "update_law": "fixed increment law", + "update_gain_per_boundary": 1., "measured_joint_position_rad": measured, + "joint_targets_rad": None if final else target, "frozen_jaw_target_m": jaw, + "raw_pose_joint_targets_rad": raw, "fixed_bias_rad": None, + "pose_error_increment_rad": increment, "correction_before_update_rad": before, + "proposed_cumulative_correction_rad": proposal, + "accumulated_correction_rad": accumulated, + "applied_cumulative_correction_rad": applied, + "correction_rad": accumulated, "correction_update_accepted": accepted, + "correction_rate_limited_axes": [], "refused_bounds": None, + "active_regulation": True, + **({"preceding_issued_joint_targets_rad": actions[-1]["joint_targets_rad"]} if final else {})} + + records = [ + record(1., issued.copy(), issued.copy(), [0.] * 6, [0.] * 6, [0.] * 6, + raw=issued.copy(), increment=None, accepted=False, applied=[0.] * 6), + record(1.02, [.101, .1, .1, .1, .1, .1], actions[1]["joint_targets_rad"], + [0.] * 6, [-.001, 0., 0., 0., 0., 0.], [-.001, 0., 0., 0., 0., 0.], + raw=issued.copy(), increment=[-.001, 0., 0., 0., 0., 0.], accepted=True, + applied=[-.001, 0., 0., 0., 0., 0.]), + record(1.04, [.101, .1, .1, .1, .1, .1], None, + [-.001, 0., 0., 0., 0., 0.], None, [-.001, 0., 0., 0., 0., 0.], + raw=None, increment=None, accepted=False, final=True), + ] + policy = {"variant": "incremental", "candidate": "incremental", "admitted": True, + "frozen_jaw_target_m": jaw, + "events": [{"type": "evidence_qualified", "t_seconds": 1., "reason": "sampled"}], + "retention": {"hold_reference": reference, "hold_records": records, "input_refusals": [], + "native_joint_control_limits_rad": [[-2., 2.]] * 6, + "update_law": "fixed increment law", "update_gain_per_boundary": 1.}} + return policy, actions, arrays + + +class NativePredicateTests(unittest.TestCase): + def test_100_sample_streak_and_strict_force_floor(self): + n = 240 + arrays = {name: np.zeros((n,) if width == 1 else (n, width), dtype=np.float64) + for name, width in audit.FIELDS.items()} + arrays["t_seconds"] = np.arange(1, n + 1, dtype=np.float64) * .001 + arrays["object_position_m"][:] = [.3, 0., .08] + arrays["object_minus_tcp_local_m"][:] = [0., 0., -.1] + arrays["object_position_m"][20:, 2] += .006 + arrays["grasp_contact_counts"][:] = 1. + arrays["pad_normal_forces_n"][:] = .02 + probe = {"start_time_s": .02, "object_origin_m": [.3, 0., .08], + "local_offset_origin_m": [0., 0., -.1]} + _, streak, qualified, _, _, _ = audit.native_predicate(arrays, probe) + self.assertEqual((streak[118], bool(qualified[118])), (99, False)) + self.assertEqual((streak[119], bool(qualified[119])), (100, True)) + arrays["pad_normal_forces_n"][60, 0] = .01 + _, streak, qualified, _, _, _ = audit.native_predicate(arrays, probe) + self.assertEqual(streak[60], 0) + self.assertFalse(qualified[119]) + self.assertTrue(qualified[160]) + + def test_stationarity_is_pose_based_and_quaternion_sign_invariant(self): + arrays = {"tcp_position_m": np.zeros((3, 3), dtype=np.float64), + "tcp_quaternion_wxyz": np.tile([1., 0., 0., 0.], (3, 1))} + arrays["tcp_quaternion_wxyz"][1:] *= -1. + self.assertTrue(audit.stationarity(arrays, 0, 2, True, True)["grip_retention_and_stationarity"]) + arrays["tcp_position_m"][1, 0] = .004001 + self.assertFalse(audit.stationarity(arrays, 0, 2, True, True)["tcp_pose_within_bounds"]) + + def test_native_endpoint_completion_is_builtin_bool(self): + n = 5200 + arrays = {name: np.zeros((n,) if width == 1 else (n, width), dtype=np.float64) + for name, width in audit.FIELDS.items()} + arrays["t_seconds"] = np.arange(1, n + 1, dtype=np.float64) * .001 + arrays["object_position_m"][:] = [.3, 0., .08] + arrays["object_position_m"][20:, 2] += .006 + arrays["tcp_position_m"][:] = [.3, 0., .08] + arrays["object_minus_tcp_local_m"][:] = 0. + arrays["tcp_quaternion_wxyz"][:] = [1., 0., 0., 0.] + arrays["grasp_contact_counts"][:] = 1. + arrays["pad_normal_forces_n"][:] = .02 + policy = {"probes": [{"probe_index": 0, "start_time_s": .02, + "object_origin_m": [.3, 0., .08], + "local_offset_origin_m": [0., 0., 0.]}]} + result = audit.native_success(arrays, policy, {"probe_index": 0, "t_seconds": .1}) + self.assertIs(type(result["complete_5s"]), bool) + + +class PrefixTests(unittest.TestCase): + def test_prefix_contains_cutoff_sample_and_excludes_that_command(self): + cutoff = {"time_s": .04, "reason": "admission", "treatment_boundaries": []} + arrays = {"t_seconds": np.arange(1, 20001, dtype=np.float64) * .001, + "signal": np.arange(20000, dtype=np.float64)} + actions = [{"t_seconds": i * .02, "command": i} for i in range(1000)] + result = audit.prefix_identity(arrays, actions, cutoff) + self.assertEqual(result["native_rows"], 40) + self.assertEqual(result["commands"], 2) + + def test_invalid_member_keeps_complete_quartet_unknown(self): + attempts = {variant: {"valid": True} for variant in audit.VARIANTS} + attempts["incremental"] = {"valid": False, "reason": "partial", "native_rows_available": 12} + result = audit.paired_prefix("scene-1", attempts) + self.assertFalse(result["verified"]) + self.assertEqual(result["reason"], "invalid_attempt_prevents_complete_quartet") + self.assertEqual(result["variants"][3]["native_rows_available"], 12) + + +class IncrementalRecordAuditTests(unittest.TestCase): + def test_valid_record_reconstructs_increment_and_final_no_command(self): + policy, actions, arrays = correction_fixture() + result = audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + self.assertTrue(result["first_admitted_target_continuous"]) + self.assertTrue(result["final_observation_no_ik_or_command"]) + self.assertAlmostEqual(result["maximum_abs_pose_error_increment_rad"], .001) + self.assertAlmostEqual(result["maximum_abs_active_correction_rad"], .001) + + def test_rejects_wrong_increment_first_target_and_hypothetical_final_action(self): + policy, actions, arrays = correction_fixture() + policy["retention"]["hold_records"][1]["pose_error_increment_rad"][0] = .001 + with self.assertRaises(AssertionError): + audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + + def test_total_bound_refusal_preserves_prior_correction_and_issues_measured_hold(self): + policy, actions, arrays = correction_fixture() + refusal = policy["retention"]["hold_records"][1] + refusal.update( + measured_joint_position_rad=[.101, .1, .1, .1, .1, .1], + raw_pose_joint_targets_rad=[.132, .1, .1, .1, .1, .1], + pose_error_increment_rad=[.031, 0., 0., 0., 0., 0.], + proposed_cumulative_correction_rad=[.031, 0., 0., 0., 0., 0.], + correction_before_update_rad=[0.] * 6, accumulated_correction_rad=[0.] * 6, + applied_cumulative_correction_rad=None, correction_rad=None, + correction_update_accepted=False, refused_bounds="joint_correction_exhausted", + active_regulation=False, joint_targets_rad=[.101, .1, .1, .1, .1, .1]) + policy["events"].append({"type": "hold_invalidated", "reason": "joint_correction_exhausted", + "t_seconds": 1.02}) + policy["admitted"] = False + actions[1]["joint_targets_rad"] = refusal["joint_targets_rad"] + final = policy["retention"]["hold_records"][-1] + final.update(active_regulation=False, correction_before_update_rad=[0.] * 6, + accumulated_correction_rad=[0.] * 6, correction_rad=None, + preceding_issued_joint_targets_rad=actions[-1]["joint_targets_rad"]) + result = audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + self.assertEqual(result["bound_refusal_counts"], {"joint_correction_exhausted": 1}) + + def test_measured_joint_margin_refusal_must_precede_pose_solve(self): + policy, actions, arrays = correction_fixture() + records = policy["retention"]["hold_records"] + row = records[1] + row.update(measured_joint_position_rad=[2., .1, .1, .1, .1, .1], + raw_pose_joint_targets_rad=None, pose_error_increment_rad=None, + proposed_cumulative_correction_rad=None, applied_cumulative_correction_rad=None, + correction_update_accepted=False, refused_bounds="measured_joint_margin_exhausted", + active_regulation=False, joint_targets_rad=[2., .1, .1, .1, .1, .1], + accumulated_correction_rad=[0.] * 6) + arrays["joint_position_rad"][1019:, 0] = 2. + arrays["joint_position_rad"][-1, 0] = .1 + policy["events"].append({"type": "hold_invalidated", "reason": "measured_joint_margin_exhausted", + "t_seconds": 1.02}) + policy["admitted"] = False + actions[1]["joint_targets_rad"] = row["joint_targets_rad"] + final = records[-1] + final.update(measured_joint_position_rad=[.1] * 6, + active_regulation=False, correction_before_update_rad=[0.] * 6, + accumulated_correction_rad=[0.] * 6, correction_rad=None, + preceding_issued_joint_targets_rad=actions[-1]["joint_targets_rad"]) + result = audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + self.assertEqual(result["bound_refusal_counts"], {"measured_joint_margin_exhausted": 1}) + + def test_terminal_fallback_latches_once_and_holds_through_measured_drift(self): + policy, actions, arrays = correction_fixture() + refusal = policy["retention"]["hold_records"][1] + refusal.update( + measured_joint_position_rad=[.101, .1, .1, .1, .1, .1], + raw_pose_joint_targets_rad=[.132, .1, .1, .1, .1, .1], + pose_error_increment_rad=[.031, 0., 0., 0., 0., 0.], + proposed_cumulative_correction_rad=[.031, 0., 0., 0., 0., 0.], + correction_before_update_rad=[0.] * 6, accumulated_correction_rad=[0.] * 6, + applied_cumulative_correction_rad=None, correction_rad=None, + correction_update_accepted=False, refused_bounds="joint_correction_exhausted", + active_regulation=False, joint_targets_rad=[.101, .1, .1, .1, .1, .1]) + policy["events"].append({"type": "hold_invalidated", "reason": "joint_correction_exhausted", + "t_seconds": 1.02}) + policy["admitted"] = False + actions[1]["joint_targets_rad"] = refusal["joint_targets_rad"] + arrays["joint_position_rad"] = np.concatenate( + [arrays["joint_position_rad"], np.tile([.102, .1, .1, .1, .1, .1], (20, 1)), + np.tile([.1] * 6, (20, 1))], axis=0) + arrays["t_seconds"] = np.arange(1, len(arrays["joint_position_rad"]) + 1) * .001 + arrays["joint_position_rad"][audit.row_at(1.04, arrays["t_seconds"])] = [.102, .1, .1, .1, .1, .1] + arrays["joint_position_rad"][audit.row_at(1.06, arrays["t_seconds"])] = [.1] * 6 + terminal = dict(refusal) + terminal.update(t_seconds=1.04, final_observation=False, command_issued=True, + measured_joint_position_rad=[.102, .1, .1, .1, .1, .1], + raw_pose_joint_targets_rad=None, pose_error_increment_rad=None, + proposed_cumulative_correction_rad=None, refused_bounds=None, + correction_update_accepted=False, joint_targets_rad=refusal["joint_targets_rad"], + preceding_issued_joint_targets_rad=None) + final = policy["retention"]["hold_records"][-1] + final.update(t_seconds=1.06, measured_joint_position_rad=[.1] * 6, + active_regulation=False, correction_before_update_rad=[0.] * 6, + accumulated_correction_rad=[0.] * 6, correction_rad=None, + preceding_issued_joint_targets_rad=refusal["joint_targets_rad"]) + policy["retention"]["hold_records"].insert(-1, terminal) + actions.append({"t_seconds": 1.04, "joint_targets_rad": refusal["joint_targets_rad"], + "gripper_position_m": policy["frozen_jaw_target_m"]}) + policy["retention"]["hold_records"][-1]["preceding_issued_joint_targets_rad"] = actions[-1]["joint_targets_rad"] + audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + + +class LegacyHoldAuditTests(unittest.TestCase): + def test_admission_target_matches_each_legacy_variant_contract(self): + issued, measured = [.1] * 6, [.11] * 6 + for variant, expected in (("measured_joint", measured), ("issued_joint", issued), + ("cartesian", issued)): + with self.subTest(variant=variant): + policy = {"variant": variant, "frozen_jaw_target_m": .01, + "retention": {"hold_reference": { + "issued_joint_targets_rad": issued, + "measured_joint_position_rad": measured, + "frozen_jaw_target_m": .01}, + "hold_records": [ + {"t_seconds": 1., "command_issued": True, + "joint_targets_rad": expected, "frozen_jaw_target_m": .01}, + {"t_seconds": 1.02, "command_issued": False, + "final_observation": True, "raw_pose_joint_targets_rad": None, + "joint_targets_rad": expected}]}} + actions = [{"t_seconds": 1., "joint_targets_rad": expected, + "gripper_position_m": .01}] + boundary = {"time_s": 1., "reason": "admission"} + audit.baseline_hold_audit(policy, actions, {}, boundary, variant) + + def test_no_admission_has_no_fabricated_final_observation(self): + result = audit.correction_audit( + {"variant": "incremental", "candidate": "incremental", "admitted": False, + "retention": {"hold_records": [], "hold_reference": None, + "input_refusals": [], "native_joint_control_limits_rad": [[-2., 2.]] * 6, + "update_law": "fixed increment law", "update_gain_per_boundary": 1.}}, + [], {"t_seconds": np.array([.001])}, + {"time_s": .001, "reason": "pre_admission_bound_refusal", + "detail": "measured_joint_margin_exhausted"}, "incremental") + self.assertFalse(result["final_observation_no_ik_or_command"]) + + +class SlewAuditTests(unittest.TestCase): + def test_requested_to_applied_slew_reconstruction_rejects_tampered_native_control(self): + n = 20000 + arrays = {"arm_actuator_controls_rad": np.zeros((n, 6), dtype=np.float64), + "applied_jaw_target_m": np.zeros(n, dtype=np.float64), + "requested_jaw_target_m": np.zeros(n, dtype=np.float64), + "object_position_m": np.zeros((n, 3), dtype=np.float64), + "joint_position_rad": np.zeros((n, 6), dtype=np.float64), + "tcp_position_m": np.zeros((n, 3), dtype=np.float64)} + action = {"t_seconds": 0., "joint_targets_rad": [0.] * 6, "gripper_position_m": 0.} + actions = [dict(action, t_seconds=i * .02) for i in range(1000)] + requested = [dict(action, t_seconds=i * .02, index=i) for i in range(1000)] + trajectory = [] + for index in range(1001): + trajectory.append({"command": actions[index] if index < 1000 else None, + "issued_joint_targets_rad": actions[index]["joint_targets_rad"] if index < 1000 else None, + "issued_gripper_position_m": 0. if index < 1000 else None, + "object_position_m": [0., 0., 0.], "joint_position_rad": [0.] * 6, + "tcp_position_m": [0., 0., 0.]}) + case = {"config": {"joint_target_rate_rad_s": .12, "jaw_target_rate_m_s": .02}} + actual = audit.reconstruct_controls(arrays, actions, requested, trajectory, [0.] * 7, case) + self.assertEqual(actual["native_substeps_reconstructed"], 20000) + arrays["arm_actuator_controls_rad"][50, 0] = 1e-6 + with self.assertRaises(AssertionError): + audit.reconstruct_controls(arrays, actions, requested, trajectory, [0.] * 7, case) + policy, actions, arrays = correction_fixture() + policy["retention"]["hold_records"][0]["joint_targets_rad"][0] += .001 + actions[0]["joint_targets_rad"][0] += .001 + with self.assertRaises(AssertionError): + audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + policy, actions, arrays = correction_fixture() + policy["retention"]["hold_records"][-1]["command_issued"] = True + with self.assertRaises(AssertionError): + audit.correction_audit(policy, actions, arrays, audit.treatment_boundary(policy), "incremental") + + +if __name__ == "__main__": + unittest.main()