From 175852f5bde568d8131cc96cfaf89f5e582f12ed Mon Sep 17 00:00:00 2001 From: mosheabr <257371078+mosheabr@users.noreply.github.com> Date: Mon, 17 Aug 2026 02:30:09 +0000 Subject: [PATCH] chore: sync skills (Jetson Device,NeMo MBridge) --- README.md | 2 +- skills/jetson-video-benchmark/BENCHMARK.md | 98 + skills/jetson-video-benchmark/SKILL.md | 259 ++ .../jetson-video-benchmark/evals/evals.json | 89 + .../references/benchmark-output-contract.md | 155 + .../references/benchmark-workflow.md | 155 + .../documented-performance-estimates.md | 249 ++ .../scripts/benchmark_controller.py | 2573 ++++++++++++++++ .../scripts/benchmark_provenance.py | 1514 ++++++++++ .../scripts/benchmark_runtime.py | 854 ++++++ skills/jetson-video-benchmark/skill-card.md | 83 + skills/jetson-video-benchmark/skill.oms.sig | 1 + skills/jetson-video-capability/BENCHMARK.md | 98 + skills/jetson-video-capability/SKILL.md | 317 ++ .../jetson-video-capability/evals/evals.json | 53 + .../references/capability-queries.md | 583 ++++ .../references/surface-selection-contract.md | 127 + .../scripts/_capability_contract.py | 851 ++++++ .../scripts/query_decoder_caps.py | 418 +++ .../scripts/query_encoder_caps.py | 363 +++ .../scripts/query_native_sample_reports.py | 1045 +++++++ .../scripts/validate_appenc_av1_ivf.py | 356 +++ skills/jetson-video-capability/skill-card.md | 83 + skills/jetson-video-capability/skill.oms.sig | 1 + skills/jetson-video-pipeline/BENCHMARK.md | 98 + skills/jetson-video-pipeline/SKILL.md | 243 ++ skills/jetson-video-pipeline/evals/evals.json | 29 + .../references/official-sample-contract.md | 61 + .../references/pipeline-workflow.md | 224 ++ .../scripts/_pipeline_provenance.py | 1560 ++++++++++ .../scripts/_pipeline_runtime.py | 1561 ++++++++++ .../scripts/encode_controller.py | 1153 +++++++ .../scripts/pipeline_controller.py | 2685 +++++++++++++++++ ...validate_representative_content_summary.py | 597 ++++ skills/jetson-video-pipeline/skill-card.md | 83 + skills/jetson-video-pipeline/skill.oms.sig | 1 + skills/jetson-video-recipe/BENCHMARK.md | 98 + skills/jetson-video-recipe/SKILL.md | 207 ++ skills/jetson-video-recipe/evals/evals.json | 41 + .../recipes-knobs-and-constraints.md | 160 + .../references/recipes-workflow.md | 210 ++ .../recipes/data/encoder-intent-catalog.json | 52 + .../scripts/recipes/recipe_model.py | 1620 ++++++++++ skills/jetson-video-recipe/skill-card.md | 82 + skills/jetson-video-recipe/skill.oms.sig | 1 + skills/jetson-video-setup/BENCHMARK.md | 99 + skills/jetson-video-setup/SKILL.md | 181 ++ skills/jetson-video-setup/evals/evals.json | 53 + .../references/setup-install.md | 244 ++ .../references/setup-output-contract.md | 334 ++ .../references/setup-workflow.md | 244 ++ .../references/video-content.md | 83 + .../scripts/setup/lock_pip_reports.py | 1000 ++++++ .../scripts/setup/plan_install.py | 1080 +++++++ .../scripts/setup/probe_nvcodec.py | 998 ++++++ .../scripts/setup/setup_contract.py | 1003 ++++++ .../scripts/setup/verify_native.py | 785 +++++ .../scripts/setup/verify_pynvc_sample.py | 1027 +++++++ skills/jetson-video-setup/skill-card.md | 84 + skills/jetson-video-setup/skill.oms.sig | 1 + .../BENCHMARK.md | 127 +- .../SKILL.md | 377 ++- .../card.yaml | 341 ++- .../evals/evals.json | 43 +- .../skill-card.md | 68 +- .../skill.oms.sig | 2 +- 66 files changed, 28929 insertions(+), 338 deletions(-) create mode 100644 skills/jetson-video-benchmark/BENCHMARK.md create mode 100644 skills/jetson-video-benchmark/SKILL.md create mode 100644 skills/jetson-video-benchmark/evals/evals.json create mode 100644 skills/jetson-video-benchmark/references/benchmark-output-contract.md create mode 100644 skills/jetson-video-benchmark/references/benchmark-workflow.md create mode 100644 skills/jetson-video-benchmark/references/documented-performance-estimates.md create mode 100644 skills/jetson-video-benchmark/scripts/benchmark_controller.py create mode 100644 skills/jetson-video-benchmark/scripts/benchmark_provenance.py create mode 100644 skills/jetson-video-benchmark/scripts/benchmark_runtime.py create mode 100644 skills/jetson-video-benchmark/skill-card.md create mode 100644 skills/jetson-video-benchmark/skill.oms.sig create mode 100644 skills/jetson-video-capability/BENCHMARK.md create mode 100644 skills/jetson-video-capability/SKILL.md create mode 100644 skills/jetson-video-capability/evals/evals.json create mode 100644 skills/jetson-video-capability/references/capability-queries.md create mode 100644 skills/jetson-video-capability/references/surface-selection-contract.md create mode 100644 skills/jetson-video-capability/scripts/_capability_contract.py create mode 100644 skills/jetson-video-capability/scripts/query_decoder_caps.py create mode 100644 skills/jetson-video-capability/scripts/query_encoder_caps.py create mode 100644 skills/jetson-video-capability/scripts/query_native_sample_reports.py create mode 100644 skills/jetson-video-capability/scripts/validate_appenc_av1_ivf.py create mode 100644 skills/jetson-video-capability/skill-card.md create mode 100644 skills/jetson-video-capability/skill.oms.sig create mode 100644 skills/jetson-video-pipeline/BENCHMARK.md create mode 100644 skills/jetson-video-pipeline/SKILL.md create mode 100644 skills/jetson-video-pipeline/evals/evals.json create mode 100644 skills/jetson-video-pipeline/references/official-sample-contract.md create mode 100644 skills/jetson-video-pipeline/references/pipeline-workflow.md create mode 100644 skills/jetson-video-pipeline/scripts/_pipeline_provenance.py create mode 100644 skills/jetson-video-pipeline/scripts/_pipeline_runtime.py create mode 100644 skills/jetson-video-pipeline/scripts/encode_controller.py create mode 100644 skills/jetson-video-pipeline/scripts/pipeline_controller.py create mode 100644 skills/jetson-video-pipeline/scripts/validate_representative_content_summary.py create mode 100644 skills/jetson-video-pipeline/skill-card.md create mode 100644 skills/jetson-video-pipeline/skill.oms.sig create mode 100644 skills/jetson-video-recipe/BENCHMARK.md create mode 100644 skills/jetson-video-recipe/SKILL.md create mode 100644 skills/jetson-video-recipe/evals/evals.json create mode 100644 skills/jetson-video-recipe/references/recipes-knobs-and-constraints.md create mode 100644 skills/jetson-video-recipe/references/recipes-workflow.md create mode 100644 skills/jetson-video-recipe/scripts/recipes/data/encoder-intent-catalog.json create mode 100644 skills/jetson-video-recipe/scripts/recipes/recipe_model.py create mode 100644 skills/jetson-video-recipe/skill-card.md create mode 100644 skills/jetson-video-recipe/skill.oms.sig create mode 100644 skills/jetson-video-setup/BENCHMARK.md create mode 100644 skills/jetson-video-setup/SKILL.md create mode 100644 skills/jetson-video-setup/evals/evals.json create mode 100644 skills/jetson-video-setup/references/setup-install.md create mode 100644 skills/jetson-video-setup/references/setup-output-contract.md create mode 100644 skills/jetson-video-setup/references/setup-workflow.md create mode 100644 skills/jetson-video-setup/references/video-content.md create mode 100644 skills/jetson-video-setup/scripts/setup/lock_pip_reports.py create mode 100644 skills/jetson-video-setup/scripts/setup/plan_install.py create mode 100644 skills/jetson-video-setup/scripts/setup/probe_nvcodec.py create mode 100644 skills/jetson-video-setup/scripts/setup/setup_contract.py create mode 100644 skills/jetson-video-setup/scripts/setup/verify_native.py create mode 100644 skills/jetson-video-setup/scripts/setup/verify_pynvc_sample.py create mode 100644 skills/jetson-video-setup/skill-card.md create mode 100644 skills/jetson-video-setup/skill.oms.sig diff --git a/README.md b/README.md index 68fd0e1e..3a54e859 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ For non-interactive installs, global installs, agent-specific installs, updates, | **Holoscan Sensor Bridge** | Agent-ready skills for Holoscan Sensor Bridge devkit workflows, including demo environment bring-up, FPGA flashing for Lattice and VB1940 hardware, example application execution, QA test-plan automation, and support for configuring and using the Holoscan Sensor Bridge FPGA intellectual property (IP) core. | [`hsb-setup`](skills/hsb-setup), [`hsb-flash`](skills/hsb-flash), [`hsb-app`](skills/hsb-app), [`hsb-test`](skills/hsb-test), [`hsb-ip-def`](skills/hsb-ip-def), [`hsb-ip-packetizer`](skills/hsb-ip-packetizer), [`hsb-ip-create-top`](skills/hsb-ip-create-top) | | **Isaac for Healthcare Workflows** | Agent-ready skills for Isaac for Healthcare agentic and catheter-navigation workflows, covering task authoring, data pipelines, policy training and validation, CT-derived digital twins, DRR rendering, and interactive catheter simulation. | [`i4h-workflow`](skills/i4h-workflow), [`i4h-workflow-setup`](skills/i4h-workflow-setup), [`i4h-workflow-create`](skills/i4h-workflow-create), [`i4h-workflow-scene-edit`](skills/i4h-workflow-scene-edit), [`i4h-workflow-dataset-teleop`](skills/i4h-workflow-dataset-teleop), [`i4h-workflow-dataset-replay`](skills/i4h-workflow-dataset-replay), [`i4h-workflow-dataset-mimic`](skills/i4h-workflow-dataset-mimic), [`i4h-workflow-dataset-annotate`](skills/i4h-workflow-dataset-annotate), [`i4h-workflow-dataset-convert`](skills/i4h-workflow-dataset-convert), [`i4h-workflow-finetune`](skills/i4h-workflow-finetune), [`i4h-workflow-validate`](skills/i4h-workflow-validate), [`i4h-workflow-e2e`](skills/i4h-workflow-e2e), [`i4h-lerobot-viz`](skills/i4h-lerobot-viz), [`i4h-catheter-navigation`](skills/i4h-catheter-navigation), [`i4h-catheter-navigation-setup`](skills/i4h-catheter-navigation-setup), [`i4h-catheter-navigation-digital-twin`](skills/i4h-catheter-navigation-digital-twin), [`i4h-catheter-navigation-render-drr`](skills/i4h-catheter-navigation-render-drr), [`i4h-catheter-navigation-viewport`](skills/i4h-catheter-navigation-viewport), [`i4h-catheter-navigation-smoke`](skills/i4h-catheter-navigation-smoke), [`i4h-catheter-navigation-e2e`](skills/i4h-catheter-navigation-e2e) | | **Jetson BSP** | Agentic skills for setting up and customizing an NVIDIA Jetson Linux Board Support Package (BSP) — pick a target, prepare image and sources, customize IO (camera, PCIe, USB, pinmux, clocks, and more), then promote, flash, and validate. | [`jetson-build-source`](skills/jetson-build-source), [`jetson-customize-camera`](skills/jetson-customize-camera), [`jetson-customize-clocks`](skills/jetson-customize-clocks), [`jetson-customize-fan`](skills/jetson-customize-fan), [`jetson-customize-mgbe`](skills/jetson-customize-mgbe), [`jetson-customize-nvpmodel`](skills/jetson-customize-nvpmodel), [`jetson-customize-pcie`](skills/jetson-customize-pcie), [`jetson-customize-pinmux`](skills/jetson-customize-pinmux), [`jetson-customize-uphy`](skills/jetson-customize-uphy), [`jetson-customize-usb`](skills/jetson-customize-usb), [`jetson-derive-carrier`](skills/jetson-derive-carrier), [`jetson-download-bsp`](skills/jetson-download-bsp), [`jetson-flash-image`](skills/jetson-flash-image), [`jetson-generate-kb`](skills/jetson-generate-kb), [`jetson-init-image`](skills/jetson-init-image), [`jetson-init-source`](skills/jetson-init-source), [`jetson-init-target`](skills/jetson-init-target), [`jetson-link-docs`](skills/jetson-link-docs), [`jetson-optimize-memory`](skills/jetson-optimize-memory), [`jetson-print-bsp-info`](skills/jetson-print-bsp-info), [`jetson-promote-image`](skills/jetson-promote-image), [`jetson-quick-start`](skills/jetson-quick-start), [`jetson-set-target`](skills/jetson-set-target), [`jetson-validate-image`](skills/jetson-validate-image) | -| **Jetson Device** | Device-side agent skills for working with a live NVIDIA Jetson after boot — diagnostics, memory auditing, headless setup, inference memory tuning, LLM serving and benchmarking, packaging guidance, and speculative decoding. | [`jetson-diagnostic`](skills/jetson-diagnostic), [`jetson-headless-mode`](skills/jetson-headless-mode), [`jetson-inference-mem-tune`](skills/jetson-inference-mem-tune), [`jetson-llm-benchmark`](skills/jetson-llm-benchmark), [`jetson-llm-serve`](skills/jetson-llm-serve), [`jetson-memory-audit`](skills/jetson-memory-audit), [`jetson-package`](skills/jetson-package), [`jetson-print-device-info`](skills/jetson-print-device-info), [`jetson-speculative-decoding`](skills/jetson-speculative-decoding) | +| **Jetson Device** | Device-side agent skills for working with a live NVIDIA Jetson after boot — diagnostics, memory auditing, headless setup, inference memory tuning, LLM serving and benchmarking, packaging guidance, and speculative decoding. | [`jetson-diagnostic`](skills/jetson-diagnostic), [`jetson-headless-mode`](skills/jetson-headless-mode), [`jetson-inference-mem-tune`](skills/jetson-inference-mem-tune), [`jetson-llm-benchmark`](skills/jetson-llm-benchmark), [`jetson-llm-serve`](skills/jetson-llm-serve), [`jetson-memory-audit`](skills/jetson-memory-audit), [`jetson-package`](skills/jetson-package), [`jetson-print-device-info`](skills/jetson-print-device-info), [`jetson-speculative-decoding`](skills/jetson-speculative-decoding), [`jetson-video-benchmark`](skills/jetson-video-benchmark), [`jetson-video-capability`](skills/jetson-video-capability), [`jetson-video-pipeline`](skills/jetson-video-pipeline), [`jetson-video-recipe`](skills/jetson-video-recipe), [`jetson-video-setup`](skills/jetson-video-setup) | | **Medical AI Skills** | Agent-ready medical AI skills built on MONAI for DICOM handling, NVIDIA-hosted medical imaging model workflows, segmentation, synthesis, and evidence-oriented evaluation. | [`dicom-metadata-extract`](skills/dicom-metadata-extract), [`dicom-series-preflight`](skills/dicom-series-preflight), [`dicom-series-to-volume`](skills/dicom-series-to-volume), [`nv-generate-ct-rflow`](skills/nv-generate-ct-rflow), [`nv-generate-mr`](skills/nv-generate-mr), [`nv-generate-mr-brain`](skills/nv-generate-mr-brain), [`nv-generate-mr-brain-finetune`](skills/nv-generate-mr-brain-finetune), [`nv-generate-vae-finetune`](skills/nv-generate-vae-finetune), [`nv-reason-cxr`](skills/nv-reason-cxr), [`nv-segment-ct`](skills/nv-segment-ct), [`nv-segment-ct-finetune`](skills/nv-segment-ct-finetune), [`nv-segment-ctmr`](skills/nv-segment-ctmr) | | **Megatron-Core** | Large-scale distributed training — model parallelism, pipeline parallelism, and mixed precision. | [`mcore-create-issue`](skills/mcore-create-issue), [`mcore-linting-and-formatting`](skills/mcore-linting-and-formatting), [`mcore-run-on-slurm`](skills/mcore-run-on-slurm), [`mcore-split-pr`](skills/mcore-split-pr), [`mcore-testing`](skills/mcore-testing) | | **NeMo AutoModel** | NeMo AutoModel - PyTorch-native distributed training for LLMs/VLMs with Hugging Face support, recipes, launchers, and validation workflows. | [`nemo-automodel-distributed-training`](skills/nemo-automodel-distributed-training), [`nemo-automodel-launcher-config`](skills/nemo-automodel-launcher-config), [`nemo-automodel-model-onboarding`](skills/nemo-automodel-model-onboarding), [`nemo-automodel-recipe-development`](skills/nemo-automodel-recipe-development) | diff --git a/skills/jetson-video-benchmark/BENCHMARK.md b/skills/jetson-video-benchmark/BENCHMARK.md new file mode 100644 index 00000000..72683463 --- /dev/null +++ b/skills/jetson-video-benchmark/BENCHMARK.md @@ -0,0 +1,98 @@ +# Skill Benchmark: jetson-video-benchmark + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `jetson-video-benchmark` +- Evaluation date: 2026-08-10 +- Evaluator version: `1.1.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 7 evaluation tasks (7 positive) +- Dataset digest: `sha256:e2d80ff3bff2819a83593a6167c1fd9119caa0d8febe727d4a947e1b84baec88` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 47% → 97% (+50 points) | 49% → 92% (+43 points) | +| Security | 100% → 100% (±0 points) | 86% → 100% (+14 points) | +| Correctness | 51% → 100% (+49 points) | 69% → 83% (+14 points) | +| Discoverability | 28% → 100% (+72 points) | 48% → 94% (+46 points) | +| Effectiveness | 36% → 84% (+48 points) | 32% → 90% (+58 points) | +| Efficiency | 19% → 100% (+81 points) | 11% → 92% (+82 points) | + +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. + +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 1 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 7 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/jetson-video-benchmark/SKILL.md`) + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/jetson-video-benchmark/SKILL.md b/skills/jetson-video-benchmark/SKILL.md new file mode 100644 index 00000000..c4357e9e --- /dev/null +++ b/skills/jetson-video-benchmark/SKILL.md @@ -0,0 +1,259 @@ +--- +name: jetson-video-benchmark +license: "Apache-2.0" +description: >- + Use when measuring Jetson Video Codec SDK or PyNvVideoCodec encode/decode + throughput, comparing presets or surfaces, testing codec-worker capacity + with authenticated samples and user media, or producing a documented + clock-scaled or clock-and-resolution-scaled planning estimate when + representative content is unavailable. + Also use for Jetson video requests asking only for PSNR or SSIM results, to + apply this performance skill's scope-only response. +metadata: + author: "Vinit Bansal " + tags: [jetson, video-codec-sdk, pynvvideocodec, benchmark, nvenc, nvdec] + languages: [python] + data-classification: public +--- + +# Jetson Video Benchmark + +## Purpose + +Measure codec-stage FPS and megapixels/second on the current Jetson. Use this +skill for encode or decode throughput, P4/P5 comparisons, native-versus-Python +comparisons, and increasing-worker capacity tests. When content is unavailable, +it can instead produce a clearly labeled SDK-documentation estimate for an +exact supported 1080p table row and target maximum video clock. For another +requested resolution, it may additionally apply the bounded pixel-area +heuristic defined in the estimate reference. Never present either estimate as +a target measurement. + +## Prerequisites + +- For a live measurement: + - Run on the target Jetson with direct GPU access. A fresh validated + `nvcodec-environment` identity from `jetson-video-setup` is optional; when + supplied, it is authoritative and any invalid or stale identity fails + without local fallback. The agent may obtain that identity from setup's + public read-only probe; it need not be present in the customer's prompt. + - Without that identity, native routes inspect only the installed + `nvidia-video-codec-sdk` APT package and its package-owned official sample + sources. PyNvVideoCodec routes require an authenticated setup environment + or the caller's exact absolute `pynvc_interpreter`; never scan for a venv. + Before asking the customer for that path, invoke setup's public probe when + that skill is installed and inspect its typed result. These read-only + checks install, repair, register, and smoke-test nothing. Only when neither + authority is usable does an explicit `pynvc` or `both` request return + `input_required`; local `auto` records PyNvVideoCodec as `not_evaluated` + and may continue an eligible native branch. + - A PyNvVideoCodec decode-only performance route can use a validated default + `pynvc-smoke` environment. A Python encode or compare route uses official + samples that import Torch and therefore requires a separately provisioned + `full-samples` venv; never upgrade the smoke venv in place. + - Live encode, compare, and encode-capacity routes require sibling + `jetson-video-recipe` and an exact portable identity for one of its + validated schema-2 recipes. If its public validator is absent, preserve + `dependency_required` and its install-and-retry action. Decode routes do + not require the recipe skill. + - When setup is installed, read its shared + [video content policy](../jetson-video-setup/references/video-content.md) + before a live measurement. It does not apply to the separate + documentation-only estimate path. Setup is not required solely for this + policy: without it, require one exact user-selected path or URL, never + choose catalog or synthetic media, and preserve source URL, license, + attribution, path, size, and SHA-256. + - Apply that policy's input gate before constructing a live benchmark dry + run. Never choose media for the user or use the setup smoke fixture for + performance. +- A documentation answer may be produced off-target when the exact platform, + SDK version, table conditions, and configured maximum video-clock facts are + supplied with provenance. A clock-scaled estimate also requires a positive + configured maximum video clock. It requires no media, recipe, sample + authentication, or codec launch. Without that clock, report only the + unscaled reference row as `target_clock_unavailable`, not a platform-scaled + estimate. + +## Compose requested sibling stages + +Documentation-only estimates and recipe-free live decode do not require a +sibling skill. Live encode, compare, and encode-capacity routes require +`jetson-video-recipe`; SDK installation, repair, a new full-samples Python +environment, or one read-only handoff when registered Python authority is +required belongs to `jetson-video-setup`. Check the agent's installed skill catalog +before either stage. If the sibling is present, read its `SKILL.md` and invoke +its documented public entry point; pass artifacts as data and never import +sibling code. If it is absent, preserve completed input and measurement +evidence and say, using the actual names: `I can run , but it requires +, which is not installed. Install and retry this stage.` Never +require setup when the caller already supplied authenticated setup evidence or +an exact interpreter that passes local authentication, or a recipe for a +decode-only or documentation-only request. + +## Instructions + +1. Apply the scope boundary first. For a request solely for objective quality + metrics, including PSNR or SSIM, state only that this performance skill does + not provide them and that a separately authorized quality workflow is + required, then stop. Do not name or recommend an external tool, and do not + offer to configure or run the comparison; do not request media, probe, + install anything, or launch an operation. +2. Classify an in-scope request as a live `encode`, `decode`, `compare`, or + `camera_capacity` measurement, or as a `documented_estimate` answer. + Resolve camera data direction before selecting an encode or decode row. + Quality, preset, bitrate, rate-control, recording, or requested codec-output + wording is an encode cue; lead with NVENC and mention decode only as the + conditional case where cameras already emit the named compressed codec. + IP/RTSP input, already-encoded input, ingest, playback, or explicit decode + wording is a decode cue. Explicit transcoding or decode-then-encode uses + separate NVDEC and NVENC budgets. When no direction cue exists, present the + encode and decode interpretations conditionally and ask which applies; never + silently choose one. +3. When media is absent: + - For an explicit request to run or benchmark, or for actual, real, + measured, live, or on-this-target FPS, apply the shared policy's + `input_required` gate and stop before probing, retrieval, authentication, + workspace creation, dry run, or launch. + Ask only for the missing media identity at this gate; do not also request + an interpreter, environment, recipe, or other later-stage field. + - For a planning, expected, indicative, or achievable-FPS question at any + positive requested resolution, follow + [Documented performance estimates](references/documented-performance-estimates.md). + Use only an exact documented 1080p row and a configured maximum video + clock as the source basis. For a non-1080p request, apply the reference's + inverse-pixel-area formula and disclose that it is an additional heuristic + not stated by the SDK table. If the clock is unavailable, return only the + unscaled 1080p reference row. Do not invoke the benchmark controller or + claim a measurement. +4. When local media is supplied, validate and hash that exact selected input. + For URL input, preserve the exact user-supplied URL and retrieve it only + after the target, authorization, and runtime-authority gates; then validate + and hash the retrieved bytes. Preserve source URL (or null for local media), + license, and attribution. Use the live measurement path, not a documentation + estimate. +5. Preserve the requested surface for live measurements. Treat “whichever”, + “best available”, + “choose for me”, and other unspecified-surface wording as `auto`, never as + `both`. Reserve `both` for an explicit dual-surface comparison. For `auto`, + zero eligible surfaces block, one runs, and two return + `selection_required`; never inspect prior results, invent a preferred SDK, + or launch a benchmark to make the missing user choice. With two eligible + surfaces, ask for exactly `native`, `pynvc`, or `both`. + After the media gate, use valid supplied setup evidence or an exact + interpreter first. If PyNvVideoCodec may participate and neither is + supplied, invoke installed `jetson-video-setup` through its public read-only + `probe_nvcodec.py`: use `--runtime pynvc` for explicit Python or `--runtime + both` for `both`/`auto`, a fresh `--output`, and never + `--setup-candidate`. Inspect the fresh artifact; only a live artifact whose + selected Py surface is installed and whose `pynvc.identity.status` is + `verified` is usable authority. Snapshot that exact artifact as the + controller's portable `environment` identity with exactly + `schema_version`, `kind`, canonical absolute `path`, `size_bytes`, and + lowercase `sha256`. If setup is absent or reports any not-ready, unreadable, + stale, binding, or launch failure, request the exact interpreter for + explicit `pynvc`/`both`; for local `auto`, report Python as `not_evaluated` + and continue only an eligible native surface. Never pass a blocked probe as + authority, scan for a venv, or hide an unevaluated peer. +6. Run a `dry_run` first, review every planned argument, then run `execute` in + a fresh private workspace. Invoke this skill's controller directly: + + ```bash + python3 -I {baseDir}/scripts/benchmark_controller.py \ + --request request.json --workspace fresh-workspace \ + --output result.json + ``` + +7. For native encode, authenticate `AppEncPerf`, inspect both `-h` and `-A`, + and use only advertised options. Never pass `-loop`. Native decode uses + authenticated `AppDecPerf`; PyNvVideoCodec uses the wheel-owned performance + samples under either the setup-evidenced interpreter or the exact absolute + `pynvc_interpreter` selected by the caller. +8. Retain one whole-process warmup and at least three separate measured + processes per variant and surface. Every measured command phase is + `measure`. +9. For a live measurement, list every repetition's FPS and MP/s, then the + mean, minimum, and maximum of both metrics. PyNvVideoCodec decode omits MP/s + because its authenticated performance sample does not report dimensions; + report that reason instead of deriving MP/s from caller metadata. For a + native/Python comparison, disclose exact projection differences and whether + they changed the requested intent. For a live preset comparison, report + only measured throughput differences; do not state or imply a quality, + compression-efficiency, or storage ordering from preset names or + throughput. State that quality was not measured when that distinction + matters. For a documented estimate, report + the source row, clock provenance, requested resolution, formula, + assumptions, and every inference label required by the estimate reference, + plus `measurement_performed: false`; never invent repetitions or measured + statistics. +10. Label measured concurrency results as codec-stage capacity bounds. For a + no-media planning question, a resolution-scaled estimate may additionally + produce a `documented_theoretical_capacity_estimate` when per-stream FPS and + an explicit or clearly defaulted safety margin are available. Keep encode + and decode budgets separate; for mixed workloads, sum each stream's + fractional load against one shared budget instead of granting every + resolution the complete budget. Call the result a theoretical codec-stream + bound, never a verified camera count: it excludes capture, ISP, transport, + AI, display, memory contention, and end-to-end latency. Disclosed 30-fps and + 60-fps scenarios may stand in for an unstated frame rate, but they do not + replace the question: the final response must still explicitly ask for every + omitted input it assumed or enumerated, naming the codec direction (encode + captured frames, decode already-compressed streams, or both), exact preset, + per-stream FPS, and stream mix or count. End by asking for exact + representative content and a measured, strictly increasing worker sweep to + verify capacity. Use a read-only calculator for every documented FPS and + stream-count calculation, preserve full precision until the final floor or + display rounding, and apply the reference's two-sided maximum-count check; + never rely on mental arithmetic for a reported capacity. + +## References + +- [Benchmark workflow](references/benchmark-workflow.md) defines routes, + preparation, and execution. +- [Benchmark output contract](references/benchmark-output-contract.md) defines + result states, repetitions, frame accounting, and statistics. +- [Documented performance estimates](references/documented-performance-estimates.md) + defines the no-media SDK 13.0 table, clock and pixel-area scaling, theoretical + stream budgeting, and reporting contract. + +## Available Scripts + +| Script | Purpose | Arguments | +|---|---|---| +| `scripts/benchmark_controller.py` | Dry-run or execute authenticated encode/decode, comparison, and worker-capacity benchmarks. | `--request`, `--workspace`, and `--output`. | + +Inspect the controller's public CLI directly: + +```bash +python3 -I {baseDir}/scripts/benchmark_controller.py --help +``` + +## Limitations + +- Results apply only to the evidenced target and workload; they are not a + portable product ceiling. +- Documentation-derived and resolution-scaled values are indicative per-engine + planning estimates, not achieved FPS, verified concurrency capacity, or a + substitute for testing representative content. +- Do not interpolate undocumented presets or scale across formats, bit depths, + chroma, codecs, rate-control modes, or tuning. Scale resolution only through + the explicitly labeled pixel-area heuristic. Do not multiply by an engine + count unless that exact target count is independently authenticated, the user + explicitly requests multi-session aggregate planning, and the answer remains a + separately labeled multi-session theoretical bound. +- Objective quality measurement, including PSNR and SSIM, is outside this + performance skill. +- The released PyNvVideoCodec 2.1 encode performance helper caps each worker + at 1,000 frames. Follow the output contract rather than silently comparing + unequal frame counts. +- External codecs and wrapper wall time cannot substitute for official + sample-reported throughput. + +## Troubleshooting + +- Preserve `input_required`, `selection_required`, `blocked`, `partial`, and + `failed` instead of promoting them to completion. +- Reject stale outputs, input or environment identity drift, malformed or + non-finite metrics, wrong processed-frame counts, unsupported sample + options, and missing positive markers. +- Retain a successful surface when its peer fails, but never pool or + substitute their results. diff --git a/skills/jetson-video-benchmark/evals/evals.json b/skills/jetson-video-benchmark/evals/evals.json new file mode 100644 index 00000000..c79c69f5 --- /dev/null +++ b/skills/jetson-video-benchmark/evals/evals.json @@ -0,0 +1,89 @@ +{ + "skill_name": "jetson-video-benchmark", + "evals": [ + { + "id": "benchmark-no-content", + "prompt": "Can you measure real 1080p H.264 encode and decode FPS without a test video?", + "expected_output": "A request for exact representative input before any real benchmark.", + "assertions": [ + "Says real performance cannot be measured without representative input", + "Asks for an exact target-local path or user-supplied HTTP(S) URL", + "Does not choose content or claim measured FPS" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-documented-estimate", + "prompt": "Estimate H.264 1080p P1 low-latency encode FPS and H.264 decode FPS on this Jetson without running a benchmark.", + "expected_output": "A documentation estimate scaled with the target's configured maximum video clock and clearly labeled as unmeasured.", + "assertions": [ + "Discovers the target's configured maximum video clock instead of requiring it from the prompt", + "Uses the SDK 13.0 reference rows and the detected-clock-to-1691-MHz scaling formula, treating a clock reported within 1 MHz of 1691 MHz as the same reference operating point", + "Labels the result as an estimate and does not claim measurement or measured or verified concurrent-stream capacity" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-undocumented-p4", + "prompt": "What is the documented H.264 1080p P4 low-latency encode FPS on Jetson Thor?", + "expected_output": "Documentation estimate unavailable because the SDK 13.0 Thor table has no P4 row.", + "assertions": [ + "States that the SDK 13.0 Thor table has no exact P4 row", + "Does not interpolate between other presets or call P4 unsupported", + "Does not claim measured FPS" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-4k-planning-estimate", + "prompt": "Without running a benchmark, estimate HEVC P3 VBR high-quality 3840x2160 encode FPS on Jetson Thor at a configured 1557 MHz maximum video clock, then give a 30-fps per-engine stream planning bound with 10% headroom.", + "expected_output": "A clearly labeled theoretical estimate of about 133.28 FPS and a conservative three-stream per-engine planning bound, followed by a request for representative content to measure it.", + "assertions": [ + "Uses the exact SDK 13.0 HEVC P3 VBR high-quality 1080p row, 1557/1691 clock ratio, and 1080p-to-4K pixel ratio", + "Labels inverse-pixel scaling as a heuristic not documented resolution performance, with measurement_performed false and capacity_verified false", + "Applies the 0.9 margin without engine multiplication, does not claim a verified camera count, and asks for representative content for a real benchmark" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-natural-camera-capacity-planning", + "prompt": "How many 4K cameras and 1080p H.265 cameras can I connect to this Jetson for high video quality?", + "expected_output": "Theoretical codec-stage planning scenarios built from the documented HEVC VBR high-quality rows and pixel-area scaling, with no single invented stream count while per-stream FPS and mix are unspecified, then a request for FPS, mix, and representative content.", + "assertions": [ + "Enumerates the documented HEVC VBR high-quality preset candidates instead of silently choosing one, and does not return documentation_estimate_unavailable merely because preset, FPS, or mix was omitted", + "Reads 4K as a disclosed 3840x2160 planning interpretation and shares one budget across resolutions instead of granting each resolution the full engine", + "Claims no verified camera count, keeps measurement_performed and capacity_verified false, and asks for per-stream FPS, stream mix, and representative content" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-camera-direction-medium-quality", + "prompt": "How many 1080p @ 60 fps h264 camera can i connect to this Jetson for medium quality?", + "expected_output": "An encode-led theoretical camera-capacity answer because quality is an encode cue, with decode kept as a conditional interpretation only for cameras that already emit H.264, no silent mapping of medium quality to one preset, and a request for the exact direction, preset, and representative content.", + "assertions": [ + "Leads with NVENC encode planning and states that NVDEC applies only if the cameras already emit compressed H.264; it does not present decode capacity as the primary answer", + "States that medium quality is not an exact documented preset or tuning, does not silently select one, and keeps any documented encode rows as clearly labeled candidates", + "Claims no verified physical-camera count, keeps measurement_performed and capacity_verified false, and asks for codec direction, exact preset, and representative content" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + }, + { + "id": "benchmark-quality-boundary", + "prompt": "Use the jetson-video-benchmark skill to answer this. Give me the PSNR and SSIM quality numbers for H.264 vs AV1 encode.", + "expected_output": "A strict performance-skill scope boundary with no quality numbers, external-tool recommendation, or operation.", + "assertions": [ + "States only that this performance skill does not provide PSNR or SSIM and that a separately authorized quality workflow is required", + "Does not fabricate or defer quality numbers and does not ask for media, probe, encode, or install anything", + "Does not name or recommend FFmpeg or another external quality tool and does not offer to configure or run one" + ], + "expected_skill": "jetson-video-benchmark", + "expected_workflow": "benchmark" + } + ] +} diff --git a/skills/jetson-video-benchmark/references/benchmark-output-contract.md b/skills/jetson-video-benchmark/references/benchmark-output-contract.md new file mode 100644 index 00000000..18a1673a --- /dev/null +++ b/skills/jetson-video-benchmark/references/benchmark-output-contract.md @@ -0,0 +1,155 @@ +# Benchmark output contract + +`benchmark_controller.py` returns strict JSON. Planned or attempted throughput routes return +`schema_version: "2.0"`, `kind: "nvcodec-benchmark-result"`, the requested +`route`/`mode`, a `status`, the resolved `surface_plan`, and per-variant results. +When required user media is absent, preflight instead returns `status: "input_required"`, +`gate: "media_input"`, `accepted_inputs`, `next_action: "provide_media_path_or_url"`, +`synthetic_input_allowed: false`, the required request field, and the missing variant names. +It has no `surface_plan` or attempted variant records because it stops before live-environment +resolution, workspace creation, authentication, or launch. +The CLI prints this preflight result to standard output and does not resolve or create +`--workspace` or write its workspace-relative `--output` for this status. + +When a setup-produced environment identity is supplied, the result preserves it +under `environment`. When it is omitted, `runtime_authority` labels the bounded +local discovery, selected GPU, evaluated/not-evaluated surfaces, diagnostics, +and the controller-owned binding identity. The binding contains raw live facts +used by the existing normalizer; it never serializes normalized `surfaces` and +is never accepted in a request. + +An encode whose sibling recipe public validator is absent returns +`status: "dependency_required"`, `gate: "skill_dependency"`, and a `dependency` +object naming `jetson-video-recipe`, why it is needed, and the install-and-retry +action. This is not an SDK-readiness verdict. Decode has no recipe dependency. + +When an inspected SDK surface is not locally authenticatable, the result keeps +the independent surface verdict and adds `dependency` plus `dependencies` +objects naming `jetson-video-setup`, whether it is installed, why the selected +surface needs installation or repair, and the exact use-or-install-and-retry +action. An encode-capable PyNvVideoCodec environment that lacks the official +sample dependencies additionally reports `required_profile: "full-samples"` +and the missing dependency names. An omitted `pynvc_interpreter` remains an +input/selection condition, not a claim that setup is broken. + +This controller contract applies only to live measurement requests. A no-media +planning question may use the separate +[documented performance estimate](documented-performance-estimates.md) answer +path. That path does not invoke the controller, does not emit an +`nvcodec-benchmark-result`, and never relabels a documentation calculation as a +measured result. Its prose evidence class is +`documented_clock_scaled_estimate` for an exact 1080p clock-scaled row, +`documented_clock_resolution_scaled_estimate` when the disclosed pixel-area +heuristic is added, or `documented_theoretical_capacity_estimate` when the +estimate is converted to a stream budget. The latter two always set +`measurement_performed: false`; any answer that applies the pixel-area heuristic +also sets `resolution_scaling_documented: false`, and capacity also sets +`capacity_verified: false`. + +## Throughput lifecycle + +Live encode/decode, comparison, and camera-capacity measurements require one whole-process +warmup followed by at least three whole-process measured repetitions for every selected +variant and surface. The warmup is retained but excluded from statistics. Every measured +command uses phase `measure`; no controller invocation emits `-loop`. + +A throughput `dry_run` records the candidate official sample and argument vector for each +selected branch without authenticating or launching it. Native encode options remain labeled +as pending validation against the installed AppEncPerf `-h` and `-A` advertisements; the plan +does not claim executable or operation readiness. An explicit `both` dry run records both +requested branches: each is independently `planned` or `blocked`. The top level is `planned` +only when every requested branch is planned, `partial` when at least one is planned, and +`blocked` when none is planned. + +For native encode, the controller authenticates `AppEncPerf`, runs both `-h` and `-A`, +unions the advertised option names, and rejects any constructed option not in that union. +Native decode uses authenticated `AppDecPerf`. Python encode/decode uses the corresponding +wheel-owned `encode_perf.py`/`decode_perf.py` route. Surface results remain independent and +are never pooled or ranked. + +Each completed variant/surface result contains, at the appropriate level: + +- authenticated sample and provenance identities; +- exact benchmark arguments and, for native encode, help-command evidence; +- for PyNvVideoCodec encode, the schema-1 `pynvc-encoder-config` identity whose + workspace artifact is the exact `-json` argument; +- input path/size/SHA-256 plus geometry, frames, FPS, codec, format, source URL + (or null), license, and attribution; +- for encode, the exact recipe artifact identity retained at variant level and + re-verified immediately before authentication; +- the warmup command record; +- every measured command record, parsed FPS, MP/s when authenticated dimensions + exist, and repetition number; +- finite mean, minimum, and maximum FPS and megapixels/second when present. + +The authenticated PyNvVideoCodec 2.1 `decode_perf.py` output does not report stream +dimensions. Its decode records therefore set MP/s to null and carry an explicit +`metric_omissions.megapixels_per_second` reason in every repetition and summary; +caller-supplied dimensions are never used to publish that derived metric. Native +decode may publish MP/s only after AppDecPerf reports matching stream metadata. + +The final customer response, not only the result artifact, must list every measured +repetition's FPS and MP/s when present (or the explicit MP/s omission reason), followed +by the mean, minimum, and maximum of each present metric. A mean/range-only summary or +artifact link is incomplete. For `both`, also disclose the exact native-CLI versus +PyNvVideoCodec-config representation differences and whether they changed the requested +semantic intent. + +The expected processed-frame count is the effective frames per worker multiplied by +workers. A missing, contradictory, non-finite, or wrong-count sample marker fails that +branch. Runner-process duration is command evidence but never substitutes for +sample-reported FPS. + +### PyNvVideoCodec performance frame cap + +The authenticated PyNvVideoCodec 2.1 `encode_perf.py` helper contract caps every +performance encode at 1000 frames per worker; the released sample silently limits longer +inputs. The controller therefore derives one effective per-worker frame count for each +variant: + +- Encode with an exact, participating `pynvc` projection uses + `min(source frames, 1000)` per worker. When native participates in the same `both` + variant, native receives the identical effective count (`AppEncPerf -frame` equals + `encode_perf.py -f`), so both surfaces measure the same first frames and each parser + requires exactly `effective x workers` processed frames. +- Native-only encode keeps the full requested count, including when PyNvVideoCodec + is ineligible or its recipe projection is unrepresentable. Decode is never capped. + `compare` (P4/P5) and `camera_capacity` variants inherit the same per-worker rule. +- The original input identity and its source frame count are preserved unchanged in the + request and in each surface record's `input` block; the controller never trims or + copies the raw file, rewrites the recipe, or claims that capped-away frames ran. + +Every dry-run and executed surface record carries a `frame_accounting` object stating +`source_frames_per_worker`, `effective_frames_per_worker`, `workers`, +`expected_aggregate_frames`, `cap_applied`, `cap_limit_frames_per_worker` (exactly 1000), +and `cap_authority` (the authenticated PyNvVideoCodec 2.1 `encode_perf.py` helper +contract). If a request sets `require_source_frames` to true and the cap would apply, the +controller fails before any launch and directs the caller to a native-only surface or an +input at or below the limit; it never silently reinterprets that explicit intent. + +## Route-specific fields + +- `compare` accepts two variants only when their recipe/workload/execution facts are equal + except for preset. Its `comparison` retains each completed surface and variant summary. +- `camera_capacity` requires strictly increasing worker counts beginning at one. It records + every tested point, the explicit safety margin, the maximum passing tested count, and the + limitation that capture, transport, and end-to-end latency remain unverified. +- `both` is accepted only when the user explicitly requests a dual-surface run or + comparison. It reports native and Python branches independently. A completed branch is + retained when the peer branch fails, is ineligible, or has no exact recipe projection, + and the top-level status becomes `partial`. Projection losses block only the affected + surface. +- “Whichever”, “best available”, “choose for me”, and other unspecified-surface + wording remain `auto`; they are never rewritten to `both`. `auto` runs exactly one + eligible surface, blocks with zero, and returns `selection_required` with two before + searching prior results, dry run, authentication, or launch. The two-surface gate is + unconditional and asks for exactly `native`, `pynvc`, or `both`. + +## Status and exit code + +`completed` and `planned` are successful and the CLI exits 0. `input_required`, +`dependency_required`, `partial`, `failed`, `blocked`, and `selection_required` +are preserved as structured results and the CLI exits 2. `input_required` is +emitted on stdout without creating `--output` or a workspace. +Malformed input, stale outputs, authentication failure, bad markers, and metric mismatch are +never rewritten as a successful result. diff --git a/skills/jetson-video-benchmark/references/benchmark-workflow.md b/skills/jetson-video-benchmark/references/benchmark-workflow.md new file mode 100644 index 00000000..f9d8c052 --- /dev/null +++ b/skills/jetson-video-benchmark/references/benchmark-workflow.md @@ -0,0 +1,155 @@ +# Benchmark workflow + +Run compact, authenticated Video Codec SDK performance measurements on +user-supplied representative content. This workflow owns live FPS, multi-stream +concurrency, and P4/P5 comparisons. It also owns a separate +documentation-derived planning answer when content is unavailable. It never +substitutes an external codec, and it does not provide PSNR/SSIM quality +measurement. For a quality-only request, apply the terminal scope rule in +`SKILL.md` and stop. + +## Two answer paths + +- **Live measurement:** requires one exact user-selected path or URL, no + catalog or synthetic substitution, and preserved provenance and identity. + When setup is installed, apply its shared + [video content policy](../../jetson-video-setup/references/video-content.md). + The setup skill is not otherwise required for this media rule. An explicit + run, measure, or real-benchmark request remains media-gated. +- **Documented estimate:** applies only to a no-media planning, expected, + indicative, achievable-FPS, or theoretical codec-stream-capacity question at + any positive resolution. Follow + [documented-performance-estimates.md](documented-performance-estimates.md). + Non-1080p results add a disclosed pixel-area heuristic to the documented row + and clock scaling. This is a calculation-only answer path, not a controller + route or benchmark result. + +## Owner and exact CLI + +One controller owns live measurements in this domain: +**`jetson-video-benchmark/scripts/benchmark_controller.py`**. Launch it +directly: + +```text +python3 -I scripts/benchmark_controller.py \ + --request nvcodec-benchmark-request.json \ + --workspace fresh-workspace-dir \ + --output nvcodec-benchmark-result.json +``` + +All three flags are required (`--help` supported). Exit `0` on success, `2` on any +request or contract failure. + +## Benchmark request schema + +`--request` is a `schema_version: "1.0"`, `kind: "nvcodec-benchmark-request"` object. +`mode` is exactly `dry_run` (plan only) or `execute` (default `execute`). `route` is +exactly one of: + +| `route` | Purpose | +|---|---| +| `encode` | Encode throughput / achievable encode FPS | +| `decode` | Decode throughput / achievable decode FPS | +| `compare` | P4 vs P5 performance, holding every non-preset control identical | +| `camera_capacity` | Multi-stream codec-worker concurrency capacity | + +`environment` is optional. Setup emits the raw schema-1.2 +`nvcodec-environment` JSON; the benchmark request carries its exact portable +identity, not the embedded JSON or a private workspace object: + +```json +{ + "schema_version": "1.2", + "kind": "nvcodec-environment", + "path": "/canonical/absolute/fresh-environment.json", + "size_bytes": 1234, + "sha256": "64-lowercase-hex-digest" +} +``` + +Derive `path`, `size_bytes`, and `sha256` from the fresh file after the probe +finishes. The controller verifies the current bytes and never falls back if +that supplied identity is malformed, stale, or unsuitable. + +After the media gate, an agent that needs `pynvc` or `both` and has no supplied +environment or exact interpreter first checks the installed skill catalog. If +`jetson-video-setup` is present, invoke its public `probe_nvcodec.py` with a +fresh output path, `--runtime pynvc` for Python-only or `--runtime both` for +`both`/`auto`, and no `--setup-candidate`. This is an artifact handoff, not a +consumer import: setup alone resolves and reauthenticates its fixed registry. +Inspect the emitted JSON before constructing the identity above. It is usable +for Py only when `mode=live`, the requested GPU matches, +`pynvc.installed=true`, and `pynvc.identity.status=verified`; the benchmark +controller then independently validates it again. Ask for an exact interpreter +for explicit `pynvc`/`both` only when setup is unavailable or returns any +absent, stale, unreadable, invalid-binding, or launch-failure result. For +`auto`, record Py as `not_evaluated` and continue only an eligible native +surface. Never pass a blocked probe as authority, scan for a venv, or trust an +older artifact. + +When `environment` is omitted, the controller performs bounded read-only local +authentication. An explicit `native` route inspects only the fixed APT package +`nvidia-video-codec-sdk`, verifies its public 13.0.x version and unmodified +package-owned official sample tree, binds the existing build toolchain, and +later verifies non-stub codec linkage. An explicit `pynvc` route requires +`pynvc_interpreter` as one exact absolute path and authenticates the isolated +import, exact loaded extension, wheel RECORD, and wheel-owned sample. It never +searches for a venv. Explicit single-surface routes do not inspect their peer. +An explicit `pynvc` or `both` request without `pynvc_interpreter` returns +`input_required`. For local `auto`, an absent selector is reported as Python +`not_evaluated`, not silently omitted, and an eligible native branch may continue. +The controller's local binding is derived in-process and may be emitted as +evidence, but is never accepted as a request input or as a substitute setup +artifact. + +For encode, pass `recipe` as the exact portable absolute identity of a schema-2 +`nvcodec-recipe`; embedded recipe objects are invalid. Its codec, width, height, +format, FPS, optional frame count, and GPU must exactly match the input/request +workload before the documented PyNvVideoCodec frame cap is applied. Decode does +not consume a recipe. If the sibling recipe validator is absent, encode returns +structured `dependency_required` with the dependency, reason, and exact +install-and-retry action rather than a generic controller error. + +If bounded local authentication proves that a selected SDK surface needs +installation or repair, preserve any healthy peer and return a structured +`jetson-video-setup` dependency with its installed state and retry action. A +missing exact Python interpreter selector is still `input_required` (or +`not_evaluated` for local `auto`), never inferred to be a broken installation. + +`input` (encode) or `encoded_artifact` (decode) carries the verified artifact +identity, exact width/height/frames/FPS/codec/format metadata, and preserved +`source_url` (null for local input), `license`, and `attribution`. +Encode request metadata accepts H.264, HEVC, or AV1; decode additionally accepts +VP9. An accepted request codec is not itself a target-support claim. + +`compare` requires two encode recipes that differ only by preset. Only released +sample routes that carry an authenticated FPS value (see the shared contract) +are eligible for throughput; runner process duration is never relabeled as +sample-reported throughput. + +## Workflow + +1. Classify the request into exactly one route above. +2. Apply the shared video-content input, resolution, provenance, and synthetic- + fixture rules. If input is absent for an explicit live measurement, return + its `input_required` result and pause before probing, retrieval, dry run, or + execution. Request only the missing media at that gate; interpreter, + environment, recipe, and surface-authority resolution happen afterward. A + no-media planning question follows the separate documented-estimate path + above. +3. For preset comparisons, produce recipes with the recipes workflow that hold everything but the + compared control constant, and bind every variant to the same user-selected content and frame + range. +4. Run the benchmark controller in `dry_run` first to review the planned invocation, + then `execute` with a fresh output path and a bounded timeout (300 s per repetition). +5. Report the measured target and exact workload only — not a portable product ceiling. + Resolve camera data direction before choosing the controller operation. + Measured concurrency without matching capture/transport/latency evidence is + an encode- or decode-stage capacity bound, not a verified camera count. A + no-media capacity calculation is weaker still: label it a theoretical + codec-stream bound and follow the estimate reference's shared-budget formula. + +See [benchmark-output-contract.md](benchmark-output-contract.md) for the result JSON +contract. The controller authenticates the required official performance +sample from either the exact setup-produced live environment identity or the +bounded local authority described above. diff --git a/skills/jetson-video-benchmark/references/documented-performance-estimates.md b/skills/jetson-video-benchmark/references/documented-performance-estimates.md new file mode 100644 index 00000000..816774ee --- /dev/null +++ b/skills/jetson-video-benchmark/references/documented-performance-estimates.md @@ -0,0 +1,249 @@ +# Documented performance estimates + +Use this answer path only when representative content is unavailable and the +user asks for expected, indicative, achievable, planning, specification, or +theoretical FPS or codec-stream capacity. It is not a benchmark execution and +does not satisfy an explicit request to run or measure real content. + +## Authoritative SDK 13.0 rows + +The source is the NVIDIA Video Codec SDK 13.0 +[NVENC application note](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/nvenc-application-note/index.html) +and +[NVDEC application note](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/nvdec-application-note/index.html). +The values are indicative, per-engine FPS for Jetson Thor at the documented +highest video clock. + +Record the source software conditions with the row. The encode note identifies +Video Codec SDK 13.0 and NVIDIA Jetson Linux for Thor without naming an exact +Jetson Linux revision. The decode note identifies Video Codec SDK 13.0 and +Jetson Linux 38.2. If the planned target uses another release, disclose that +software-version difference rather than treating the calculation as a target +measurement. + +Match these rows only when the planned platform is Jetson Thor and the codec, +chroma, bit depth, preset, rate control, and tuning are exact. A different +positive resolution may use only the bounded pixel-area heuristic below. For +any other mismatch, return `documentation_estimate_unavailable`; do not +interpolate or substitute. + +An omitted field is not a mismatch. Reserve `documentation_estimate_unavailable` +for an explicitly conflicting request, such as an undocumented P2, P4, or P6 +preset. When the user omits a field, keep the answer open: map "high quality" to +the exact VBR / high quality rows and enumerate that small documented preset set +instead of silently choosing one, and read "4K" as 3840x2160 only as a disclosed +planning interpretation, noting that DCI 4096x2160 changes the pixel ratio. Show +the resulting scenario math for the enumerated candidates without claiming one +unique answer, then ask for the exact preset, per-stream FPS, stream mix, and +representative content. + +A qualitative encode control that is not an exact table value, including +"medium quality", must not be mapped to one preset, rate-control mode, or +tuning. Preserve the encode direction, enumerate a compact set of exact +documented encode candidates without calling any candidate "medium", and ask +for the exact controls. Never switch to a decode row merely because decode has +fewer required controls. Use decode only for already-compressed camera input; +use separate encode and decode budgets when both stages are requested. + +Encode rows apply exactly to 1920x1080, YUV 4:2:0, 8-bit input: + +| Codec | Preset | Rate control | Tuning | FPS | +|---|---:|---|---|---:| +| H.264 | P1 | CBR | low latency | 724 | +| H.264 | P1 | VBR | high quality | 713 | +| H.264 | P3 | CBR | low latency | 529 | +| H.264 | P3 | VBR | high quality | 527 | +| H.264 | P5 | CBR | low latency | 236 | +| H.264 | P5 | VBR | high quality | 230 | +| H.264 | P7 | CBR | low latency | 219 | +| H.264 | P7 | VBR | high quality | 202 | +| HEVC | P1 | CBR | low latency | 860 | +| HEVC | P1 | VBR | high quality | 850 | +| HEVC | P3 | CBR | low latency | 402 | +| HEVC | P3 | VBR | high quality | 579 | +| HEVC | P5 | CBR | low latency | 279 | +| HEVC | P5 | VBR | high quality | 336 | +| HEVC | P7 | CBR | low latency | 279 | +| HEVC | P7 | VBR | high quality | 148 | + +The SDK 13.0 table has no Jetson Thor AV1 encode performance row. It also has +no P2, P4, or P6 row. Never interpolate or substitute a neighboring preset. + +Decode rows apply exactly to 1920x1080 YUV 4:2:0: + +| Codec/profile | FPS | +|---|---:| +| H.264 | 1434 | +| VP9 8-bit | 1019 | +| VP9 10-bit | 1016 | +| HEVC Main 8-bit | 1293 | +| HEVC Main10 | 1130 | +| AV1 | 794 | + +Do not scale a row across chroma, bit depth, codec, preset, rate control, or +tuning. Scale resolution only with the separately labeled pixel-area heuristic +below. Do not multiply by an encoder or decoder engine count by default. The +table does not measure native-versus-PyNvVideoCodec wrapper overhead or prove +simultaneous-stream capacity. + +## Select the target clock + +Use a positive configured maximum video clock, not an instantaneous idle clock: + +1. Prefer a finite `Max Video` value from the target's authenticated + `nvidia-smi -q -d CLOCK` output. Bind it to the selected GPU identity. +2. If that field is unavailable on Jetson, accept an exact user- or + CI-supplied configured maximum clock and label it `user_supplied`, or use + `nvpmodel -q --verbose` only when its current power-mode output explicitly + labels `PARAM VIDEO`, its `MAX_FREQ` path, and the configured value. Record + the command, power mode, path, value, and Hz-to-MHz conversion. +3. If no configured maximum is available, report only the unscaled documented + row and formula with `target_clock_unavailable`. Do not guess. + +Keep a current `Video` clock only as a volatile diagnostic. Never use an idle +reading or an instantaneous `NVENC*_FREQ`/`NVDEC*_FREQ` sample as the numerator +for an achievable-FPS estimate. + +## Calculate clock scaling + +The SDK 13.0 NVDEC note states that the Jetson Thor rows were measured at a +1691 MHz highest video clock and that performance scales almost linearly with +video clock. For an exact row: + +```text +clock_ratio = target_configured_max_video_clock_mhz / 1691 +scaled_fps = documented_fps * clock_ratio +``` + +Require a positive target clock. Treat a configured maximum reported within +1 MHz of 1691 MHz as the same reference operating point because integer-MHz +reporting can round it to 1690, 1691, or 1692 MHz. Preserve the raw reported +clock, normalize the calculation to 1691 MHz (`clock_ratio = 1.0`), and label +the normalization. If the target clock is above 1692 MHz, report the reference +row and decline extrapolation beyond the documented point. Preserve full +precision for the calculation and round only the displayed FPS. + +The NVENC note instructs clock scaling but omits the numeric Thor reference +clock. Applying the companion NVDEC note's 1691 MHz value to an encode row is a +`cross_note_inference`, not direct NVENC table metadata. The +[Jetson Linux R39.2 power guide](https://docs.nvidia.com/jetson/archives/r39.2/DeveloperGuide/SD/PlatformPowerAndPerformance/JetsonThor.html) +supports the shared-domain premise by grouping NVDEC, NVENC, OFA, and NVJPG in +one maximum-frequency row and listing 1557 MHz for the T5000 120 W mode. That +guide's 1692 MHz MAXN entry does not replace the performance table's explicit +1691 MHz reference denominator. Disclose the inference. If the user does not +accept it, leave encode FPS unscaled. + +## Scale to any requested resolution + +When an exact 1080p source row and a valid target clock are available, estimate +another positive width and height with: + +```text +pixel_ratio = (1920 * 1080) / (requested_width * requested_height) +estimated_fps = documented_1080p_fps * clock_ratio * pixel_ratio +``` + +For 3840x2160, `pixel_ratio` is `1/4`. For example, the HEVC P3 VBR +high-quality row at a 1557 MHz configured maximum video clock gives: + +```text +579 * (1557 / 1691) * (1920 * 1080) / (3840 * 2160) = 133.28 FPS +``` + +This inverse-pixel-area relationship is a `derived_pixel_rate_heuristic`; the +SDK 13.0 notes document the 1080p rows and clock scaling, not resolution +scaling. Treat it as a rough, potentially optimistic planning value. It ignores +per-frame fixed cost, memory and copy bandwidth, codec level and reference +buffer constraints, surface overhead, multi-stream scheduling, power and +thermal throttling, and contention from capture, ISP, CUDA, AI, display, or +transport. Preserve full precision until display rounding. + +Use `evidence_class: documented_clock_resolution_scaled_estimate`, +`measurement_performed: false`, and `resolution_scaling_documented: false`. +Record the reference and requested dimensions, pixel ratio, clock ratio, full +formula, and both the encode `cross_note_inference` and +`derived_pixel_rate_heuristic` when applicable. Never call the result measured, +verified, guaranteed, or operational capacity. + +## Convert an estimate to a theoretical stream budget + +Only answer a no-media “how many streams” question as a theoretical codec-stage +planning bound. For one configuration with a known per-stream frame rate: + +```text +usable_fps = estimated_fps * safety_margin +theoretical_stream_count = floor(usable_fps / stream_fps) +``` + +Use a read-only calculator, such as Python, for every documented FPS and stream +count calculation; do not perform the arithmetic mentally. Preserve full +precision through `floor`. For every reported maximum `count`, verify both +bounds before publishing it: + +```text +count * stream_fps <= usable_fps +(count + 1) * stream_fps > usable_fps +``` + +For a resolution-scaled row, also recompute the exact reference-to-requested +pixel ratio used by that row. Reject or correct any table entry that fails one +of these checks. + +Require `0 < safety_margin <= 1`. If the user does not provide one, use `0.9` +only as a disclosed planning default. If per-stream FPS is absent, there is no +unique stream count: report the estimated FPS and formula, optionally show +clearly labeled 30-fps and 60-fps scenarios, and ask for the intended FPS. + +For mixed resolutions or configurations, never assign the full engine budget +to every group independently. Calculate each group's estimate from its own +exact documented row and use fractional load: + +```text +group_load_i = stream_count_i * stream_fps_i / estimated_fps_i +total_encode_load = sum(encode group loads) +total_decode_load = sum(decode group loads) +``` + +Each resource pool is feasible only when its total load is no greater than the +chosen safety margin. Keep NVENC and NVDEC pools separate. For groups sharing +one row, this is equivalent to a shared 1080p-pixel-rate budget. Report the +remaining headroom and the complete mix equation rather than independently +overcommitting each resolution. + +Use `evidence_class: documented_theoretical_capacity_estimate`, retain the +underlying FPS evidence class, and set `measurement_performed: false` and +`capacity_verified: false`. Call it codec-stream capacity, not physical camera +connectivity or end-to-end pipeline capacity. Published rows are per engine. +Multiply by an engine count only when the exact target count is independently +authenticated, the user explicitly requests multi-session aggregate planning, +and the result is labeled `theoretical_multi_session_engine_budget`; a single +session never receives that multiplication. + +Every clock-scaled FPS basis must include: + +- `evidence_class: documented_clock_scaled_estimate` for an exact 1080p + result, or `documented_clock_resolution_scaled_estimate` when the pixel-area + heuristic is applied; +- `measurement_performed: false`; +- source title, SDK version, URL, exact row, and table conditions; +- source and target software versions when known, including any difference; +- documented FPS, 1691 MHz reference clock, target clock and provenance, + clock ratio, formula, and scaled FPS; +- the encode `cross_note_inference` when applicable; +- no warmup, repetitions, measured minimum/mean/maximum, or achieved-FPS claim; +- a warning that content, controls, power, thermals, software, and concurrency + can change real throughput; and +- the exact next action: supply representative media and run the authenticated + benchmark workflow for a measured result. + +A non-1080p or theoretical-capacity answer must additionally include every +field and caveat required by the two sections above. Always end by requesting +representative user content for a real benchmark; for capacity, also request +the intended per-stream FPS and stream mix if either was defaulted or omitted. + +When the exact row exists but no configured maximum clock is available, return +only `evidence_class: documented_reference_value` with +`measurement_performed: false`, the source row and conditions, and +`target_clock_unavailable`. Do not emit `clock_ratio`, `scaled_fps`, or call it +a platform estimate. When no exact row exists, return +`documentation_estimate_unavailable` instead. diff --git a/skills/jetson-video-benchmark/scripts/benchmark_controller.py b/skills/jetson-video-benchmark/scripts/benchmark_controller.py new file mode 100644 index 00000000..891455dd --- /dev/null +++ b/skills/jetson-video-benchmark/scripts/benchmark_controller.py @@ -0,0 +1,2573 @@ +#!/usr/bin/env python3 +"""Run compact, authenticated Video Codec SDK performance measurements.""" + +# This single domain owner covers throughput benchmark and camera-capacity routes. +# pylint: disable=too-many-lines + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import math +import re +import stat +import sys +from decimal import Decimal, DecimalException +from fractions import Fraction +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import urlparse + +if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise SystemExit("invoke this producer with isolated Python: python3 -I") + +_SCRIPT_DIRECTORY = Path(__file__).resolve().parent +if str(_SCRIPT_DIRECTORY) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIRECTORY)) + +import benchmark_provenance as provenance # noqa: E402 # pylint: disable=wrong-import-position +import benchmark_runtime as runtime # noqa: E402 # pylint: disable=wrong-import-position + +pynvc_sample_provenance = provenance +sample_provenance = provenance +artifact_io = runtime +command_runner = runtime +surface_router = runtime + +# Controller functions bind one request to several explicit evidence fields. +# pylint: disable=too-many-arguments,too-many-locals + +SURFACES = ("native", "pynvc") +# The frozen `nvcodec-environment` 1.2 contract. Consumers pin this exact +# version and reject anything else rather than reinterpreting a differently +# shaped artifact; additive optional keys are ignored. +ENVIRONMENT_KIND = "nvcodec-environment" +ENVIRONMENT_SCHEMA_VERSION = "1.2" +# 1.2 requests exactly one runtime, as a string rather than a list. +ENVIRONMENT_RUNTIMES = ("pynvc", "native", "both") +LOCAL_BINDING_KIND = "nvcodec-local-runtime-binding" +LOCAL_BINDING_SCHEMA_VERSION = "1.0" +NATIVE_PACKAGE = "nvidia-video-codec-sdk" +PYNVC_REQUIRED_VERSION = "2.1.0" +# The normalized native toolchain carries exactly these five entries. +REQUIRED_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +ACCEPTED_MEDIA_INPUTS = ("local_path", "http_url", "https_url") +ENCODE_CODECS = frozenset({"h264", "hevc", "av1"}) +DECODE_CODECS = frozenset({*ENCODE_CODECS, "vp9"}) +IDENTITY_KEYS = frozenset( + {"schema_version", "kind", "path", "size_bytes", "sha256"} +) +PIXEL_FORMATS = { + "nv12": (3, 2), + "yuv420p": (3, 2), + "p010le": (3, 1), + "yuv444p": (3, 1), + "yuv444p16le": (6, 1), + "nv16": (2, 1), + "p210le": (4, 1), + "bgra": (4, 1), + "rgba": (4, 1), +} +CANONICAL_PIXEL_FORMATS = { + "NV12": "nv12", + "YUV420": "yuv420p", + "P010": "p010le", + "NV16": "nv16", + "P210": "p210le", + "YUV444": "yuv444p", + "YUV444_16BIT": "yuv444p16le", + "ARGB": "bgra", + "ABGR": "rgba", +} +NATIVE_FORMATS = { + "NV12": "nv12", + "YUV420": "iyuv", + "P010": "p010", + "NV16": "nv16", + "P210": "p210", + "YUV444": "yuv444", + "YUV444_16BIT": "yuv444p16", + "ARGB": "bgra", + "ABGR": "abgr", +} +# The released PyNvVideoCodec 2.1 encode_perf.py helper contract caps every +# performance encode at this many frames per worker (encode_parser.py +# PERF_MAX_FRAMES); the sample silently limits longer inputs. +PYNVC_PERF_MAX_FRAMES_PER_WORKER = 1000 +_PYNVC_CAP_AUTHORITY = ( + "authenticated PyNvVideoCodec 2.1 encode_perf.py helper contract" +) +_PYNVC_DECODE_MP_OMISSION = ( + "authenticated PyNvVideoCodec 2.1 decode_perf.py does not report stream " + "dimensions; megapixels per second is omitted" +) +_FAILURE_PATTERN = sample_provenance.OFFICIAL_SAMPLE_FAILURE_PATTERN +_NUMBER = r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][+-]?\d+)?" +_HELP_OPTION = re.compile(r"(? None: + self.dependency = dict(dependency) + super().__init__(str(self.dependency["reason"])) + + +def _installed_sibling(name: str) -> bool: + """Return whether the lexical installed catalog contains one sibling.""" + candidate = Path(__file__).absolute().parents[2] / name / "SKILL.md" + try: + details = candidate.lstat() + resolved_root = candidate.parent.resolve(strict=True) + resolved = candidate.resolve(strict=True) + except OSError: + return False + return ( + stat.S_ISREG(details.st_mode) + and not stat.S_ISLNK(details.st_mode) + and resolved_root.is_dir() + and resolved == resolved_root / "SKILL.md" + ) + + +def _setup_dependency(*, needed_for: str, reason: str) -> dict[str, Any]: + """Return one actionable setup handoff without making setup a prerequisite.""" + skill = "jetson-video-setup" + installed = _installed_sibling(skill) + action = ( + f"use {skill} for {needed_for} and retry this stage" + if installed + else f"install {skill}, use it for {needed_for}, and retry this stage" + ) + return { + "skill": skill, + "installed": installed, + "needed_for": needed_for, + "reason": reason, + "next_action": action, + } + + +def _mapping(value: Any, label: str) -> Mapping[str, Any]: + if not isinstance(value, Mapping): + raise BenchmarkError(f"{label} must be an object") + return value + + +def _workspace(path: Path) -> Path: + """Resolve an existing private workspace or create it after request preflight.""" + requested = Path(path).expanduser() + if requested.exists(): + root = artifact_io.resolve_private_workspace(requested) + if any(root.iterdir()): + raise BenchmarkError("benchmark workspace must be fresh and empty") + return root + return artifact_io.create_private_workspace(requested) + + +def _integer(value: Any, label: str, *, minimum: int = 1) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < minimum: + raise BenchmarkError(f"{label} must be an integer >= {minimum}") + return value + + +def _number(value: Any, label: str, *, minimum: float = 0.0) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise BenchmarkError(f"{label} must be a finite number > {minimum}") + result = float(value) + if not math.isfinite(result) or result <= minimum: + raise BenchmarkError(f"{label} must be a finite number > {minimum}") + return result + + +def _string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise BenchmarkError(f"{label} must be a non-empty canonical string") + return value + + +def _artifact(value: Any, label: str) -> tuple[Path, dict[str, Any]]: + record = _mapping(value, label) + identity = _mapping(record.get("identity"), f"{label}.identity") + identity_copy = dict(identity) + locator = identity_copy.get("path") + if not isinstance(locator, str): + raise BenchmarkError(f"{label}.identity.path must be a string") + if Path(locator).is_absolute(): + path = artifact_io.verify_external_artifact(identity_copy) + else: + workspace_value = record.get("workspace") + if not isinstance(workspace_value, str): + raise BenchmarkError( + f"{label}.workspace is required for a workspace-relative identity" + ) + path = artifact_io.verify_artifact(Path(workspace_value), identity_copy) + return path, identity_copy + + +def _external_identity( + value: Any, + label: str, + *, + schema_version: str, + kinds: Sequence[str], +) -> tuple[Path, dict[str, Any]]: + """Verify one exact portable identity and its current external bytes.""" + if not isinstance(value, Mapping) or set(value) != IDENTITY_KEYS: + raise BenchmarkError(f"{label} must be an exact portable artifact identity") + identity = dict(value) + if identity.get("schema_version") != schema_version: + raise BenchmarkError(f"{label} must use schema {schema_version}") + if identity.get("kind") not in kinds: + raise BenchmarkError(f"{label} kind must be one of {list(kinds)}") + try: + path = artifact_io.verify_external_artifact(identity) + except (OSError, ValueError) as exc: + raise BenchmarkError(f"{label} identity is not current: {exc}") from exc + return path, identity + + +def _environment( + request: Mapping[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + _path, identity = _external_identity( + request.get("environment"), + "environment", + schema_version=ENVIRONMENT_SCHEMA_VERSION, + kinds=(ENVIRONMENT_KIND,), + ) + try: + value = artifact_io.read_verified_external_json(identity) + except (OSError, ValueError) as exc: + raise BenchmarkError(f"environment identity is not current: {exc}") from exc + if not isinstance(value, dict): + raise BenchmarkError("environment must contain one JSON object") + version = value.get("schema_version") + if version != ENVIRONMENT_SCHEMA_VERSION: + raise BenchmarkError( + f"environment schema_version must be exactly {ENVIRONMENT_SCHEMA_VERSION!r}; " + f"refusing unknown environment version {version!r}" + ) + # Required subset only: unknown additive keys are ignored by design. + if ( + value.get("kind") != ENVIRONMENT_KIND + or value.get("mode") != "live" + or not isinstance(value.get("installation"), Mapping) + or not isinstance(value.get("pynvc"), Mapping) + ): + raise BenchmarkError( + f"environment must contain live {ENVIRONMENT_KIND} " + f"{ENVIRONMENT_SCHEMA_VERSION} installation and pynvc facts" + ) + if value.get("requested_runtime") not in ENVIRONMENT_RUNTIMES: + raise BenchmarkError( + "environment requested_runtime must be exactly one of " + f"{list(ENVIRONMENT_RUNTIMES)}" + ) + selected_gpu = value.get("selected_gpu") + if ( + isinstance(selected_gpu, bool) + or not isinstance(selected_gpu, int) + or selected_gpu < 0 + ): + raise BenchmarkError("environment selected_gpu must be a non-negative integer") + return identity, value + + +def _is_text(value: Any) -> bool: + return isinstance(value, str) and bool(value) and value == value.strip() + + +def _is_absolute(value: Any) -> bool: + return _is_text(value) and Path(str(value)).is_absolute() + + +def _is_version(value: Any) -> bool: + """Accept one non-advisory version token.""" + return _is_text(value) and str(value)[0].isdigit() + + +def _numpy_requirement_satisfied(value: Any) -> bool: + """Implement the released NumPy >=1.24 runtime floor without extra imports.""" + if not isinstance(value, str): + return False + match = re.fullmatch( + r"(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(?:\.(0|[1-9][0-9]*))?" + r"(?:\.post[0-9]+)?(?:\+[0-9A-Za-z.-]+)?", + value, + ) + return match is not None and tuple(int(part) for part in match.groups()[:2]) >= (1, 24) + + +def _is_hash_identity(value: Any) -> bool: + return ( + isinstance(value, Mapping) + and _is_absolute(value.get("path")) + and _is_text(value.get("sha256")) + ) + + +def _is_tool(record: Any, name: str) -> bool: + """One normalized build tool: {path, sha256} (+ generator name). + + 1.2 binds a build tool by path and content hash and publishes no tool + version, so a version is honoured when present and never required. + """ + return ( + _is_hash_identity(record) + and (record.get("version") is None or _is_version(record.get("version"))) + and (name != "generator" or _is_text(record.get("name"))) + ) + + +def _native_reasons(native: Mapping[str, Any]) -> list[str]: + """Structurally validate the required native subset at planning time. + + Additive keys are ignored and no exact-set equality is used. A malformed + native surface is blocked here rather than at execute-time authentication, + so it can never consume a run slot or re-couple the two surfaces by timing. + """ + reasons: list[str] = [] + if native.get("installed") is not True: + reasons.append("live environment does not report the native SDK installed") + package = native.get("package") + if ( + not isinstance(package, Mapping) + or package.get("name") != NATIVE_PACKAGE + or package.get("status") != "installed" + or not _is_version(package.get("version")) + ): + reasons.append( + f"live environment does not report an installed {NATIVE_PACKAGE} package" + ) + if not _is_absolute(native.get("sdk_root")): + reasons.append("live environment carries no canonical native sdk_root") + cuda = native.get("cuda") + if ( + not isinstance(cuda, Mapping) + or cuda.get("status") != "installed" + or not _is_version(cuda.get("version")) + ): + reasons.append( + "live environment does not report an installed CUDA toolkit version" + ) + tools = native.get("tools") + incomplete = sorted( + name + for name in REQUIRED_TOOLS + if not _is_tool(tools.get(name) if isinstance(tools, Mapping) else None, name) + ) + if incomplete: + reasons.append( + "live environment build tool identities are incomplete: " + + ", ".join(incomplete) + ) + return reasons + + +def _pynvc_encode_dependency_gaps(pynvc: Mapping[str, Any]) -> list[str]: + """Return missing or incompatible dependencies for the official encode sample.""" + dependencies = pynvc.get("dependencies") + dependencies = dependencies if isinstance(dependencies, Mapping) else {} + incomplete = [] + for name in ("numpy", "pycuda", "torch"): + record = dependencies.get(name) + if ( + not isinstance(record, Mapping) + or record.get("status") != "installed" + or record.get("ready") is not True + ): + incomplete.append(name) + torch = dependencies.get("torch") + numpy = dependencies.get("numpy") + if ( + isinstance(numpy, Mapping) + and not _numpy_requirement_satisfied(numpy.get("version")) + and "numpy" not in incomplete + ): + incomplete.append("numpy") + pycuda = dependencies.get("pycuda") + if ( + isinstance(pycuda, Mapping) + and pycuda.get("version") != "2026.1" + and "pycuda" not in incomplete + ): + incomplete.append("pycuda") + if ( + isinstance(torch, Mapping) + and ( + torch.get("version") != "2.9.1+cu130" + or torch.get("cuda_build") != "13.0" + or torch.get("cuda_available") is not True + ) + and "torch" not in incomplete + ): + incomplete.append("torch") + return sorted(incomplete) + + +def _pynvc_reasons(pynvc: Mapping[str, Any], operation: str) -> list[str]: + """Structurally validate the required Py subset at planning time. + + Schema 1.2 authenticates the extension by wheel-member identity and does + not publish which file the interpreter actually loaded, so + `extension.loaded_path` is additive: when a document carries it, it must + equal the authenticated `extension.path`, and it is never inferred when + absent. The imported `module.version` must equal the distribution version; + both comparisons fail closed. + """ + reasons: list[str] = [] + version = pynvc.get("version") + if ( + pynvc.get("installed") is not True + or version != PYNVC_REQUIRED_VERSION + or not _is_hash_identity(pynvc.get("interpreter_identity")) + or not _is_absolute(pynvc.get("interpreter")) + or not _is_absolute(pynvc.get("sys_prefix")) + ): + reasons.append( + "live environment does not report installed PyNvVideoCodec " + f"{PYNVC_REQUIRED_VERSION} with an interpreter identity" + ) + extension = pynvc.get("extension") + loaded_path = extension.get("loaded_path") if isinstance(extension, Mapping) else None + if not _is_hash_identity(extension) or ( + loaded_path is not None + and (not _is_absolute(loaded_path) or loaded_path != extension.get("path")) + ): + reasons.append( + "live environment does not carry one authenticated PyNvVideoCodec " + "extension identity" + ) + module = pynvc.get("module") + imported_version = module.get("version") if isinstance(module, Mapping) else None + if ( + not isinstance(module, Mapping) + or not _is_absolute(module.get("path")) + or imported_version != version + ): + reasons.append( + "live environment imported PyNvVideoCodec module version " + f"{imported_version!r} differs from distribution version {version!r}" + ) + if operation == "encode": + incomplete = _pynvc_encode_dependency_gaps(pynvc) + if incomplete: + reasons.append( + "Py encode_perf dependencies are incomplete: " + + ", ".join(sorted(incomplete)) + + ". A jetson-video-setup pynvc-smoke environment deliberately carries no" + " Torch. Remedy: provision a NEW full-samples PyNvVideoCodec venv through" + " jetson-video-setup (plan_install.py --profile full-samples with a new" + " --venv). A smoke venv is never installed into or upgraded in place." + ) + return reasons + + +def _stage_environment( + workspace: Path, environment: Mapping[str, Any] +) -> tuple[Path, dict[str, Any]]: + """Stage live facts only because official provenance consumes workspace identities.""" + root = artifact_io.create_private_workspace(workspace / "environment-facts") + identity = artifact_io.write_fresh_json( + root, root / "nvcodec-environment.json", dict(environment) + ) + return root, identity + + +def _stage_local_binding( + workspace: Path, binding: Mapping[str, Any] +) -> tuple[Path, dict[str, Any]]: + """Stage controller-owned local authority without claiming setup provenance.""" + root = artifact_io.create_private_workspace(workspace / "local-runtime-binding") + identity = artifact_io.write_fresh_json( + root, root / "runtime-binding.json", dict(binding) + ) + return root, identity + + +def _runtime_authority( + request: Mapping[str, Any], + *, + operation: str, + selected_gpu: int, + workspace: Path, + runner: Any, + timeout_seconds: float, +) -> dict[str, Any]: + """Use supplied setup evidence or discover only explicitly selected local surfaces.""" + if "environment" in request: + identity, value = _environment(request) + environment_gpu = int(value["selected_gpu"]) + if environment_gpu != selected_gpu: + raise BenchmarkError( + f"request/recipe gpu {selected_gpu} differs from environment " + f"selected_gpu {environment_gpu}" + ) + return { + "source": "setup_environment", + "selected_gpu": environment_gpu, + "value": value, + "external_identity": identity, + "diagnostics": {}, + } + requested = request.get("surface", "auto") + if not isinstance(requested, str) or requested not in (*SURFACES, "auto", "both"): + raise BenchmarkError("surface must be native, pynvc, auto, or both") + targets = [requested] if requested in SURFACES else list(SURFACES) + surfaces: dict[str, Any] = {} + diagnostics: dict[str, str] = {} + evaluations: dict[str, dict[str, Any]] = { + surface: { + "status": ( + "pending" if surface in targets else "not_inspected" + ) + } + for surface in SURFACES + } + installation: dict[str, Any] = { + "native_sdk": {}, + "native_build_prerequisites": {}, + "cuda_toolkit": {}, + "build_tool_identities": {}, + "python": {}, + } + pynvc_evidence: dict[str, Any] = {} + sample = "AppEncPerf" if operation == "encode" else "AppDecPerf" + for surface in targets: + try: + if surface == "native": + discovery = provenance.discover_local_native_surface( + workspace=workspace, + samples=(sample,), + runner=runner, + timeout_seconds=timeout_seconds, + ) + else: + interpreter = request.get("pynvc_interpreter") + if interpreter is None: + reason = ( + "pynvc_interpreter is required for local PyNvVideoCodec " + "benchmarking; provide one exact absolute interpreter path" + ) + diagnostics[surface] = reason + evaluations[surface] = { + "status": "not_evaluated", + "reason": reason, + "next_action": "provide pynvc_interpreter and retry", + } + continue + discovery = provenance.discover_local_pynvc_surface( + interpreter=interpreter, + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + ) + surfaces[surface] = dict(discovery["surface"]) + evidence = _mapping(discovery.get("evidence"), "local discovery evidence") + if surface == "native": + installation.update(dict(evidence)) + else: + installation["python"] = dict( + _mapping(evidence.get("python"), "local Python evidence") + ) + pynvc_evidence = dict( + _mapping(evidence.get("pynvc"), "local PyNvVideoCodec evidence") + ) + evaluations[surface] = {"status": "eligible"} + except (OSError, ValueError) as exc: + diagnostics[surface] = f"{type(exc).__name__}: {exc}" + evaluations[surface] = { + "status": "failed", + "reason": diagnostics[surface], + } + return { + "source": "local_discovery", + "selected_gpu": selected_gpu, + "value": { + "schema_version": LOCAL_BINDING_SCHEMA_VERSION, + "kind": LOCAL_BINDING_KIND, + "mode": "live", + "source": "benchmark-local-discovery", + "requested_runtime": ( + requested if requested in SURFACES else "both" + ), + "selected_gpu": selected_gpu, + "installation": installation, + "pynvc": pynvc_evidence, + }, + "normalized_surfaces": surfaces, + "diagnostics": diagnostics, + "evaluations": evaluations, + } + + +def _runtime_result_fields(authority: Mapping[str, Any]) -> dict[str, Any]: + """Preserve legacy environment output and label local binding separately.""" + if authority.get("source") == "setup_environment": + return {"environment": dict(authority["external_identity"])} + _mapping(authority.get("value"), "local runtime binding") + normalized = _mapping( + authority.get("normalized_surfaces", {}), "normalized local surfaces" + ) + result: dict[str, Any] = { + "runtime_authority": { + "source": "local_discovery", + "selected_gpu": authority["selected_gpu"], + "surfaces": sorted(normalized), + "diagnostics": dict(authority.get("diagnostics", {})), + "evaluations": dict(authority.get("evaluations", {})), + } + } + identity = authority.get("staged_identity") + if isinstance(identity, Mapping): + result["runtime_authority"]["binding"] = dict(identity) + result["runtime_authority"]["binding_workspace"] = str( + authority["staged_workspace"] + ) + return result + + +def _surface_plan( + request: Mapping[str, Any], + environment: Mapping[str, Any], + operation: str, + diagnostics: Mapping[str, str] | None = None, + normalized_surfaces: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + requested = request.get("surface", "auto") + if not isinstance(requested, str): + raise BenchmarkError("surface must be a string") + if requested not in (*SURFACES, "auto", "both"): + raise BenchmarkError("surface must be native, pynvc, auto, or both") + inspected = {requested} if requested in SURFACES else set(SURFACES) + failures = diagnostics or {} + eligibility: dict[str, dict[str, Any]] = {} + for surface in SURFACES: + if surface not in inspected: + reasons = [f"{surface} was not inspected for explicit {requested} routing"] + elif surface in failures: + reasons = [failures[surface]] + elif surface == "native": + value = ( + normalized_surfaces.get(surface, {}) + if normalized_surfaces is not None + else (runtime.environment_surface(environment, surface) or {}) + ) + reasons = _native_reasons(value if isinstance(value, Mapping) else {}) + else: + value = ( + normalized_surfaces.get(surface, {}) + if normalized_surfaces is not None + else (runtime.environment_surface(environment, surface) or {}) + ) + reasons = _pynvc_reasons( + value if isinstance(value, Mapping) else {}, operation + ) + eligibility[surface] = {"eligible": not reasons, "reasons": reasons} + try: + return surface_router.build_surface_plan(requested, eligibility) + except ValueError as exc: + raise BenchmarkError(str(exc)) from exc + + +def _runtime_setup_dependencies( + request: Mapping[str, Any], + authority: Mapping[str, Any], + environment: Mapping[str, Any], + operation: str, + plan: Mapping[str, Any], +) -> list[dict[str, Any]]: + """Describe only inspected, ineligible SDK surfaces as setup handoffs.""" + requested = request.get("surface", "auto") + inspected = {requested} if requested in SURFACES else set(SURFACES) + eligible = set(plan.get("eligible_surfaces", [])) + diagnostics = authority.get("diagnostics", {}) + diagnostics = diagnostics if isinstance(diagnostics, Mapping) else {} + evaluations = authority.get("evaluations", {}) + evaluations = evaluations if isinstance(evaluations, Mapping) else {} + normalized = authority.get("normalized_surfaces", {}) + normalized = normalized if isinstance(normalized, Mapping) else {} + dependencies: list[dict[str, Any]] = [] + for surface in SURFACES: + if surface not in inspected or surface in eligible: + continue + evaluation = evaluations.get(surface) + if ( + isinstance(evaluation, Mapping) + and evaluation.get("status") == "not_evaluated" + ): + # An omitted exact Py interpreter is a selector request, not evidence + # that installation or repair is required. + continue + if surface in diagnostics: + reasons = [str(diagnostics[surface])] + value: Mapping[str, Any] = {} + else: + candidate = ( + normalized.get(surface, {}) + if authority.get("source") == "local_discovery" + else (runtime.environment_surface(environment, surface) or {}) + ) + value = candidate if isinstance(candidate, Mapping) else {} + reasons = ( + _native_reasons(value) + if surface == "native" + else _pynvc_reasons(value, operation) + ) + if not reasons: + continue + needed_for = f"{surface} SDK installation or repair" + dependency = _setup_dependency( + needed_for=needed_for, + reason="; ".join(reasons), + ) + dependency["surface"] = surface + if ( + surface == "pynvc" + and operation == "encode" + and value.get("installed") is True + ): + gaps = _pynvc_encode_dependency_gaps(value) + if gaps: + dependency["required_profile"] = "full-samples" + dependency["missing_dependencies"] = gaps + dependency["needed_for"] = ( + "PyNvVideoCodec full-samples environment provisioning" + ) + dependency["next_action"] = ( + "use jetson-video-setup to provision a new full-samples " + "PyNvVideoCodec environment, then retry this stage" + if dependency["installed"] + else "install jetson-video-setup, provision a new full-samples " + "PyNvVideoCodec environment, then retry this stage" + ) + dependencies.append(dependency) + return dependencies + + +def _dependency_result_fields( + dependencies: Sequence[Mapping[str, Any]], +) -> dict[str, Any]: + """Expose one convenient primary dependency plus the complete independent list.""" + if not dependencies: + return {} + values = [dict(value) for value in dependencies] + return {"dependency": values[0], "dependencies": values} + + +def _mode(request: Mapping[str, Any]) -> str: + value = request.get("mode", "execute") + if value not in {"dry_run", "execute"}: + raise BenchmarkError("mode must be exactly 'dry_run' or 'execute'") + return str(value) + + +def _operation(request: Mapping[str, Any], route: str) -> str: + default = route if route in {"encode", "decode"} else "encode" + value = request.get("operation", default) + if value not in {"encode", "decode"}: + raise BenchmarkError("operation must be exactly 'encode' or 'decode'") + return str(value) + + +def _process_model(request: Mapping[str, Any]) -> str: + value = request.get("process_model", "thread") + if value not in {"thread", "process"}: + raise BenchmarkError("process_model must be exactly 'thread' or 'process'") + return str(value) + + +def _input(request: Mapping[str, Any], operation: str) -> dict[str, Any]: + field = "input" if operation == "encode" else "encoded_artifact" + record = _mapping(request.get(field), field) + path, identity = _artifact(record, field) + metadata = _mapping(record.get("metadata"), f"{field}.metadata") + width = _integer(metadata.get("width"), f"{field}.metadata.width") + height = _integer(metadata.get("height"), f"{field}.metadata.height") + frames = _integer(metadata.get("frames"), f"{field}.metadata.frames") + fps = _number(metadata.get("fps"), f"{field}.metadata.fps") + codec = _string(metadata.get("codec"), f"{field}.metadata.codec").lower() + allowed_codecs = ENCODE_CODECS if operation == "encode" else DECODE_CODECS + if codec not in allowed_codecs: + raise BenchmarkError(f"unsupported codec for {operation}: {codec}") + fmt = _string(metadata.get("format"), f"{field}.metadata.format").upper() + if fmt not in NATIVE_FORMATS: + raise BenchmarkError(f"unsupported raw input format: {fmt}") + if "source_url" not in record: + raise BenchmarkError(f"{field}.source_url is required") + raw_source_url = record.get("source_url") + source_url = ( + None + if raw_source_url is None + else _string(raw_source_url, f"{field}.source_url") + ) + if source_url is not None: + try: + parsed = urlparse(source_url) + port = parsed.port + valid_source_url = bool( + parsed.scheme in {"http", "https"} + and parsed.netloc + and parsed.hostname + and parsed.username is None + and parsed.password is None + and "\\" not in source_url + and not any( + character.isspace() + or ord(character) < 0x20 + or ord(character) == 0x7F + for character in source_url + ) + and (port is None or 1 <= port <= 65535) + ) + except (UnicodeError, ValueError): + valid_source_url = False + if not valid_source_url: + raise BenchmarkError( + f"{field}.source_url must be null or one exact HTTP(S) URL" + ) + license_value = _string(record.get("license"), f"{field}.license") + attribution = _string(record.get("attribution"), f"{field}.attribution") + if operation == "encode": + factor = PIXEL_FORMATS.get(CANONICAL_PIXEL_FORMATS[fmt]) + if factor is None: + raise BenchmarkError(f"unsupported raw input format: {fmt}") + numerator, denominator = factor + pixels = width * height + if pixels * numerator % denominator: + raise BenchmarkError("raw input geometry does not form whole frames") + expected_size = pixels * numerator // denominator * frames + if identity.get("size_bytes") != expected_size: + raise BenchmarkError( + f"raw input has {identity.get('size_bytes')} bytes; expected {expected_size}" + ) + return { + "path": str(path), + "identity": identity, + "width": width, + "height": height, + "frames": frames, + "fps": fps, + "codec": codec, + "format": fmt, + "source_url": source_url, + "license": license_value, + "attribution": attribution, + } + + +def _recipe( + request: Mapping[str, Any], + *, + workspace: Path, + timeout_seconds: float, + validation_index: int, +) -> tuple[dict[str, Any], dict[str, Any]]: + if not _RECIPE_CLI.exists(): + skill = "jetson-video-recipe" + needed_for = "encode recipe replay validation" + raise DependencyRequired( + { + "skill": skill, + "installed": False, + "public_cli": str(_RECIPE_CLI), + "needed_for": needed_for, + "reason": ( + f"I can run {needed_for}, but it requires the {skill} skill, " + f"which is not installed. Install {skill} and retry this stage." + ), + "next_action": ( + f"install {skill} beside this skill and retry the unchanged " + "benchmark request" + ), + } + ) + _recipe_path, identity = _external_identity( + request.get("recipe"), + "recipe", + schema_version="2.0", + kinds=("nvcodec-recipe",), + ) + try: + value = artifact_io.read_verified_external_json(identity) + except (OSError, ValueError) as exc: + raise BenchmarkError(f"recipe identity is not current: {exc}") from exc + if not isinstance(value, dict): + raise BenchmarkError("recipe identity must contain one schema-2 recipe object") + try: + recipe_root = _RECIPE_CLI.parents[2].resolve(strict=True) + recipe_details = _RECIPE_CLI.lstat() + recipe_cli = _RECIPE_CLI.resolve(strict=True) + if stat.S_ISLNK(recipe_details.st_mode) or not stat.S_ISREG(recipe_details.st_mode): + raise BenchmarkError("recipe public CLI must be a regular file") + if recipe_cli != recipe_root / "scripts" / "recipes" / "recipe_model.py": + raise BenchmarkError("recipe public CLI path is not canonical") + python = _SYSTEM_PYTHON.resolve(strict=True) + validation_workspace = artifact_io.create_private_workspace( + workspace / f"recipe-validation-{validation_index:04d}" + ) + validation_runner = command_runner.CommandRunner(validation_workspace) + validation_identity = artifact_io.write_fresh_json( + validation_workspace, + validation_workspace / "recipe.json", + value, + ) + validation_recipe = artifact_io.verify_artifact( + validation_workspace, validation_identity + ) + result = validation_runner.run( + [ + str(python), + "-I", + str(recipe_cli), + "validate", + "--recipe", + str(validation_recipe), + ], + cwd=validation_workspace, + env=_COMMAND_ENV, + timeout_seconds=timeout_seconds, + stage="validate-recipe", + phase="verify", + ) + stdout = artifact_io.read_verified_text( + validation_workspace, result["stdout"] + ) + stderr = artifact_io.read_verified_text( + validation_workspace, result["stderr"] + ) + if ( + result.get("timed_out") + or result.get("launch_error") + or result.get("exit_code") != 0 + ): + detail = stderr.strip() or stdout.strip() + raise BenchmarkError( + f"recipe validator failed with exit {result.get('exit_code')}: " + f"{detail[-500:]}" + ) + outcome = artifact_io.strict_json_loads(stdout) + if outcome != {"valid": True, "errors": []}: + raise BenchmarkError("recipe validator returned an invalid success result") + artifact_io.verify_artifact(validation_workspace, validation_identity) + except (OSError, ValueError, TypeError) as exc: + raise BenchmarkError(f"invalid benchmark recipe: {exc}") from exc + try: + artifact_io.verify_external_artifact(identity) + except (OSError, ValueError) as exc: + raise BenchmarkError(f"recipe identity is not current: {exc}") from exc + return identity, value + + +def _bind_recipe_workload( + data: Mapping[str, Any], + recipe: Mapping[str, Any], + request: Mapping[str, Any], + environment_gpu: int, +) -> None: + """Reject any request that would silently rebind a validated recipe.""" + intent = _mapping(recipe.get("encoder_intent"), "recipe.encoder_intent") + for field in ("codec", "width", "height", "format", "fps"): + if data[field] != intent.get(field): + raise BenchmarkError( + f"recipe {field} {intent.get(field)!r} differs from " + f"input metadata {data[field]!r}" + ) + recipe_frames = intent.get("frame_count") + if recipe_frames is not None and recipe_frames != data["frames"]: + raise BenchmarkError( + f"recipe frame_count {recipe_frames!r} differs from " + f"input metadata frames {data['frames']!r}" + ) + recipe_gpu = _integer(intent.get("gpu"), "recipe.encoder_intent.gpu", minimum=0) + requested_gpu = _integer( + request.get("gpu", recipe_gpu), + "gpu", + minimum=0, + ) + if requested_gpu != recipe_gpu: + raise BenchmarkError( + f"request gpu {requested_gpu} differs from recipe gpu {recipe_gpu}" + ) + if recipe_gpu != environment_gpu: + raise BenchmarkError( + f"recipe gpu {recipe_gpu} differs from environment selected_gpu " + f"{environment_gpu}" + ) + + +def _authentication( + operation: str, + selected: Sequence[str], + *, + environment_workspace: Path, + environment_identity: Mapping[str, Any], + workspace: Path, + runner: Any, + timeout_seconds: float, +) -> dict[str, dict[str, Any]]: + """Authenticate every selected leaf before any benchmark leaf launches.""" + authenticated: dict[str, dict[str, Any]] = {} + if "native" in selected: + sample = "AppEncPerf" if operation == "encode" else "AppDecPerf" + try: + value = sample_provenance.authenticate_native_samples( + environment_workspace=environment_workspace, + environment_identity=environment_identity, + workspace=workspace, + samples=(sample,), + runner=runner, + timeout_seconds=timeout_seconds, + report_path=workspace / f"native-{operation}-provenance.json", + ) + authenticated["native"] = {"status": "authenticated", "value": value} + except (OSError, ValueError) as exc: + authenticated["native"] = { + "status": "failed", + "reason": f"{type(exc).__name__}: {exc}", + } + if "pynvc" in selected: + sample = ( + "samples/advanced/encode_perf.py" + if operation == "encode" + else "samples/advanced/decode_perf.py" + ) + try: + value = pynvc_sample_provenance.authenticate_pynvc_samples( + environment_workspace=environment_workspace, + environment_identity=environment_identity, + workspace=workspace, + samples=(sample,), + runner=runner, + timeout_seconds=timeout_seconds, + report_path=workspace / f"pynvc-{operation}-provenance.json", + ) + authenticated["pynvc"] = {"status": "authenticated", "value": value} + except (OSError, ValueError) as exc: + authenticated["pynvc"] = { + "status": "failed", + "reason": f"{type(exc).__name__}: {exc}", + } + return authenticated + + +def _token(authentication: Any, operation: str, surface: str) -> Any: + if surface == "native": + return authentication.token("AppEncPerf" if operation == "encode" else "AppDecPerf") + return authentication.token( + "samples/advanced/encode_perf.py" + if operation == "encode" + else "samples/advanced/decode_perf.py" + ) + + +def _token_evidence(token: Any) -> dict[str, Any]: + return { + "surface": token.surface, + "sample": token.sample, + "launcher": list(token.launcher), + "protected_files": [dict(item.identity) for item in token.protected_files], + "runtime_libraries": [dict(item.identity) for item in token.runtime_libraries], + } + + +def _run_sample( + token: Any, + arguments: Sequence[str], + *, + workspace: Path, + runner: Any, + timeout_seconds: float, + stage: str, + phase: str, +) -> dict[str, Any]: + if "-loop" in arguments: + raise BenchmarkError("benchmark repetition must never use -loop") + try: + return sample_provenance.run_authenticated( + token, + tuple(arguments), + workspace=workspace, + runner=runner, + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=stage, + phase=phase, + ) + except (ValueError, OSError) as exc: + raise BenchmarkError(str(exc)) from exc + + +def _successful( + result: Mapping[str, Any], + label: str, + *, + complete_text: str | None = None, +) -> None: + if ( + result.get("exit_code") != 0 + or result.get("timed_out") is not False + or result.get("launch_error") is not None + ): + raise BenchmarkError(f"{label} command did not complete successfully") + combined = ( + complete_text + if complete_text is not None + else f"{result.get('stdout_tail', '')}\n{result.get('stderr_tail', '')}" + ) + if _FAILURE_PATTERN.search(combined): + raise BenchmarkError(f"{label} command reported a failure marker") + + +def _command_text(workspace: Path, result: Mapping[str, Any]) -> str: + """Read complete captured logs when command identities are available.""" + stdout = result.get("stdout") + stderr = result.get("stderr") + if stdout is None and stderr is None: + return f"{result.get('stdout_tail', '')}\n{result.get('stderr_tail', '')}" + if not isinstance(stdout, Mapping) or not isinstance(stderr, Mapping): + raise BenchmarkError("command result has incomplete log identities") + try: + return ( + artifact_io.read_verified_text(workspace, stdout) + + "\n" + + artifact_io.read_verified_text(workspace, stderr) + ) + except ValueError as exc: + raise BenchmarkError(f"command log identity is invalid: {exc}") from exc + + +def _native_help( + token: Any, + *, + workspace: Path, + runner: Any, + timeout_seconds: float, +) -> tuple[set[str], list[dict[str, Any]]]: + evidence: list[dict[str, Any]] = [] + advertised: set[str] = set() + for option, name in (("-h", "h"), ("-A", "all")): + result = _run_sample( + token, + (option,), + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + stage=f"benchmark-help-{name}", + phase="probe", + ) + _successful(result, f"native help {option}") + evidence.append(result) + advertised.update(_HELP_OPTION.findall(_command_text(workspace, result))) + return advertised, evidence + + +def _option_names(arguments: Sequence[str]) -> set[str]: + return { + token for token in arguments if token.startswith("-") and token != "-" + } + + +def _projection_plan( + operation: str, + selected_surfaces: Sequence[str], + recipe: Mapping[str, Any] | None, +) -> tuple[list[str], dict[str, dict[str, Any]]]: + """Separate exact recipe surfaces from independently blocked projections.""" + if operation == "decode": + return list(selected_surfaces), {} + if recipe is None: + raise BenchmarkError("encode benchmark recipe is absent") + projections = _mapping(recipe.get("projections"), "recipe.projections") + losses = _mapping(recipe.get("projection_losses"), "recipe.projection_losses") + selected: list[str] = [] + blocked: dict[str, dict[str, Any]] = {} + for surface in selected_surfaces: + projection = _mapping(projections.get(surface), f"recipe.projections.{surface}") + if projection.get("status") == "exact": + selected.append(surface) + continue + raw_losses = losses.get(surface) + projection_losses = ( + [dict(_mapping(item, f"recipe.projection_losses.{surface}")) for item in raw_losses] + if isinstance(raw_losses, list) + else [] + ) + blocked[surface] = { + "status": "blocked", + "surface": surface, + "reason": f"recipe has no exact {surface} projection", + "projection_losses": projection_losses, + } + return selected, blocked + + +def _frame_plan( + operation: str, + data: Mapping[str, Any], + selected_surfaces: Sequence[str], + request: Mapping[str, Any], +) -> dict[str, Any]: + """Return the per-worker frame plan honoring the PyNv 2.1 perf encode cap.""" + source = int(data["frames"]) + capped = ( + operation == "encode" + and "pynvc" in selected_surfaces + and source > PYNVC_PERF_MAX_FRAMES_PER_WORKER + ) + if capped and request.get("require_source_frames") is True: + raise BenchmarkError( + "request requires every source frame, but the " + f"{_PYNVC_CAP_AUTHORITY} caps performance encode at " + f"{PYNVC_PERF_MAX_FRAMES_PER_WORKER} frames per worker; use a " + "native-only surface or an input at or below the limit" + ) + return { + "source_frames_per_worker": source, + "effective_frames_per_worker": ( + PYNVC_PERF_MAX_FRAMES_PER_WORKER if capped else source + ), + "cap_applied": capped, + "cap_limit_frames_per_worker": PYNVC_PERF_MAX_FRAMES_PER_WORKER, + "cap_authority": _PYNVC_CAP_AUTHORITY, + } + + +def _effective_data( + data: Mapping[str, Any], frame_plan: Mapping[str, Any] +) -> Mapping[str, Any]: + effective = int(frame_plan["effective_frames_per_worker"]) + if effective == int(data["frames"]): + return data + return {**data, "frames": effective} + + +def _frame_accounting( + frame_plan: Mapping[str, Any], workers: int +) -> dict[str, Any]: + return { + **frame_plan, + "workers": workers, + "expected_aggregate_frames": ( + int(frame_plan["effective_frames_per_worker"]) * workers + ), + } + + +def _native_arguments( + operation: str, + data: Mapping[str, Any], + recipe: Mapping[str, Any] | None, + request: Mapping[str, Any], +) -> list[str]: + workers = _integer(request.get("workers", 1), "workers") + if operation == "decode": + gpu = _integer(request.get("gpu", 0), "gpu", minimum=0) + arguments = ["-i", data["path"], "-gpu", str(gpu), "-thread", str(workers)] + if request.get("single_context") is True: + arguments.append("-single") + if request.get("host_memory") is True: + arguments.append("-host") + return arguments + assert recipe is not None + intent = _mapping(recipe.get("encoder_intent"), "recipe.encoder_intent") + gpu = _integer(intent.get("gpu"), "recipe.encoder_intent.gpu", minimum=0) + projection = _mapping(recipe["projections"].get("native"), "recipe.projections.native") + if projection.get("status") != "exact": + raise BenchmarkError("recipe has no exact native projection") + raw_options = projection.get("cli_options") + if not isinstance(raw_options, list) or any(not isinstance(item, str) for item in raw_options): + raise BenchmarkError("native recipe cli_options must be a list of strings") + skip_with_value = {"-s", "-if", "-gpu", "-codec"} + controls: list[str] = [] + index = 0 + while index < len(raw_options): + option = raw_options[index] + if not option.startswith("-"): + raise BenchmarkError("native recipe projection contains a value without an option") + has_value = index + 1 < len(raw_options) and not raw_options[index + 1].startswith("-") + value = raw_options[index + 1] if has_value else None + if option in skip_with_value and value is None: + raise BenchmarkError(f"native recipe projection option {option} lacks its value") + if option not in skip_with_value: + controls.append(option) + if value is not None: + controls.append(value) + index += 2 if has_value else 1 + fmt = NATIVE_FORMATS.get(str(data["format"])) + if fmt is None: + raise BenchmarkError(f"native AppEncPerf does not map format {data['format']}") + return [ + "-i", data["path"], "-s", f"{data['width']}x{data['height']}", + "-if", fmt, "-gpu", str(gpu), "-frame", str(data["frames"]), + "-thread", str(workers), "-codec", str(data["codec"]), *controls, + ] + + +def _pynvc_arguments( + operation: str, + data: Mapping[str, Any], + recipe: Mapping[str, Any] | None, + request: Mapping[str, Any], + *, + workspace: Path, + variant: str, +) -> list[str]: + workers = _integer(request.get("workers", 1), "workers") + process_model = _process_model(request) + if operation == "decode": + gpu = _integer(request.get("gpu", 0), "gpu", minimum=0) + return [ + "-i", data["path"], "-n", str(workers), "-m", str(process_model), + "-g", str(gpu), "-f", str(data["frames"]), + ] + assert recipe is not None + intent = _mapping(recipe.get("encoder_intent"), "recipe.encoder_intent") + gpu = _integer(intent.get("gpu"), "recipe.encoder_intent.gpu", minimum=0) + projection = _mapping(recipe["projections"].get("pynvc"), "recipe.projections.pynvc") + if projection.get("status") != "exact": + raise BenchmarkError("recipe has no exact pynvc projection") + config = _mapping(projection.get("config"), "recipe.projections.pynvc.config") + safe_name = re.sub(r"[^A-Za-z0-9_.-]", "-", variant)[:48] or "benchmark" + config_path = workspace / f"pynvc-{safe_name}-config.json" + artifact_io.write_fresh_bytes( + workspace, + config_path, + artifact_io.canonical_json_bytes(dict(config)), + schema_version="1", + kind="pynvc-encoder-config", + ) + return [ + "-m", str(process_model), "-i", data["path"], + "-s", f"{data['width']}x{data['height']}", "-if", str(data["format"]), + "-n", str(workers), "-f", str(data["frames"]), "-g", str(gpu), + "-c", str(data["codec"]), "-json", str(config_path), + ] + + +def _pynvc_config_identity( + arguments: Sequence[str], workspace: Path +) -> dict[str, Any]: + """Bind the one generated encoder config to the exact -json operand.""" + if arguments.count("-json") != 1: + raise BenchmarkError("PyNvVideoCodec encode argv must contain one -json option") + index = arguments.index("-json") + if index + 1 == len(arguments): + raise BenchmarkError("PyNvVideoCodec encode -json option lacks its value") + return artifact_io.snapshot_artifact( + workspace, + Path(arguments[index + 1]), + schema_version="1", + kind="pynvc-encoder-config", + ) + + +def _generated_config_result( + workspace: Path, identity: Mapping[str, Any] | None +) -> dict[str, Any]: + """Return config evidence only after its generated bytes remain current.""" + if identity is None: + return {} + artifact_io.verify_artifact(workspace, identity) + return {"generated_config": dict(identity)} + + +def _values(pattern: str, text: str, label: str) -> list[float]: + raw = re.findall(pattern, text, flags=re.MULTILINE) + if not raw: + raise BenchmarkError(f"{label} marker is absent") + values = [float(item) for item in raw] + if any(not math.isfinite(item) or item <= 0 for item in values): + raise BenchmarkError(f"{label} marker is non-finite or non-positive") + if any(not math.isclose(item, values[-1], rel_tol=1e-9, abs_tol=1e-9) for item in values[:-1]): + raise BenchmarkError(f"{label} markers contradict one another") + return values + + +def _quantized_value( + pattern: str, + text: str, + label: str, +) -> tuple[Decimal, Decimal]: + """Return a numeric marker and half of its displayed rounding interval.""" + raw = re.findall(pattern, text, flags=re.MULTILINE) + if not raw: + raise BenchmarkError(f"{label} marker is absent") + try: + values = [Decimal(item) for item in raw] + except (DecimalException, ValueError) as exc: + raise BenchmarkError(f"{label} marker is invalid") from exc + if any(not item.is_finite() or item <= 0 for item in values): + raise BenchmarkError(f"{label} marker is non-finite or non-positive") + try: + float_values = [float(item) for item in values] + except (OverflowError, ValueError) as exc: + raise BenchmarkError(f"{label} marker exceeds the supported numeric range") from exc + if any(not math.isfinite(item) or item <= 0 for item in float_values): + raise BenchmarkError(f"{label} marker exceeds the supported numeric range") + if any(item != values[-1] for item in values[:-1]): + raise BenchmarkError(f"{label} markers contradict one another") + try: + half_widths = [ + Decimal(1).scaleb(item.as_tuple().exponent) / 2 + for item in values + ] + except DecimalException as exc: + raise BenchmarkError(f"{label} display precision is invalid") from exc + if any( + not item.is_finite() or item <= 0 + for item in half_widths + ): + raise BenchmarkError(f"{label} display precision is invalid") + if any(item != half_widths[-1] for item in half_widths[:-1]): + raise BenchmarkError(f"{label} display precisions contradict one another") + return values[-1], half_widths[-1] + + +def _quantized_rate_matches( + frames: int, + elapsed: Decimal, + elapsed_half_width: Decimal, + fps: Decimal, + fps_half_width: Decimal, +) -> bool: + """Return whether rounded elapsed/FPS intervals can describe one rate.""" + elapsed_value = Fraction(elapsed) + elapsed_width = Fraction(elapsed_half_width) + fps_value = Fraction(fps) + fps_width = Fraction(fps_half_width) + elapsed_low = elapsed_value - elapsed_width + elapsed_high = elapsed_value + elapsed_width + reported_low = max(fps_value - fps_width, Fraction(0)) + reported_high = fps_value + fps_width + # Compare the positive intervals without division. Fraction preserves every + # displayed digit, independent of the process-global Decimal precision. + return ( + reported_low * elapsed_low + <= frames + <= reported_high * elapsed_high + ) + + +def _frames(pattern: str, text: str, label: str) -> int: + raw = re.findall(pattern, text, flags=re.MULTILINE) + if not raw: + raise BenchmarkError(f"{label} marker is absent") + values = [int(item) for item in raw] + if any(item <= 0 for item in values) or any(item != values[-1] for item in values[:-1]): + raise BenchmarkError(f"{label} markers are invalid or contradictory") + return values[-1] + + +def _native_decode_metadata( + text: str, + expected: Mapping[str, Any], + workers: int, +) -> dict[str, Any]: + """Bind AppDecPerf's repeated official input description to the artifact.""" + patterns = { + "codec": r"^\s*Codec\s*:\s*([^\r\n]+?)\s*$", + "frame_rate": r"^\s*Frame rate\s*:\s*([^\r\n]+?)\s*$", + "coded_size": r"^\s*Coded size\s*:\s*\[(\d+),\s*(\d+)\]\s*$", + "display_area": ( + r"^\s*Display area\s*:\s*\[(\d+),\s*(\d+),\s*(\d+),\s*(\d+)\]\s*$" + ), + "chroma": r"^\s*Chroma\s*:\s*([^\r\n]+?)\s*$", + "bit_depth": r"^\s*Bit depth\s*:\s*(\d+)\s*$", + } + matches = { + field: re.findall(pattern, text, flags=re.MULTILINE) + for field, pattern in patterns.items() + } + for field, values in matches.items(): + if len(values) != workers: + raise BenchmarkError( + f"AppDecPerf expected {workers} {field} marker(s); found {len(values)}" + ) + if any(value != values[-1] for value in values[:-1]): + raise BenchmarkError(f"AppDecPerf {field} markers contradict one another") + codec = _NATIVE_CODECS.get(str(matches["codec"][-1]).strip()) + coded_width, coded_height = (int(item) for item in matches["coded_size"][-1]) + left, top, right, bottom = (int(item) for item in matches["display_area"][-1]) + chroma = str(matches["chroma"][-1]).strip() + bit_depth = int(matches["bit_depth"][-1]) + pixel_format = _NATIVE_LAYOUTS.get((chroma, bit_depth)) + rate = re.fullmatch( + rf"({_NUMBER})\s*/\s*({_NUMBER})\s*=\s*({_NUMBER})\s+fps", + str(matches["frame_rate"][-1]).strip(), + ) + if rate is None: + raise BenchmarkError("AppDecPerf frame-rate marker is malformed") + numerator, denominator, reported_fps = (float(item) for item in rate.groups()) + if ( + denominator == 0 + or not all(math.isfinite(item) for item in (numerator, denominator, reported_fps)) + or not math.isclose( + numerator / denominator, reported_fps, rel_tol=1e-4, abs_tol=1e-3 + ) + ): + raise BenchmarkError("AppDecPerf frame-rate marker is contradictory") + observed = { + "codec": codec, + "coded_width": coded_width, + "coded_height": coded_height, + "display_area": [left, top, right, bottom], + "width": right - left, + "height": bottom - top, + "format": pixel_format, + "bit_depth": bit_depth, + "chroma": chroma, + "fps": reported_fps, + "instances": workers, + } + for field in ("codec", "width", "height", "format"): + if observed[field] != expected[field]: + raise BenchmarkError( + f"AppDecPerf reported {field} {observed[field]}; expected {expected[field]}" + ) + if not math.isclose( + reported_fps, float(expected["fps"]), rel_tol=1e-4, abs_tol=1e-3 + ): + raise BenchmarkError( + f"AppDecPerf reported fps {reported_fps}; expected {expected['fps']}" + ) + if not (0 <= left < right <= coded_width and 0 <= top < bottom <= coded_height): + raise BenchmarkError("AppDecPerf reported an invalid display area") + return observed + + +def _parse_performance( + surface: str, + operation: str, + result: Mapping[str, Any], + *, + workspace: Path, + expected_frames: int, + expected_metadata: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + # Four official samples have four source-defined terminal marker grammars. + # pylint: disable=too-many-branches + text = _command_text(workspace, result) + _successful( + result, + f"{surface} {operation} benchmark", + complete_text=text, + ) + elapsed_decimal = None + elapsed_half_width = None + fps_decimal = None + fps_half_width = None + if surface == "native" and operation == "encode": + matches = re.findall( + rf"nTotal=(\d+),\s*time=({_NUMBER}) seconds,\s*FPS=({_NUMBER})", text + ) + if not matches: + raise BenchmarkError("AppEncPerf result marker is absent") + parsed = [ + (int(frames), float(elapsed), float(fps)) + for frames, elapsed, fps in matches + ] + if any(item != parsed[-1] for item in parsed[:-1]): + raise BenchmarkError("AppEncPerf result markers contradict one another") + frames, elapsed, fps = parsed[-1] + elif surface == "native": + frames = _frames( + r"Total Frames Decoded=(\d+)\s+FPS\s*=", text, "AppDecPerf frames" + ) + fps = _values( + rf"Total Frames Decoded=\d+\s+FPS\s*=\s*({_NUMBER})", + text, + "AppDecPerf FPS", + )[-1] + elapsed = None + elif operation == "encode": + frames = _frames(r"^Total frames processed:\s*(\d+)\s*$", text, "Py encode frames") + elapsed_decimal, elapsed_half_width = _quantized_value( + rf"^Duration:\s*({_NUMBER}) seconds\s*$", + text, + "Py encode duration", + ) + fps_decimal, fps_half_width = _quantized_value( + rf"^Total FPS:\s*({_NUMBER})\s*$", + text, + "Py encode FPS", + ) + elapsed = float(elapsed_decimal) + fps = float(fps_decimal) + else: + frames = _frames(r"^Total frames decoded:\s*(\d+)\s*$", text, "Py decode frames") + fps = _values(rf"^Total FPS:\s*({_NUMBER})\s*$", text, "Py decode FPS")[-1] + elapsed = _values(rf"^Total wall time:\s*({_NUMBER})s\s*$", text, "Py decode wall time")[-1] + if frames != expected_frames: + raise BenchmarkError(f"sample reported {frames} frames; expected {expected_frames}") + if not math.isfinite(fps) or fps <= 0: + raise BenchmarkError("sample FPS is non-finite or non-positive") + if elapsed is not None and not (surface == "pynvc" and operation == "decode"): + if not math.isfinite(elapsed) or elapsed <= 0: + raise BenchmarkError("sample elapsed time is non-finite or non-positive") + derived = frames / elapsed + if ( + elapsed_decimal is not None + and elapsed_half_width is not None + and fps_decimal is not None + and fps_half_width is not None + ): + consistent = _quantized_rate_matches( + frames, + elapsed_decimal, + elapsed_half_width, + fps_decimal, + fps_half_width, + ) + else: + consistent = math.isclose( + fps, + derived, + rel_tol=0.05, + abs_tol=0.1, + ) + if not consistent: + raise BenchmarkError( + f"sample FPS {fps} contradicts frames/elapsed derived FPS {derived}" + ) + parsed_result = {"frames": frames, "elapsed_seconds": elapsed, "fps": fps} + if surface == "native" and operation == "decode": + if expected_metadata is None: + raise BenchmarkError("native decode metadata expectation is absent") + parsed_result["input_media"] = _native_decode_metadata( + text, + expected_metadata, + _integer(expected_frames // int(expected_metadata["frames"]), "workers"), + ) + return parsed_result + + +def _summarize(runs: Sequence[Mapping[str, Any]]) -> dict[str, Any]: + fps = [float(run["fps"]) for run in runs] + raw_megapixels = [run.get("megapixels_per_second") for run in runs] + if not fps or any(not math.isfinite(item) or item <= 0 for item in fps): + raise BenchmarkError("benchmark statistics contain a non-finite or non-positive metric") + megapixels: list[float] | None + omission_reason = None + if raw_megapixels and all(item is None for item in raw_megapixels): + reasons = { + _mapping(run.get("metric_omissions"), "metric_omissions").get( + "megapixels_per_second" + ) + for run in runs + } + if len(reasons) != 1: + raise BenchmarkError("benchmark repetitions contain contradictory metric omissions") + omission_reason = _string( + reasons.pop(), + "metric_omissions.megapixels_per_second", + ) + megapixels = None + elif any(item is None for item in raw_megapixels): + raise BenchmarkError("benchmark repetitions mix present and omitted MP/s metrics") + else: + megapixels = [float(item) for item in raw_megapixels] + if any(not math.isfinite(item) or item <= 0 for item in megapixels): + raise BenchmarkError( + "benchmark statistics contain a non-finite or non-positive metric" + ) + fps_scale = max(fps) + summary = { + "repetitions": len(runs), + "fps": { + "mean": ( + fps_scale + * (math.fsum(item / fps_scale for item in fps) / len(fps)) + ), + "minimum": min(fps), + "maximum": fps_scale, + }, + } + if megapixels is None: + summary["megapixels_per_second"] = None + summary["metric_omissions"] = { + "megapixels_per_second": omission_reason, + } + else: + megapixels_scale = max(megapixels) + summary["megapixels_per_second"] = { + "mean": ( + megapixels_scale + * ( + math.fsum(item / megapixels_scale for item in megapixels) + / len(megapixels) + ) + ), + "minimum": min(megapixels), + "maximum": megapixels_scale, + } + summary_values = list(summary["fps"].values()) + if megapixels is not None: + summary_values += list(summary["megapixels_per_second"].values()) + if any(not math.isfinite(item) or item <= 0 for item in summary_values): + raise BenchmarkError("benchmark statistics overflowed the supported numeric range") + return summary + + +def _megapixels_per_second(fps: float, width: int, height: int) -> float: + value = fps * width * height / 1_000_000 + if not math.isfinite(value) or value <= 0: + raise BenchmarkError("megapixels per second exceeds the supported numeric range") + return value + + +def _run_surface( + surface: str, + operation: str, + request: Mapping[str, Any], + *, + variant: str, + data: Mapping[str, Any], + recipe: Mapping[str, Any] | None, + authentication: Any, + workspace: Path, + runner: Any, + timeout_seconds: float, + frame_plan: Mapping[str, Any], + native_help: tuple[set[str], list[dict[str, Any]]] | None = None, +) -> dict[str, Any]: + token = _token(authentication, operation, surface) + effective_data = _effective_data(data, frame_plan) + if surface == "native": + arguments = _native_arguments(operation, effective_data, recipe, request) + if operation == "encode": + if native_help is None: + raise BenchmarkError("native benchmark help evidence is absent") + advertised, help_evidence = native_help + emitted = _option_names(arguments) + if "-loop" in emitted: + raise BenchmarkError("AppEncPerf argv must never contain -loop") + missing = sorted(emitted - advertised) + if missing: + raise BenchmarkError( + f"installed native sample help does not advertise {missing}" + ) + else: + advertised = set() + help_evidence = [] + else: + help_evidence = [] + advertised = set() + arguments = _pynvc_arguments( + operation, effective_data, recipe, request, workspace=workspace, variant=variant + ) + generated_config = ( + _pynvc_config_identity(arguments, workspace) + if surface == "pynvc" and operation == "encode" + else None + ) + workers = _integer(request.get("workers", 1), "workers") + expected_frames = int(effective_data["frames"]) * workers + repetitions = _integer( + request.get("measured_repetitions", 3), + "measured_repetitions", + minimum=3, + ) + warmup_result = _run_sample( + token, + arguments, + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + stage=f"benchmark-{surface}-{variant}", + phase="warmup", + ) + _parse_performance( + surface, + operation, + warmup_result, + workspace=workspace, + expected_frames=expected_frames, + expected_metadata=effective_data, + ) + measured: list[dict[str, Any]] = [] + for index in range(repetitions): + command = _run_sample( + token, + arguments, + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + stage=f"benchmark-{surface}-{variant}-{index + 1}", + phase="measure", + ) + metrics = _parse_performance( + surface, + operation, + command, + workspace=workspace, + expected_frames=expected_frames, + expected_metadata=effective_data, + ) + if surface == "pynvc" and operation == "decode": + megapixels_per_second = None + metric_omissions = { + "megapixels_per_second": _PYNVC_DECODE_MP_OMISSION, + } + else: + metric_width = int(data["width"]) + metric_height = int(data["height"]) + if surface == "native" and operation == "decode": + observed = _mapping(metrics.get("input_media"), "input_media") + metric_width = int(observed["width"]) + metric_height = int(observed["height"]) + megapixels_per_second = _megapixels_per_second( + float(metrics["fps"]), + metric_width, + metric_height, + ) + metric_omissions = None + metrics.update( + { + "repetition": index + 1, + "megapixels_per_second": megapixels_per_second, + "command": command, + } + ) + if metric_omissions is not None: + metrics["metric_omissions"] = metric_omissions + measured.append(metrics) + result = { + "status": "completed", + "surface": surface, + "sample": token.sample, + "sample_authority": _token_evidence(token), + "provenance_report": dict(authentication.report_identity), + "input": { + key: data[key] + for key in ( + "identity", "width", "height", "frames", "fps", "codec", "format", + "source_url", "license", "attribution", + ) + }, + "help_commands": help_evidence, + "advertised_options": sorted(advertised), + "benchmark_arguments": list(arguments), + "frame_accounting": _frame_accounting(frame_plan, workers), + "warmup": warmup_result, + "measured_runs": measured, + "summary": _summarize(measured), + } + result.update(_generated_config_result(workspace, generated_config)) + return result + + +def _variants(request: Mapping[str, Any], route: str) -> list[tuple[str, dict[str, Any]]]: + if route == "camera_capacity": + counts = request.get("worker_counts") + if ( + not isinstance(counts, list) + or len(counts) < 2 + or any( + not isinstance(item, int) or isinstance(item, bool) or item <= 0 + for item in counts + ) + or counts[0] != 1 + or any(right <= left for left, right in zip(counts, counts[1:])) + ): + raise BenchmarkError( + "camera_capacity worker_counts must be a strictly increasing " + "integer series beginning at 1" + ) + return [ + (f"workers-{count}", {**request, "workers": count}) for count in counts + ] + if route != "compare": + return [(str(request.get("name", route)), dict(request))] + raw = request.get("variants") + if not isinstance(raw, list) or len(raw) < 2: + raise BenchmarkError("compare requires at least two variants") + result: list[tuple[str, dict[str, Any]]] = [] + for index, item in enumerate(raw): + variant = dict(_mapping(item, f"variants[{index}]")) + name = _string(variant.pop("name", None), f"variants[{index}].name") + merged = dict(request) + merged.pop("variants", None) + merged.update(variant) + result.append((name, merged)) + if len({name for name, _item in result}) != len(result): + raise BenchmarkError("compare variant names must be unique") + return result + + +def _media_input_preflight( + request: Mapping[str, Any], route: str +) -> dict[str, Any] | None: + """Identify absent user media before workspace, environment, or sample work.""" + operation = _operation(request, route) + mode = _mode(request) + field = "input" if operation == "encode" else "encoded_artifact" + variants = _variants(request, route) + missing = [name for name, variant in variants if variant.get(field) is None] + if not missing: + return None + return { + "schema_version": "2.0", + "kind": "nvcodec-benchmark-result", + "route": route, + "mode": mode, + "operation": operation, + "status": "input_required", + "gate": "media_input", + "accepted_inputs": list(ACCEPTED_MEDIA_INPUTS), + "next_action": "provide_media_path_or_url", + "synthetic_input_allowed": False, + "required_field": field, + "missing_variants": missing, + } + + +def _pynvc_interpreter_preflight( + request: Mapping[str, Any], route: str +) -> dict[str, Any] | None: + """Require one exact selector only when the local request selects Py.""" + if request.get("environment") is not None: + return None + surface = request.get("surface", "auto") + interpreter = request.get("pynvc_interpreter") + missing_for_selected = surface in {"pynvc", "both"} and interpreter is None + selector_is_valid = False + if isinstance(interpreter, str): + candidate = Path(interpreter) + if ( + interpreter + and interpreter == interpreter.strip() + and candidate.is_absolute() + and str(candidate) == interpreter + ): + try: + details = candidate.stat() + except OSError: + pass + else: + selector_is_valid = bool( + stat.S_ISREG(details.st_mode) and details.st_mode & 0o111 + ) + malformed_for_considered = ( + surface in {"pynvc", "both", "auto"} + and interpreter is not None + and not selector_is_valid + ) + if not missing_for_selected and not malformed_for_considered: + return None + return { + "schema_version": "2.0", + "kind": "nvcodec-benchmark-result", + "route": route, + "mode": _mode(request), + "operation": _operation(request, route), + "status": "input_required", + "gate": "surface_selector", + "requested_surface": surface, + "surface": "pynvc", + "required_field": "pynvc_interpreter", + "reason": ( + "provide one canonical absolute path to an existing executable regular-file " + "Python interpreter for the PyNvVideoCodec environment; no environment " + "scan is performed" + ), + "next_action": "provide_pynvc_interpreter_and_retry", + "next_actions": [ + "provide_pynvc_interpreter_and_retry", + "set_surface_native_if_only_native_is_required", + ], + "variants": [], + "no_mutation_performed": True, + } + + +def _validate_compare( + prepared: Sequence[ + tuple[ + str, + Mapping[str, Any], + Mapping[str, Any], + Mapping[str, Any] | None, + Mapping[str, Any] | None, + ] + ], +) -> None: + if len(prepared) != 2 or {name.lower() for name, *_rest in prepared} != {"p4", "p5"}: + raise BenchmarkError("compare requires exactly P4 and P5 variants") + canonical: dict[str, bytes] = {} + for name, variant, data, _recipe_identity, recipe in prepared: + if recipe is None: + raise BenchmarkError("P4/P5 comparison requires encode recipes") + # _recipe() has already replayed the recipe from its original intent and + # current catalog. The effective encoder intent is therefore sufficient + # here; projection/default provenance cannot conceal a semantic change. + intent = dict(_mapping(recipe.get("encoder_intent"), "recipe.encoder_intent")) + preset = str(intent.pop("preset", "")).lower() + if preset != name.lower(): + raise BenchmarkError(f"{name} variant recipe preset is {preset!r}") + workload = { + key: data[key] + for key in ( + "path", "identity", "width", "height", "frames", "fps", "codec", + "format", "source_url", "license", "attribution", + ) + } + facts = dict(variant) + facts["input"] = { + "request": variant.get("input"), + "effective": workload, + } + facts["recipe"] = intent + facts.update( + { + "mode": _mode(variant), + "operation": _operation(variant, "compare"), + "surface": variant.get("surface", "auto"), + "workers": _integer(variant.get("workers", 1), "workers"), + "gpu": _integer( + variant.get("gpu", intent.get("gpu")), + "gpu", + minimum=0, + ), + "process_model": _process_model(variant), + "single_context": variant.get("single_context") is True, + "host_memory": variant.get("host_memory") is True, + "measured_repetitions": _integer( + variant.get("measured_repetitions", 3), + "measured_repetitions", + minimum=3, + ), + "require_source_frames": ( + variant.get("require_source_frames") is True + ), + } + ) + canonical[name.lower()] = artifact_io.canonical_json_bytes(facts) + if canonical["p4"] != canonical["p5"]: + raise BenchmarkError( + "P4/P5 comparison must keep workload and every non-preset control identical" + ) + + +def _planned_operation( + surface: str, + operation: str, + request: Mapping[str, Any], + *, + variant: str, + data: Mapping[str, Any], + recipe: Mapping[str, Any] | None, + workspace: Path, +) -> dict[str, Any]: + """Describe one candidate invocation without authentication or execution.""" + sample = ( + ("AppEncPerf" if operation == "encode" else "AppDecPerf") + if surface == "native" + else ( + "samples/advanced/encode_perf.py" + if operation == "encode" + else "samples/advanced/decode_perf.py" + ) + ) + generated_config = None + if surface == "native": + arguments = _native_arguments(operation, data, recipe, request) + else: + arguments = _pynvc_arguments( + operation, + data, + recipe, + request, + workspace=workspace, + variant=variant, + ) + if operation == "encode": + generated_config = _pynvc_config_identity(arguments, workspace) + result = { + "status": "planned", + "surface": surface, + "sample": sample, + "candidate_arguments": arguments, + "sample_authentication": "deferred_until_execute", + "operation_verification": "not_attempted", + } + if surface == "native" and operation == "encode": + result["option_validation"] = ( + "pending_installed_AppEncPerf_-h_and_-A_advertisement" + ) + if generated_config is not None: + result["generated_config"] = generated_config + return result + + +def _benchmark_route( + request: Mapping[str, Any], + *, + route: str, + workspace: Path, + runner: Any, + timeout_seconds: float, +) -> dict[str, Any]: + # One route-level transaction preserves independent surface outcomes. + # pylint: disable=too-many-branches,too-many-statements + operation = _operation(request, route) + mode = _mode(request) + variants = _variants(request, route) + requested_surface = request.get("surface", "auto") + prepared: list[ + tuple[ + str, + dict[str, Any], + dict[str, Any], + dict[str, Any] | None, + dict[str, Any] | None, + ] + ] = [] + selected_gpus: list[int] = [] + for validation_index, (name, variant) in enumerate(variants, start=1): + data = _input(variant, operation) + if operation == "encode": + recipe_identity, recipe = _recipe( + variant, + workspace=workspace, + timeout_seconds=timeout_seconds, + validation_index=validation_index, + ) + intent = _mapping(recipe.get("encoder_intent"), "recipe.encoder_intent") + recipe_gpu = _integer( + intent.get("gpu"), "recipe.encoder_intent.gpu", minimum=0 + ) + # First bind request and recipe without requiring setup-owned evidence. + _bind_recipe_workload(data, recipe, variant, recipe_gpu) + selected_gpus.append(recipe_gpu) + else: + recipe_identity, recipe = None, None + selected_gpus.append( + _integer(variant.get("gpu", 0), "gpu", minimum=0) + ) + prepared.append((name, variant, data, recipe_identity, recipe)) + if len(set(selected_gpus)) != 1: + raise BenchmarkError("all benchmark variants must select the same GPU") + selected_gpu = selected_gpus[0] + authority = _runtime_authority( + request, + operation=operation, + selected_gpu=selected_gpu, + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + ) + environment = _mapping(authority["value"], "runtime authority") + plan = _surface_plan( + request, + environment, + operation, + _mapping(authority.get("diagnostics", {}), "runtime diagnostics"), + ( + None + if authority["source"] == "setup_environment" + else _mapping( + authority.get("normalized_surfaces", {}), + "normalized local surfaces", + ) + ), + ) + if authority["source"] == "local_discovery" and authority.get( + "normalized_surfaces" + ): + binding_workspace, binding_identity = _stage_local_binding( + workspace, environment + ) + authority["staged_workspace"] = binding_workspace + authority["staged_identity"] = binding_identity + authority_fields = _runtime_result_fields(authority) + setup_dependencies = _runtime_setup_dependencies( + request, authority, environment, operation, plan + ) + dependency_fields = _dependency_result_fields(setup_dependencies) + if not plan["selected_surfaces"] and (requested_surface != "both" or mode == "execute"): + return { + "schema_version": "2.0", "kind": "nvcodec-benchmark-result", + "route": route, "status": plan["classification"], "mode": mode, + **authority_fields, + **dependency_fields, + "surface_plan": plan, "variants": [], + } + variant_surfaces: dict[str, list[str]] = {} + projection_blocks: dict[str, dict[str, dict[str, Any]]] = {} + for name, _variant, _data, _recipe_identity, recipe_value in prepared: + active, blocked = _projection_plan( + operation, + plan["selected_surfaces"], + recipe_value, + ) + variant_surfaces[name] = active + projection_blocks[name] = blocked + frame_plans = { + name: _frame_plan(operation, data, variant_surfaces[name], variant) + for name, variant, data, _recipe_identity, _recipe_value in prepared + } + if route == "compare": + _validate_compare(prepared) + if route == "camera_capacity": + camera_fps = _number(request.get("camera_fps"), "camera_fps") + safety_margin = _number(request.get("safety_margin"), "safety_margin") + if safety_margin > 1: + raise BenchmarkError("safety_margin must be greater than zero and at most 1") + reported_surfaces = ( + SURFACES if requested_surface == "both" else tuple(plan["selected_surfaces"]) + ) + if mode == "dry_run": + planned_variants: list[dict[str, Any]] = [] + branch_states: list[str] = [] + for name, variant, data, recipe_identity, recipe_value in prepared: + planned_workers = _integer(variant.get("workers", 1), "workers") + accounting = _frame_accounting(frame_plans[name], planned_workers) + planned_operations: list[dict[str, Any]] = [] + for surface in reported_surfaces: + if surface not in plan["selected_surfaces"]: + planned = { + "status": "blocked", + "surface": surface, + "reason": ( + "surface is not eligible in the current live environment" + ), + } + elif surface in projection_blocks[name]: + planned = dict(projection_blocks[name][surface]) + else: + try: + planned = _planned_operation( + surface, + operation, + variant, + variant=name, + data=_effective_data(data, frame_plans[name]), + recipe=recipe_value, + workspace=workspace, + ) + except (OSError, ValueError) as exc: + planned = { + "status": "blocked", + "surface": surface, + "reason": f"{type(exc).__name__}: {exc}", + } + planned["frame_accounting"] = dict(accounting) + planned_operations.append(planned) + branch_states.append(str(planned["status"])) + planned_variant = { + "name": name, + "operation": operation, + "surfaces": list(reported_surfaces), + "frame_accounting": accounting, + "planned_operations": planned_operations, + } + if recipe_identity is not None: + planned_variant["recipe"] = dict(recipe_identity) + planned_variants.append(planned_variant) + if ( + branch_states + and all(state == "planned" for state in branch_states) + and plan["classification"] == "ready" + ): + dry_run_status = "planned" + elif "planned" in branch_states: + dry_run_status = "partial" + else: + dry_run_status = "blocked" + return { + "schema_version": "2.0", "kind": "nvcodec-benchmark-result", + "route": route, "status": dry_run_status, "mode": mode, + **authority_fields, **dependency_fields, "surface_plan": plan, + "variants": planned_variants, + } + # Recheck supplied setup evidence; local authority was discovered and staged here. + if authority["source"] == "setup_environment": + environment_source = authority["external_identity"] + artifact_io.verify_external_artifact(environment_source) + environment_workspace, environment_identity = _stage_environment( + workspace, environment + ) + else: + environment_workspace = authority["staged_workspace"] + environment_identity = authority["staged_identity"] + for _name, _variant, _data, recipe_identity, _recipe_value in prepared: + if recipe_identity is not None: + artifact_io.verify_external_artifact(recipe_identity) + authentication_surfaces = [ + surface + for surface in SURFACES + if any( + surface in variant_surfaces[name] + for name, _variant, _data, _recipe_identity, _recipe_value in prepared + ) + ] + authentication = _authentication( + operation, + authentication_surfaces, + environment_workspace=environment_workspace, + environment_identity=environment_identity, + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + ) + native_help: tuple[set[str], list[dict[str, Any]]] | None = None + native_authority = authentication.get("native") + if ( + operation == "encode" + and native_authority + and native_authority["status"] == "authenticated" + ): + try: + native_help = _native_help( + _token(native_authority["value"], operation, "native"), + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + ) + except (OSError, ValueError) as exc: + native_authority.clear() + native_authority.update({ + "status": "failed", + "reason": f"{type(exc).__name__}: {exc}", + }) + results: list[dict[str, Any]] = [] + for name, variant, data, recipe_identity, recipe in prepared: + planned_workers = _integer(variant.get("workers", 1), "workers") + accounting = _frame_accounting(frame_plans[name], planned_workers) + branches: dict[str, Any] = {} + for surface in reported_surfaces: + if surface not in plan["selected_surfaces"]: + branches[surface] = { + "status": "blocked", + "surface": surface, + "reason": "surface is not eligible in the current live environment", + } + continue + if surface in projection_blocks[name]: + branches[surface] = dict(projection_blocks[name][surface]) + continue + authority = authentication[surface] + if authority["status"] != "authenticated": + branches[surface] = { + "status": "failed", + "surface": surface, + "reason": authority["reason"], + } + continue + try: + branches[surface] = _run_surface( + surface, + operation, + variant, + variant=name, + data=data, + recipe=recipe, + authentication=authority["value"], + workspace=workspace, + runner=runner, + timeout_seconds=timeout_seconds, + frame_plan=frame_plans[name], + native_help=native_help, + ) + except (OSError, ValueError) as exc: + branches[surface] = { + "status": "failed", + "surface": surface, + "reason": f"{type(exc).__name__}: {exc}", + } + for branch in branches.values(): + branch["frame_accounting"] = dict(accounting) + result_variant = {"name": name, "operation": operation, "surfaces": branches} + if recipe_identity is not None: + result_variant["recipe"] = dict(recipe_identity) + results.append(result_variant) + branch_states = [ + branch["status"] + for item in results + for branch in item["surfaces"].values() + ] + completed = branch_states.count("completed") + if completed and completed == len(branch_states) and plan["classification"] == "ready": + status = "completed" + elif completed: + status = "partial" + elif branch_states and all(state == "blocked" for state in branch_states): + status = "blocked" + else: + status = "failed" + output: dict[str, Any] = { + "schema_version": "2.0", "kind": "nvcodec-benchmark-result", + "route": route, "status": status, "mode": mode, + **authority_fields, **dependency_fields, "surface_plan": plan, + "variants": results, + } + if route == "compare": + output["comparison"] = { + surface: { + item["name"]: item["surfaces"][surface]["summary"] + for item in results + if item["surfaces"].get(surface, {}).get("status") == "completed" + } + for surface in SURFACES + if any(surface in item["surfaces"] for item in results) + } + if route == "camera_capacity": + capacity: dict[str, Any] = {} + for surface in SURFACES: + points = [] + for item in results: + workers = int(item["name"].removeprefix("workers-")) + branch = item["surfaces"].get(surface, {"status": "blocked"}) + mean_fps = ( + branch["summary"]["fps"]["mean"] + if branch["status"] == "completed" + else None + ) + points.append( + { + "workers": workers, + "status": branch["status"], + "mean_fps": mean_fps, + "margin_adjusted_fps": ( + mean_fps * safety_margin if mean_fps is not None else None + ), + "meets_requested_load": ( + mean_fps is not None + and mean_fps * safety_margin >= workers * camera_fps + ), + } + ) + if any(point["status"] != "blocked" for point in points): + passing = [ + point["workers"] for point in points if point["meets_requested_load"] + ] + capacity[surface] = { + "requested_fps_per_camera": camera_fps, + "safety_margin": safety_margin, + "tested_points": points, + "safe_tested_camera_count": max(passing, default=0), + "scope": ( + "codec-only measured estimate; camera capture and end-to-end " + "latency remain unverified" + ), + } + output["camera_capacity"] = capacity + return output + + +ROUTE_HANDLERS: dict[str, Callable[..., dict[str, Any]]] = { + "encode": _benchmark_route, + "decode": _benchmark_route, + "compare": _benchmark_route, + "camera_capacity": _benchmark_route, +} + + +def run_benchmark_request( + request: Mapping[str, Any], + *, + workspace: Path, + runner: Any = None, + timeout_seconds: float = 300, +) -> dict[str, Any]: + """Execute one benchmark-domain request in a private fresh workspace.""" + value = _mapping(request, "request") + if value.get("schema_version") != "1.0" or value.get("kind") != "nvcodec-benchmark-request": + raise BenchmarkError("request must be a schema-1.0 nvcodec-benchmark-request") + route = value.get("route") + if route not in ROUTE_HANDLERS: + raise BenchmarkError(f"route must be exactly one of {sorted(ROUTE_HANDLERS)}") + timeout = _number(timeout_seconds, "timeout_seconds") + input_required = _media_input_preflight(value, str(route)) + if input_required is not None: + return input_required + selector_required = _pynvc_interpreter_preflight(value, str(route)) + if selector_required is not None: + return selector_required + root = _workspace(Path(workspace)) + service = runner or command_runner.CommandRunner(root) + handler = ROUTE_HANDLERS[str(route)] + try: + return handler( + value, + route=str(route), + workspace=root, + runner=service, + timeout_seconds=timeout, + ) + except DependencyRequired as exc: + return { + "schema_version": "2.0", + "kind": "nvcodec-benchmark-result", + "route": route, + "mode": _mode(value), + "operation": _operation(value, str(route)), + "status": "dependency_required", + "gate": "skill_dependency", + "dependency": exc.dependency, + "variants": [], + } + + +def main(argv: Sequence[str] | None = None) -> int: + """CLI for one request JSON and one fresh private workspace.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + request = artifact_io.strict_json_loads(args.request.read_bytes()) + result = run_benchmark_request(request, workspace=args.workspace) + if result.get("status") == "input_required": + print( + artifact_io.canonical_json_bytes(result).decode("utf-8"), + end="", + ) + else: + workspace = artifact_io.resolve_private_workspace(args.workspace) + artifact_io.write_fresh_json(workspace, args.output, result) + except (OSError, ValueError) as exc: + print(f"benchmark request failed: {exc}", file=sys.stderr) + return 2 + return 0 if result.get("status") in {"completed", "planned"} else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-benchmark/scripts/benchmark_provenance.py b/skills/jetson-video-benchmark/scripts/benchmark_provenance.py new file mode 100644 index 00000000..1abfa18e --- /dev/null +++ b/skills/jetson-video-benchmark/scripts/benchmark_provenance.py @@ -0,0 +1,1514 @@ +#!/usr/bin/env python3 +"""Private authentication for official native and PyNvVideoCodec samples.""" + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Authentication keeps exact, auditable conjunctions rather than permissive helpers. +# pylint: disable=too-many-arguments,too-many-boolean-expressions +# pylint: disable=too-many-instance-attributes,too-many-locals +# This single module owns native and PyNvVideoCodec sample authentication end to end. +# pylint: disable=too-many-lines + +from __future__ import annotations + +import base64 +import csv +import hashlib +import io +import re +import stat +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence + +import benchmark_runtime as runtime + +_COMMAND_ENV = { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} +_DEBIAN_VERSION = re.compile( + r"^(?:(?:[0-9]+):)?(?P13\.0(?:\.[0-9]+)*)" + r"(?:\+[0-9A-Za-z.]+)?(?:-[0-9A-Za-z.+]+)?$" +) +# The frozen `nvcodec-environment` 1.2 contract. Authentication pins this exact +# version and rejects anything else; additive optional keys are ignored. +ENVIRONMENT_KIND = "nvcodec-environment" +ENVIRONMENT_SCHEMA_VERSION = "1.2" +LOCAL_BINDING_KIND = "nvcodec-local-runtime-binding" +LOCAL_BINDING_SCHEMA_VERSION = "1.0" +NATIVE_PACKAGE = "nvidia-video-codec-sdk" +PYNVC_REQUIRED_VERSION = "2.1.0" +# The normalized native toolchain carries exactly these five entries, each +# {path, sha256}; `generator` adds the CMake generator `name`. +_BUILD_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +_CMAKE_GENERATORS = ("Ninja", "Unix Makefiles") +_VERSION_TOKEN = re.compile(r"[0-9][0-9A-Za-z.+:~-]*") +_RELEASE = re.compile(r"(?:[0-9]+:)?([0-9]+)\.([0-9]+)") +OFFICIAL_SAMPLE_FAILURE_PATTERN = re.compile( + r"""(?imx)^\s*(?: + traceback\ \(most\ recent\ call\ last\) + | error\b + | an\ (?:unexpected\ )?error\ occurred(?:\ with\ [^:\r\n]+)?: + | operation\ or\ configuration\ not\ supported: + | decode\ error\ occurred\ for\ picture\b + | create(?:encoder|decoder)\ failure: + | (?:worker\ [^:\r\n]+:\s*)?(?:an\ unexpected\ error\ occurred|setup\ failed): + | (?:thread|process)\ [^:\r\n]+\ error: + | encoding\ aborted:\ setup\ failed\b + | nvdecoder::~nvdecoder\(\):\ exception\ during\ cleanup\ \(suppressed\) + | (?:[a-z_][a-z0-9_:~]*\s*:\s*)?[a-z_][a-z0-9_.]* + (?:\([^\r\n]*\))?\s+returned\ error\b + | (?:cuda(?:\ (?:driver|runtime)\ api)?|nvenc|hresult|glenum|general|ffmpeg) + \ error\b + | cudacheckerror\(\)\ failed\b + | \[(?:error|fatal)\s*\](?:\[[^]\r\n]+\])? + | failed\b + | failure\b + | (?:operation|command|sample|encode|decode|encoding|decoding|encoder| + decoder|setup|test)\b[^\r\n]*\b(?:failed|failure)\b + )""" +) +_NATIVE_ROUTES = { + "AppEncPerf": ( + "AppEncode/AppEncPerf/AppEncPerf", + "Samples/AppEncode/AppEncPerf/AppEncPerf.cpp", + "Samples/AppEncode/AppEncPerf/CMakeLists.txt", + ("libcuda.so.1", "libnvidia-encode.so.1"), + ), + "AppDecPerf": ( + "AppDecode/AppDecPerf/AppDecPerf", + "Samples/AppDecode/AppDecPerf/AppDecPerf.cpp", + "Samples/AppDecode/AppDecPerf/CMakeLists.txt", + ("libcuda.so.1", "libnvcuvid.so.1"), + ), +} +_PYNVC_ROUTES = { + "samples/advanced/encode_perf.py": ( + "samples/utils/__init__.py", + "samples/utils/Utils.py", + "samples/utils/encode_parser.py", + "samples/utils/frame_utils.py", + "samples/utils/encode_parallel_utils.py", + ), + "samples/advanced/decode_perf.py": ( + "samples/utils/__init__.py", + "samples/utils/decode_parser.py", + ), +} +_NATIVE_EXTENSION_MEMBER = re.compile( + r"PyNvVideoCodec/_?PyNvVideoCodec(?:_(?:121|130))?\.[^/]*\.so" +) +_PROBE_PROGRAM = r""" +import importlib +import importlib.metadata +import json +import pathlib +import re +import sys + +dist = importlib.metadata.distribution("PyNvVideoCodec") +name = str(dist.metadata.get("Name", "")) +files = [str(item).replace("\\", "/") for item in (dist.files or [])] +records = [item for item in files if item.endswith(".dist-info/RECORD")] +if len(records) != 1: + raise RuntimeError("expected exactly one PyNvVideoCodec RECORD") +paths = { + item: str(pathlib.Path(dist.locate_file(pathlib.Path(item))).resolve()) + for item in files +} +# Metadata alone cannot prove which extension the interpreter actually loaded, +# so import the package and report what is live. +package = importlib.import_module("PyNvVideoCodec") +loaded = sorted({ + str(pathlib.Path(module.__file__).resolve()) + for key, module in sys.modules.items() + if (key == "_PyNvVideoCodec" or key.endswith("._PyNvVideoCodec")) + and getattr(module, "__file__", None) +}) +dependencies = {} +for distribution, module_name in ( + ("numpy", "numpy"), + ("pycuda", "pycuda.driver"), + ("torch", "torch"), +): + record = {"status": "missing", "ready": False} + try: + module = importlib.import_module(module_name) + record = { + "status": "installed", + "ready": True, + "version": str(importlib.metadata.version(distribution)), + } + if distribution == "torch": + record["cuda_build"] = str(getattr(module.version, "cuda", "")) + record["cuda_available"] = bool(module.cuda.is_available()) + record["ready"] = record["cuda_available"] + except Exception as exc: + record["error"] = f"{type(exc).__name__}: {exc}" + dependencies[distribution] = record +print(json.dumps({ + "schema_version": "1", + "kind": "pynvc-record-probe", + "interpreter": str(pathlib.Path(sys.executable).resolve()), + "sys_prefix": str(pathlib.Path(sys.prefix).resolve()), + "name": name, + "normalized_name": re.sub(r"[-_.]+", "-", name).lower(), + "version": str(dist.version), + "imported_version": str(getattr(package, "__version__", "")), + "module_path": str(pathlib.Path(package.__file__).resolve()), + "loaded_extensions": loaded, + "record_entry": records[0], + "paths": paths, + "dependencies": dependencies, +}, sort_keys=True)) +""".strip() + + +class ProvenanceError(ValueError): + """Raised when official-sample authority cannot be established.""" + + +@dataclass(frozen=True) +class FileSeal: + """One protected external file identity.""" + + identity: Mapping[str, Any] + + +@dataclass(frozen=True) +class AuthenticatedSample: + """Frozen identity and launcher for one authenticated sample.""" + + surface: str + sample: str + launcher: tuple[str, ...] + workspace: str + protected_files: tuple[FileSeal, ...] + runtime_libraries: tuple[FileSeal, ...] + environment_workspace: str + environment_identity: Mapping[str, Any] + + +@dataclass(frozen=True) +class Authentication: + """Authenticated sample set and its report identity.""" + + samples: tuple[AuthenticatedSample, ...] + report_identity: Mapping[str, Any] + label: str + + def token(self, sample: str) -> AuthenticatedSample: + """Return the unique authenticated token for a sample.""" + matches = [item for item in self.samples if item.sample == sample] + if len(matches) != 1: + raise ProvenanceError(f"{self.label} sample is not authenticated: {sample!r}") + return matches[0] + + +def _identity(path: Path, kind: str) -> dict[str, Any]: + return runtime.snapshot_external_artifact(path, schema_version="1", kind=kind) + + +def _sample_record( + *, + surface: str, + sample: str, + launcher: Sequence[str], + workspace: Path, + protected: Sequence[Mapping[str, Any]], + runtime_libraries: Sequence[Mapping[str, Any]], + environment_workspace: Path, + environment_identity: Mapping[str, Any], +) -> AuthenticatedSample: + return AuthenticatedSample( + surface=surface, + sample=sample, + launcher=tuple(launcher), + workspace=str(workspace), + protected_files=tuple(FileSeal(dict(item)) for item in protected), + runtime_libraries=tuple(FileSeal(dict(item)) for item in runtime_libraries), + environment_workspace=str(environment_workspace), + environment_identity=dict(environment_identity), + ) + + +def _environment(workspace: Path, identity: Mapping[str, Any]) -> dict[str, Any]: + value = runtime.read_verified_json(workspace, identity) + if not isinstance(value, dict): + raise ProvenanceError("authentication requires one environment object") + kind = value.get("kind") + version = value.get("schema_version") + if kind == ENVIRONMENT_KIND: + if version != ENVIRONMENT_SCHEMA_VERSION: + raise ProvenanceError( + "environment schema_version must be exactly " + f"{ENVIRONMENT_SCHEMA_VERSION!r}; refusing unknown environment " + f"version {version!r}" + ) + # Preserve the frozen setup-environment contract unchanged. + if ( + value.get("mode") != "live" + or not isinstance(value.get("installation"), Mapping) + or not isinstance(value.get("pynvc"), Mapping) + ): + raise ProvenanceError( + f"authentication requires one live {ENVIRONMENT_KIND} " + f"{ENVIRONMENT_SCHEMA_VERSION} with installation and pynvc facts" + ) + elif kind == LOCAL_BINDING_KIND: + selected_gpu = value.get("selected_gpu") + installation = value.get("installation") + if ( + set(value) != { + "schema_version", + "kind", + "mode", + "source", + "requested_runtime", + "selected_gpu", + "installation", + "pynvc", + } + or + version != LOCAL_BINDING_SCHEMA_VERSION + or value.get("mode") != "live" + or value.get("source") != "benchmark-local-discovery" + or value.get("requested_runtime") not in {"native", "pynvc", "both"} + or isinstance(selected_gpu, bool) + or not isinstance(selected_gpu, int) + or selected_gpu < 0 + or "surfaces" in value + or not isinstance(installation, Mapping) + or not all( + isinstance(installation.get(name), Mapping) + for name in ( + "native_sdk", + "native_build_prerequisites", + "cuda_toolkit", + "build_tool_identities", + "python", + ) + ) + or not isinstance(value.get("pynvc"), Mapping) + ): + raise ProvenanceError("local runtime binding is malformed") + else: + raise ProvenanceError( + "authentication requires a setup environment or benchmark-local " + "runtime binding" + ) + return value + + +def _version(value: Any, label: str) -> str: + """Validate one required, non-advisory version token.""" + if not isinstance(value, str) or _VERSION_TOKEN.fullmatch(value) is None: + raise ProvenanceError(f"environment {label} version is missing or malformed") + return value + + +def _upstream(version: str) -> str: + """Return the upstream part of a Debian version: 1:13.2.1-1 -> 13.2.1.""" + return re.sub(r"^[0-9]+:", "", version).rsplit("-", 1)[0] + + +def _release(version: str, label: str) -> tuple[int, int]: + """Return the (major, minor) release pair of one version token.""" + match = _RELEASE.match(version) + if match is None: + raise ProvenanceError(f"environment {label} version has no major.minor release") + return int(match.group(1)), int(match.group(2)) + + +def _run( + service: Any, + commands: list[dict[str, Any]], + argv: Sequence[str], + *, + cwd: Path, + timeout_seconds: float, + stage: str, +) -> tuple[dict[str, Any], str, str]: + result = service.run( + list(argv), + cwd=cwd, + env=_COMMAND_ENV, + timeout_seconds=timeout_seconds, + stage=stage, + phase="prepare", + ) + if not isinstance(result, dict): + raise ProvenanceError("command runner returned a non-object result") + commands.append(result) + stdout = runtime.read_verified_text(service.workspace, result["stdout"]) + stderr = runtime.read_verified_text(service.workspace, result["stderr"]) + if result.get("timed_out") or result.get("launch_error") or result.get("exit_code") != 0: + raise ProvenanceError( + f"command failed during {stage}: exit={result.get('exit_code')} " + f"timeout={result.get('timed_out')}: {stderr[-500:]}" + ) + return result, stdout, stderr + + +def _native_root(environment: Mapping[str, Any]) -> tuple[Path, str]: + """Resolve the one canonical SDK root and its declared package version. + + The normalized native surface carries a single `sdk_root`, taken from the + one complete root 1.2 publishes. Package ownership of this root, and of + every file built against it, is re-authenticated live in `_package_files` + rather than replayed from the artifact's inventory. + """ + native = runtime.environment_surface(environment, "native") or {} + package = native.get("package") + root_value = native.get("sdk_root") + if ( + native.get("installed") is not True + or not isinstance(package, Mapping) + or package.get("name") != NATIVE_PACKAGE + or package.get("status") != "installed" + or not isinstance(root_value, str) + or not Path(root_value).is_absolute() + ): + raise ProvenanceError( + f"environment does not identify one installed {NATIVE_PACKAGE} root" + ) + raw_version = package.get("version") + match = _DEBIAN_VERSION.fullmatch( + raw_version.strip() if isinstance(raw_version, str) else "" + ) + if ( + match is None + or "~" in str(raw_version) + or "really" in str(raw_version).lower() + ): + raise ProvenanceError("native SDK package is not a stable public 13.0.x release") + root = Path(root_value).resolve(strict=True) + if not root.is_dir(): + raise ProvenanceError("native SDK root is not an existing directory") + return root, str(raw_version) + + +def _live_package_inventory( + service: Any, + commands: list[dict[str, Any]], + *, + cwd: Path, + timeout_seconds: float, +) -> tuple[str, set[Path], set[Path]]: + """Return the verified live package version, directories, and files.""" + _, output, _ = _run( + service, + commands, + [ + "/usr/bin/dpkg-query", + "-W", + "-f=${db:Status-Abbrev}\\t${Version}\\t${binary:Package}\\n", + NATIVE_PACKAGE, + ], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-version", + ) + fields = output.rstrip("\n").split("\t") + if len(fields) != 3 or fields[0] != "ii " or fields[2] != NATIVE_PACKAGE: + raise ProvenanceError(f"live dpkg does not report installed {NATIVE_PACKAGE}") + version = fields[1] + match = _DEBIAN_VERSION.fullmatch(version) + if match is None or "~" in version or "really" in version.lower(): + raise ProvenanceError("native SDK package is not a stable public 13.0.x release") + _, listing, _ = _run( + service, + commands, + ["/usr/bin/dpkg-query", "-L", NATIVE_PACKAGE], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-files", + ) + _, verified, _ = _run( + service, + commands, + ["/usr/bin/dpkg", "--verify", NATIVE_PACKAGE], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-verify", + ) + if verified.strip(): + raise ProvenanceError("dpkg --verify reported modified native SDK files") + entries = [Path(line) for line in listing.splitlines() if line.startswith("/")] + owned_directories = {item.resolve(strict=True) for item in entries if item.is_dir()} + owned_files = {item.resolve(strict=True) for item in entries if item.is_file()} + return version, owned_directories, owned_files + + +def _package_files( + service: Any, + commands: list[dict[str, Any]], + *, + cwd: Path, + timeout_seconds: float, + version: str, + root: Path, +) -> set[Path]: + """Re-authenticate live package ownership and return every owned file.""" + live_version, owned_directories, owned_files = _live_package_inventory( + service, + commands, + cwd=cwd, + timeout_seconds=timeout_seconds, + ) + if live_version != version: + raise ProvenanceError("live dpkg package identity differs from runtime binding") + if root not in owned_directories: + raise ProvenanceError( + f"live {NATIVE_PACKAGE} listing does not own the bound SDK root: {root}" + ) + if not any(item.is_relative_to(root) for item in owned_files): + raise ProvenanceError( + f"live {NATIVE_PACKAGE} listing owns no file under the SDK root" + ) + return owned_files + + +def _tool_record(environment: Mapping[str, Any], name: str) -> Mapping[str, Any]: + native = runtime.environment_surface(environment, "native") or {} + tools = native.get("tools") + record = tools.get(name) if isinstance(tools, Mapping) else None + if not isinstance(record, Mapping): + raise ProvenanceError(f"environment did not record required build tool {name}") + return record + + +def _tool(environment: Mapping[str, Any], name: str) -> dict[str, Any]: + """Rehash one normalized build tool against its recorded identity. + + Schema 1.2 binds a build tool by path, size and content hash and publishes + no tool version, so the binding is the live content hash itself. Every + identity field the artifact did publish must still match; a version is + validated only when one is present. + """ + record = _tool_record(environment, name) + path = record.get("path") + digest = record.get("sha256") + if record.get("version") is not None: + _version(record.get("version"), f"tools.{name}") + if not isinstance(path, str) or not isinstance(digest, str): + raise ProvenanceError( + f"environment has no canonical identity for build tool {name}" + ) + observed = _identity(Path(path), f"native-build-tool-{name}") + if any( + record.get(field) not in (None, observed[field]) + for field in ("path", "size_bytes", "sha256") + ): + raise ProvenanceError(f"build tool {name} changed after environment probe") + return observed + + +def _generator(environment: Mapping[str, Any]) -> str: + """Return the exact CMake generator named by `tools.generator.name`.""" + name = _tool_record(environment, "generator").get("name") + if name not in _CMAKE_GENERATORS: + raise ProvenanceError( + f"environment generator name must be exactly one of {list(_CMAKE_GENERATORS)}" + ) + return str(name) + + +def _cuda(environment: Mapping[str, Any]) -> tuple[Path, str]: + """Return the installed CUDA root and validated version from `cuda`, not `tools`. + + Both the toolkit status and its version are required, so both are validated + here and the version is cross-checked against the live root marker by + `_cuda_marker`. + """ + native = runtime.environment_surface(environment, "native") or {} + cuda = native.get("cuda") + if not isinstance(cuda, Mapping) or cuda.get("status") != "installed": + raise ProvenanceError("environment does not report an installed CUDA toolkit") + version = _version(cuda.get("version"), "cuda") + root = cuda.get("root") + if not isinstance(root, str): + raise ProvenanceError("environment did not record the selected CUDA root") + return Path(root).resolve(strict=True), version + + +def _cuda_marker(root: Path, version: str) -> Path: + """Validate the recorded CUDA version against the live root version marker.""" + marker = next( + ( + candidate + for candidate in (root / "version.json", root / "version.txt") + if candidate.is_file() + ), + None, + ) + if marker is None: + raise ProvenanceError("selected CUDA root has no version marker") + try: + payload = marker.read_text(encoding="utf-8", errors="strict") + except (OSError, UnicodeDecodeError) as exc: + raise ProvenanceError(f"selected CUDA version marker is unreadable: {exc}") from exc + if _upstream(version) not in payload: + raise ProvenanceError( + f"CUDA root marker does not declare environment CUDA version {version!r}" + ) + return marker + + +def _root_from_suffix(path: Path, suffix: str) -> Path | None: + """Remove one exact relative suffix from an absolute path.""" + parts = Path(suffix).parts + if len(path.parts) < len(parts) or path.parts[-len(parts):] != parts: + return None + return Path(*path.parts[:-len(parts)]) + + +def _fixed_executable(candidates: Sequence[str], label: str) -> Path: + """Resolve one executable from a bounded, ordered path allowlist.""" + for value in candidates: + path = Path(value) + try: + resolved = path.resolve(strict=True) + details = resolved.stat() + except OSError: + continue + if stat.S_ISREG(details.st_mode) and details.st_mode & 0o111: + return resolved + raise ProvenanceError(f"required native build tool is unavailable: {label}") + + +def discover_local_native_surface( # pylint: disable=too-many-locals + *, + workspace: Path, + samples: Sequence[str], + runner: Any = None, + timeout_seconds: float = 600, +) -> dict[str, Any]: + """Discover only the fixed APT SDK package and its official sample build path.""" + root_workspace = runtime.resolve_private_workspace(workspace) + requested = tuple(samples) + if not requested or len(set(requested)) != len(requested): + raise ProvenanceError("native sample list must be non-empty and unique") + unknown = sorted(set(requested) - set(_NATIVE_ROUTES)) + if unknown: + raise ProvenanceError(f"native sample is not allowlisted: {unknown}") + service = runner or runtime.CommandRunner(root_workspace) + commands: list[dict[str, Any]] = [] + version, owned_directories, owned_files = _live_package_inventory( + service, + commands, + cwd=root_workspace, + timeout_seconds=timeout_seconds, + ) + roots: set[Path] = set() + first_suffix = _NATIVE_ROUTES[requested[0]][1] + for owned in owned_files: + candidate = _root_from_suffix(owned, first_suffix) + if candidate is None or candidate not in owned_directories: + continue + required = {candidate / "Samples/CMakeLists.txt"} + for sample in requested: + _binary, source_suffix, cmake_suffix, _libraries = _NATIVE_ROUTES[sample] + required.update({candidate / source_suffix, candidate / cmake_suffix}) + if {item.resolve(strict=True) for item in required} <= owned_files: + roots.add(candidate.resolve(strict=True)) + if len(roots) != 1: + raise ProvenanceError( + "live package does not expose exactly one complete official sample root" + ) + sdk_root = next(iter(roots)) + cmake = _fixed_executable(("/usr/bin/cmake",), "cmake") + cxx = _fixed_executable(("/usr/bin/g++",), "g++") + pkg_config = _fixed_executable(("/usr/bin/pkg-config",), "pkg-config") + try: + generator = _fixed_executable(("/usr/bin/ninja",), "ninja") + generator_name = "Ninja" + except ProvenanceError: + generator = _fixed_executable(("/usr/bin/make",), "make") + generator_name = "Unix Makefiles" + nvcc = _fixed_executable( + ( + "/usr/local/cuda/bin/nvcc", + "/usr/local/cuda-13.2/bin/nvcc", + "/usr/local/cuda-13.1/bin/nvcc", + "/usr/local/cuda-13.0/bin/nvcc", + ), + "nvcc", + ) + _result, nvcc_stdout, nvcc_stderr = _run( + service, + commands, + [str(nvcc), "--version"], + cwd=root_workspace, + timeout_seconds=timeout_seconds, + stage="native-nvcc-version", + ) + match = re.search(r"\brelease\s+([0-9]+\.[0-9]+)\b", nvcc_stdout + nvcc_stderr) + if match is None: + raise ProvenanceError("nvcc did not report a canonical CUDA release") + cuda_version = match.group(1) + cuda_root = nvcc.parent.parent.resolve(strict=True) + _cuda_marker(cuda_root, cuda_version) + tools = { + "cmake": _identity(cmake, "native-build-tool-cmake"), + "cxx": _identity(cxx, "native-build-tool-cxx"), + "nvcc": { + **_identity(nvcc, "native-build-tool-nvcc"), + "version": cuda_version, + }, + "pkg_config": _identity(pkg_config, "native-build-tool-pkg_config"), + "generator": { + **_identity(generator, "native-build-tool-generator"), + "name": generator_name, + }, + } + published_tools = { + "cmake": dict(tools["cmake"]), + "g++": dict(tools["cxx"]), + "nvcc": dict(tools["nvcc"]), + "pkg-config": dict(tools["pkg_config"]), + ("ninja" if generator_name == "Ninja" else "make"): dict( + tools["generator"] + ), + } + package = { + "name": NATIVE_PACKAGE, + "status": "installed", + "version": version, + } + return { + "surface": { + "installed": True, + "package": package, + "sdk_root": str(sdk_root), + "build_prerequisites": { + "status": "complete", + "unresolved_modules": [], + }, + "cuda": { + "status": "installed", + "version": cuda_version, + "root": str(cuda_root), + }, + "tools": tools, + }, + "evidence": { + "native_sdk": { + "status": "installed", + "package": package, + "complete_roots": [str(sdk_root)], + }, + "native_build_prerequisites": { + "status": "complete", + "missing_modules": [], + "unknown_modules": [], + }, + "cuda_toolkit": { + "status": "available", + "version": cuda_version, + "nvcc_discovery": {"root": str(cuda_root)}, + }, + "build_tool_identities": published_tools, + }, + "commands": commands, + "authority": "live-dpkg-package-and-package-owned-official-sources", + } + + +def _compile_sources(path: Path) -> set[Path]: + value = runtime.strict_json_loads(path.read_bytes()) + if not isinstance(value, list) or not value: + raise ProvenanceError("CMake compile_commands.json must be non-empty") + sources: set[Path] = set() + for entry in value: + if not isinstance(entry, dict) or not isinstance(entry.get("file"), str): + raise ProvenanceError("compile_commands.json entry has no source") + sources.add(Path(entry["file"]).resolve(strict=True)) + return sources + + +def _runtime_libraries(output: str, required: Sequence[str]) -> list[dict[str, Any]]: + if re.search(r"(?m)=>\s+not found\s*$", output): + raise ProvenanceError("ldd reported an unresolved runtime library") + resolved: dict[str, Path] = {} + for line in output.splitlines(): + match = re.match(r"\s*(\S+)\s+=>\s+(\/\S+)\s+\(", line) + if match: + library, path = match.groups() + candidate = Path(path).resolve(strict=True) + if "stubs" in candidate.parts: + raise ProvenanceError(f"ldd resolved a CUDA stub library: {candidate}") + resolved[library] = candidate + missing = [name for name in required if name not in resolved] + if missing: + raise ProvenanceError(f"ldd did not resolve required codec libraries: {missing}") + return [_identity(resolved[name], f"native-runtime-library-{name}") for name in required] + + +def _prepare_native( # pylint: disable=too-many-arguments,too-many-locals + sample: str, + *, + root: Path, + owned: set[Path], + environment: Mapping[str, Any], + environment_identity: Mapping[str, Any], + environment_workspace: Path, + workspace: Path, + service: Any, + commands: list[dict[str, Any]], + timeout_seconds: float, +) -> tuple[AuthenticatedSample, dict[str, Any]]: + binary_suffix, source_suffix, cmake_suffix, libraries = _NATIVE_ROUTES[sample] + source = (root / source_suffix).resolve(strict=True) + cmake_file = (root / cmake_suffix).resolve(strict=True) + top_cmake = (root / "Samples/CMakeLists.txt").resolve(strict=True) + if not {source, cmake_file, top_cmake} <= owned: + raise ProvenanceError(f"{sample} source/CMake files are not package-owned") + tools = {name: _tool(environment, name) for name in _BUILD_TOOLS} + generator = _generator(environment) + cuda_root, cuda_version = _cuda(environment) + if not Path(tools["nvcc"]["path"]).is_relative_to(cuda_root): + raise ProvenanceError("environment nvcc is outside selected CUDA root") + nvcc_version = _version(_tool_record(environment, "nvcc").get("version"), "tools.nvcc") + if _release(nvcc_version, "tools.nvcc") != _release( + _upstream(cuda_version), "cuda" + ): + raise ProvenanceError("environment nvcc release differs from selected CUDA release") + marker = _cuda_marker(cuda_root, cuda_version) + build = runtime.create_private_workspace(workspace / f"native-build-{sample}") + configure = [ + tools["cmake"]["path"], + "-S", + str(root / "Samples"), + "-B", + str(build), + "-G", + generator, + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + f"-DCMAKE_CXX_COMPILER={tools['cxx']['path']}", + f"-DCMAKE_CUDA_COMPILER={tools['nvcc']['path']}", + f"-DCMAKE_MAKE_PROGRAM={tools['generator']['path']}", + f"-DCMAKE_PREFIX_PATH={cuda_root}", + ] + _run( + service, + commands, + configure, + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"configure-{sample.lower()}", + ) + _run( + service, + commands, + [tools["cmake"]["path"], "--build", str(build), "--target", sample, "--parallel", "1"], + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"build-{sample.lower()}", + ) + cache_path = build / "CMakeCache.txt" + compile_path = build / "compile_commands.json" + cache: dict[str, str] = {} + for line in cache_path.read_text(encoding="utf-8").splitlines(): + if not line.startswith(("//", "#")) and "=" in line and ":" in line: + name_type, value = line.split("=", 1) + cache[name_type.split(":", 1)[0]] = value + expected = { + "CMAKE_HOME_DIRECTORY": str(root / "Samples"), + "CMAKE_CXX_COMPILER": tools["cxx"]["path"], + "CMAKE_CUDA_COMPILER": tools["nvcc"]["path"], + "CMAKE_MAKE_PROGRAM": tools["generator"]["path"], + "CMAKE_GENERATOR": generator, + } + for name, value in expected.items(): + observed = cache.get(name) + differs = ( + observed != value + if name == "CMAKE_GENERATOR" + else not observed or Path(observed).resolve() != Path(value).resolve() + ) + if differs: + raise ProvenanceError(f"CMake cache disagrees for {name}") + compiled = _compile_sources(compile_path) + if source not in compiled or not compiled <= owned: + raise ProvenanceError(f"{sample} is not bound to package-owned sources") + executable = (build / binary_suffix).resolve(strict=True) + if not stat.S_ISREG(executable.stat().st_mode) or executable.stat().st_mode & 0o111 == 0: + raise ProvenanceError(f"built {sample} output is not executable") + _, linkage, _ = _run( + service, + commands, + ["/usr/bin/ldd", str(executable)], + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"linkage-{sample.lower()}", + ) + linked = _runtime_libraries(linkage, libraries) + protected = [ + _identity(executable, "native-official-sample-executable"), + *(_identity(path, "native-package-source") for path in sorted(compiled)), + _identity(cache_path, "native-cmake-cache"), + _identity(compile_path, "native-compile-commands"), + _identity(marker, "cuda-root-version-marker"), + *tools.values(), + ] + token = _sample_record( + surface="native", + sample=sample, + launcher=(str(executable),), + workspace=workspace, + protected=protected, + runtime_libraries=linked, + environment_workspace=environment_workspace, + environment_identity=environment_identity, + ) + return token, { + "sample": sample, + "launcher": list(token.launcher), + "source": str(source), + "configure_argv": configure, + "compiled_sources": [str(path) for path in sorted(compiled)], + "protected_files": protected, + "runtime_libraries": linked, + } + + +def authenticate_native_samples( # pylint: disable=too-many-arguments,too-many-locals + *, + environment_workspace: Path, + environment_identity: Mapping[str, Any], + workspace: Path, + samples: Sequence[str], + runner: Any = None, + timeout_seconds: float = 600, + report_path: Path | None = None, +) -> Authentication: + """Build and authenticate the benchmark's allowlisted native samples.""" + root_workspace = runtime.resolve_private_workspace(workspace) + requested = tuple(samples) + if not requested or len(set(requested)) != len(requested): + raise ProvenanceError("native sample list must be non-empty and unique") + unknown = sorted(set(requested) - set(_NATIVE_ROUTES)) + if unknown: + raise ProvenanceError(f"native sample is not allowlisted: {unknown}") + destination = report_path or root_workspace / "native-sample-provenance.json" + commands: list[dict[str, Any]] = [] + evidence: list[dict[str, Any]] = [] + records: list[AuthenticatedSample] = [] + service = runner or runtime.CommandRunner(root_workspace) + report: dict[str, Any] = { + "schema_version": "1", + "kind": "native-sample-provenance", + "status": "failed", + "environment": dict(environment_identity), + "environment_workspace": str( + runtime.resolve_private_workspace(environment_workspace) + ), + "requested_samples": list(requested), + "samples": evidence, + "commands": commands, + } + try: + environment = _environment(environment_workspace, environment_identity) + sdk_root, version = _native_root(environment) + owned = _package_files( + service, + commands, + cwd=root_workspace, + timeout_seconds=timeout_seconds, + version=version, + root=sdk_root, + ) + for sample in requested: + token, item = _prepare_native( + sample, + root=sdk_root, + owned=owned, + environment=environment, + environment_identity=environment_identity, + environment_workspace=runtime.resolve_private_workspace( + environment_workspace + ), + workspace=root_workspace, + service=service, + commands=commands, + timeout_seconds=timeout_seconds, + ) + records.append(token) + evidence.append(item) + report["status"] = "authenticated" + report["package"] = { + "name": NATIVE_PACKAGE, + "version": version, + "root": str(sdk_root), + "ownership": "re-authenticated-live", + } + except Exception as exc: + report["failure"] = f"{type(exc).__name__}: {exc}" + runtime.write_fresh_json(root_workspace, destination, report) + raise + identity = runtime.write_fresh_json(root_workspace, destination, report) + return Authentication(tuple(records), identity, "native") + + +def _pynvc_environment( + workspace: Path, identity: Mapping[str, Any] +) -> tuple[Path, Path, Mapping[str, Any]]: + """Bind the normalized Py surface identity to the live venv it names. + + The surface carries identity only: the lexical `interpreter` as invoked, + the resolved `interpreter_identity`, `sys_prefix`, the PyNv `version`, the + `module` and the `extension`. Every binding below is proven against the + live filesystem rather than replayed from the artifact. + """ + value = _environment(workspace, identity) + pynvc = runtime.environment_surface(value, "pynvc") or {} + interpreter_identity = pynvc.get("interpreter_identity") + if ( + pynvc.get("installed") is not True + or pynvc.get("version") != PYNVC_REQUIRED_VERSION + or not isinstance(interpreter_identity, Mapping) + ): + raise ProvenanceError( + f"environment lacks installed PyNvVideoCodec {PYNVC_REQUIRED_VERSION}" + ) + launcher_value = pynvc.get("interpreter") + resolved_value = interpreter_identity.get("path") + prefix_value = pynvc.get("sys_prefix") + digest = interpreter_identity.get("sha256") + if not all( + isinstance(item, str) + for item in (launcher_value, resolved_value, prefix_value, digest) + ) or not Path(str(launcher_value)).is_absolute(): + raise ProvenanceError("environment lacks selected PyNvVideoCodec venv") + launcher = Path(str(launcher_value)) + resolved = launcher.resolve(strict=True) + if ( + not launcher.is_file() + or launcher.stat().st_mode & 0o111 == 0 + or resolved != Path(str(resolved_value)).resolve(strict=True) + or launcher.parent.parent.resolve(strict=True) + != Path(str(prefix_value)).resolve(strict=True) + ): + raise ProvenanceError("selected venv launcher binding is invalid") + if _identity(resolved, "pynvc-venv-python")["sha256"] != digest: + raise ProvenanceError("selected interpreter changed after environment probe") + # The imported module version must equal the distribution version. + module = pynvc.get("module") + if ( + not isinstance(module, Mapping) + or not isinstance(module.get("path"), str) + or module.get("version") != PYNVC_REQUIRED_VERSION + ): + raise ProvenanceError( + "environment imported PyNvVideoCodec module version differs from " + f"distribution version {PYNVC_REQUIRED_VERSION}" + ) + return launcher, resolved, pynvc + + +def _probe_pynvc( + launcher: Path, + *, + workspace: Path, + service: Any, + timeout_seconds: float, +) -> dict[str, Any]: + result = service.run( + [str(launcher), "-I", "-c", _PROBE_PROGRAM], + cwd=workspace, + env=_COMMAND_ENV, + timeout_seconds=timeout_seconds, + stage="pynvc-record-probe", + phase="probe", + ) + stderr = runtime.read_verified_text(service.workspace, result["stderr"]) + if result.get("timed_out") or result.get("launch_error") or result.get("exit_code") != 0: + raise ProvenanceError(f"isolated PyNvVideoCodec RECORD probe failed: {stderr[-500:]}") + value = runtime.strict_json_loads( + runtime.read_verified_text(service.workspace, result["stdout"]) + ) + if ( + not isinstance(value, dict) + or value.get("schema_version") != "1" + or value.get("kind") != "pynvc-record-probe" + or value.get("interpreter") != str(launcher.resolve(strict=True)) + or value.get("normalized_name") != "pynvvideocodec" + or value.get("version") != PYNVC_REQUIRED_VERSION + or value.get("imported_version") != PYNVC_REQUIRED_VERSION + or not isinstance(value.get("paths"), dict) + or not isinstance(value.get("module_path"), str) + ): + raise ProvenanceError("isolated probe did not select PyNvVideoCodec 2.1.0") + # Exactly one extension must actually be loaded by the interpreter. + loaded = value.get("loaded_extensions") + if not isinstance(loaded, list) or len(loaded) != 1 or not isinstance(loaded[0], str): + raise ProvenanceError( + "isolated probe did not load exactly one PyNvVideoCodec extension" + ) + prefix = Path(value.get("sys_prefix", "")).resolve(strict=True) + if launcher.parent.parent.resolve(strict=True) != prefix: + raise ProvenanceError("isolated probe sys.prefix differs from selected venv") + for member, path_value in value["paths"].items(): + if ( + not isinstance(member, str) + or member != member.replace("\\", "/") + or not isinstance(path_value, str) + or not Path(path_value).resolve(strict=True).is_relative_to(prefix) + ): + raise ProvenanceError("wheel member resolved outside selected venv") + value["command"] = result + return value + + +def discover_local_pynvc_surface( + *, + interpreter: str, + workspace: Path, + runner: Any = None, + timeout_seconds: float = 600, +) -> dict[str, Any]: + """Probe one exact caller-selected interpreter; never search for a venv.""" + if ( + not isinstance(interpreter, str) + or not interpreter + or interpreter != interpreter.strip() + or not Path(interpreter).is_absolute() + or str(Path(interpreter)) != interpreter + ): + raise ProvenanceError( + "pynvc_interpreter must be one exact canonical absolute path" + ) + launcher = Path(interpreter) + try: + details = launcher.stat() + resolved = launcher.resolve(strict=True) + except OSError as exc: + raise ProvenanceError(f"pynvc_interpreter is unavailable: {exc}") from exc + if not stat.S_ISREG(details.st_mode) or details.st_mode & 0o111 == 0: + raise ProvenanceError("pynvc_interpreter must be an executable regular file") + root = runtime.resolve_private_workspace(workspace) + service = runner or runtime.CommandRunner(root) + probe = _probe_pynvc( + launcher, + workspace=root, + service=service, + timeout_seconds=timeout_seconds, + ) + extension = _loaded_extension_identity(probe) + dependencies = probe.get("dependencies") + if not isinstance(dependencies, Mapping): + raise ProvenanceError("isolated PyNvVideoCodec probe omitted dependencies") + surface = { + "installed": True, + "version": probe["version"], + "interpreter": interpreter, + "interpreter_identity": _identity(resolved, "pynvc-venv-python"), + "sys_prefix": probe["sys_prefix"], + "extension": extension, + "module": { + "path": probe["module_path"], + "version": probe["imported_version"], + }, + "dependencies": dict(dependencies), + } + package_records = { + name: { + **dict(record), + "requirement_satisfied": record.get("ready") is True, + } + for name, record in dependencies.items() + if isinstance(record, Mapping) + } + return { + "surface": surface, + "evidence": { + "python": { + "interpreter_identities": { + "running": dict(surface["interpreter_identity"]) + }, + "packages": package_records, + }, + "pynvc": { + "imported": True, + "distribution_version": probe["version"], + "identity": { + "status": "verified", + "interpreter": interpreter, + "sys_prefix": probe["sys_prefix"], + "extension": extension, + "module": dict(surface["module"]), + }, + }, + }, + "command": probe["command"], + "authority": "exact-interpreter-wheel-record-and-loaded-extension", + } + + +def _bind_probe(probe: Mapping[str, Any], pynvc: Mapping[str, Any]) -> Path: + """Bind the live probe to the recorded Py surface and return the venv prefix. + + What this interpreter actually imports must equal what the environment + authenticated, not merely what distribution metadata locates on disk. The + interpreter's single loaded extension is compared against the authenticated + `extension.path`; an environment that additively publishes `loaded_path` + is bound to that instead, and both must name the same file. + """ + prefix = Path(str(probe["sys_prefix"])).resolve(strict=True) + if ( + prefix != Path(str(pynvc["sys_prefix"])).resolve(strict=True) + or probe.get("version") != pynvc.get("version") + ): + raise ProvenanceError("isolated RECORD probe differs from environment") + module = pynvc["module"] + extension = pynvc.get("extension") if isinstance(pynvc.get("extension"), Mapping) else {} + loaded_path = extension.get("loaded_path") + if loaded_path is None: + loaded_path = extension.get("path") + if not isinstance(loaded_path, str): + raise ProvenanceError("environment lacks the authenticated PyNvVideoCodec extension") + if ( + probe.get("imported_version") != module.get("version") + or Path(str(probe.get("module_path"))) + != Path(str(module.get("path"))).resolve(strict=True) + or probe.get("loaded_extensions") + != [str(Path(loaded_path).resolve(strict=True))] + ): + raise ProvenanceError( + "isolated probe loaded a different PyNvVideoCodec module or extension " + "than the environment recorded" + ) + return prefix + + +def _record_rows( + probe: Mapping[str, Any] +) -> tuple[dict[str, tuple[str, str]], dict[str, Any]]: + paths = probe.get("paths") + member = probe.get("record_entry") + if not isinstance(paths, dict) or not isinstance(member, str): + raise ProvenanceError("record probe shape is invalid") + path_value = paths.get(member) + if not isinstance(path_value, str): + raise ProvenanceError("record probe did not resolve RECORD") + prefix = Path(str(probe["sys_prefix"])).resolve(strict=True) + record_path = Path(path_value).resolve(strict=True) + if not record_path.is_relative_to(prefix): + raise ProvenanceError("wheel RECORD is outside selected venv") + try: + rows = list( + csv.reader( + io.StringIO(record_path.read_bytes().decode("utf-8", errors="strict")) + ) + ) + except (UnicodeDecodeError, csv.Error) as exc: + raise ProvenanceError(f"wheel RECORD is invalid: {exc}") from exc + parsed: dict[str, tuple[str, str]] = {} + for row in rows: + if len(row) != 3 or not row[0] or row[0] != row[0].replace("\\", "/"): + raise ProvenanceError("wheel RECORD row is not canonical") + if row[0] in parsed: + raise ProvenanceError(f"wheel RECORD duplicate member: {row[0]!r}") + parsed[row[0]] = (row[1], row[2]) + return parsed, _identity(record_path, "pynvc-wheel-record") + + +def _loaded_extension_identity(probe: Mapping[str, Any]) -> dict[str, Any]: + """Prove the one loaded native extension is owned by the selected wheel.""" + loaded = probe.get("loaded_extensions") + paths = probe.get("paths") + if ( + not isinstance(loaded, list) + or len(loaded) != 1 + or not isinstance(loaded[0], str) + or not isinstance(paths, Mapping) + ): + raise ProvenanceError("probe did not expose one loaded wheel extension") + prefix = Path(str(probe["sys_prefix"])).resolve(strict=True) + extension_path = Path(loaded[0]).resolve(strict=True) + if not extension_path.is_relative_to(prefix): + raise ProvenanceError("loaded PyNvVideoCodec extension is outside selected venv") + members = [ + member + for member, value in paths.items() + if isinstance(member, str) + and isinstance(value, str) + and Path(value).resolve(strict=True) == extension_path + ] + rows, _record_identity = _record_rows(probe) + if len(members) != 1 or members[0] not in rows: + raise ProvenanceError("loaded extension is not one unique wheel RECORD member") + member = members[0] + encoded_hash, size_text = rows[member] + if not encoded_hash.startswith("sha256=") or not size_text.isdecimal(): + raise ProvenanceError("loaded extension has no wheel RECORD SHA-256/size") + payload = extension_path.read_bytes() + actual = ( + base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + .rstrip(b"=") + .decode() + ) + matches_record = ( + encoded_hash.removeprefix("sha256=") == actual + and int(size_text) == len(payload) + ) + known_stale = ( + probe.get("version") == PYNVC_REQUIRED_VERSION + and _NATIVE_EXTENSION_MEMBER.fullmatch(member) is not None + ) + if not matches_record and not known_stale: + raise ProvenanceError("loaded extension differs from wheel RECORD") + identity = _identity(extension_path, "pynvc-extension") + return { + **identity, + "loaded_path": identity["path"], + "record_member": member, + "record_consistency": ( + "matches_record" if matches_record else "known_upstream_stale_record" + ), + } + + +def _record_members( + probe: Mapping[str, Any], members: Sequence[str] +) -> tuple[dict[str, Any], ...]: + rows, _identity_value = _record_rows(probe) + paths = probe["paths"] + prefix = Path(str(probe["sys_prefix"])).resolve(strict=True) + result: list[dict[str, Any]] = [] + for member in members: + if member not in rows or member not in paths: + raise ProvenanceError(f"wheel RECORD member is missing: {member!r}") + encoded_hash, size_text = rows[member] + if not encoded_hash.startswith("sha256=") or not size_text.isdecimal(): + raise ProvenanceError(f"wheel RECORD member has no SHA-256/size: {member}") + path = Path(paths[member]).resolve(strict=True) + if not path.is_relative_to(prefix): + raise ProvenanceError("wheel member resolved outside selected venv") + payload = path.read_bytes() + actual = ( + base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + .rstrip(b"=") + .decode() + ) + if encoded_hash.removeprefix("sha256=") != actual or int(size_text) != len(payload): + raise ProvenanceError(f"installed wheel member differs from RECORD: {member}") + result.append( + { + "member": member, + "record_hash": encoded_hash, + "identity": _identity(path, "pynvc-wheel-member"), + } + ) + return tuple(result) + + +def _extension_seal( + pynvc: Mapping[str, Any], prefix: Path +) -> list[dict[str, Any]]: + """Seal the extension the interpreter actually loaded. + + Hashing a file found through distribution metadata would prove only that a + matching file exists on disk, so the live bytes must hash to the recorded + digest inside the selected venv prefix, and `_bind_probe` has already + required the interpreter's one loaded extension to be this exact file. An + environment that additively publishes `extension.loaded_path` must name the + same file; 1.2 publishes none and nothing is inferred from its absence. + """ + extension = pynvc.get("extension") if isinstance(pynvc.get("extension"), Mapping) else {} + path_value = extension.get("path") + digest = extension.get("sha256") + loaded_path = extension.get("loaded_path") + if loaded_path is None: + loaded_path = path_value + if ( + not isinstance(path_value, str) + or not isinstance(digest, str) + or not isinstance(loaded_path, str) + ): + raise ProvenanceError("environment lacks a PyNvVideoCodec extension identity") + if loaded_path != path_value: + raise ProvenanceError( + "environment loaded extension differs from the authenticated extension" + ) + identity = _identity(Path(path_value), "pynvc-extension") + if identity["path"] != path_value or identity["sha256"] != digest: + raise ProvenanceError("installed extension differs from the environment identity") + if not Path(identity["path"]).is_relative_to(prefix): + raise ProvenanceError("PyNvVideoCodec extension resolved outside selected venv") + return [identity] + + +def authenticate_pynvc_samples( # pylint: disable=too-many-arguments,too-many-locals + *, + environment_workspace: Path, + environment_identity: Mapping[str, Any], + workspace: Path, + samples: Sequence[str], + runner: Any = None, + timeout_seconds: float = 600, + report_path: Path | None = None, +) -> Authentication: + """Authenticate the benchmark's wheel-owned Python performance samples.""" + root = runtime.resolve_private_workspace(workspace) + env_root = runtime.resolve_private_workspace(environment_workspace) + requested = tuple(samples) + if not requested or len(set(requested)) != len(requested): + raise ProvenanceError("PyNvVideoCodec sample list must be non-empty and unique") + unknown = sorted(set(requested) - set(_PYNVC_ROUTES)) + if unknown: + raise ProvenanceError(f"PyNvVideoCodec sample is not allowlisted: {unknown}") + destination = report_path or root / "pynvc-sample-provenance.json" + report: dict[str, Any] = { + "schema_version": "1", + "kind": "pynvc-sample-provenance", + "status": "failed", + "environment": dict(environment_identity), + "requested_samples": list(requested), + "samples": [], + } + try: + launcher, interpreter, pynvc = _pynvc_environment( + env_root, environment_identity + ) + service = runner or runtime.CommandRunner(root) + probe = _probe_pynvc( + launcher, + workspace=root, + service=service, + timeout_seconds=timeout_seconds, + ) + prefix = _bind_probe(probe, pynvc) + record_member = probe.get("record_entry") + if not isinstance(record_member, str) or not record_member.endswith( + ".dist-info/RECORD" + ): + raise ProvenanceError("wheel RECORD member is invalid") + metadata = record_member.removesuffix("RECORD") + "METADATA" + wheel = record_member.removesuffix("RECORD") + "WHEEL" + # The package module is proven live rather than replayed: + # `_record_members` requires it to be an exact wheel RECORD member + # resolving inside the selected venv prefix. + common = _record_members( + probe, ("PyNvVideoCodec/__init__.py", metadata, wheel) + ) + _rows, record_identity = _record_rows(probe) + extension = _extension_seal(pynvc, prefix) + interpreter_identity = _identity(interpreter, "pynvc-venv-python") + records: list[AuthenticatedSample] = [] + for sample in requested: + members = (sample, *_PYNVC_ROUTES[sample]) + owned = _record_members(probe, members) + protected = [ + interpreter_identity, + record_identity, + *(item["identity"] for item in common), + *(item["identity"] for item in owned), + *extension, + ] + token = _sample_record( + surface="pynvc", + sample=sample, + launcher=(str(launcher), "-I", probe["paths"][sample]), + workspace=root, + protected=protected, + runtime_libraries=(), + environment_workspace=env_root, + environment_identity=environment_identity, + ) + records.append(token) + report["samples"].append( + { + "sample": sample, + "launcher": list(token.launcher), + "record_members": list(members), + "protected_files": protected, + } + ) + report["status"] = "authenticated" + report["distribution"] = { + "reported_name": probe["name"], + "normalized_name": probe["normalized_name"], + "version": probe["version"], + } + report["record_probe_command"] = probe["command"] + report["record"] = record_identity + report["record_member"] = record_member + except Exception as exc: + report["failure"] = f"{type(exc).__name__}: {exc}" + runtime.write_fresh_json(root, destination, report) + raise + identity = runtime.write_fresh_json(root, destination, report) + return Authentication(tuple(records), identity, "PyNvVideoCodec") + + +def run_authenticated( # pylint: disable=too-many-arguments + token: AuthenticatedSample, + arguments: Sequence[str], + *, + workspace: Path, + runner: Any, + cwd: Path, + timeout_seconds: float, + stage: str, + phase: str, +) -> dict[str, Any]: + """Immediately rehash every seal, then launch the exact sample argv.""" + if not isinstance(token, AuthenticatedSample): + raise ProvenanceError("official sample has no authenticated record") + root = runtime.resolve_private_workspace(workspace) + if root != Path(token.workspace).resolve(strict=True): + raise ProvenanceError("launch workspace differs from authenticated workspace") + command_cwd = Path(cwd).resolve(strict=True) + if not command_cwd.is_relative_to(root): + raise ProvenanceError("official sample cwd must be inside private workspace") + suffix = tuple(arguments) + if any(not isinstance(item, str) or "\0" in item for item in suffix): + raise ProvenanceError("official sample arguments must be NUL-free strings") + runtime.verify_artifact( + Path(token.environment_workspace), token.environment_identity + ) + for item in (*token.protected_files, *token.runtime_libraries): + runtime.verify_external_artifact(item.identity) + executable = Path(token.launcher[0]) + if executable.stat().st_mode & 0o111 == 0: + raise ProvenanceError("authenticated launcher is no longer executable") + expected_kind = { + "native": "native-official-sample-executable", + "pynvc": "pynvc-venv-python", + }.get(token.surface) + seals = [ + seal + for seal in token.protected_files + if seal.identity.get("kind") == expected_kind + ] + if expected_kind is None or len(seals) != 1: + raise ProvenanceError("authenticated launcher seal is absent or duplicated") + if executable.resolve(strict=True) != Path(seals[0].identity["path"]): + raise ProvenanceError("authenticated launcher resolves outside sealed target") + argv = [*token.launcher, *suffix] + result = runner.run( + argv, + cwd=command_cwd, + env=_COMMAND_ENV, + timeout_seconds=timeout_seconds, + stage=stage, + phase=phase, + ) + expected = { + "argv": argv, + "cwd": str(command_cwd), + "stage": stage, + "phase": phase, + "timeout_seconds": float(timeout_seconds), + } + if not isinstance(result, dict) or any( + result.get(key) != value for key, value in expected.items() + ): + raise ProvenanceError("command evidence does not match authenticated launch") + return result diff --git a/skills/jetson-video-benchmark/scripts/benchmark_runtime.py b/skills/jetson-video-benchmark/scripts/benchmark_runtime.py new file mode 100644 index 00000000..eecdcc0b --- /dev/null +++ b/skills/jetson-video-benchmark/scripts/benchmark_runtime.py @@ -0,0 +1,854 @@ +#!/usr/bin/env python3 +"""Private artifact, command-evidence, and surface-routing runtime.""" + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# The routing truth table and exact command shape are intentionally explicit. +# pylint: disable=too-many-boolean-expressions,too-many-branches + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import signal +import stat +import subprocess +import time +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Mapping, Sequence + +IDENTITY_KEYS = frozenset( + {"schema_version", "kind", "path", "size_bytes", "sha256"} +) +MAX_JSON_BYTES = 64 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 * 1024 +_HASH_CHUNK_BYTES = 1024 * 1024 +_TAIL_BYTES = 4096 +_STAGE_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") +_PHASES = { + "probe", + "plan", + "prepare", + "install", + "warmup", + "measure", + "verify", + "execute", +} +_SURFACES = ("native", "pynvc") +# 1.2 publishes `installation.build_tool_identities` under the executable's own +# name; these are the internal names the toolchain is consumed under. +_PUBLISHED_BUILD_TOOLS = ( + ("cmake", "cmake"), + ("cxx", "g++"), + ("nvcc", "nvcc"), + ("pkg_config", "pkg-config"), +) +# CMake generator preference, highest first, with its exact generator name. +_PUBLISHED_GENERATORS = (("ninja", "Ninja"), ("make", "Unix Makefiles")) + + +class ArtifactError(ValueError): + """Raised when an artifact fails its portable content contract.""" + + +class CommandError(ValueError): + """Raised when a command violates the execution-evidence contract.""" + + +def _reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON number is not allowed: {value}") + + +def _object_from_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError(f"duplicate JSON object name: {key!r}") + value[key] = item + return value + + +def strict_json_loads(payload: bytes | str) -> Any: + """Decode strict UTF-8 JSON without duplicates or non-finite values.""" + if isinstance(payload, bytes): + try: + text = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ArtifactError(f"JSON is not UTF-8: {exc}") from exc + elif isinstance(payload, str): + text = payload + else: + raise ArtifactError("JSON payload must be bytes or text") + try: + return json.loads( + text, + object_pairs_hook=_object_from_pairs, + parse_constant=_reject_constant, + ) + except (json.JSONDecodeError, ValueError) as exc: + raise ArtifactError(f"invalid strict JSON: {exc}") from exc + + +def canonical_json_bytes(value: Any) -> bytes: + """Encode deterministic strict JSON with one trailing newline.""" + try: + rendered = json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + except (TypeError, ValueError) as exc: + raise ArtifactError( + f"value cannot be represented as strict JSON: {exc}" + ) from exc + return (rendered + "\n").encode("utf-8") + + +def _contract_string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ArtifactError(f"{label} must be a non-empty canonical string") + return value + + +def _contract_path(value: Any) -> str: + locator = _contract_string(value, "path") + if any(ord(character) < 32 or ord(character) == 127 for character in locator): + raise ArtifactError("path must not contain control characters") + return locator + + +def _bound(value: Any, label: str, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ArtifactError(f"{label} must be a non-negative integer") + if value > maximum: + raise ArtifactError(f"{label} exceeds the {maximum}-byte bound") + return value + + +def _existing_regular(path: Path) -> Path: + try: + candidate = Path(_contract_path(str(path))).expanduser().resolve(strict=True) + details = candidate.stat() + except (OSError, RuntimeError) as exc: + raise ArtifactError(f"artifact is not an accessible file: {path}") from exc + if not stat.S_ISREG(details.st_mode): + raise ArtifactError(f"artifact is not a regular file: {candidate}") + return candidate + + +def create_private_workspace(path: Path) -> Path: + """Create one fresh canonical workspace with exact mode 0700.""" + requested = Path(_contract_path(str(path))).expanduser() + try: + parent = requested.parent.resolve(strict=True) + candidate = parent / requested.name + os.mkdir(candidate, mode=0o700) + os.chmod(candidate, 0o700) + except (OSError, RuntimeError) as exc: + raise ArtifactError(f"cannot create fresh workspace {requested}: {exc}") from exc + return resolve_private_workspace(candidate) + + +def resolve_private_workspace(path: Path) -> Path: + """Resolve an existing workspace and require exact mode 0700.""" + try: + candidate = Path(_contract_path(str(path))).expanduser().resolve(strict=True) + details = candidate.stat() + except (OSError, RuntimeError) as exc: + raise ArtifactError(f"workspace is unavailable: {path}") from exc + if not stat.S_ISDIR(details.st_mode): + raise ArtifactError(f"workspace is not a directory: {candidate}") + if stat.S_IMODE(details.st_mode) != 0o700: + raise ArtifactError(f"workspace mode must be exactly 0700: {candidate}") + return candidate + + +def _workspace_path(workspace: Path, path: Path, *, must_exist: bool) -> Path: + root = resolve_private_workspace(workspace) + requested = Path(_contract_path(str(path))) + candidate = requested if requested.is_absolute() else root / requested + try: + resolved = candidate.resolve(strict=must_exist) + resolved.relative_to(root) + except (OSError, RuntimeError, ValueError) as exc: + raise ArtifactError(f"artifact is outside or unavailable: {candidate}") from exc + if must_exist: + return _existing_regular(resolved) + try: + parent = resolved.parent.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ArtifactError( + f"artifact output parent is unavailable: {resolved.parent}" + ) from exc + if not parent.is_dir(): + raise ArtifactError(f"artifact output parent is not a directory: {parent}") + return parent / resolved.name + + +def _hash(path: Path, maximum: int = MAX_ARTIFACT_BYTES) -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + try: + with path.open("rb") as stream: + while chunk := stream.read(_HASH_CHUNK_BYTES): + total += len(chunk) + if total > maximum: + raise ArtifactError( + f"artifact exceeds the {maximum}-byte bound: {path}" + ) + digest.update(chunk) + except OSError as exc: + raise ArtifactError(f"cannot read artifact {path}: {exc}") from exc + return total, digest.hexdigest() + + +def _identity( + locator: str, + size_bytes: int, + sha256: str, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + return { + "schema_version": _contract_string(schema_version, "schema_version"), + "kind": _contract_string(kind, "kind"), + "path": _contract_path(locator), + "size_bytes": size_bytes, + "sha256": sha256, + } + + +def _validate_identity(identity: Mapping[str, Any]) -> tuple[str, int, str]: + if not isinstance(identity, Mapping) or set(identity) != IDENTITY_KEYS: + raise ArtifactError( + f"artifact identity fields must be exactly {sorted(IDENTITY_KEYS)}" + ) + _contract_string(identity["schema_version"], "schema_version") + _contract_string(identity["kind"], "kind") + locator = _contract_path(identity["path"]) + size_bytes = _bound(identity["size_bytes"], "size_bytes", MAX_ARTIFACT_BYTES) + digest = identity["sha256"] + if ( + not isinstance(digest, str) + or len(digest) != 64 + or any(character not in "0123456789abcdef" for character in digest) + ): + raise ArtifactError("sha256 must be 64 lowercase hexadecimal characters") + return locator, size_bytes, digest + + +def snapshot_external_artifact( + path: Path, *, schema_version: str, kind: str +) -> dict[str, Any]: + """Snapshot an external file with a canonical absolute locator.""" + candidate = _existing_regular(path) + size_bytes, digest = _hash(candidate) + return _identity( + str(candidate), + size_bytes, + digest, + schema_version=schema_version, + kind=kind, + ) + + +def verify_external_artifact(identity: Mapping[str, Any]) -> Path: + """Verify an external identity and return its current canonical path.""" + locator, expected_size, expected_digest = _validate_identity(identity) + if not Path(locator).is_absolute(): + raise ArtifactError("external artifact path must be absolute") + candidate = _existing_regular(Path(locator)) + if str(candidate) != locator: + raise ArtifactError("external artifact path must be canonical") + if _hash(candidate) != (expected_size, expected_digest): + raise ArtifactError(f"external artifact changed: {candidate}") + return candidate + + +def snapshot_artifact( + workspace: Path, + path: Path, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + """Snapshot a workspace file with a portable relative locator.""" + root = resolve_private_workspace(workspace) + candidate = _workspace_path(root, path, must_exist=True) + size_bytes, digest = _hash(candidate) + return _identity( + candidate.relative_to(root).as_posix(), + size_bytes, + digest, + schema_version=schema_version, + kind=kind, + ) + + +def _workspace_identity_path( + workspace: Path, identity: Mapping[str, Any] +) -> Path: + locator, _size, _digest = _validate_identity(identity) + pure = PurePosixPath(locator) + if ( + pure.is_absolute() + or not pure.parts + or any(part in {"", ".", ".."} for part in pure.parts) + or pure.as_posix() != locator + ): + raise ArtifactError( + "workspace artifact path must be a canonical relative locator" + ) + return _workspace_path(workspace, Path(*pure.parts), must_exist=True) + + +def verify_artifact(workspace: Path, identity: Mapping[str, Any]) -> Path: + """Verify one workspace identity and return its current path.""" + candidate = _workspace_identity_path(workspace, identity) + _locator, expected_size, expected_digest = _validate_identity(identity) + if _hash(candidate) != (expected_size, expected_digest): + raise ArtifactError(f"workspace artifact changed: {candidate}") + return candidate + + +def write_fresh_bytes( + workspace: Path, + path: Path, + payload: bytes, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + """Write one new 0600 workspace artifact and return its identity.""" + if not isinstance(payload, bytes): + raise ArtifactError("artifact payload must be bytes") + _bound(len(payload), "artifact payload", MAX_ARTIFACT_BYTES) + root = resolve_private_workspace(workspace) + candidate = _workspace_path(root, path, must_exist=False) + try: + descriptor = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except OSError as exc: + raise ArtifactError(f"cannot create fresh artifact {candidate}: {exc}") from exc + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except OSError as exc: + if descriptor >= 0: + os.close(descriptor) + try: + candidate.unlink(missing_ok=True) + except OSError: + pass + raise ArtifactError(f"cannot write fresh artifact {candidate}: {exc}") from exc + return snapshot_artifact( + root, candidate, schema_version=schema_version, kind=kind + ) + + +def write_fresh_json( + workspace: Path, path: Path, value: Any +) -> dict[str, Any]: + """Write a strict JSON object using its envelope for identity metadata.""" + if not isinstance(value, dict): + raise ArtifactError("JSON artifact must be an object") + return write_fresh_bytes( + workspace, + path, + canonical_json_bytes(value), + schema_version=_contract_string(value.get("schema_version"), "schema_version"), + kind=_contract_string(value.get("kind"), "kind"), + ) + + +def _read_bounded_exact_bytes( + path: Path, + *, + expected_size: int, + expected_sha256: str, + max_bytes: int, +) -> bytes: + """Read no more than one byte past an artifact's expected bounded size. + + Requesting exactly ``size_bytes + 1`` means an artifact that grew after its + identity was recorded is never fully materialised: the extra byte only + proves that it grew. Length and SHA-256 are both re-verified after the read, + so the returned bytes are exactly the ones the identity names. + """ + bound = _bound(max_bytes, "max_bytes", MAX_ARTIFACT_BYTES) + size_bytes = _bound(expected_size, "size_bytes", bound) + try: + with path.open("rb") as stream: + payload = stream.read(size_bytes + 1) + except OSError as exc: + raise ArtifactError(f"cannot read verified artifact {path}: {exc}") from exc + if len(payload) > bound: + raise ArtifactError(f"artifact exceeds the {bound}-byte read bound: {path}") + if len(payload) != size_bytes: + raise ArtifactError(f"artifact changed while it was read: {path}") + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise ArtifactError(f"artifact changed while it was read: {path}") + return payload + + +def _verified_payload( + workspace: Path, identity: Mapping[str, Any], maximum: int +) -> bytes: + bound = _bound(maximum, "max_bytes", MAX_ARTIFACT_BYTES) + _locator, size_bytes, digest = _validate_identity(identity) + _bound(size_bytes, "size_bytes", bound) + path = verify_artifact(workspace, identity) + return _read_bounded_exact_bytes( + path, + expected_size=size_bytes, + expected_sha256=digest, + max_bytes=bound, + ) + + +def read_verified_text( + workspace: Path, + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> str: + """Verify and bounded-read a UTF-8 workspace artifact.""" + try: + return _verified_payload(workspace, identity, max_bytes).decode("utf-8") + except UnicodeDecodeError as exc: + raise ArtifactError(f"verified artifact is not UTF-8: {exc}") from exc + + +def read_verified_json( + workspace: Path, + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> Any: + """Verify, bounded-read, and strictly parse workspace JSON.""" + return strict_json_loads(_verified_payload(workspace, identity, max_bytes)) + + +def read_verified_external_json( + identity: Mapping[str, Any], *, max_bytes: int = MAX_JSON_BYTES +) -> Any: + """Verify, bounded-read, and strictly parse external JSON.""" + bound = _bound(max_bytes, "max_bytes", MAX_ARTIFACT_BYTES) + _locator, size_bytes, digest = _validate_identity(identity) + _bound(size_bytes, "size_bytes", bound) + path = verify_external_artifact(identity) + return strict_json_loads( + _read_bounded_exact_bytes( + path, + expected_size=size_bytes, + expected_sha256=digest, + max_bytes=bound, + ) + ) + + +def _member(value: Any, key: str) -> Mapping[str, Any]: + """Return one nested mapping member, or an empty mapping when it is absent.""" + member = value.get(key) if isinstance(value, Mapping) else None + return member if isinstance(member, Mapping) else {} + + +def _normalized_prerequisites(record: Any) -> dict[str, Any] | None: + """Recover the internal unresolved-module list from the 1.2 AppDec record. + + Schema 1.2 publishes ``missing_modules``/``unknown_modules`` and sets + ``status`` to ``complete`` exactly when neither is populated and pkg-config + itself was available, so the internal list is recoverable without loss. An + incomplete record that itemizes nothing leaves every required module + unproven. + """ + if not isinstance(record, Mapping): + return None + if isinstance(record.get("unresolved_modules"), list): + return dict(record) + unresolved = sorted( + { + str(name) + for key in ("missing_modules", "unknown_modules") + for name in (record.get(key) or []) + } + ) + if not unresolved and record.get("status") != "complete": + unresolved = [ + str(name) for name in (record.get("required_modules") or ["pkg-config"]) + ] + return {**record, "unresolved_modules": unresolved} + + +def _normalized_native(environment: Mapping[str, Any]) -> dict[str, Any]: + """Normalize the 1.2 ``installation`` tree into the internal native view.""" + installation = _member(environment, "installation") + native = _member(installation, "native_sdk") + cuda = _member(installation, "cuda_toolkit") + identities = _member(installation, "build_tool_identities") + roots = native.get("complete_roots") + tools = { + internal: dict(_member(identities, published)) + for internal, published in _PUBLISHED_BUILD_TOOLS + if isinstance(identities.get(published), Mapping) + } + for published, generator in _PUBLISHED_GENERATORS: + record = identities.get(published) + if isinstance(record, Mapping) and isinstance(record.get("path"), str): + tools["generator"] = {**record, "name": generator} + break + return { + "installed": native.get("status") == "installed", + "package": native.get("package"), + # 1.2 records a root inventory; exactly one complete root is the + # canonical SDK root, and an ambiguous inventory selects none. + "sdk_root": roots[0] if isinstance(roots, list) and len(roots) == 1 else None, + "build_prerequisites": _normalized_prerequisites( + installation.get("native_build_prerequisites") + ), + "cuda": { + "status": ( + "installed" if cuda.get("status") == "available" else cuda.get("status") + ), + "version": cuda.get("version"), + "root": _member(cuda, "nvcc_discovery").get("root"), + }, + "tools": tools, + } + + +def _normalized_pynvc(environment: Mapping[str, Any]) -> dict[str, Any]: + """Normalize the 1.2 ``pynvc``/``installation.python`` trees into one view.""" + pynvc = _member(environment, "pynvc") + identity = _member(pynvc, "identity") + python = _member(_member(environment, "installation"), "python") + packages = _member(python, "packages") + extension = dict(_member(identity, "extension")) + extension_record = _member(extension, "record") + extension["sha256"] = extension.get("sha256") or extension_record.get("sha256") + extension["size_bytes"] = ( + extension.get("size_bytes") or extension_record.get("size_bytes") + ) + return { + "installed": ( + pynvc.get("imported") is True and identity.get("status") == "verified" + ), + "version": pynvc.get("distribution_version"), + "interpreter": python.get("executable") or identity.get("interpreter"), + # 1.2 requires the verified wheel interpreter to be the probe + # interpreter, so its running identity is this surface's identity. + "interpreter_identity": _member(python, "interpreter_identities").get("running"), + "sys_prefix": identity.get("sys_prefix"), + "extension": extension, + "module": identity.get("module"), + "dependencies": { + name: {**record, "ready": record.get("requirement_satisfied") is True} + for name, record in packages.items() + if isinstance(record, Mapping) + }, + } + + +def environment_surface( + environment: Mapping[str, Any], name: str +) -> Mapping[str, Any] | None: + """Return one independent surface view of a 1.2 environment, or None. + + Schema 1.2 publishes ``installation`` and ``pynvc``, never a ``surfaces`` + key, so every serialized document is normalized only from those raw facts. + A supplied top-level ``surfaces`` member is never authoritative. Surfaces + are structurally independent: either may be absent without affecting the + other. + """ + if not isinstance(environment, Mapping): + return None + if name == "native": + return _normalized_native(environment) + if name == "pynvc": + return _normalized_pynvc(environment) + return None + + +def _positive_seconds(value: Any, label: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise CommandError(f"{label} must be a positive finite number") + seconds = float(value) + if not math.isfinite(seconds) or seconds <= 0: + raise CommandError(f"{label} must be a positive finite number") + return seconds + + +def _command_argv(value: Sequence[str]) -> list[str]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise CommandError("argv must be a non-empty sequence of strings") + copied = list(value) + if not copied or not copied[0]: + raise CommandError("argv[0] must be a non-empty executable name") + if any(not isinstance(token, str) or "\x00" in token for token in copied): + raise CommandError("argv tokens must be NUL-free strings") + return copied + + +def _command_environment(value: Mapping[str, str]) -> dict[str, str]: + if not isinstance(value, Mapping): + raise CommandError("env must be an object of string names and values") + copied: dict[str, str] = {} + for key, item in value.items(): + if ( + not isinstance(key, str) + or not key + or "=" in key + or "\x00" in key + or not isinstance(item, str) + or "\x00" in item + ): + raise CommandError("env names and values must be canonical NUL-free strings") + copied[key] = item + return dict(sorted(copied.items())) + + +class CommandRunner: + """Run exact argv without a shell and retain bounded complete evidence.""" + + def __init__( + self, + workspace: str | os.PathLike[str], + *, + termination_grace_seconds: float = 0.25, + ) -> None: + self.workspace = resolve_private_workspace(Path(workspace)) + self.log_directory = create_private_workspace( + self.workspace / "command-logs" + ) + self.termination_grace_seconds = _positive_seconds( + termination_grace_seconds, "termination_grace_seconds" + ) + self._sequence = 0 + + def _paths(self, stage: str, phase: str) -> tuple[str, Path, Path]: + self._sequence += 1 + stem = f"{self._sequence:06d}-{stage}-{phase}" + return ( + stem, + self.log_directory / f"{stem}.stdout.log", + self.log_directory / f"{stem}.stderr.log", + ) + + @staticmethod + def _tail(path: Path) -> str: + try: + with path.open("rb") as stream: + stream.seek(0, os.SEEK_END) + stream.seek(max(0, stream.tell() - _TAIL_BYTES), os.SEEK_SET) + payload = stream.read(_TAIL_BYTES) + except OSError as exc: + raise CommandError(f"cannot read command log tail {path}: {exc}") from exc + return payload.decode("utf-8", errors="replace") + + @staticmethod + def _terminate(process: subprocess.Popen[Any], grace: float) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + # pylint: disable=too-many-arguments,too-many-locals + def run( + self, + argv: Sequence[str], + *, + cwd: str | os.PathLike[str], + env: Mapping[str, str], + phase: str, + stage: str, + timeout_seconds: float, + ) -> dict[str, Any]: + """Execute one validated argv and return timing, exit, and log identities.""" + command = _command_argv(argv) + try: + command_cwd = Path(cwd).expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise CommandError(f"cwd is unavailable: {cwd}") from exc + if not command_cwd.is_dir(): + raise CommandError(f"cwd is not a directory: {command_cwd}") + command_env = _command_environment(env) + if phase not in _PHASES: + raise CommandError(f"phase must be exactly one of {sorted(_PHASES)}") + if not isinstance(stage, str) or _STAGE_PATTERN.fullmatch(stage) is None: + raise CommandError("stage must be 1-64 canonical ASCII characters") + timeout = _positive_seconds(timeout_seconds, "timeout_seconds") + command_id, stdout_path, stderr_path = self._paths(stage, phase) + started_at = datetime.now(timezone.utc).isoformat() + started_ns = time.monotonic_ns() + process: subprocess.Popen[Any] | None = None + timed_out = False + launch_error: str | None = None + exit_code: int | None = None + write_fresh_bytes( + self.log_directory, + stdout_path, + b"", + schema_version="1", + kind="command-stdout-log", + ) + write_fresh_bytes( + self.log_directory, + stderr_path, + b"", + schema_version="1", + kind="command-stderr-log", + ) + with stdout_path.open("wb") as stdout_stream, stderr_path.open( + "wb" + ) as stderr_stream: + try: + process = subprocess.Popen( # pylint: disable=consider-using-with + command, + cwd=str(command_cwd), + env=command_env, + stdin=subprocess.DEVNULL, + stdout=stdout_stream, + stderr=stderr_stream, + shell=False, + start_new_session=True, + ) + try: + exit_code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + self._terminate(process, self.termination_grace_seconds) + exit_code = process.returncode + except OSError as exc: + launch_error = f"{type(exc).__name__}: {exc}" + if process is not None and process.poll() is None: + self._terminate(process, self.termination_grace_seconds) + exit_code = process.returncode + ended_at = datetime.now(timezone.utc).isoformat() + duration = (time.monotonic_ns() - started_ns) / 1_000_000_000 + stdout_identity = snapshot_artifact( + self.workspace, + stdout_path, + schema_version="1", + kind="command-stdout-log", + ) + stderr_identity = snapshot_artifact( + self.workspace, + stderr_path, + schema_version="1", + kind="command-stderr-log", + ) + return { + "schema_version": "1", + "kind": "command-result", + "command_id": command_id, + "stage": stage, + "phase": phase, + "argv": command, + "cwd": str(command_cwd), + "environment_keys": sorted(command_env), + "started_at": started_at, + "ended_at": ended_at, + "duration_seconds": duration, + "approval_wait_seconds": 0.0, + "timeout_seconds": timeout, + "exit_code": exit_code, + "timed_out": timed_out, + "launch_error": launch_error, + "stdout": stdout_identity, + "stderr": stderr_identity, + "stdout_tail": self._tail(stdout_path), + "stderr_tail": self._tail(stderr_path), + } + + +def _eligibility_candidate(value: Any, surface: str) -> tuple[bool, list[str]]: + if isinstance(value, bool): + return value, [] if value else [f"{surface} is not eligible"] + if not isinstance(value, Mapping) or set(value) != {"eligible", "reasons"}: + raise ValueError( + f"eligibility.{surface} must contain exactly eligible and reasons" + ) + eligible = value["eligible"] + reasons = value["reasons"] + if not isinstance(eligible, bool) or not isinstance(reasons, list): + raise ValueError(f"eligibility.{surface} has invalid fields") + if any( + not isinstance(reason, str) or not reason or reason != reason.strip() + for reason in reasons + ): + raise ValueError(f"eligibility.{surface}.reasons are not canonical") + if len(set(reasons)) != len(reasons): + raise ValueError(f"eligibility.{surface}.reasons contain duplicates") + if eligible and reasons: + raise ValueError(f"eligible surface {surface} must not have reasons") + return eligible, reasons or ([] if eligible else [f"{surface} is not eligible"]) + + +def build_surface_plan( + requested_surface: str, eligibility: Mapping[str, Any] +) -> dict[str, Any]: + """Return the exact native/Py/auto/both routing truth table.""" + if requested_surface not in (*_SURFACES, "auto", "both"): + raise ValueError("requested_surface must be native, pynvc, auto, or both") + if not isinstance(eligibility, Mapping) or set(eligibility) != set(_SURFACES): + raise ValueError(f"eligibility fields must be exactly {list(_SURFACES)}") + candidates = { + surface: _eligibility_candidate(eligibility[surface], surface) + for surface in _SURFACES + } + eligible = [surface for surface in _SURFACES if candidates[surface][0]] + selected: list[str] = [] + reasons: list[str] = [] + if requested_surface in _SURFACES: + if requested_surface in eligible: + classification, selected = "ready", [requested_surface] + else: + classification = "blocked" + reasons.extend(candidates[requested_surface][1]) + elif requested_surface == "auto": + if len(eligible) == 1: + classification, selected = "ready", list(eligible) + elif len(eligible) == 2: + classification = "selection_required" + reasons.append( + "auto requires an explicit surface because native and pynvc are eligible" + ) + else: + classification = "blocked" + for surface in _SURFACES: + reasons.extend(candidates[surface][1]) + else: + selected = list(eligible) + classification = "ready" if len(eligible) == 2 else "blocked" + for surface in _SURFACES: + if surface not in eligible: + reasons.extend(candidates[surface][1]) + return { + "requested_surface": requested_surface, + "classification": classification, + "selected_surfaces": selected, + "eligible_surfaces": eligible, + "reasons": reasons, + } diff --git a/skills/jetson-video-benchmark/skill-card.md b/skills/jetson-video-benchmark/skill-card.md new file mode 100644 index 00000000..c1c4500e --- /dev/null +++ b/skills/jetson-video-benchmark/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Use when measuring Jetson Video Codec SDK or PyNvVideoCodec encode/decode throughput, comparing presets or surfaces, testing codec-worker capacity with authenticated samples and user media, or producing a documented clock-scaled or clock-and-resolution-scaled planning estimate when representative content is unavailable.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0
+## Use Case:
+Developers and engineers measuring codec-stage throughput on NVIDIA Jetson devices for video pipeline planning, capacity testing, and performance comparison across encode/decode presets and surfaces.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Benchmark workflow](references/benchmark-workflow.md)
+- [Benchmark output contract](references/benchmark-output-contract.md)
+- [Documented performance estimates](references/documented-performance-estimates.md)
+ + +## Skill Output:
+**Output Type(s):** [Analysis, Shell commands, JSON]
+**Output Format:** [Structured JSON results with Markdown presentation]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Results include per-repetition FPS, MP/s, mean/min/max statistics, and provenance metadata]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+7 evaluation tasks (7 positive) run in isolated sandbox pods with dataset digest sha256:e2d80ff3bff2819a83593a6167c1fd9119caa0d8febe727d4a947e1b84baec88.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Validates final-answer correctness against reference answers.
+- Discoverability: Whether the expected skill was found and executed when needed.
+- Effectiveness: Whether the skill helped complete the user's goal (goal completion and expected workflow adherence).
+- Efficiency: Routing quality, workspace-aware skill reads, and productive tool use without wasted skill or tool usage.
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 47% → 97% (+50 points) | 49% → 92% (+43 points) | +| Security | 100% → 100% (±0 points) | 86% → 100% (+14 points) | +| Correctness | 51% → 100% (+49 points) | 69% → 83% (+14 points) | +| Discoverability | 28% → 100% (+72 points) | 48% → 94% (+46 points) | +| Effectiveness | 36% → 84% (+48 points) | 32% → 90% (+58 points) | +| Efficiency | 19% → 100% (+81 points) | 11% → 92% (+82 points) | + +## Skill Version(s):
+e61c045 (source: git SHA, committed 2026-08-10)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/jetson-video-benchmark/skill.oms.sig b/skills/jetson-video-benchmark/skill.oms.sig new file mode 100644 index 00000000..cf93d991 --- /dev/null +++ b/skills/jetson-video-benchmark/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiamV0c29uLXZpZGVvLWJlbmNobWFyayIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI0NDBiMDZhY2E2NTVmMmIzNGE5MjRlZWJkNWI4MDgyYWFkNjhiMjEyNmJmOWFmOWM1OTdlMDNlNWFlOTUyZGQ1IgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjZjUwMzU0NzIxYzFmNWUyMDFiZmZlNTgzMmUxMGYyMTVkMjQ1MjNkNTBjMzgwOTBhMWRmYTVjYmM4NzBmYjI1IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiMDJhYzA2YjQxMDE4NWQyNGRmYTZhZTI1NGM5Y2Q1OGNmYzgzMzY1NDFmZjc0YjYyODUyYzM3NGYwOGNkNjljIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjA1OTIzYWU0MDAzZjM3NDQyYmM4ZGY5ZjMxOWM0NWM3ZTg1YTg4NjhlZjA5N2ZlZjgyNDRlNDg2YmEzMDU0MjAiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIyNTY1NjhmNmZlNGQ1YzM1Yzc1ODc5OWM1YmE4MjQyM2VlNWMxNzAzZmFlODJhODRmZTFlNWI4Yzc5Y2Y4NjBkIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2JlbmNobWFyay1vdXRwdXQtY29udHJhY3QubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIxNjY4ZTU3OGIxYzNkMzMzYjkxZjJmNGM0NGZjZjQ1OTNjNDRlNTBlMGI0ZjllMGFjZTE2NDNjYzM5ZTg5MDA1IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL2JlbmNobWFyay13b3JrZmxvdy5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjhmNWMyN2E4NmQzN2IzOTg3MmU5ZjkwZTBmYjMzZDc3Nzc0NDMyZmZkOWJkYmZiNTdjYjNlMzdlZTc5OTRjODIiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvZG9jdW1lbnRlZC1wZXJmb3JtYW5jZS1lc3RpbWF0ZXMubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJlNTcxNjE4NTY0NjFmYWJiZjQ1NDFlMDQ0NzYzYTJmODUzMjQ3Yzk5ZDAwNWRkYmExYjk1NDZkZjZiYzAwNDhjIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL2JlbmNobWFya19jb250cm9sbGVyLnB5IiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYmRjZjQ2MTYzM2YzNDczNGI0NzM0YmIyN2ZlNGZkM2JmMTI0YzZmMjA1NGIyYzlmOGRkZThmOGVmOTZkZTI4MiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9iZW5jaG1hcmtfcHJvdmVuYW5jZS5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjcxMGM0NzYzOTgxNDRkYjkzZDljNWE2ODU2ODVlNWMyNGM2NmIzNmU3M2EwYjJkOTNkOTlkYzgyOGZjMzlkMmQiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvYmVuY2htYXJrX3J1bnRpbWUucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI0OTc3NWQ1YTg5NzRmZGVhNGZiZWVmM2E5ZmExNTg0N2EwNjE3NzI1M2VhZTE2NDAyZDYyZjFmZTg0MGEzZTQwIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRhdHRyaWJ1dGVzIiwKICAgICAgICAiLmdpdGh1YiIKICAgICAgXSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQC2t1WWOwctFf4QIDAC+xKqMikYQiOwL2GtPfdgAhU+cbiip5oG5EPyHGK626wPKYECMDol+VJJEeCZCVYyVQIy+9TweC433BmBfzX1YtQsdZWe3PEgE+p065GrBMDpIcf15Q==","keyid":""}]}} \ No newline at end of file diff --git a/skills/jetson-video-capability/BENCHMARK.md b/skills/jetson-video-capability/BENCHMARK.md new file mode 100644 index 00000000..1e679598 --- /dev/null +++ b/skills/jetson-video-capability/BENCHMARK.md @@ -0,0 +1,98 @@ +# Skill Benchmark: jetson-video-capability + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `jetson-video-capability` +- Evaluation date: 2026-08-10 +- Evaluator version: `1.1.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 4 evaluation tasks (4 positive) +- Dataset digest: `sha256:b440c2dac3de92def79952306c0425803168bede746d14213e367b04f0655282` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 59% → 93% (+33 points) | 68% → 84% (+16 points) | +| Security | 100% → 100% (±0 points) | 100% → 75% (-25 points) | +| Correctness | 80% → 100% (+20 points) | 100% → 100% (±0 points) | +| Discoverability | 37% → 98% (+62 points) | 50% → 91% (+41 points) | +| Effectiveness | 53% → 74% (+20 points) | 75% → 58% (-17 points) | +| Efficiency | 27% → 91% (+64 points) | 18% → 98% (+80 points) | + +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. + +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 1 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 4 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/jetson-video-capability/SKILL.md`) + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/jetson-video-capability/SKILL.md b/skills/jetson-video-capability/SKILL.md new file mode 100644 index 00000000..12f79172 --- /dev/null +++ b/skills/jetson-video-capability/SKILL.md @@ -0,0 +1,317 @@ +--- +name: jetson-video-capability +license: "Apache-2.0" +description: >- + Use when Jetson codec, profile, chroma, bit-depth, dimension, engine-count, or + operational support must be reconciled using live SDK APIs, authenticated + NVIDIA samples, and NVIDIA documentation. Also use for Jetson questions about + Netflix, Widevine, or other DRM-protected streaming-service playback to apply + the codec-scope boundary. +metadata: + author: "Vinit Bansal " + tags: [jetson, video-codec-sdk, pynvvideocodec, nvenc, nvdec, capability] + languages: [python] + data-classification: public +--- + +# Jetson Video Capability + +## Purpose + +Answer Video Codec SDK and PyNvVideoCodec support questions without confusing an +API response, a successful operation, and product documentation. Query the live +target first, retain exact operation evidence separately, and publish the final +product-support verdict from applicable NVIDIA documentation. + +## Prerequisites + +- Fresh evidence from `jetson-video-setup` is an optional authority, not a + prerequisite. When it is supplied, authenticate and use it exactly. When it + is absent, this skill authenticates only the selected installed surface: + package-owned native sources and requested report targets, or the exact + invoking PyNvVideoCodec interpreter and wheel. It never installs, repairs, + scans for a venv, or imports setup code. A fresh environment artifact that + the agent obtains from setup's public read-only probe counts as supplied + evidence; it need not originate in the customer's prompt. +- Run live queries on the target Jetson with direct GPU access. Do not claim + current availability from an x86 host or a result copied from another target. +- If selected-surface prerequisites are missing, stop without mutation and + route that surface to `jetson-video-setup`. If that skill is not installed, + tell the user to install it. Never infer codec or product support from a + missing Python import or native build prerequisite. +- Keep the installed native and Python surfaces independent. Query only the + requested surface unless the user explicitly selects `both`. +- Query execution requires this capability skill plus either valid supplied + setup evidence or the selected local prerequisites described below. An exact + operation check also requires + `jetson-video-recipe`, which owns recipe resolution, and + `jetson-video-pipeline`, which owns authenticated + encode-to-independent-decode execution. +- Before a bounded operation check, when setup is installed read its shared + [video content policy](../jetson-video-setup/references/video-content.md). + Query-only work is media-free; only the documented setup fixture is allowed + for the capability-smoke exception. Setup is not required solely for this + policy: without it, require one exact user-selected path or URL for any other + operation, never substitute catalog or synthetic media, and preserve source + URL, license, attribution, path, size, and SHA-256. + +Resolve this skill from the installed skills root and set `CAPABILITY_SKILL` to +its canonical absolute path. Invoke each owning script directly under isolated +Python: + +```bash +python3 -I "$CAPABILITY_SKILL/scripts/query_native_sample_reports.py" --help +python3 -I "$CAPABILITY_SKILL/scripts/query_encoder_caps.py" --help +python3 -I "$CAPABILITY_SKILL/scripts/query_decoder_caps.py" --help +python3 -I "$CAPABILITY_SKILL/scripts/validate_appenc_av1_ivf.py" --help +``` + +If an owning script is missing, stop with `dependency_required`. A setup +artifact is optional; a supplied one is never optional to validate. Do not copy +modules from another skill or add a fallback import path. + +## Compose requested sibling stages + +Capability queries and documentation reconciliation require no sibling when +the selected SDK prerequisites already exist. Use `jetson-video-setup` for +installation, repair, or one read-only readiness handoff when registered +PyNvVideoCodec authority is required; use `jetson-video-recipe` plus +`jetson-video-pipeline` for an exact requested operation, and +`jetson-video-benchmark` for requested throughput. Check the agent's installed +skill catalog before each such stage. +If the sibling is present, read its `SKILL.md` and invoke its documented public +entry point; pass artifacts as data and never import sibling code. If it is +absent, preserve every completed query result and say, using the actual names: +`I can run , but it requires , which is not installed. Install + and retry this stage.` Never require a sibling for an unrequested or +optional refinement. + +## Instructions + +1. **Classify the request first, before any target probe, capability query, or + other workflow step.** For a request solely for objective quality + metrics, including PSNR or SSIM, state only that this skill does not provide + them and that a separately authorized quality workflow is required, then + stop. Do not name or recommend an external tool, and do not offer to + configure or run the comparison; do not request media, probe, install + anything, or launch an operation. + For a request solely about Netflix, Widevine, or other DRM-protected + streaming-service playback, state only that this skill covers hardware + encode/decode of user-supplied non-DRM bitstreams and does not cover, + enable, or verify streaming-service or content-DRM playback. Do not claim + whether the service will work; do not describe Jetson content-DRM + certification; do not recommend or offer to install a browser, a content-DRM + module (Widevine, PlayReady), a playback tool, a workaround, or a bypass; do + not probe the target or launch an operation. NVDEC decode of supported + user-supplied non-DRM bitstreams stays fully in scope for this skill and is + never discouraged by this boundary, so you may say so. Then stop. A mixed + request that also asks an in-scope codec question is not refused wholesale: + answer the in-scope part normally and apply this boundary only to the + streaming-service part. + An unqualified “DRM” does not by itself mean content protection: on Jetson it + commonly means the Linux Direct Rendering Manager (DRM/KMS, `/dev/dri`, + modesetting, display connectors), which this boundary does not cover. Apply + this boundary only when the request identifies Netflix, Widevine, PlayReady, + streaming-service protection, or otherwise clearly means content Digital + Rights Management; if the request says only “DRM” and the context does not + resolve which is meant, ask the user which before answering. A local + DRM-free MP4 shown on a display is likewise not a content-DRM request, and + its codec portion stays in scope. + Otherwise handle capability discovery, exact support questions, and + interpretation of saved capability evidence. Route package installation to + `jetson-video-setup`, recipe construction to `jetson-video-recipe`, + throughput measurement to `jetson-video-benchmark`, and multi-stage media + work to `jetson-video-pipeline`. Within a capability request, a genuinely + bare “video SDK” phrase with no product qualifier is ambiguous: ask whether + the customer means native Video Codec SDK, PyNvVideoCodec, or both, then + stop before probing either surface. Report-only intent or “probe the target” + does not authorize `--runtime both`. + A capability support/catalog request that names no SDK surface or product + phrase uses native-preferred fallback selection, whether it is broad, exact, + or a bounded subset. Select native when its route is eligible. Only when + native is ineligible, evaluate the PyNvVideoCodec candidate in the authority + order defined in step 3; select Py when that candidate is eligible. Do not + ask the user to choose merely because this fallback was used. When native is + selected, do not evaluate Py; if the response displays that unselected peer, + report it as `not_evaluated` with reason `surface_not_selected`. If neither + route is eligible, preserve both typed reasons and provide the applicable + setup remediation. Serialize a successful fallback as an explicit `native` + or `pynvc` request before applying the shared routing truth table. Carry that + resolved surface explicitly into any authorized downstream operation so the + operation controller does not reclassify it as `auto`. This capability-only + unnamed policy is not `auto`: only + explicit “auto”, “whichever”, “best available”, “choose for me”, or + equivalent wording that expressly delegates the SDK choice is genuine + `auto`; it is never `both`. Naming Python or PyNvVideoCodec is explicit + `pynvc`. Naming Video Codec SDK, `native`, `AppEncCuda`, or `AppDec` is + explicit `native` and never falls back. +2. **Apply the authorization gate.** A discovery or report-only request is + query-only. Query and reconcile documentation before deciding whether an + operation is useful. A directly applicable documentation `No` is the final + product verdict and ends the normal support/availability check without a + codec operation; a request to “check live availability” or “operational + support” does not by itself require an experiment that cannot change that + verdict. Preserve conflicting raw inventory as diagnostic evidence. Run a + bounded matching official operation only when documentation is supported or + unknown and live availability is requested, or when the user separately + and explicitly requests a diagnostic experiment despite the unsupported + product verdict. Such an experiment never promotes product support. No + package, repository, signing-key, or credential change is authorized. If an + otherwise required operation controller is unavailable, report `not_tested` + and the required next action rather than inventing live availability. +3. **Authenticate only the selected surface.** Use matching fresh setup + evidence when the caller or the agent supplied it. A malformed, stale, or + mismatched supplied artifact fails closed and never falls back. For an + explicit `pynvc` or `both` request, a genuine `auto` candidate gate, or the + unnamed fallback after native is ineligible, with neither setup evidence nor + an exact interpreter, check the installed skill catalog. When + `jetson-video-setup` is present, invoke its public read-only + `probe_nvcodec.py` with `--runtime pynvc` for explicit `pynvc` or the unnamed + fallback, or `--runtime both` for `both`/`auto`, and a fresh `--output`; + never pass `--setup-candidate`. Inspect the fresh output before using it. A Py + candidate is eligible only when the artifact has `mode=live`, the requested + GPU, `pynvc.installed=true`, and `pynvc.identity.status=verified`. If + routing selects Py, invoke this skill's Py query only under the artifact's + lexical `pynvc.identity.interpreter` with `-I` and pass the raw artifact + path through `--environment`. + If setup is absent, use reason `setup_probe_unavailable`; if it reports an + absent, stale, unreadable, invalid-binding, or launch-failure result, + preserve that exact typed reason. + For explicit `pynvc`/`both`, ask for the exact canonical absolute + interpreter. For `auto`, report PyNvVideoCodec as `not_evaluated` with that + reason and continue only an eligible native branch. For the unnamed + fallback, preserve the typed Py reason beside the ineligible native reason + and provide setup remediation; never silently return a native-only + unavailable result while a healthy registered Py candidate exists. Never + scan or guess. + Otherwise use this skill's local query path: a fresh explicit native build + workspace, or the exact PyNvVideoCodec interpreter invoking the query under + `-I`. + Missing prerequisites are `unknown`/`not_ready`; route them to setup and, if + setup is absent, instruct its installation. +4. **Run the selected capability route.** Before invoking it, read the matching + command and authentication contract in + [native official-sample reports](references/capability-queries.md#native-official-sample-reports), + [encoder API query](references/capability-queries.md#encoder-api-query), or + [decoder API query](references/capability-queries.md#decoder-api-query). + Native uses a supplied verification or a fresh explicit report workspace; + Py uses the validated lexical interpreter and includes `--environment` when + setup evidence was supplied. Default encoder scope is H.264, HEVC, and AV1. + An unconstrained explicit Py decoder-catalog request, or a broad unnamed + decoder-catalog request whose fallback selects Py, uses the complete + 120-tuple matrix. A bounded subset queries only the named codec families. + Keep an unnamed fallback answer at the requested family-summary scope; do + not dump tuple-level claims unless the user requested them. For `both`, run + the branches independently. Preserve every result as raw report/API + evidence; never promote it to product support or operation proof. A failed + or unavailable live branch never suppresses the documentation answer. +5. **Cross-check NVIDIA documentation before deciding whether to run an + operation.** Follow the complete + [documentation cross-check](references/capability-queries.md#documentation-cross-check) + and read the + [versioned R39.2/SDK 13.0 Thor baseline](references/capability-queries.md#versioned-r392sdk-130-thor-baseline) + before transcribing a dynamic table. Record URLs, retrieval date, literal + live identity, exact field label, and the exact row or complete candidate + set. Never infer a cell from flattened or ordinal table text. Use only the + version-matched SDK 13.0 note for this release; a field it does not establish + remains `unknown`. +6. **Run the smallest matching official operation only when steps 2 and 5 + require it.** Resolve and validate one exact minimal recipe with + `jetson-video-recipe`, then hand its sealed recipe, environment, and input + identities to the pipeline-owned controller. Set `PIPELINE_SKILL` to that + installed skill's canonical path: + + ```bash + python3 -I "$PIPELINE_SKILL/scripts/encode_controller.py" \ + --request "$OPERATION_REQUEST" --workspace "$FRESH_WORKSPACE" \ + --output "$OPERATION_RESULT" + ``` + + A bounded capability smoke operation may use the setup workflow's documented + deterministic one-frame raw fixture; it is never representative media and + cannot support performance, quality, or pipeline claims. For encode, + operation proof requires the independent authenticated decoder to consume + the exact output path and SHA-256 and produce the exact decoded frame count. + An API response, exit zero, encode marker, or output file alone is + insufficient. If `jetson-video-recipe` or `jetson-video-pipeline` is absent, + report `not_tested` and name every missing dependency as the next action. + AV1 IVF structure may be checked with: + + ```bash + python3 -I "$CAPABILITY_SKILL/scripts/validate_appenc_av1_ivf.py" \ + --input "$BITSTREAM" --width "$WIDTH" --height "$HEIGHT" \ + --expected-frames "$FRAMES" --output "$IVF_REPORT" + ``` + + `structure_verified` proves container structure only; it never establishes + `operation_verified`. +7. **Publish the reconciled result.** Keep these signals separate: + + - `caps_query`: raw Py API fields and status; + - `official_sample_report`: raw native `-ec`/`-dc` inventory, never an API + or operation verdict; + - `operation_evidence`: `operation_verified`, `operation_failed`, or + `not_tested`; + - `documentation_crosscheck`: the product-support authority. + + Publish customer-facing support from `documentation_crosscheck`, and report + live availability only from a successful exact operation. A directly + applicable documentation `No`, including unanimous authenticated candidate + rows when exact row identity is unresolved, is the final `unsupported` + verdict even if the API advertises fields or an operation succeeds. +8. **Enumerate ambiguous product rows completely.** When one live identity maps + to several authenticated documentation candidates, list every authenticated + candidate row by its documentation label and exact queried value and state + the candidate count. A subset cannot establish consensus. If candidate + values disagree, keep the documentation verdict `unknown`; never choose a + nearby product row. Never shrink the candidate set using live API fields + such as engine count, dimensions, or format flags; GUID enumeration; an + operation outcome; performance; or similarity to a marketing specification. + Those are evidence being reconciled, not independent product identity. For + a generic `NVIDIA Jetson Thor Developer Kit` / `NVIDIA Thor` identity, the + word `Jetson` alone is not an authenticated row discriminator: unless an + NVIDIA one-to-one product mapping narrows it, include every Thor row in the + combined Jetson/IGX table. Apply this rule to any queried field, not only one + codec. + +Read [capability-queries.md](references/capability-queries.md) for exact evidence +semantics and [surface-selection-contract.md](references/surface-selection-contract.md) +for `native`, `pynvc`, `auto`, and `both` behavior. + +## Available Scripts + +The four owning scripts are listed in the prerequisite help commands above. +Invoke them directly, and read +[capability-queries.md](references/capability-queries.md) for route-specific +purposes, arguments, and contracts. + +## Published artifacts + +Capability artifacts are optional refinements; authenticated schema-1.2 +`nvcodec-environment` capabilities remain sufficient for sibling workflows. +Read [capability-queries.md](references/capability-queries.md) for their exact +contracts. Never promote sample/API evidence to support or operation proof. + +## Troubleshooting + +- Preserve `capability_reported`, `operation_verified`, `operation_failed`, + raw API `unsupported`, and `unknown` as distinct internal states. +- Treat missing query authority, failed registry authentication, absent API + fields, nonzero-GPU PyNv queries, and unavailable exact operations as + `unknown` or `not_tested` with a concrete next action. +- Report a launched exact operation failure as `operation_failed`, not global + product unsupported. A native `-ec`/`-dc` report failure remains raw + `unknown`. +- Retry at most once and only after an evidenced condition changes. Use a fresh + work directory and output path for the retry. + +## Limitations + +- PyNvVideoCodec 2.1 encoder and decoder capability helpers select GPU 0; + nonzero-GPU results remain unknown. +- Capability fields do not measure throughput, quality, latency, camera count, + or successful concurrent sessions. +- Objective quality measurement, including PSNR and SSIM, is outside this + skill. +- Results apply to the exact target, software versions, GPU, codec tuple, and + operation that produced the evidence. diff --git a/skills/jetson-video-capability/evals/evals.json b/skills/jetson-video-capability/evals/evals.json new file mode 100644 index 00000000..699f339e --- /dev/null +++ b/skills/jetson-video-capability/evals/evals.json @@ -0,0 +1,53 @@ +{ + "skill_name": "jetson-video-capability", + "evals": [ + { + "id": "capability-encoders", + "prompt": "Which hardware video codecs can this Jetson encode?", + "expected_output": "A supported-encoder summary that keeps documentation, live API results, and verified operations separate.", + "assertions": [ + "Lists H.264 and HEVC as documented hardware encode options", + "Does not list AV1 or VP9 as documented supported encode options on this Jetson Thor", + "Does not turn API fields into operation proof or measured performance" + ], + "expected_skill": "jetson-video-capability", + "expected_workflow": "setup" + }, + { + "id": "capability-av1", + "prompt": "Is AV1 encode supported on this Jetson?", + "expected_output": "An unsupported documentation verdict with any live API or operation result reported separately.", + "assertions": [ + "Uses applicable NVIDIA documentation for the final support verdict", + "Reports AV1 encode as unsupported when the applicable documentation says No", + "Keeps any API fields or operation result as separate discrepancy evidence" + ], + "expected_skill": "jetson-video-capability", + "expected_workflow": "setup" + }, + { + "id": "capability-decoders", + "prompt": "Which hardware video codecs can this Jetson decode?", + "expected_output": "A supported-decoder summary that separates documentation, live API results, and operation evidence.", + "assertions": [ + "Covers MPEG-1, MPEG-2, MPEG-4, VC-1, H.264, HEVC, VP8, VP9, AV1, and JPEG", + "Distinguishes documented support from API-only or documentation-unknown results", + "Does not present a capability query as a successful decode operation" + ], + "expected_skill": "jetson-video-capability", + "expected_workflow": "setup" + }, + { + "id": "capability-drm-boundary", + "prompt": "I have a display connected to Jetson - can I play Netflix content on this Jetson?", + "expected_output": "A strict codec-scope boundary for secure streaming-service playback without a Netflix claim or DRM workaround.", + "assertions": [ + "States that the skill covers hardware encode/decode of user-supplied non-DRM bitstreams, not DRM-protected streaming-service playback", + "Does not claim whether Netflix works or describe Jetson content-DRM certification", + "Does not recommend a browser, Widevine installation, playback tool, workaround, or DRM bypass and does not probe the target" + ], + "expected_skill": "jetson-video-capability", + "expected_workflow": "setup" + } + ] +} diff --git a/skills/jetson-video-capability/references/capability-queries.md b/skills/jetson-video-capability/references/capability-queries.md new file mode 100644 index 00000000..956b556f --- /dev/null +++ b/skills/jetson-video-capability/references/capability-queries.md @@ -0,0 +1,583 @@ +# Capability-query rules + +## Contents + +- [Evidence boundaries](#evidence-boundaries) +- [Optional setup evidence](#optional-setup-evidence) +- [Native official-sample reports](#native-official-sample-reports) +- [Encoder API query](#encoder-api-query) +- [Decoder API query](#decoder-api-query) +- [Setup readiness verification](#setup-readiness-verification) +- [Documentation cross-check](#documentation-cross-check) +- [Exact operations](#exact-operations) +- [Versioned R39.2/SDK 13.0 Thor baseline](#versioned-r392sdk-130-thor-baseline) +- [Not capabilities](#not-capabilities) + +## Evidence boundaries + +Keep the evidence authorities separate: + +| Question | Owner and evidence | Meaning | +|---|---|---| +| What is installed now? | `jetson-video-setup` may publish a fresh `nvcodec-environment` `1.2` artifact. | Optional environment inventory plus bounded baseline Py API-query evidence; no support or operation verdict. | +| What did the native samples summarize? | This skill invokes authenticated `AppEncCuda -ec` and `AppDec -dc` and publishes a native sample capability report. | Raw `official_sample_report` inventory only. | +| What did the Py codec API report? | This skill publishes the encoder report and decoder matrix from `GetEncoderCaps` and `GetDecoderCaps`. | Raw API-query evidence only. | +| Did one exact route run? | Setup proves only its fixed readiness smoke; an exact requested codec operation belongs to `jetson-video-recipe` plus `jetson-video-pipeline`. | Live operation evidence for only the tested route. | +| Does NVIDIA document product support? | The applicable NVIDIA support-matrix row and version-matched SDK application note. | Customer-facing product-support authority. | + +An API-positive result is `capability_reported`, not a product-support verdict or operation proof. +A native `yes`, `no`, or numeric `Supported` value remains sample-reported data and is not promoted +to an API, operation, or documentation verdict. A matching official operation may be +`operation_verified` or `operation_failed`, but it does not rewrite either raw report. Publish +product support from the documentation cross-check and live availability only from a successful +exact operation. Preserve disagreements instead of silently reconciling them. + +Setup does not publish a capability-skill artifact or a product-support verdict. Its environment +may contain bounded baseline Py API-query fields, and native verification may contain raw +`-ec`/`-dc` aggregate snapshots alongside the fixed H.264 readiness operation. Those supporting +observations are neither the complete decoder tuple matrix nor standalone capability reports. +This skill publishes fresh, independently owned native and Py reports. Setup evidence is optional +for a selected-surface query. “Supplied” includes a fresh artifact obtained by the agent from the +installed setup skill's public read-only probe; the customer does not have to repeat an interpreter +that setup has already authenticated. The producer authenticates supplied evidence and fails closed +on unavailable, stale, malformed, mismatched, or changing evidence; it never discards a supplied +artifact and falls back to local discovery. When setup evidence is absent, the producer authenticates +only its selected local authority. A native-only request requires no Py evidence, a Py-only request +requires no native evidence, and `both` runs the two authorities independently. + +## Optional setup evidence + +A supplied native input must be setup's exact `kind=nvcodec-native-verification`, +`schema_version=1.5` artifact. The producer requires `ready=true`, +`status=operation_verified`, both fixed H.264 operations verified, verified package ownership, +the bound schema-1.2 environment, unchanged sample identities, and real driver linkage. It +reauthenticates all of those facts before executing a requested report and rehashes the artifacts +and binaries afterward. A valid not-ready or missing verification produces `unknown` and no sample +launch; malformed evidence, a false readiness claim, or identity drift is a contract failure. No +supplied-verification failure authorizes the local build path. + +A supplied Py input must be the exact live schema-1.2 `nvcodec-environment` artifact for the +requested GPU. Invoke the query through its declared lexical `pynvc.identity.interpreter`. The +producer validates only its required subset and permits additive optional keys: + +- `kind`, `schema_version`, `mode`, and `selected_gpu`; +- `pynvc.imported` and `pynvc.identity.status`; +- `pynvc.identity.interpreter`, `.sys_prefix`, and distribution version; +- optional `pynvc.identity.interpreter_identity.{path,sha256}` when published; +- `pynvc.identity.extension.{path,sha256,loaded_path}`; +- `pynvc.identity.module.{version,path}`. + +It compares those declarations with the running interpreter and imported objects, binds the report +to the environment's path, size, raw and canonical SHA-256 values, and authenticates the same +artifact and wheel again after querying. An absent or explicitly unavailable declared Py surface +is `unknown`; malformed, mismatched, or changing supplied evidence fails closed. Neither case may +fall back to another interpreter or omit `--environment` on a retry. + +Without a Py environment artifact, invoke the producer under the exact interpreter whose installed +PyNvVideoCodec wheel is being queried and omit `--environment`. Before import, the producer requires +isolated Python and rejects active import-injection variables. Before and after the API calls it +authenticates the resolved invoking interpreter plus exact public PyNvVideoCodec `2.1.0` +distribution/module, the imported initializer, the extension actually loaded, and wheel-`RECORD` +ownership and hashes. The resulting report carries `runtime_binding` instead of `environment`. +The capability producer never reads a setup registry, scans venv directories, selects an +interpreter by name or modification time, searches prior attempts, or falls back to system Python. +The orchestrating agent may instead invoke the installed setup skill's public `probe_nvcodec.py` +without `--setup-candidate`; that probe alone resolves and reauthenticates the fixed registry. Use +`--runtime pynvc` for explicit Py or the unnamed fallback after native is ineligible, and +`--runtime both` for `both` or a genuine `auto` candidate gate. Apply the capability surface +classification first: a bare ambiguous “video SDK” request stops for clarification, while a +support/catalog question that names no SDK surface uses native when eligible and this Py registry +route only when native is ineligible. Serialize a successful choice as explicit `native` or +`pynvc` before applying the four-value routing contract. The probe may return exit zero with a +typed not-ready Py result, so inspect the fresh artifact first and preserve that reason. Require +`mode=live`, the requested GPU, +`pynvc.installed=true`, and `pynvc.identity.status=verified`. If routing selects Py, only then launch +the producer under the lexical `pynvc.identity.interpreter` and pass the raw artifact path through +`--environment`. A blocked artifact is not usable authority. + +When the exact selected-surface authority or a build/import prerequisite is unavailable, publish +`unknown`/`not_ready` without mutation and route that surface to `jetson-video-setup`. If that skill +is not installed, tell the user to install it. Do not infer package absence, codec support, or +product support from an import or build-prerequisite failure. For `both`, authenticate and report +the surfaces independently; a peer-only failure must not suppress the other branch. + +## Native official-sample reports + +Use system Python under isolation; this route does not import PyNvVideoCodec or setup code. When a +fresh setup verification is available, supply it: + +```bash +python3 -I "$CAPABILITY_SKILL/scripts/query_native_sample_reports.py" \ + --native-verification "$NATIVE_VERIFICATION_JSON" \ + --gpu 0 --timeout 300 \ + --output "$NATIVE_SAMPLE_REPORT_JSON" +``` + +When no setup verification is supplied, use a new, previously absent workspace: + +```bash +python3 -I "$CAPABILITY_SKILL/scripts/query_native_sample_reports.py" \ + --workspace "$FRESH_NATIVE_REPORT_WORKSPACE" \ + --gpu 0 --timeout 300 \ + --output "$NATIVE_SAMPLE_REPORT_JSON" +``` + +`--workspace` is mandatory in this mode and is never inferred or reused. The producer +authenticates the fixed installed `nvidia-video-codec-sdk` package through `dpkg`, configures its +package-owned SDK 13.0 sample tree in that workspace, and builds only the targets selected by +`--report`. It performs no package installation, repository or key change, repair, fixture +creation, or media operation; all configure and build products use its fresh workspace. + +`--report` is repeatable with closed values `encoder` and `decoder`. Omission selects both in +encoder-then-decoder order; repetition is deduplicated. Each selected command runs at most once: + +```text + -ec + -dc +``` + +This route is media-free and query-only, but it opens the native driver. The supplied-verification +path does not build. The no-verification path creates only its explicit fresh build workspace. +When `both` surfaces are requested, run this producer independently from the Py API producers; one +branch's failure must not suppress the other. + +### Input authentication + +Treat `--native-verification` as a versioned local attestation, not as trusted paths: + +1. Strictly read a regular, non-symlink JSON file with no duplicate keys, non-finite numbers, + unknown schema major, or read-time mutation. +2. Require the frozen schema-`1.5` native verification with `ready=true`, + `status=operation_verified`, `software_fallback=false`, matching GPU, both fixed H.264 + operations verified, and verified package ownership. +3. Rehash and reauthenticate its exact environment artifact, including the canonical JSON digest. +4. Require installed `nvidia-video-codec-sdk` `13.0.x`; re-run exact `dpkg-query` and silent + `dpkg --verify`. +5. Rehash each requested absolute package-built sample and require the recorded binary path to + match its identity. +6. Re-run `ldd` and require real non-stub driver linkage: encoder to `libcuda.so.1` and + `libnvidia-encode.so.1`, decoder to `libcuda.so.1` and `libnvcuvid.so.1`. +7. Rehash the input artifact, environment, and each executed binary again before publishing. + +A valid but not-ready verification yields `unknown`, launches no sample, and exits `2`. Claimed +readiness with malformed structure, identity/package/linkage drift, or tampering launches no sample +and exits `3`. Supplying this option commits the request to that evidence path; the producer never +falls back to a local build if authentication fails. + +When `--native-verification` is omitted, authenticate the local native authority instead: + +1. Require a fresh explicit `--workspace` whose canonical, non-symlink parent already exists. +2. Resolve exactly `/usr/bin/dpkg-query` and `/usr/bin/dpkg`; require the fixed + `nvidia-video-codec-sdk` package installed at public SDK `13.0.x`, enumerate its owned files, and + require silent `dpkg --verify`. +3. Resolve and record the fixed compiler/configuration tools and the first executable `nvcc` in its + bounded preference order, then require that selected compiler to be CUDA 13.x. Resolve the first + usable generator in its bounded preference order. Require FFmpeg development modules only when + decoder reporting is requested. +4. Configure the package-owned `Samples` tree inside the fresh workspace and issue one bounded + `cmake --build ... --target ` for each requested report: `AppEncCuda` for `encoder` and + `AppDec` for `decoder`. Do not build an unrequested target. +5. Rehash the tools and built binaries, require real non-stub driver linkage, run each requested + report once, then reauthenticate the fixed package and binary identities before publishing. + +An unavailable package, tool, development module, configure/build prerequisite, or requested +authority launches no report and exits `2`; apply the shared missing-prerequisite remediation in +**Optional setup evidence** above. Malformed package ownership, unexpected SDK version, integrity +drift, unsafe paths, or tampering fails closed with exit `3`. + +### Artifact and classification + +With a supplied verification, the producer publishes this deterministic outer contract: + +```json +{ + "kind": "nvcodec-native-sample-capability-report", + "schema_version": "1.0", + "status": "complete", + "authority": "native_official_samples", + "gpu": 0, + "requested_reports": ["encoder", "decoder"], + "native_verification": { + "initial_identity": {}, + "terminal_identity": {}, + "canonical_sha256": "...", + "kind": "nvcodec-native-verification", + "schema_version": "1.5", + "ready": true, + "status": "operation_verified", + "environment_identity": {}, + "package": {}, + "samples": {} + }, + "aggregate_encoder_capabilities": {}, + "aggregate_decoder_capabilities": {}, + "summary": {"completed": 2, "unknown": 0, "not_requested": 0} +} +``` + +Without a verification, the same outer report records +`native_verification.status=not_supplied`, `native_verification.ready=false`, and a +`local_authentication` block containing the authenticated package, fresh build, requested targets, +tools, built sample identities, and linkage. That `not_supplied` value is not a readiness failure: +the report's authority comes from the separately recorded local package authentication. It does not +promote the sample output to setup readiness or operation proof. + +Top-level `status` is `complete` when all requested reports parse, `partial` when at least one +completes and one is unknown, and `unknown` when none completes. An unrequested family remains +present with `status=not_requested`. A requested family is only `completed` or `unknown`. +`complete` means the official sample's report grammar was captured; it does not mean setup ready, +API-reported support, exact operation success, throughput, or documented product support. + +Both aggregate records use: + +```json +{ + "evidence_source_type": "official_sample_report", + "official_sample": true, + "scope": "sample_reported" +} +``` + +The encoder classification names `AppEncCuda` and `-ec`; the decoder classification names `AppDec` +and `-dc`. Do not emit `supported=true`, `capability_reported`, `operation_verified`, or a +documentation verdict from either aggregate. + +### Accepted report grammars + +Every requested report requires exit `0`, no timeout, complete bounded stdout/stderr evidence, no +recognized CUDA/NVENC/NVDEC/error/failure marker, the selected GPU where the format can identify +one, and unchanged binary identity. Preserve the complete streams, sizes, SHA-256 values, display +tails, parsed raw records, and the recognized `format_variant`. + +For `AppEncCuda -ec`, accept either: + +- the legacy summary grammar: exactly one `Encoder Capability Summary`, one detail hint, unique + `GPU - ` blocks containing the selected GPU, exactly one codec-support section and + one capability-summary-table section per GPU, and exactly one `H264`, `HEVC`, and `AV1` row; or +- the released R39.2 compact grammar: exactly one `Encoder Capability`, unique + `GPU - ` blocks containing the selected GPU, and exactly one basic `H264`, `HEVC`, + and `AV1` `yes`/`no` row per GPU. This real format has no synthetic sections or detail hint. + +Preserve the basic encoder row values as lowercase `yes` or `no`; they remain +`sample_reported_codec_rows`, never support. + +For `AppDec -dc`, accept either: + +- the legacy summary grammar: unique `GPU - ` blocks containing the selected GPU, + exactly one `GPU Decoder Capabilities` and `Codec Support Summary` per GPU, and one detail hint; + or +- the released R39.2 compact grammar: exactly one `Decoder Capability`, exactly one + `GPU in use: `, and at least one strict + `Codec ... BitDepth ... ChromaFormat ... Supported ...` row. Because that format carries no GPU + ordinal, it is eligible only for requested GPU `0`. + +Preserve every compact decoder row and its raw numeric `Supported` value. Do not synthesize the +complete `cuvidGetDecoderCaps` tuple matrix or turn any value into product support. The Py decoder +producer remains the distinct API authority for that matrix. + +Encoder and decoder reports are independent inventory commands: failure of one does not suppress +the other. + +### Exit codes + +| Condition | rc | +|---|---:| +| All requested reports completed | `0` | +| Selected native authority or prerequisite unavailable, or any requested report unknown | `2` | +| Malformed arguments/artifact, identity or linkage drift, unsafe output, write failure, or internal contract failure | `3` | + +Every handled path prints exactly one strict JSON document. Child exit codes remain inside their +aggregate records and are never returned directly. + +## Encoder API query + +PyNvVideoCodec exposes `GetEncoderCaps(gpuid, codec)`. The binding selects device 0 internally, so +nonzero-GPU encoder capability remains `unknown`. + +Invoke this skill's producer directly under the exact PyNvVideoCodec interpreter to query. Setup +evidence is optional, so the baseline local-authentication command omits `--environment`: + +```bash +"$PYNVC_PYTHON" -I \ + "$CAPABILITY_SKILL/scripts/query_encoder_caps.py" \ + --gpu 0 \ + --output "$ENCODER_REPORT_JSON" +``` + +The producer authenticates that exact lexical and resolved interpreter, `sys.prefix`, public +PyNvVideoCodec `2.1.0` distribution and imported module, initializer, and loaded extension before +and after the query. It requires wheel-`RECORD` ownership and records a `runtime_binding`; it never +scans for a venv or selects another interpreter. + +When the caller supplies a setup environment, use its declared lexical interpreter and pass the +artifact explicitly: + +```bash +"$VALIDATED_PYNVC_PYTHON" -I \ + "$CAPABILITY_SKILL/scripts/query_encoder_caps.py" \ + --environment "$ENVIRONMENT_JSON" --gpu 0 \ + --output "$ENCODER_REPORT_JSON" +``` + +The producer must authenticate and bind that exact artifact before and after the query. A supplied +environment is never ignored and an authentication failure never falls back to the local-only +route. + +With no repeated `--codec`, the producer queries `h264`, `hevc`, and `av1`. It publishes: + +- `kind=nvcodec-encoder-capability-report`; +- `schema_version=1.0`; +- `authority=pynvc`; +- `evidence_classification.encode` with + `evidence_source_type=api_query_helper`, `official_sample=false`, and + `api_symbol=PyNvVideoCodec.GetEncoderCaps`; +- the observed linked NVENC API and one `encode.` record per requested codec. + +`GetEncoderCaps` returns capability fields but no explicit boolean codec-support result. A +successful record is therefore `status=capability_reported`, `supported=null`, +`session_status=opened_by_GetEncoderCaps`, and `operation_status=not_tested`; never promote it to +`supported=true`. The loaded extension may expose NVENC API 12.1 or 13.0. Fields absent from a 12.1 +header remain unqueryable/`unknown`, not unsupported. + +Use the returned minimum and maximum dimensions before planning an exact operation. NV12 dimensions +must be even. Producer exit codes are `0` for complete, `2` for partial/unknown, and `3` for +authentication, malformed-input, or safe-write failure. A declared absent, unimported, or +unverified `pynvc` authority, a local import failure, or unavailable local wheel authority is +`unknown` with exit `2`, not a codec or product-support verdict. Route missing prerequisites to +`jetson-video-setup` without mutation and tell the user to install that skill if it is absent. + +## Decoder API query + +PyNvVideoCodec exposes `GetDecoderCaps(gpuid, codec, chromaformat, bitdepth)`. It also routes the +query to device 0, so nonzero-GPU results remain `unknown`. + +Invoke this skill's producer directly under the exact PyNvVideoCodec interpreter. Omit +`--environment` when no setup artifact is supplied: + +```bash +"$PYNVC_PYTHON" -I \ + "$CAPABILITY_SKILL/scripts/query_decoder_caps.py" \ + --gpu 0 \ + --output "$DECODER_MATRIX_JSON" +``` + +This path performs the same before-and-after local interpreter, wheel-`RECORD`, initializer, and +loaded-extension authentication as the encoder producer and publishes `runtime_binding`. Never +scan for, guess, or silently change the invoking interpreter. If a setup environment is supplied, +invoke its declared lexical interpreter and include `--environment "$ENVIRONMENT_JSON"`; the +producer must validate and bind that artifact and may not fall back if it fails. + +The default matrix contains all ten decoder families: + +`mpeg1`, `mpeg2`, `mpeg4`, `vc1`, `h264`, `hevc`, `vp8`, `vp9`, `av1`, and `jpeg`. + +Each family is queried across `monochrome`, `420`, `422`, and `444` chroma at 8-, 10-, and 12-bit +depth, for 120 tuples. Keep JPEG exactly as the live API reports it; never pre-assume its state. +Repeat `--codec` to request a subset in one fresh artifact: + +```bash +"$PYNVC_PYTHON" -I \ + "$CAPABILITY_SKILL/scripts/query_decoder_caps.py" \ + --gpu 0 \ + --codec h264 --codec hevc --codec av1 \ + --output "$DECODER_SUBSET_JSON" +``` + +Use the complete matrix for an unconstrained explicit Py decoder catalog. Also use it when a broad +unnamed decoder-catalog request reaches Py only because native was ineligible, but summarize that +fallback at family level and do not dump unsolicited tuple claims. For a bounded request such as +H.264, HEVC, and AV1, repeat `--codec` only for those named families; that example produces 36 +tuples rather than expanding to all 120. + +The producer publishes `kind=nvcodec-decoder-capability-matrix`, its current +`schema_version=1.2`, and the exact classification +`evidence_source_type=api_query_helper`, `official_sample=false`, and +`api_symbol=PyNvVideoCodec.GetDecoderCaps`. + +Gate every returned field on `bIsSupported`: + +- `bIsSupported=1` becomes raw `status=capability_reported`, `supported=true`, and applicable + limits. Here `supported=true` records only the API flag; it is not the documented product verdict. +- `bIsSupported=0` becomes raw `status=unsupported`, `supported=false`, with the remaining zeroed + output fields inapplicable. +- A missing enum, failed call, absent flag, or missing result is `unknown`. + +The encoder report and decoder matrix are separate artifacts with separate classifications. Neither +may be substituted by setup inventory or sample output. Missing local import or wheel prerequisites +produce `unknown` plus non-mutating `jetson-video-setup` remediation; if setup is not installed, +tell the user to install it. Malformed or mismatched supplied evidence fails closed and never +authorizes a retry without that evidence. + +## Setup readiness verification + +Setup verification is an optional fixed installation-readiness proof, not a prerequisite for +capability capture. The PyNv verifier runs the wheel-owned official CPU-buffer encoder and a +profile-selected decoder for exactly one 640x360 NV12 frame of H.264: + +```bash +"$VALIDATED_PYNVC_PYTHON" -I \ + "$SETUP_SKILL/scripts/setup/verify_pynvc_sample.py" \ + --environment "$ENVIRONMENT_JSON" --gpu 0 \ + --work-dir "$FRESH_SETUP_WORK_DIR" \ + --output "$FRESH_SETUP_REPORT_JSON" +``` + +It always requires exactly one encoded-frame marker and a fresh nonempty H.264 bitstream. The +default `pynvc-smoke` profile runs `advanced/decode_perf.py` and requires both anchored one-frame +production markers exactly once, with no worker warning, error, or traceback; that sample writes no +raw output. The explicit `full-samples` profile runs `advanced/decode.py` and additionally requires +one decoded-frame marker plus an exact 345,600-byte decoded NV12 output. The native verifier uses +the same fixed one-frame scope and exact raw-output proof with its package-owned samples. None of +these setup proofs enumerates codecs, emits capability records, proves another tuple, or publishes +product support. + +Every verification attempt needs a fresh work directory and output path. A failed attempt consumes +both paths; do not overwrite or reinterpret it as package absence. + +## Documentation cross-check + +After preserving the raw query, cross-check the target before deciding whether an exact operation +is useful. Use NVIDIA's public +[Video Encode and Decode Support Matrix](https://developer.nvidia.com/video-encode-decode-support-matrix) +and the version-matched Video Codec SDK 13.0 +[NVENC](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/nvenc-application-note/index.html) +or +[NVDEC](https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/nvdec-application-note/index.html) +application note. + +Obtain that live model evidence read-only, never by guessing a SKU. Use a supplied authenticated +`nvcodec-environment` artifact only when it actually carries literal exact model evidence; its +platform record normally reports system, release, machine, Jetson Linux version, and GPU name, +none of which identify an exact board model or SKU. Otherwise read the fixed device-tree identity +sources directly: + +```bash +tr -d '\000' Path: + """Return the setup marker in the lexical installed-skill catalog.""" + return Path(__file__).absolute().parents[2] / "jetson-video-setup" / "SKILL.md" + + +def setup_dependency(*, surface: str, needed_for: str) -> dict[str, Any]: + """Describe whether the canonical setup sibling can handle one remediation.""" + if surface not in {"native", "pynvc"}: + raise ValueError("setup dependency surface must be native or pynvc") + candidate = _setup_skill_path() + try: + details = candidate.lstat() + resolved_root = candidate.parent.resolve(strict=True) + resolved = candidate.resolve(strict=True) + except OSError: + installed = False + else: + installed = bool( + stat.S_ISREG(details.st_mode) + and not stat.S_ISLNK(details.st_mode) + and resolved_root.is_dir() + and resolved == resolved_root / "SKILL.md" + ) + action = ( + f"use jetson-video-setup to configure or repair the {surface} surface, " + "then retry this capability query" + if installed + else f"install jetson-video-setup as a sibling skill, use it to configure " + f"or repair the {surface} surface, then retry this capability query" + ) + return { + "skill": "jetson-video-setup", + "installed": installed, + "needed_for": needed_for, + "next_action": action, + } + + +def add_setup_remediation( + result: dict[str, Any], *, surface: str, reason: str +) -> None: + """Attach one additive, non-mutating, customer-actionable setup handoff.""" + result["remediation"] = { + "route": "jetson-video-setup", + "surface": surface, + "mutation_performed": False, + "reason": reason, + "dependency": setup_dependency( + surface=surface, + needed_for=f"{surface} capability-query prerequisites", + ), + } + + +def _sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-RFC JSON numeric constant is forbidden: {value}") + + +def strict_json_loads(value: str | bytes) -> Any: + return json.loads(value, parse_constant=_reject_json_constant) + + +def _require_finite_json(value: Any, *, location: str = "$") -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"non-finite numeric value is forbidden at {location}") + if isinstance(value, dict): + for key, item in value.items(): + _require_finite_json(item, location=f"{location}.{key}") + elif isinstance(value, (list, tuple)): + for index, item in enumerate(value): + _require_finite_json(item, location=f"{location}[{index}]") + + +def strict_json_dumps(value: Any, **kwargs: Any) -> str: + _require_finite_json(value) + return json.dumps(value, allow_nan=False, **kwargs) + + +def exclusive_write_bytes(path: Path, content: bytes, *, mode: int = 0o600) -> Path: + """Create one fresh regular file without following any symlink component.""" + requested = path.expanduser().absolute() + if not requested.parent.is_dir(): + raise FileNotFoundError( + f"evidence parent directory does not exist: {requested.parent}" + ) + current = Path(requested.anchor) + for part in requested.parent.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError(f"evidence parent contains a symlink component: {current}") + resolved = requested.parent.resolve(strict=True) / requested.name + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(resolved, flags, mode) + try: + with os.fdopen(descriptor, "wb") as handle: + descriptor = -1 + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + except Exception: + resolved.unlink(missing_ok=True) + raise + finally: + if descriptor >= 0: + os.close(descriptor) + return resolved + + +def exclusive_write_text(path: Path, content: str) -> None: + exclusive_write_bytes(path, content.encode("utf-8")) + + +def _canonical_json_sha256(value: Any) -> str: + rendered = strict_json_dumps(value, sort_keys=True, separators=(",", ":")) + return _sha256_bytes(rendered.encode("utf-8")) + + +def _record_entry(distribution: Any, path: Path) -> tuple[Path, str, str, str]: + resolved = path.expanduser().resolve() + matching = [ + entry + for entry in distribution.files or [] + if Path(distribution.locate_file(entry)).resolve() == resolved + ] + if len(matching) != 1: + raise ValueError( + f"expected one distribution file entry for {resolved}, found {len(matching)}" + ) + record_text = distribution.read_text("RECORD") + if not record_text: + raise ValueError("installed PyNvVideoCodec distribution has no readable RECORD") + entry_name = str(matching[0]).replace("\\", "/") + rows = [ + row + for row in csv.reader(io.StringIO(record_text)) + if row and row[0].replace("\\", "/") == entry_name + ] + if len(rows) != 1: + raise ValueError(f"expected one wheel RECORD row for {entry_name}, found {len(rows)}") + record_path, encoded_hash, recorded_size = (rows[0] + ["", ""])[:3] + return resolved, record_path, encoded_hash, recorded_size + + +def _wheel_owned_identity(distribution: Any, path: Path, *, label: str) -> dict[str, Any]: + """Prove one executed file is a wheel-RECORD member and hash it locally. + + Ownership is the assertion: the loaded path must resolve to exactly one + installed distribution entry and exactly one ``RECORD`` row. The recorded + digest is then re-derived here with :mod:`hashlib` and the comparison is + published as evidence rather than silently trusted, so a known upstream + ``RECORD`` inconsistency stays visible instead of requiring an external + attestation to excuse it. + """ + resolved, record_path, encoded_hash, recorded_size = _record_entry(distribution, path) + entry_name = record_path.replace("\\", "/") + observed_sha256 = _sha256_file(resolved) + observed_size = resolved.stat().st_size + record: dict[str, Any] = { + "record_path": record_path, + "hash": encoded_hash or None, + "size_bytes": int(recorded_size) if str(recorded_size).isdigit() else None, + } + if not encoded_hash or "=" not in encoded_hash: + record["consistency"] = "record_hash_absent" + else: + algorithm, expected = encoded_hash.split("=", 1) + if algorithm.lower() != "sha256": + raise ValueError( + f"{label} wheel RECORD hash algorithm must be sha256, got {algorithm}" + ) + actual = ( + base64.urlsafe_b64encode(bytes.fromhex(observed_sha256)) + .rstrip(b"=") + .decode("ascii") + ) + size_agrees = record["size_bytes"] in (None, observed_size) + record["consistency"] = ( + "matches_record" if actual == expected and size_agrees else "upstream_mismatch" + ) + return { + "path": str(resolved), + "sha256": observed_sha256, + "size_bytes": observed_size, + "wheel_member": entry_name, + "record": record, + } + + +def _require_record_consistency( + identity: dict[str, Any], *, label: str, version: str, allow_native_stale: bool = False +) -> None: + """Fail on a RECORD mismatch, permitting only the evidenced native-extension row. + + The exception is scoped by exact package version and by exact RECORD member: + the public PyNvVideoCodec 2.1.0 native extension. Ownership of the executed + file is still proven independently, so the exception narrows to the recorded + digest alone and is published in the report rather than hidden. + """ + record = identity["record"] + if record["consistency"] == "matches_record": + return + member = identity["wheel_member"] + if ( + allow_native_stale + and record["consistency"] == "upstream_mismatch" + and version == EXPECTED_PYNVC_VERSION + and _NATIVE_EXTENSION_MEMBER.fullmatch(member) is not None + ): + record["consistency"] = KNOWN_STALE_RECORD + record["exception"] = ( + f"known upstream PyNvVideoCodec {EXPECTED_PYNVC_VERSION} stale RECORD row for" + " its native extension; wheel ownership of the executed file is still proven" + ) + return + raise ValueError( + f"PyNvVideoCodec {label} does not match its wheel RECORD" + f" ({record['consistency']}): {member}" + ) + + +def scalar_attributes(value: Any) -> dict[str, Any]: + """Preserve every finite scalar field of a capability result, mapping or object.""" + if isinstance(value, Mapping): + if any(isinstance(item, float) and not math.isfinite(item) for item in value.values()): + raise ValueError("capability API returned a non-finite numeric field") + return { + str(key): item + for key, item in value.items() + if (isinstance(item, (str, int, bool)) or item is None) + or (isinstance(item, float) and math.isfinite(item)) + } + result: dict[str, Any] = {} + for name in dir(value): + if name.startswith("_"): + continue + try: + item = getattr(value, name) + except Exception: # pylint: disable=broad-exception-caught + continue + if isinstance(item, float) and not math.isfinite(item): + raise ValueError("capability API returned a non-finite numeric field") + if (isinstance(item, (str, int, bool)) or item is None) or ( + isinstance(item, float) and math.isfinite(item) + ): + result[name] = item + return result + + +def linked_nvenc_api(module: Any) -> dict[str, Any]: + """Read the NVENC API level the loaded extension is linked against.""" + suffix = getattr(module, "module_suffix", None) + extension_file = getattr(getattr(module, "_PyNvVideoCodec", None), "__file__", None) + combined = " ".join(str(item) for item in (suffix, extension_file) if item) + value = "13.0" if "_130" in combined else ("12.1" if "_121" in combined else None) + return { + "status": "observed" if value else "unknown", + "value": value, + "module_suffix": str(suffix) if suffix else None, + "extension_file": str(extension_file) if extension_file else None, + "provenance": "loaded PyNvVideoCodec extension filename/module_suffix", + } + + +def _active_import_environment() -> tuple[tuple[str, str], ...]: + return tuple( + sorted((name, value) for name in _IMPORT_VARIABLES if (value := os.environ.get(name))) + ) + + +def _process_context() -> tuple[int, str, str, int]: + return ( + os.getpid(), + str(Path(sys.executable).resolve()), + str(Path(sys.prefix).resolve()), + id(os.environ), + ) + + +class _ImportValidation: + """Opaque process-local proof that import inputs were validated.""" + + __slots__ = ("_seal", "_context", "_module_id", "_environment") + + def __init__(self) -> None: + self._seal = _IMPORT_SEAL + self._context = _process_context() + self._module_id: int | None = None + self._environment: tuple[tuple[str, str], ...] | None = None + + +def _assert_isolated() -> None: + if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise ValueError( + "Python authentication must run isolated with environment/user-site disabled" + " (python -I)" + ) + + +def assert_safe_python_import_environment() -> _ImportValidation: + ambient = _active_import_environment() + if ambient: + names = ", ".join(name for name, _value in ambient) + raise ValueError( + "ambient Python import injection variables must be absent during authentication: " + + names + ) + _assert_isolated() + return _ImportValidation() + + +def _assert_preimport(validation: object, module: Any) -> _ImportValidation: + if type(validation) is not _ImportValidation or validation._seal is not _IMPORT_SEAL: + raise ValueError("PyNvVideoCodec pre-import environment validation is invalid") + _assert_isolated() + if validation._context != _process_context(): + raise ValueError( + "PyNvVideoCodec pre-import environment validation belongs to a different process" + " or interpreter" + ) + current = _active_import_environment() + if validation._environment is None: + validation._module_id = id(module) + validation._environment = current + elif validation._module_id != id(module): + raise ValueError( + "PyNvVideoCodec pre-import environment validation is bound to another module" + ) + elif validation._environment != current: + raise ValueError( + "Python import injection environment changed after PyNvVideoCodec authentication" + ) + return validation + + +def live_pynvc_identity( + module: Any, + *, + expected_version: str = EXPECTED_PYNVC_VERSION, + preimport_validation: object | None = None, +) -> dict[str, Any]: + """Authenticate the imported PyNvVideoCodec locally, with no external attestation. + + Three local facts are established with the standard library alone: the + import happened under an isolated interpreter, the distribution and module + both report the exact expected version, and each file the process actually + executed is a wheel-``RECORD`` member inside the running ``sys.prefix``. + + The executed extension is taken from the imported ``_PyNvVideoCodec.__file__`` + rather than located through metadata, so ``loaded_path`` proves which binary + the interpreter really ran instead of merely which one exists on disk. + """ + if preimport_validation is None: + assert_safe_python_import_environment() + else: + _assert_preimport(preimport_validation, module) + distribution = importlib.metadata.distribution("PyNvVideoCodec") + distribution_version = str(distribution.version) + module_version = str(getattr(module, "__version__", "") or "") + if distribution_version != expected_version or module_version != expected_version: + raise ValueError( + "PyNvVideoCodec distribution and imported module must both be exact public " + f"{expected_version}" + ) + module_path_value = getattr(module, "__file__", None) + extension_path_value = getattr(getattr(module, "_PyNvVideoCodec", None), "__file__", None) + if not module_path_value or not extension_path_value: + raise ValueError("imported PyNvVideoCodec package/extension paths are unavailable") + module_path = Path(module_path_value).resolve() + extension_path = Path(extension_path_value).resolve() + prefix = Path(sys.prefix).resolve() + for label, owned_path in (("module", module_path), ("extension", extension_path)): + if not owned_path.is_relative_to(prefix): + raise ValueError( + f"PyNvVideoCodec {label} path {owned_path} is outside current sys.prefix {prefix}" + ) + linked_api = linked_nvenc_api(module) + if linked_api["value"] is None: + raise ValueError("loaded PyNvVideoCodec extension API suffix is not recognized") + module_identity = _wheel_owned_identity(distribution, module_path, label="module") + _require_record_consistency( + module_identity, label="module", version=distribution_version + ) + module_identity.update({"loaded_path": str(module_path_value), "version": module_version}) + extension_identity = _wheel_owned_identity(distribution, extension_path, label="extension") + _require_record_consistency( + extension_identity, + label="extension", + version=distribution_version, + allow_native_stale=True, + ) + extension_identity.update( + { + "loaded_path": str(extension_path_value), + "linked_nvenc_api": linked_api["value"], + "module_suffix": linked_api["module_suffix"], + } + ) + interpreter = Path(sys.executable).resolve(strict=True) + return { + "status": "verified", + "authentication": "local_wheel_record_ownership", + # Lexical: exactly what was invoked. Resolved identity: what actually ran. + "interpreter": sys.executable, + "interpreter_identity": { + "path": str(interpreter), + "sha256": _sha256_file(interpreter), + "size_bytes": interpreter.stat().st_size, + }, + "sys_prefix": str(prefix), + "version": distribution_version, + "module": module_identity, + "extension": extension_identity, + } + + +def require_environment_schema(data: Any) -> str: + """Reject an unknown major first, then pin the exact supported environment.""" + if not isinstance(data, dict): + raise ValueError("environment manifest must be a JSON object") + if data.get("kind") != ENVIRONMENT_KIND: + raise ValueError(f"environment manifest kind must be {ENVIRONMENT_KIND}") + version = data.get("schema_version") + if not isinstance(version, str) or re.fullmatch(r"\d+\.\d+", version) is None: + raise ValueError("environment schema_version must be MAJOR.MINOR") + if int(version.split(".", 1)[0]) != ENVIRONMENT_SUPPORTED_MAJOR: + raise ValueError( + f"unknown {ENVIRONMENT_KIND} major in {version}; this skill implements major" + f" {ENVIRONMENT_SUPPORTED_MAJOR}" + ) + if version != ENVIRONMENT_SCHEMA_VERSION: + raise ValueError( + f"{ENVIRONMENT_KIND} {version} is not the pinned {ENVIRONMENT_SCHEMA_VERSION}" + ) + if data.get("mode") != "live": + raise ValueError("capability queries require a live environment manifest") + return version + + +def _require_same_file(value: Any, expected: str, *, label: str) -> None: + if not isinstance(value, str) or not value: + raise ValueError(f"environment pynvc.identity.{label} is missing or malformed") + if str(Path(value).expanduser().resolve()) != expected: + raise ValueError( + f"environment pynvc.identity.{label} is not the file this process actually loaded" + ) + + +def declared_pynvc_identity(data: dict[str, Any]) -> dict[str, Any]: + """Return the declared ``pynvc.identity`` block of a 1.2 manifest. + + Schema 1.2 publishes the Python surface as the top-level ``pynvc`` member, + whose ``identity`` block is what this skill authenticated at probe time. An + absent or unverified authority is a classification, not a hard failure, so it + raises :class:`CapabilityAuthorityUnavailable` rather than ``ValueError``. + """ + surface = data.get("pynvc") + if not isinstance(surface, dict) or surface.get("imported") is not True: + raise CapabilityAuthorityUnavailable( + "environment manifest reports no imported pynvc surface" + ) + identity = surface.get("identity") + if not isinstance(identity, dict) or identity.get("status") != "verified": + raise CapabilityAuthorityUnavailable( + "environment pynvc.identity is not a verified authority" + ) + return identity + + +def declared_distribution_version(declared: dict[str, Any]) -> Any: + """Read the manifest's PyNvVideoCodec distribution version by either spelling. + + The established 1.2 producer carries it as ``identity.distribution``; this + skill additionally publishes the flattened ``identity.version``. Both mean the + installed distribution version, so either satisfies the cross-check and + neither is required on its own. ``distribution`` is accepted as the bare + version or as a mapping carrying one, which keeps this a required-subset read. + """ + if "version" in declared: + return declared.get("version") + distribution = declared.get("distribution") + if isinstance(distribution, Mapping): + return distribution.get("version") + return distribution + + +def require_pynvc_identity(data: dict[str, Any], identity: dict[str, Any]) -> dict[str, Any]: + """Validate only the required ``pynvc.identity`` subset against the live process. + + Additive optional keys anywhere in the block are ignored by construction: + every check names one required field, and nested blocks are compared key by + required key rather than by set equality. ``extension.loaded_path`` and + ``module.{version,path}`` are compared against the imported objects, so a + manifest that describes a different binary than the one executed fails closed. + + ``identity.version`` and ``identity.interpreter_identity`` are additive + refinements, not requirements: the established producer closes ``identity`` to + ``{status, interpreter, sys_prefix, dist_info_path, distribution, module, + extension}``. The distribution version is read through whichever spelling is + present, and a resolved interpreter identity is held to the exact running + interpreter only when the manifest actually publishes one. The lexical + ``identity.interpreter`` is required either way, so interpreter agreement is + still asserted for a manifest that carries no resolved identity. + """ + declared = declared_pynvc_identity(data) + for field in ("interpreter", "sys_prefix"): + if declared.get(field) != identity[field]: + raise ValueError( + f"environment pynvc.identity.{field} does not match the live PyNvVideoCodec import" + ) + if declared_distribution_version(declared) != identity["version"]: + raise ValueError( + "environment pynvc.identity distribution version does not match the live" + " PyNvVideoCodec import" + ) + for field, additive, required in ( + ( + "interpreter_identity", + True, + { + "path": identity["interpreter_identity"]["path"], + "sha256": identity["interpreter_identity"]["sha256"], + }, + ), + ( + "extension", + False, + { + "path": identity["extension"]["path"], + "sha256": identity["extension"]["sha256"], + }, + ), + ("module", False, {"version": identity["module"]["version"]}), + ): + observed = declared.get(field) + if observed is None and additive: + continue + if not isinstance(observed, dict): + raise ValueError(f"environment pynvc.identity.{field} is missing or malformed") + for key, value in required.items(): + if observed.get(key) != value: + raise ValueError( + f"environment pynvc.identity.{field}.{key} differs from the live process" + ) + _require_same_file( + declared["extension"].get("loaded_path"), + identity["extension"]["path"], + label="extension.loaded_path", + ) + _require_same_file( + declared["module"].get("path"), identity["module"]["path"], label="module.path" + ) + return declared + + +def require_declared_pynvc_authority(path: Path, *, gpu: int) -> None: + """Classify an absent Py surface before importing its runtime authority.""" + requested = path.expanduser().absolute() + if requested.is_symlink(): + raise ValueError("environment manifest must not be a symlink") + data = strict_json_loads(requested.read_bytes()) + require_environment_schema(data) + if data.get("selected_gpu") != gpu: + raise ValueError("environment selected_gpu does not match --gpu") + declared_pynvc_identity(data) + + +def authenticated_environment( + path: Path, + *, + gpu: int, + module: Any, + preimport_validation: object | None = None, +) -> tuple[dict[str, Any], dict[str, Any]]: + """Bind capability queries to the exact live 1.2 manifest and the imported wheel.""" + requested = path.expanduser().absolute() + if requested.is_symlink(): + raise ValueError("environment manifest must not be a symlink") + data = strict_json_loads(requested.read_bytes()) + require_environment_schema(data) + if data.get("selected_gpu") != gpu: + raise ValueError("environment selected_gpu does not match --gpu") + identity = live_pynvc_identity(module, preimport_validation=preimport_validation) + require_pynvc_identity(data, identity) + # environment_binding re-reads and compares the snapshot so replacement races fail closed. + binding = environment_binding(requested, data, identity) + binding["selected_gpu"] = gpu + return data, binding + + +def local_pynvc_binding( + module: Any, + *, + gpu: int, + preimport_validation: object | None = None, +) -> dict[str, Any]: + """Authenticate the exact invoking interpreter and wheel without setup evidence.""" + identity = live_pynvc_identity( + module, preimport_validation=preimport_validation + ) + return { + "schema_version": LOCAL_BINDING_SCHEMA_VERSION, + "kind": LOCAL_BINDING_KIND, + "mode": "live", + "selected_gpu": gpu, + "interpreter": sys.executable, + "interpreter_identity": identity["interpreter_identity"], + "pynvc_identity": identity, + } + + +def require_local_pynvc_binding(binding: Any, *, gpu: int) -> None: + """Require the closed process-local binding produced above.""" + if not isinstance(binding, dict): + raise ValueError("local PyNvVideoCodec runtime binding is mandatory") + expected = { + "schema_version": LOCAL_BINDING_SCHEMA_VERSION, + "kind": LOCAL_BINDING_KIND, + "mode": "live", + "selected_gpu": gpu, + "interpreter": sys.executable, + } + if set(binding) != {*expected, "interpreter_identity", "pynvc_identity"}: + raise ValueError("local PyNvVideoCodec runtime binding shape is invalid") + for field, value in expected.items(): + if binding.get(field) != value: + raise ValueError( + f"local PyNvVideoCodec runtime binding {field} is not authenticated" + ) + identity = binding.get("pynvc_identity") + interpreter_identity = binding.get("interpreter_identity") + if ( + not isinstance(identity, dict) + or identity.get("status") != "verified" + or identity.get("authentication") != "local_wheel_record_ownership" + or not isinstance(interpreter_identity, dict) + or interpreter_identity != identity.get("interpreter_identity") + ): + raise ValueError("local PyNvVideoCodec wheel/interpreter identity is invalid") + + +def reauthenticate_local_pynvc_binding( + binding: dict[str, Any], + *, + gpu: int, + module: Any, + preimport_validation: object | None = None, +) -> None: + """Recompute and compare one local binding at a query boundary.""" + require_local_pynvc_binding(binding, gpu=gpu) + observed = local_pynvc_binding( + module, gpu=gpu, preimport_validation=preimport_validation + ) + if observed != binding: + raise ValueError( + "local PyNvVideoCodec wheel or interpreter identity changed during queries" + ) + + +def require_authenticated_binding(environment: Any, *, gpu: int) -> None: + """Require the required binding subset produced by ``authenticated_environment``.""" + if not isinstance(environment, dict): + raise ValueError("authenticated environment binding is mandatory") + expected = { + "schema_version": ENVIRONMENT_SCHEMA_VERSION, + "kind": ENVIRONMENT_KIND, + "mode": "live", + "selected_gpu": gpu, + "interpreter": sys.executable, + } + for field, value in expected.items(): + if environment.get(field) != value: + raise ValueError(f"capability environment binding {field} is not authenticated") + if not isinstance(environment.get("path"), str) or not environment["path"]: + raise ValueError("capability environment binding has no manifest path") + if type(environment.get("size_bytes")) is not int: + raise ValueError("capability environment binding has no manifest size") + for field in ("sha256", "canonical_sha256"): + if re.fullmatch(r"[0-9a-f]{64}", str(environment.get(field))) is None: + raise ValueError(f"capability environment binding {field} is not a SHA-256 digest") + identity = environment.get("interpreter_identity") + if not isinstance(identity, dict) or not isinstance(identity.get("path"), str): + raise ValueError("capability environment binding has no resolved interpreter identity") + + +def _regular_file_snapshot( + path: Path, *, label: str +) -> tuple[bytes, dict[str, Any]]: + requested = Path(os.path.abspath(os.fspath(path.expanduser()))) + if not requested.parent.is_dir(): + raise FileNotFoundError(f"{label} parent directory does not exist: {requested.parent}") + current = Path(requested.anchor) + for part in requested.parent.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError(f"{label} parent contains a symlink component: {current}") + if requested.is_symlink(): + raise ValueError(f"{label} must not be a symlink: {requested}") + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(requested, flags) + try: + opened_stat = os.fstat(descriptor) + if not stat.S_ISREG(opened_stat.st_mode): + raise ValueError(f"{label} must be a regular file: {requested}") + with os.fdopen(descriptor, "rb") as handle: + descriptor = -1 + raw = handle.read() + finally: + if descriptor >= 0: + os.close(descriptor) + terminal_stat = os.stat(requested, follow_symlinks=False) + opened_key = (opened_stat.st_dev, opened_stat.st_ino, opened_stat.st_size) + terminal_key = (terminal_stat.st_dev, terminal_stat.st_ino, terminal_stat.st_size) + if opened_key != terminal_key or not stat.S_ISREG(terminal_stat.st_mode): + raise ValueError(f"{label} identity changed while it was being read") + resolved = requested.parent.resolve(strict=True) / requested.name + return raw, { + "path": str(resolved), + "device": opened_stat.st_dev, + "inode": opened_stat.st_ino, + "size_bytes": len(raw), + "sha256": _sha256_bytes(raw), + } + + +def environment_binding( + path: Path, data: dict[str, Any], identity: dict[str, Any] +) -> dict[str, Any]: + """Bind a report to the exact environment artifact bytes it was produced from. + + The recorded ``{path, size_bytes, sha256, canonical_sha256}`` is what lets a + downstream consumer reject a report produced against a different environment + and detect an environment that changed after production. + """ + raw, initial = _regular_file_snapshot(path, label="environment manifest") + parsed = strict_json_loads(raw) + if parsed != data: + raise ValueError("environment manifest changed after it was loaded") + terminal_raw, terminal = _regular_file_snapshot(path, label="environment manifest") + if terminal_raw != raw or terminal != initial: + raise ValueError("environment manifest inode/hash changed during authentication") + return { + "path": initial["path"], + "size_bytes": initial["size_bytes"], + "sha256": _sha256_bytes(raw), + "canonical_sha256": _canonical_json_sha256(parsed), + "schema_version": data.get("schema_version"), + "kind": data.get("kind"), + "mode": data.get("mode"), + # Lexical: exactly what was invoked. Resolved identity: what actually ran. + "interpreter": sys.executable, + "interpreter_identity": identity["interpreter_identity"], + "pynvc_identity": identity, + } + + +def enum_member(module: Any, containers: tuple[str, ...], names: tuple[str, ...]) -> Any: + for container_name in containers: + container = getattr(module, container_name, None) + if container is None: + continue + for name in names: + if hasattr(container, name): + return getattr(container, name) + for name in names: + if hasattr(module, name): + return getattr(module, name) + return None diff --git a/skills/jetson-video-capability/scripts/query_decoder_caps.py b/skills/jetson-video-capability/scripts/query_decoder_caps.py new file mode 100644 index 00000000..d45ff7f6 --- /dev/null +++ b/skills/jetson-video-capability/scripts/query_decoder_caps.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Query the complete PyNvVideoCodec GPU-0 NVDEC capability matrix.""" + +# pylint: disable=missing-function-docstring + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_SCRIPT_DIR = Path(__file__).absolute().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position +from _capability_contract import ( # noqa: E402 + add_setup_remediation, + assert_safe_python_import_environment, + authenticated_environment, + CapabilityAuthorityUnavailable, + ENVIRONMENT_SCHEMA_VERSION, + enum_member, + exclusive_write_text, + local_pynvc_binding, + reauthenticate_local_pynvc_binding, + require_declared_pynvc_authority, + require_authenticated_binding, + scalar_attributes, + strict_json_dumps, +) + +SCHEMA_VERSION = "1.2" +EVIDENCE_SOURCE_TYPE = "api_query_helper" +API_SYMBOL = "PyNvVideoCodec.GetDecoderCaps" + +# Producer exit codes, frozen by the public capability contract. +RC_COMPLETE = 0 +RC_PARTIAL = 2 +RC_FAILED = 3 + +CODECS = { + "mpeg1": ("MPEG1", "MPEG_1"), + "mpeg2": ("MPEG2", "MPEG_2"), + "mpeg4": ("MPEG4", "MPEG_4"), + "vc1": ("VC1", "VC_1"), + "h264": ("H264", "H_264"), + "hevc": ("HEVC", "H265", "H_265"), + "vp8": ("VP8", "VP_8"), + "vp9": ("VP9", "VP_9"), + "av1": ("AV1",), + "jpeg": ("JPEG",), +} +CHROMAS = { + "monochrome": ("Monochrome", "MONOCHROME"), + "420": ("420", "YUV420"), + "422": ("422", "YUV422"), + "444": ("444", "YUV444"), +} +BIT_DEPTHS = (8, 10, 12) + + +def evidence_classification() -> dict[str, Any]: + return { + "evidence_source_type": EVIDENCE_SOURCE_TYPE, + "official_sample": False, + "api_symbol": API_SYMBOL, + } + + +# Preserve the historical alias while using the capability-owned writer. +atomic_write = exclusive_write_text + + +def _unknown_records(reason: str, selected_codecs: tuple[str, ...]) -> list[dict[str, Any]]: + return [ + { + "codec": codec, + "chroma": chroma, + "bit_depth": bit_depth, + "status": "unknown", + "supported": None, + "limits_applicable": None, + "values": {}, + "reason": reason, + } + for codec in selected_codecs + for chroma in CHROMAS + for bit_depth in BIT_DEPTHS + ] + + +def add_matrix_counts(result: dict[str, Any]) -> None: + queries = result.get("queries", []) + tuples = { + (record.get("codec"), record.get("chroma"), record.get("bit_depth")) + for record in queries + if isinstance(record, dict) + } + expected = result.get("matrix", {}).get("expected_query_count") + result["matrix"].update( + { + "observed_query_count": len(queries), + "unique_tuple_count": len(tuples), + "exact_matrix_complete": len(queries) == expected and len(tuples) == expected, + } + ) + + +def _query_matrix_unchecked( # pylint: disable=too-many-locals,too-many-branches + module: Any, + gpu: int, + selected_codecs: tuple[str, ...] = tuple(CODECS), + *, + environment: dict[str, Any] | None = None, + runtime_binding: dict[str, Any] | None = None, +) -> tuple[dict[str, Any], int]: + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-decoder-capability-matrix", + "generated_at": datetime.now(timezone.utc).isoformat(), + "authority": API_SYMBOL, + "evidence_classification": evidence_classification(), + "gpu": gpu, + "matrix": { + "codecs": list(selected_codecs), + "chromas": list(CHROMAS), + "bit_depths": list(BIT_DEPTHS), + "expected_query_count": len(selected_codecs) * len(CHROMAS) * len(BIT_DEPTHS), + }, + "queries": [], + } + if environment is not None: + result["environment"] = environment + else: + result["runtime_binding"] = runtime_binding + if gpu != 0: + reason = ( + "PyNvVideoCodec GetDecoderCaps routes capability queries to GPU 0; nonzero-GPU results" + " are unknown" + ) + result.update({"status": "unknown", "reason": reason}) + result["queries"] = _unknown_records(reason, selected_codecs) + result["summary"] = { + "capability_reported": 0, + "unsupported": 0, + "unknown": len(result["queries"]), + } + add_matrix_counts(result) + return result, 2 + + get_caps = getattr(module, "GetDecoderCaps", None) + if not callable(get_caps): + reason = "PyNvVideoCodec.GetDecoderCaps is unavailable" + result.update({"status": "unknown", "reason": reason}) + result["queries"] = _unknown_records(reason, selected_codecs) + result["summary"] = { + "capability_reported": 0, + "unsupported": 0, + "unknown": len(result["queries"]), + } + add_matrix_counts(result) + return result, 2 + + codec_containers = ("cudaVideoCodec", "Codec", "VideoCodec") + chroma_containers = ("cudaVideoChromaFormat", "ChromaFormat") + queries: list[dict[str, Any]] = [] + for codec_name in selected_codecs: + codec_value = enum_member(module, codec_containers, CODECS[codec_name]) + for chroma_name, chroma_aliases in CHROMAS.items(): + chroma_value = enum_member(module, chroma_containers, chroma_aliases) + for bit_depth in BIT_DEPTHS: + record: dict[str, Any] = { + "codec": codec_name, + "chroma": chroma_name, + "bit_depth": bit_depth, + "status": "unknown", + "supported": None, + "limits_applicable": None, + "values": {}, + } + if codec_value is None or chroma_value is None: + missing = "codec" if codec_value is None else "chroma" + record["reason"] = ( + f"Required {missing} enum is unavailable in the loaded module" + ) + queries.append(record) + continue + try: + values = scalar_attributes(get_caps(gpu, codec_value, chroma_value, bit_depth)) + record["values"] = values + supported = ( + values["supported"] + if "supported" in values + else values.get("bIsSupported") + ) + if supported in (0, False): + record.update( + { + "status": "unsupported", + "supported": False, + "limits_applicable": False, + "interpretation": ( + "bIsSupported=0; remaining zeroed OUT fields are inapplicable" + ), + } + ) + elif isinstance(supported, (int, bool)) and int(supported) > 0: + record.update( + { + "status": "capability_reported", + "supported": True, + "limits_applicable": True, + "interpretation": "bIsSupported=1; returned limits are applicable", + } + ) + else: + record["reason"] = "bIsSupported was absent or not interpretable" + except Exception as exc: # pylint: disable=broad-exception-caught + record.update({"reason": "GetDecoderCaps failed", "error": str(exc)}) + queries.append(record) + + result["queries"] = queries + add_matrix_counts(result) + summary = { + state: sum(1 for record in queries if record["status"] == state) + for state in ("capability_reported", "unsupported", "unknown") + } + result["summary"] = summary + if summary["unknown"]: + result.update( + {"status": "partial", "reason": "One or more requested tuples remain unknown"} + ) + return result, 2 + result["status"] = "complete" + return result, 0 + + +def query_matrix( + module: Any, + gpu: int, + selected_codecs: tuple[str, ...] = tuple(CODECS), + *, + environment: dict[str, Any] | None = None, + preimport_validation: object | None = None, +) -> tuple[dict[str, Any], int]: + """Use supplied setup evidence or derive local authority inside this process.""" + runtime_binding = None + if environment is not None: + require_authenticated_binding(environment, gpu=gpu) + _, initial_binding = authenticated_environment( + Path(environment.get("path", "")), + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + if initial_binding != environment: + raise ValueError( + "decoder environment binding differs from current manifest authority" + ) + else: + runtime_binding = local_pynvc_binding( + module, gpu=gpu, preimport_validation=preimport_validation + ) + result, code = _query_matrix_unchecked( + module, + gpu, + selected_codecs, + environment=environment, + runtime_binding=runtime_binding, + ) + if environment is not None: + _, terminal_binding = authenticated_environment( + Path(environment["path"]), + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + if terminal_binding != environment: + raise ValueError( + "decoder environment manifest or wheel identity changed during queries" + ) + else: + reauthenticate_local_pynvc_binding( + runtime_binding, + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + return result, code + + +def unknown_result( + reason: str, *, gpu: int, selected_codecs: tuple[str, ...], error: str | None = None +) -> dict[str, Any]: + queries = _unknown_records(reason, selected_codecs) + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-decoder-capability-matrix", + "generated_at": datetime.now(timezone.utc).isoformat(), + "authority": API_SYMBOL, + "evidence_classification": evidence_classification(), + "gpu": gpu, + "status": "unknown", + "reason": reason, + "queries": queries, + "matrix": { + "codecs": list(selected_codecs), + "chromas": list(CHROMAS), + "bit_depths": list(BIT_DEPTHS), + "expected_query_count": len(queries), + }, + "summary": { + "capability_reported": 0, + "unsupported": 0, + "unknown": len(queries), + }, + } + if error is not None: + result["error"] = error + add_matrix_counts(result) + return result + + +class _ExitThreeParser(argparse.ArgumentParser): + """Report malformed input with the frozen producer exit code.""" + + def error(self, message: str) -> Any: + raise ValueError(f"{self.prog}: {message}") + + +def main() -> int: + parser = _ExitThreeParser(description=__doc__) + parser.add_argument( + "--environment", + type=Path, + help=( + f"Optional validated live schema-{ENVIRONMENT_SCHEMA_VERSION} " + "nvcodec-environment manifest; when omitted authenticate the exact " + "invoking interpreter and wheel locally" + ), + ) + parser.add_argument("--gpu", type=int, default=0) + parser.add_argument("--codec", action="append", choices=tuple(CODECS), dest="codecs") + parser.add_argument("--output", type=Path) + try: + args = parser.parse_args() + if args.gpu < 0: + parser.error("--gpu must be non-negative") + except ValueError as exc: + print(strict_json_dumps({ + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-decoder-capability-matrix-error", + "status": "error", + "error": str(exc), + }, sort_keys=True)) + return RC_FAILED + selected_codecs = tuple(dict.fromkeys(args.codecs or CODECS)) + try: + if args.environment is not None: + require_declared_pynvc_authority(args.environment, gpu=args.gpu) + preimport_validation = assert_safe_python_import_environment() + import PyNvVideoCodec as nvc # pylint: disable=import-error,import-outside-toplevel + if args.environment is not None: + _, binding = authenticated_environment( + args.environment, + gpu=args.gpu, + module=nvc, + preimport_validation=preimport_validation, + ) + result, code = query_matrix( + nvc, + args.gpu, + selected_codecs, + environment=binding, + preimport_validation=preimport_validation, + ) + else: + result, code = query_matrix( + nvc, + args.gpu, + selected_codecs, + preimport_validation=preimport_validation, + ) + except (CapabilityAuthorityUnavailable, ImportError) as exc: + reason = "decoder capability authority is unavailable" + result = unknown_result( + reason, gpu=args.gpu, selected_codecs=selected_codecs, error=str(exc) + ) + add_setup_remediation(result, surface="pynvc", reason=str(exc)) + code = RC_PARTIAL + except Exception as exc: # pylint: disable=broad-exception-caught + reason = "decoder capability authority authentication failed" + result = unknown_result( + reason, gpu=args.gpu, selected_codecs=selected_codecs, error=str(exc) + ) + code = RC_FAILED + rendered = strict_json_dumps(result, indent=2, sort_keys=True) + "\n" + write_error = None + if args.output: + try: + atomic_write(args.output, rendered) + except Exception as exc: # pylint: disable=broad-exception-caught + write_error = str(exc) + print(rendered, end="") + if write_error is not None: + print(f"decoder capability report write failed: {write_error}", file=sys.stderr) + return RC_FAILED + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-capability/scripts/query_encoder_caps.py b/skills/jetson-video-capability/scripts/query_encoder_caps.py new file mode 100644 index 00000000..89e5950e --- /dev/null +++ b/skills/jetson-video-capability/scripts/query_encoder_caps.py @@ -0,0 +1,363 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Publish the capability-owned PyNvVideoCodec GPU-0 NVENC capability report.""" + +# pylint: disable=missing-function-docstring + +from __future__ import annotations + +import argparse +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_SCRIPT_DIR = Path(__file__).absolute().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position +from _capability_contract import ( # noqa: E402 + add_setup_remediation, + assert_safe_python_import_environment, + authenticated_environment, + CapabilityAuthorityUnavailable, + ENVIRONMENT_SCHEMA_VERSION, + exclusive_write_text, + linked_nvenc_api, + local_pynvc_binding, + reauthenticate_local_pynvc_binding, + require_declared_pynvc_authority, + require_authenticated_binding, + scalar_attributes, + strict_json_dumps, +) + +SCHEMA_VERSION = "1.0" +REPORT_KIND = "nvcodec-encoder-capability-report" +AUTHORITY = "pynvc" +EVIDENCE_SOURCE_TYPE = "api_query_helper" +API_SYMBOL = "PyNvVideoCodec.GetEncoderCaps" + +# GetEncoderCaps takes the codec label directly; there is no enum indirection. +CODECS = ("h264", "hevc", "av1") + +# Capabilities that exist only in the NVENC 13.0 header. When the loaded +# extension links NVENC 12.1 their absence is unqueryable, never unsupported. +API_13_ONLY_CAPS = ( + "support_yuv422_encode", + "support_mvhevc_encode", + "support_temporal_filter", + "support_lookahead_level", + "support_unidirectional_b", +) + +SESSION_STATUS = "opened_by_GetEncoderCaps" +OPERATION_STATUS = "not_tested" +INTERPRETATION = ( + "GetEncoderCaps returned capability fields but no explicit codec support result;" + " an official encode operation is still required" +) + +# Producer exit codes, frozen. +RC_COMPLETE = 0 +RC_PARTIAL = 2 +RC_FAILED = 3 +GPU_ZERO_REASON = ( + "PyNvVideoCodec GetEncoderCaps discards the requested GPU and selects device 0;" + " nonzero-GPU encoder capability support remains unknown" +) + + +def evidence_classification() -> dict[str, Any]: + """Publish the closed per-family API-helper classification recipe requires.""" + return { + "encode": { + "evidence_source_type": EVIDENCE_SOURCE_TYPE, + "official_sample": False, + "api_symbol": API_SYMBOL, + } + } + + +def _unknown_capability(reason: str) -> dict[str, Any]: + return {"status": "unknown", "supported": None, "reason": reason} + + +def _failed_capability(exc: Exception) -> dict[str, Any]: + return {"status": "unknown", "query_status": "error", "supported": None, "error": str(exc)} + + +def _unavailable_fields(values: dict[str, Any], linked_api: dict[str, Any]) -> dict[str, Any]: + if linked_api.get("value") != "12.1": + return {} + return { + key: { + "status": "unknown", + "reason": "capability is absent from the linked NVENC 12.1 header", + } + for key in API_13_ONLY_CAPS + if key not in values + } + + +def _capability_record(values: dict[str, Any], linked_api: dict[str, Any]) -> dict[str, Any]: + # A positive query is `capability_reported` with `supported: null`, never + # `supported: true`. The API reports a capability, not product support. + return { + "status": "capability_reported", + "supported": None, + "session_status": SESSION_STATUS, + "operation_status": OPERATION_STATUS, + "interpretation": INTERPRETATION, + "values": values, + "unavailable_fields": _unavailable_fields(values, linked_api), + } + + +def _summarize(result: dict[str, Any]) -> None: + records = result["encode"].values() + result["summary"] = { + state: sum(1 for record in records if record.get("status") == state) + for state in ("capability_reported", "unsupported", "unknown") + } + + +def _query_encoder_caps_unchecked( + module: Any, + gpu: int, + selected_codecs: tuple[str, ...], + *, + environment: dict[str, Any] | None = None, + runtime_binding: dict[str, Any] | None = None, +) -> tuple[dict[str, Any], int]: + linked_api = linked_nvenc_api(module) + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": REPORT_KIND, + "generated_at": datetime.now(timezone.utc).isoformat(), + "authority": AUTHORITY, + "gpu": gpu, + "evidence_classification": evidence_classification(), + "linked_nvenc_api": linked_api, + "codecs": list(selected_codecs), + "encode": {}, + } + if environment is not None: + result["environment"] = environment + else: + result["runtime_binding"] = runtime_binding + if gpu != 0: + result.update({"status": "unknown", "reason": GPU_ZERO_REASON}) + result["encode"] = { + codec: _unknown_capability(GPU_ZERO_REASON) for codec in selected_codecs + } + _summarize(result) + return result, RC_PARTIAL + + get_encoder_caps = getattr(module, "GetEncoderCaps", None) + for codec in selected_codecs: + if not callable(get_encoder_caps): + result["encode"][codec] = _unknown_capability("GetEncoderCaps is unavailable") + continue + try: + values = scalar_attributes(get_encoder_caps(gpu, codec)) + except Exception as exc: # pylint: disable=broad-exception-caught + result["encode"][codec] = _failed_capability(exc) + continue + if not values: + result["encode"][codec] = _unknown_capability("GetEncoderCaps returned no fields") + continue + result["encode"][codec] = _capability_record(values, linked_api) + + _summarize(result) + if result["summary"]["unknown"]: + result.update( + {"status": "partial", "reason": "One or more requested codecs remain unknown"} + ) + return result, RC_PARTIAL + result["status"] = "complete" + return result, RC_COMPLETE + + +def query_encoder_caps( + module: Any, + gpu: int, + selected_codecs: tuple[str, ...] = CODECS, + *, + environment: dict[str, Any] | None = None, + preimport_validation: object | None = None, +) -> tuple[dict[str, Any], int]: + """Use supplied setup evidence or derive local authority inside this process.""" + runtime_binding = None + if environment is not None: + require_authenticated_binding(environment, gpu=gpu) + _, initial_binding = authenticated_environment( + Path(environment.get("path", "")), + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + if initial_binding != environment: + raise ValueError( + "encoder environment binding differs from current manifest authority" + ) + else: + runtime_binding = local_pynvc_binding( + module, gpu=gpu, preimport_validation=preimport_validation + ) + result, code = _query_encoder_caps_unchecked( + module, + gpu, + selected_codecs, + environment=environment, + runtime_binding=runtime_binding, + ) + if environment is not None: + _, terminal_binding = authenticated_environment( + Path(environment["path"]), + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + if terminal_binding != environment: + raise ValueError( + "encoder environment manifest or wheel identity changed during queries" + ) + else: + reauthenticate_local_pynvc_binding( + runtime_binding, + gpu=gpu, + module=module, + preimport_validation=preimport_validation, + ) + return result, code + + +def unknown_result( + reason: str, *, gpu: int, selected_codecs: tuple[str, ...], error: str | None = None +) -> dict[str, Any]: + """Report an unqueried encoder surface without claiming absence.""" + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": REPORT_KIND, + "generated_at": datetime.now(timezone.utc).isoformat(), + "authority": AUTHORITY, + "gpu": gpu, + "evidence_classification": evidence_classification(), + "linked_nvenc_api": { + "status": "unknown", + "value": None, + "module_suffix": None, + "extension_file": None, + "provenance": "loaded PyNvVideoCodec extension filename/module_suffix", + }, + "codecs": list(selected_codecs), + "status": "unknown", + "reason": reason, + "encode": {codec: _unknown_capability(reason) for codec in selected_codecs}, + } + if error is not None: + result["error"] = error + _summarize(result) + return result + + +class _ExitThreeParser(argparse.ArgumentParser): + """Report malformed input with the frozen producer exit code.""" + + def error(self, message: str) -> Any: + raise ValueError(f"{self.prog}: {message}") + + +def main() -> int: + parser = _ExitThreeParser(description=__doc__) + parser.add_argument( + "--environment", + type=Path, + help=( + f"Optional validated live schema-{ENVIRONMENT_SCHEMA_VERSION} " + "nvcodec-environment manifest; when omitted authenticate the exact " + "invoking interpreter and wheel locally" + ), + ) + parser.add_argument("--gpu", type=int, default=0) + parser.add_argument("--codec", action="append", choices=CODECS, dest="codecs") + parser.add_argument("--output", type=Path) + try: + args = parser.parse_args() + if args.gpu < 0: + parser.error("--gpu must be non-negative") + except ValueError as exc: + print(strict_json_dumps({ + "schema_version": SCHEMA_VERSION, + "kind": f"{REPORT_KIND}-error", + "status": "error", + "error": str(exc), + }, sort_keys=True)) + return RC_FAILED + selected_codecs = tuple(dict.fromkeys(args.codecs or CODECS)) + try: + if args.environment is not None: + require_declared_pynvc_authority(args.environment, gpu=args.gpu) + preimport_validation = assert_safe_python_import_environment() + import PyNvVideoCodec as nvc # pylint: disable=import-error,import-outside-toplevel + if args.environment is not None: + _, binding = authenticated_environment( + args.environment, + gpu=args.gpu, + module=nvc, + preimport_validation=preimport_validation, + ) + result, code = query_encoder_caps( + nvc, + args.gpu, + selected_codecs, + environment=binding, + preimport_validation=preimport_validation, + ) + else: + result, code = query_encoder_caps( + nvc, + args.gpu, + selected_codecs, + preimport_validation=preimport_validation, + ) + except CapabilityAuthorityUnavailable as exc: + reason = "encoder capability authority is unavailable" + result = unknown_result( + reason, gpu=args.gpu, selected_codecs=selected_codecs, error=str(exc) + ) + add_setup_remediation(result, surface="pynvc", reason=str(exc)) + code = RC_PARTIAL + except ImportError as exc: + reason = "encoder capability authority is unavailable" + result = unknown_result( + reason, gpu=args.gpu, selected_codecs=selected_codecs, error=str(exc) + ) + add_setup_remediation(result, surface="pynvc", reason=str(exc)) + code = RC_PARTIAL + except Exception as exc: # pylint: disable=broad-exception-caught + reason = "encoder capability authority authentication failed" + result = unknown_result( + reason, gpu=args.gpu, selected_codecs=selected_codecs, error=str(exc) + ) + code = RC_FAILED + rendered = strict_json_dumps(result, indent=2, sort_keys=True) + "\n" + write_error = None + if args.output: + try: + exclusive_write_text(args.output, rendered) + except Exception as exc: # pylint: disable=broad-exception-caught + write_error = str(exc) + print(rendered, end="") + if write_error is not None: + print(f"encoder capability report write failed: {write_error}", file=sys.stderr) + return RC_FAILED + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-capability/scripts/query_native_sample_reports.py b/skills/jetson-video-capability/scripts/query_native_sample_reports.py new file mode 100644 index 00000000..e4a18bb0 --- /dev/null +++ b/skills/jetson-video-capability/scripts/query_native_sample_reports.py @@ -0,0 +1,1045 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Query raw native AppEncCuda/AppDec capability reports from verified binaries.""" + +# pylint: disable=missing-function-docstring,too-many-locals,too-many-lines + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import signal +import stat +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +_SCRIPT_DIR = Path(__file__).absolute().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position +from _capability_contract import ( # noqa: E402 + add_setup_remediation, + ENVIRONMENT_SCHEMA_VERSION as ENVIRONMENT_SCHEMA, + exclusive_write_bytes, +) + +KIND = "nvcodec-native-sample-capability-report" +SCHEMA = "1.0" +VERIFY_KIND = "nvcodec-native-verification" +VERIFY_SCHEMA = "1.5" +PACKAGE = "nvidia-video-codec-sdk" +MAX_JSON = 16 * 1024 * 1024 +MAX_STREAM = 8 * 1024 * 1024 +MAX_BINARY = 64 * 1024 * 1024 +SHA_RE = re.compile(r"[0-9a-f]{64}") +SDK_RE = re.compile(r"13\.0(?:\.\d+)*(?:\+[0-9A-Za-z.]+)?(?:-[0-9A-Za-z.+]+)?") +NVCC_RE = re.compile(r"\brelease\s+(13\.\d+)(?:\D|$)") +GPU_RE = re.compile(r"(?m)^GPU[ \t]+(\d+)[ \t]+-[ \t]+([^\r\n]+?)[ \t]*\r?$") +GPU_IN_USE_RE = re.compile(r"(?m)^GPU in use:[ \t]+([^\r\n]+?)[ \t]*\r?$") +CODEC_RE = re.compile(r"(?m)^[ \t]*(H264|HEVC|AV1):[ \t]*(yes|no)[ \t]*\r?$") +DECODER_ROW_RE = re.compile( + r"^Codec[ \t]+(?P[A-Z0-9]+)[ \t]+BitDepth[ \t]+(?P\d+)" + r"[ \t]+ChromaFormat[ \t]+(?P\d+:\d+:\d+)[ \t]+Supported[ \t]+" + r"(?P[01])[ \t]+MaxWidth[ \t]+(?P\d+)[ \t]+MaxHeight[ \t]+" + r"(?P\d+)[ \t]+MaxMBCount[ \t]+(?P\d+)[ \t]+MinWidth[ \t]+" + r"(?P\d+)[ \t]+MinHeight[ \t]+(?P\d+)[ \t]+SurfaceFormat" + r"[ \t]+(?P[A-Z0-9]+(?:[ \t]+[A-Z0-9]+)*)[ \t]*\r?$" +) +FAILURES = ( + ("nvenc_error", re.compile(r"(?im)^\s*(?:NV_ENC_ERR_[A-Z0-9_]+\b|NVENC error\b)")), + ("cuda_error", re.compile(r"(?im)^\s*(?:CUDA_ERROR_[A-Z0-9_]+\b|CUDA .*error\b)")), + ("error", re.compile( + r"(?im)^\s*(?:Traceback \(most recent call last\)|\[(?:FATAL|ERROR)\s*\]|" + r"(?:uncaught )?exception\b|error\b|failed\b|failure\b)" + )), +) +REQUIRED_LIBS = { + "encoder": ("libcuda.so.1", "libnvidia-encode.so.1"), + "decoder": ("libcuda.so.1", "libnvcuvid.so.1"), +} +SAMPLES = {"encoder": ("AppEncCuda", "-ec"), "decoder": ("AppDec", "-dc")} +SAFE_ENV = {"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LANG": "C", "LC_ALL": "C"} +BUILD_TOOLS = { + "cmake": (Path("/usr/bin/cmake"),), + "cxx": (Path("/usr/bin/g++"),), + "pkg_config": (Path("/usr/bin/pkg-config"),), + "nvcc": ( + Path("/usr/local/cuda/bin/nvcc"), + Path("/usr/local/cuda-13.0/bin/nvcc"), + ), +} +GENERATORS = ( + ("Ninja", Path("/usr/bin/ninja")), + ("Unix Makefiles", Path("/usr/bin/make")), +) +APPDEC_MODULES = ("libavcodec", "libavformat", "libavutil", "libswresample") + + +class _Unavailable(ValueError): + """A well-formed native authority is unavailable.""" + + +def _public_sdk_release(value: Any) -> bool: + """Accept the same stable Debian 13.0 family as native setup verification.""" + normalized = value.strip().split(":", 1)[-1] if isinstance(value, str) else "" + if not normalized or "~" in normalized or "really" in normalized.lower(): + return False + return SDK_RE.fullmatch(normalized) is not None + + +def _reject_constant(value: str) -> None: + raise ValueError(f"non-RFC JSON numeric constant is forbidden: {value}") + + +def _pairs(items: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in items: + if key in result: + raise ValueError(f"duplicate JSON key is forbidden: {key}") + result[key] = value + return result + + +def _loads(raw: bytes) -> Any: + return json.loads( + raw.decode("utf-8", errors="strict"), + object_pairs_hook=_pairs, + parse_constant=_reject_constant, + ) + + +def _dumps(value: Any, *, compact: bool = False) -> bytes: + options = {"sort_keys": True, "allow_nan": False} + text = json.dumps( + value, separators=(",", ":") if compact else None, + indent=None if compact else 2, **options + ) + return (text + ("" if compact else "\n")).encode("utf-8") + + +def _snapshot(path: Path, *, label: str, limit: int = MAX_JSON, + executable: bool = False) -> tuple[bytes, dict[str, Any]]: + requested = path.expanduser().absolute() + if requested.is_symlink(): + raise ValueError(f"{label} must not be a symlink: {requested}") + flags = os.O_RDONLY | (getattr(os, "O_NOFOLLOW", 0)) + descriptor = os.open(requested, flags) + try: + before = os.fstat(descriptor) + if not stat.S_ISREG(before.st_mode) or before.st_size > limit: + raise ValueError(f"{label} must be a regular file no larger than {limit} bytes") + chunks, total = [], 0 + while chunk := os.read(descriptor, min(1024 * 1024, limit + 1 - total)): + chunks.append(chunk) + total += len(chunk) + if total > limit: + raise ValueError(f"{label} exceeds {limit} bytes") + after = os.fstat(descriptor) + finally: + os.close(descriptor) + current = os.stat(requested, follow_symlinks=False) + before_token = (before.st_dev, before.st_ino, before.st_size, before.st_mtime_ns) + after_token = (after.st_dev, after.st_ino, after.st_size, after.st_mtime_ns) + current_token = (current.st_dev, current.st_ino, current.st_size, current.st_mtime_ns) + if before_token != after_token or after_token != current_token: + raise ValueError(f"{label} changed while it was read") + if executable and not after.st_mode & 0o111: + raise ValueError(f"{label} is not executable") + raw = b"".join(chunks) + return raw, {"path": str(requested.resolve(strict=True)), "size_bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest()} + + +def _load(path: Path, *, label: str) -> tuple[bytes, dict[str, Any], dict[str, Any]]: + raw, identity = _snapshot(path, label=label) + value = _loads(raw) + if not isinstance(value, dict): + raise ValueError(f"{label} must contain one JSON object") + return raw, identity, value + + +def _same_identity(observed: dict[str, Any], expected: Any, *, label: str) -> None: + if not isinstance(expected, dict) or any( + expected.get(key) != observed[key] for key in ("path", "size_bytes", "sha256") + ): + raise ValueError(f"{label} does not match its recorded identity") + + +def _run(argv: list[str], timeout: int) -> dict[str, Any]: + started_at = datetime.now(timezone.utc) + started_clock = time.monotonic() + try: + done = subprocess.run(argv, capture_output=True, check=False, timeout=timeout, + env=SAFE_ENV) + code, out, err, expired = done.returncode, done.stdout, done.stderr, False + except subprocess.TimeoutExpired as exc: + code, out, err, expired = None, exc.stdout or b"", exc.stderr or b"", True + out = out.encode() if isinstance(out, str) else out + err = err.encode() if isinstance(err, str) else err + ended_at = datetime.now(timezone.utc) + return {"argv": argv, "cwd": str(Path.cwd().resolve()), "exit_code": code, + "timed_out": expired, "started_at": started_at.isoformat(), + "ended_at": ended_at.isoformat(), + "duration_seconds": round(time.monotonic() - started_clock, 6), + "stdout_bytes": out, "stderr_bytes": err} + + +def _run_build(argv: list[str], timeout: int) -> dict[str, Any]: + """Run a build command and reap its whole process group on timeout.""" + started_at = datetime.now(timezone.utc) + started_clock = time.monotonic() + process = subprocess.Popen( # pylint: disable=consider-using-with + argv, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=SAFE_ENV, + start_new_session=True, + ) + expired = False + try: + out, err = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + expired = True + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + out, err = process.communicate(timeout=2) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + out, err = process.communicate() + ended_at = datetime.now(timezone.utc) + return { + "argv": argv, + "cwd": str(Path.cwd().resolve()), + "exit_code": None if expired else process.returncode, + "timed_out": expired, + "started_at": started_at.isoformat(), + "ended_at": ended_at.isoformat(), + "duration_seconds": round(time.monotonic() - started_clock, 6), + "stdout_bytes": out, + "stderr_bytes": err, + } + + +def _command_record(evidence: dict[str, Any]) -> dict[str, Any]: + return { + "argv": evidence["argv"], + "cwd": evidence["cwd"], + "exit_code": evidence["exit_code"], + "timed_out": evidence["timed_out"], + "started_at": evidence["started_at"], + "ended_at": evidence["ended_at"], + "duration_seconds": evidence["duration_seconds"], + "stdout": _stream(evidence["stdout_bytes"]), + "stderr": _stream(evidence["stderr_bytes"]), + } + + +def _fresh_workspace(path: Path) -> Path: + requested = path.expanduser().absolute() + if os.path.lexists(requested): + raise ValueError(f"native report workspace must be fresh: {requested}") + parent = requested.parent.resolve(strict=True) + if requested.parent != parent or requested.parent.is_symlink(): + raise ValueError("native report workspace parent must be canonical and non-symlink") + requested.mkdir(mode=0o700) + return requested.resolve(strict=True) + + +def _executable(candidates: tuple[Path, ...], *, label: str) -> tuple[Path, dict[str, Any]]: + """Select the first executable from one bounded preference list.""" + for candidate in candidates: + try: + path = candidate.resolve(strict=True) + except OSError: + continue + if path.is_file() and path.stat().st_mode & 0o111: + return path, _snapshot( + path, + label=f"native report tool {label}", + executable=True, + limit=MAX_BINARY, + )[1] + locations = ", ".join(str(candidate) for candidate in candidates) + raise _Unavailable( + f"required native report tool {label} is unavailable at fixed candidate paths: " + f"{locations}" + ) + + +def _local_tools(timeout: int, reports: tuple[str, ...]) -> dict[str, Any]: + tools: dict[str, Any] = {} + for name, candidates in BUILD_TOOLS.items(): + path, identity = _executable(candidates, label=name) + tools[name] = {"path": path, "identity": identity} + available_generators = [] + for name, candidate in GENERATORS: + try: + path = candidate.resolve(strict=True) + except OSError: + continue + if path.is_file() and path.stat().st_mode & 0o111: + available_generators.append((name, path)) + if not available_generators: + raise _Unavailable("no supported CMake generator is installed") + generator_name, generator_path = available_generators[0] + tools["generator"] = { + "name": generator_name, + "path": generator_path, + "identity": _snapshot( + generator_path, + label="native report build generator", + executable=True, + limit=MAX_BINARY, + )[1], + } + nvcc_version = _run([str(tools["nvcc"]["path"]), "--version"], min(timeout, 30)) + nvcc_text = (nvcc_version["stdout_bytes"] + nvcc_version["stderr_bytes"]).decode( + "utf-8", errors="replace" + ) + if nvcc_version["exit_code"] != 0 or NVCC_RE.search(nvcc_text) is None: + raise _Unavailable("native report build requires a CUDA 13.x nvcc toolchain") + tools["nvcc_version_command"] = _command_record(nvcc_version) + if "decoder" in reports: + missing = [] + for module in APPDEC_MODULES: + query = _run( + [str(tools["pkg_config"]["path"]), "--exists", module], min(timeout, 30) + ) + if query["exit_code"] != 0: + missing.append(module) + if missing: + raise _Unavailable( + "native AppDec report build prerequisites are missing: " + + ", ".join(missing) + ) + return tools + + +def _stream(raw: bytes) -> dict[str, Any]: + within = len(raw) <= MAX_STREAM + return {"capture_status": "complete" if within else "limit_exceeded", + "size_bytes": len(raw), "sha256": hashlib.sha256(raw).hexdigest(), + "content": raw.decode("utf-8", errors="replace") if within else None, + "tail": raw[-4000:].decode("utf-8", errors="replace")} + + +def _markers(stdout: str, stderr: str) -> list[str]: + combined = stdout + "\n" + stderr + return [name for name, pattern in FAILURES if pattern.search(combined)] + + +def _classification(role: str) -> dict[str, Any]: + sample, option = SAMPLES[role] + return {"evidence_source_type": "official_sample_report", "official_sample": True, + "sample_name": sample, "report_option": option, "scope": "sample_reported"} + + +def _line_count(lines: list[str], value: str) -> int: + return sum(line.rstrip(" \t\r") == value for line in lines) + + +def _encoder_parse( + stdout: str, gpu: int +) -> tuple[str | None, list[dict[str, Any]], bool, str | None]: + lines = stdout.splitlines() + legacy = _line_count(lines, "Encoder Capability Summary") + compact = _line_count(lines, "Encoder Capability") + if (legacy, compact) not in ((1, 0), (0, 1)): + return None, [], False, "encoder output has no single recognized header" + matches = list(GPU_RE.finditer(stdout)) + records = [] + for index, match in enumerate(matches): + block = stdout[match.end():matches[index + 1].start() if index + 1 < len(matches) + else len(stdout)] + rows = CODEC_RE.findall(block) + valid_rows = sorted(codec for codec, _value in rows) == ["AV1", "H264", "HEVC"] + valid_sections = True + if legacy: + valid_sections = ( + _line_count(block.splitlines(), "=== CODEC SUPPORT SUMMARY ===") == 1 + and _line_count( + block.splitlines(), "=== ENCODER CAPABILITIES SUMMARY TABLE ===" + ) == 1 + ) + records.append({"gpu": int(match.group(1)), "name": match.group(2).strip(), + "sample_reported_codec_rows": [ + {"codec": name.lower(), "value": value} for name, value in rows], + "required_sections_present": valid_sections, + "required_rows_present": valid_rows}) + ordinals = [item["gpu"] for item in records] + hint_ok = _line_count( + lines, "For detailed information about all capabilities, use -ec-detail" + ) == 1 + valid = bool(records and len(ordinals) == len(set(ordinals)) and gpu in ordinals + and all(item["required_sections_present"] and item["required_rows_present"] + for item in records) and (not legacy or hint_ok)) + return ("legacy_summary" if legacy else "sdk_13_compact"), records, gpu in ordinals, ( + None if valid else "encoder report grammar or selected GPU is incomplete") + + +def _decoder_parse( # pylint: disable=too-many-branches + stdout: str, gpu: int +) -> tuple[str | None, list[dict[str, Any]], bool, str | None]: + lines = stdout.splitlines() + legacy_headers = _line_count(lines, "GPU Decoder Capabilities") + compact_headers = _line_count(lines, "Decoder Capability") + if legacy_headers and not compact_headers: + records = [{"gpu": int(match.group(1)), "name": match.group(2).strip()} + for match in GPU_RE.finditer(stdout)] + ordinals = [item["gpu"] for item in records] + valid = bool(records and len(ordinals) == len(set(ordinals)) and gpu in ordinals + and legacy_headers == len(records) + and _line_count(lines, "Codec Support Summary") == len(records) + and _line_count( + lines, + "For detailed information about all capabilities, use -dc-detail", + ) == 1) + return "legacy_summary", records, gpu in ordinals, ( + None if valid else "legacy decoder report grammar or selected GPU is incomplete") + if compact_headers != 1 or legacy_headers: + return None, [], False, "decoder output has no single recognized header" + names = GPU_IN_USE_RE.findall(stdout) + codec_lines = [line for line in lines if line.lstrip().startswith("Codec")] + matches = [DECODER_ROW_RE.fullmatch(line) for line in codec_lines] + rows = [] + for line, match in zip(codec_lines, matches): + if match is None: + continue + rows.append({"raw_line": line, "codec": match["codec"].lower(), + "bit_depth": int(match["bit"]), "chroma_format": match["chroma"], + "supported_field_value": int(match["value"]), + "max_width": int(match["maxw"]), "max_height": int(match["maxh"]), + "max_mb_count": int(match["maxmb"]), "min_width": int(match["minw"]), + "min_height": int(match["minh"]), + "surface_formats": match["formats"].split()}) + valid = bool(gpu == 0 and len(names) == 1 and codec_lines and all(matches)) + records = [{"gpu": None, "name": names[0].strip() if len(names) == 1 else None, + "sample_reported_codec_rows": rows}] + return "sdk_13_compact", records, valid, ( + None if valid else "compact decoder report requires GPU 0, one GPU name, and strict rows") + + +def _live_linkage(binary: Path, role: str, timeout: int) -> dict[str, Any]: + evidence = _run(["/usr/bin/ldd", str(binary)], timeout) + text = (evidence["stdout_bytes"] + b"\n" + evidence["stderr_bytes"]).decode( + "utf-8", errors="replace") + libraries = {} + for name in REQUIRED_LIBS[role]: + line = next((item.strip() for item in text.splitlines() + if item.strip().startswith(name)), None) + value = line.split("=>", 1)[1].strip().split()[0] if line and "=>" in line else "" + try: + real = str(Path(value).resolve(strict=True)) if value.startswith("/") else None + except OSError: + real = None + state = "resolved" if real and "/stubs/" not in value and "/stubs/" not in real else ( + "stub" if real else "missing") + libraries[name] = {"status": state, "real_path": real, "line": line} + if evidence["exit_code"] != 0 or any(item["status"] != "resolved" + for item in libraries.values()): + raise ValueError( + f"{SAMPLES[role][0]} live driver linkage is not verified" + ) + return {"status": "verified", "libraries": libraries, "command": evidence["argv"]} + + +def _local_package(timeout: int) -> dict[str, Any]: + query = Path("/usr/bin/dpkg-query") + dpkg = Path("/usr/bin/dpkg") + if not query.is_file() or not dpkg.is_file(): + raise _Unavailable("dpkg package authority is unavailable") + status = _run( + [str(query), "-W", "-f=${db:Status-Abbrev}\t${Version}", PACKAGE], + min(timeout, 60), + ) + fields = status["stdout_bytes"].decode("utf-8", errors="replace").strip().split("\t") + if status["exit_code"] != 0 or len(fields) != 2 or not fields[0].startswith("ii"): + raise _Unavailable(f"{PACKAGE} is not installed") + version = fields[1] + if not _public_sdk_release(version): + raise ValueError(f"installed {PACKAGE} version is not public SDK 13.0.x: {version}") + listing = _run([str(query), "-L", PACKAGE], min(timeout, 60)) + if listing["exit_code"] != 0: + raise ValueError(f"cannot enumerate package-owned {PACKAGE} files") + owned = { + Path(line.strip()) + for line in listing["stdout_bytes"].decode("utf-8", errors="replace").splitlines() + if line.strip().startswith("/") + } + roots = sorted( + { + path.parent.parent + for path in owned + if path.name == "CMakeLists.txt" + and path.parent.name == "Samples" + and path.is_file() + and not path.is_symlink() + } + ) + if len(roots) != 1: + raise ValueError( + f"{PACKAGE} must own exactly one complete SDK Samples root, found {len(roots)}" + ) + sdk_root = roots[0] + samples = sdk_root / "Samples" + if ( + not sdk_root.is_dir() + or sdk_root.is_symlink() + or not samples.is_dir() + or samples.is_symlink() + or samples / "CMakeLists.txt" not in owned + ): + raise ValueError("package-owned SDK Samples root is malformed") + unexpected = [ + path + for path in samples.rglob("*") + if (path.is_file() or path.is_symlink()) and path not in owned + ] + if unexpected: + raise ValueError( + f"package-owned SDK Samples tree contains {len(unexpected)} unowned file(s)" + ) + source_records = [] + for path in sorted(item for item in owned if item.is_relative_to(samples)): + if path.is_symlink(): + raise ValueError(f"package-owned SDK source must not be a symlink: {path}") + if path.is_file(): + identity = _snapshot(path, label="package-owned SDK source")[1] + source_records.append( + (str(path.relative_to(samples)), identity["size_bytes"], identity["sha256"]) + ) + if not source_records: + raise ValueError(f"{PACKAGE} owns no regular source below its Samples tree") + source_manifest = hashlib.sha256( + _dumps(source_records, compact=True) + ).hexdigest() + integrity = _run([str(dpkg), "--verify", PACKAGE], min(timeout, 60)) + if integrity["exit_code"] != 0 or integrity["stdout_bytes"].strip(): + raise ValueError(f"dpkg verification reported a modified {PACKAGE} payload") + return { + "name": PACKAGE, + "version": version, + "sdk_root": str(sdk_root.resolve(strict=True)), + "owned_path_count": len(owned), + "source_tree": { + "regular_file_count": len(source_records), + "manifest_sha256": source_manifest, + }, + "commands": [_command_record(item) for item in (status, listing, integrity)], + } + + +def _verify_tool_identities(tools: dict[str, Any]) -> None: + for name in ("cmake", "cxx", "pkg_config", "nvcc", "generator"): + record = tools[name] + observed = _snapshot( + Path(record["path"]), + label=f"native report tool {name}", + executable=True, + limit=MAX_BINARY, + )[1] + _same_identity(observed, record["identity"], label=f"native report tool {name}") + + +def _local_build( + package: dict[str, Any], + tools: dict[str, Any], + workspace: Path, + reports: tuple[str, ...], + timeout: int, +) -> tuple[dict[str, Any], dict[str, Any]]: + samples = Path(package["sdk_root"]) / "Samples" + build_root = workspace / "build" + cuda_root = Path(tools["nvcc"]["path"]).parent.parent.resolve(strict=True) + configure_argv = [ + str(tools["cmake"]["path"]), + "-S", + str(samples), + "-B", + str(build_root), + "-G", + str(tools["generator"]["name"]), + f"-DCMAKE_MAKE_PROGRAM={tools['generator']['path']}", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_CXX_COMPILER={tools['cxx']['path']}", + f"-DCUDAToolkit_ROOT={cuda_root}", + f"-DCUDAToolkit_NVCC_EXECUTABLE={tools['nvcc']['path']}", + f"-DCMAKE_CUDA_COMPILER={tools['nvcc']['path']}", + f"-DPKG_CONFIG_EXECUTABLE={tools['pkg_config']['path']}", + ] + commands = [_run_build(configure_argv, timeout)] + failures: dict[str, str] = {} + configure_failed = ( + commands[-1]["exit_code"] != 0 or commands[-1]["timed_out"] + ) + if configure_failed: + failures.update( + {role: "official native sample configure failed" for role in reports} + ) + for role in () if configure_failed else reports: + target = SAMPLES[role][0] + commands.append( + _run_build( + [ + str(tools["cmake"]["path"]), + "--build", + str(build_root), + "--target", + target, + "--parallel", + "2", + ], + timeout, + ) + ) + if commands[-1]["exit_code"] != 0 or commands[-1]["timed_out"]: + failures[role] = f"official {target} report target build failed" + _verify_tool_identities(tools) + binaries: dict[str, Any] = {} + locations = { + "encoder": build_root / "AppEncode" / "AppEncCuda" / "AppEncCuda", + "decoder": build_root / "AppDecode" / "AppDec" / "AppDec", + } + for role in reports: + if role in failures: + continue + binary = locations[role] + resolved = binary.resolve(strict=True) + if ( + not resolved.is_relative_to(workspace) + or not resolved.is_file() + or resolved.stat().st_mode & 0o111 == 0 + ): + raise ValueError(f"official {SAMPLES[role][0]} build output is invalid") + identity = _snapshot( + resolved, + label=SAMPLES[role][0], + executable=True, + limit=MAX_BINARY, + )[1] + try: + linkage = _live_linkage(resolved, role, min(timeout, 60)) + except ValueError as exc: + failures[role] = str(exc) + continue + binaries[role] = { + "path": resolved, + "identity": identity, + "linkage": linkage, + } + evidence_tools = { + name: { + "path": str(record["path"]), + "identity": record["identity"], + **({"name": record["name"]} if name == "generator" else {}), + } + for name, record in tools.items() + if name in {"cmake", "cxx", "pkg_config", "nvcc", "generator"} + } + return binaries, { + "workspace": str(workspace), + "build_root": str(build_root.resolve(strict=True)), + "requested_targets": [SAMPLES[role][0] for role in reports], + "target_failures": failures, + "tools": evidence_tools, + "nvcc_version_command": tools["nvcc_version_command"], + "commands": [_command_record(item) for item in commands], + } + + +def _authenticate_local( + workspace_path: Path, + output_path: Path, + gpu: int, + reports: tuple[str, ...], + timeout: int, +) -> dict[str, Any]: + package = _local_package(timeout) + sdk_root = Path(package["sdk_root"]) + workspace_requested = workspace_path.expanduser().absolute() + output_requested = output_path.expanduser().absolute() + if workspace_requested == sdk_root or workspace_requested.is_relative_to(sdk_root): + raise ValueError("native report workspace must be outside the installed SDK tree") + output_parent = output_requested.parent.resolve(strict=True) + if (output_parent / output_requested.name).is_relative_to(sdk_root): + raise ValueError("native report output must be outside the installed SDK tree") + tools = _local_tools(timeout, reports) + workspace = _fresh_workspace(workspace_path) + binaries, build = _local_build(package, tools, workspace, reports, timeout) + return { + "source": "local_package_build", + "gpu": gpu, + "package": package, + "build": build, + "binaries": binaries, + "tools": tools, + } + + +def _authenticate( # pylint: disable=too-many-branches,too-many-statements + path: Path, gpu: int, reports: tuple[str, ...], timeout: int +) -> dict[str, Any]: + raw, identity, data = _load(path, label="native verification") + if data.get("kind") != VERIFY_KIND or data.get("schema_version") != VERIFY_SCHEMA: + raise ValueError(f"input must be {VERIFY_KIND} schema {VERIFY_SCHEMA}") + if not isinstance(data.get("ready"), bool) or data.get("status") not in ( + "operation_verified", "operation_failed", "unknown" + ): + raise ValueError("native verification readiness/status fields are malformed") + if data.get("ready") is not True or data.get("status") != "operation_verified": + raise _Unavailable("native verification is not operation_verified") + if data.get("software_fallback") is not False: + raise ValueError("ready native verification must prove software_fallback=false") + operations = data.get("hardware_operations") + if not isinstance(operations, dict) or any( + not isinstance(operations.get(role), dict) + or operations[role].get("status") != "operation_verified" + for role in ("encode", "decode") + ): + raise ValueError("ready native verification lacks both official operation proofs") + environment = data.get("environment") + if not isinstance(environment, dict) or environment.get("selected_gpu") != gpu: + raise ValueError("native verification selected GPU does not match --gpu") + env_raw, env_identity, env_data = _load( + Path(environment.get("identity", {}).get("path", "")), label="environment") + _same_identity(env_identity, environment.get("identity"), label="environment") + if ( + hashlib.sha256(_dumps(env_data, compact=True)).hexdigest() + != environment.get("canonical_sha256") + ): + raise ValueError("environment canonical JSON digest does not match") + if ( + env_data.get("kind") != "nvcodec-environment" + or env_data.get("schema_version") != ENVIRONMENT_SCHEMA + or env_data.get("mode") != "live" + or env_data.get("selected_gpu") != gpu + ): + raise ValueError( + f"bound environment is not the selected live schema-{ENVIRONMENT_SCHEMA} target" + ) + native = environment.get("native") + package = native.get("package") if isinstance(native, dict) else None + version = package.get("version") if isinstance(package, dict) else None + if ( + not _public_sdk_release(version) + or package.get("name") != PACKAGE + or package.get("status") != "installed" + ): + raise ValueError("native verification package is not public nvidia-video-codec-sdk 13.0.x") + ownership = data.get("package_ownership") + expected_ownership = { + "status": "verified", "package": PACKAGE, "declared_version": version, + "live_version": version, "sdk_root_owned": True, "verify_silent": True, + } + if not isinstance(ownership, dict) or any( + ownership.get(key) != value for key, value in expected_ownership.items() + ): + raise ValueError("native package ownership is not verified") + query = _run(["/usr/bin/dpkg-query", "-W", "-f=${db:Status-Abbrev}\\t${Version}", + PACKAGE], min(timeout, 60)) + fields = query["stdout_bytes"].decode(errors="replace").strip().split("\t") + if ( + query["exit_code"] != 0 or len(fields) != 2 + or not fields[0].startswith("ii") or fields[1] != version + ): + raise ValueError("live native package version does not match verification") + verify = _run(["/usr/bin/dpkg", "--verify", PACKAGE], min(timeout, 60)) + if verify["exit_code"] != 0 or verify["stdout_bytes"].strip(): + raise ValueError("live dpkg verification is not silent") + samples, binaries = data.get("samples"), {} + if not isinstance(samples, dict): + raise ValueError("native verification has no samples block") + for role in reports: + record = samples.get(role) + recorded_linkage = record.get("linkage") if isinstance(record, dict) else None + recorded_libraries = ( + recorded_linkage.get("libraries") if isinstance(recorded_linkage, dict) else None + ) + if ( + not isinstance(recorded_libraries, dict) + or recorded_linkage.get("status") != "verified" + or any( + not isinstance(recorded_libraries.get(name), dict) + or recorded_libraries[name].get("status") != "resolved" + for name in REQUIRED_LIBS[role] + ) + ): + raise ValueError(f"recorded {SAMPLES[role][0]} linkage is not verified") + expected = record.get("identity") + binary = Path(record.get("binary", "")) + _same_identity(_snapshot(binary, label=SAMPLES[role][0], executable=True, + limit=MAX_BINARY)[1], expected, + label=SAMPLES[role][0]) + if expected.get("path") != str(binary.expanduser().absolute().resolve(strict=True)): + raise ValueError(f"{SAMPLES[role][0]} binary and identity paths differ") + _live_linkage(binary, role, min(timeout, 60)) + binaries[role] = {"path": binary, "identity": expected} + return { + "source": "setup_verification", + "raw": raw, + "identity": identity, + "data": data, + "environment_raw": env_raw, + "environment_identity": env_identity, + "environment_data": env_data, + "version": version, + "binaries": binaries, + } + + +def _report(role: str, binary: dict[str, Any], gpu: int, timeout: int) -> dict[str, Any]: + path, expected = binary["path"], binary["identity"] + initial = _snapshot(path, label=SAMPLES[role][0], executable=True, + limit=MAX_BINARY)[1] + _same_identity(initial, expected, label=SAMPLES[role][0]) + evidence = _run([str(path), SAMPLES[role][1]], timeout) + terminal = _snapshot(path, label=SAMPLES[role][0], executable=True, + limit=MAX_BINARY)[1] + _same_identity(terminal, initial, label=SAMPLES[role][0]) + streams = {"stdout": _stream(evidence["stdout_bytes"]), + "stderr": _stream(evidence["stderr_bytes"])} + stdout, stderr = (streams[name]["content"] or "" for name in ("stdout", "stderr")) + variant, records, selected, parse_reason = ( + _encoder_parse(stdout, gpu) if role == "encoder" else _decoder_parse(stdout, gpu)) + markers = _markers(stdout, stderr) + reasons = [] + if evidence["timed_out"]: + reasons.append("official sample report timed out") + if evidence["exit_code"] != 0: + reasons.append(f"official sample exited {evidence['exit_code']}") + if markers: + reasons.append("explicit failure marker(s): " + ", ".join(markers)) + if any(item["capture_status"] != "complete" for item in streams.values()): + reasons.append(f"captured stream exceeded {MAX_STREAM} bytes") + if parse_reason: + reasons.append(parse_reason) + return {"status": "unknown" if reasons else "completed", "format_variant": variant, + "evidence_classification": _classification(role), + "authority": f"official {SAMPLES[role][0]} aggregate report", + "claim_scope": ( + "raw sample-reported values only; neither API/product support nor" + " operation evidence" + ), + "command": evidence["argv"], "cwd": evidence["cwd"], + "exit_code": evidence["exit_code"], "timed_out": evidence["timed_out"], + "started_at": evidence["started_at"], "ended_at": evidence["ended_at"], + "duration_seconds": evidence["duration_seconds"], + "selected_gpu": gpu, "selected_gpu_present": selected, + "reported_gpus": records, "failure_markers": markers, + "reason": "; ".join(reasons) if reasons else None, "streams": streams, + "stdout_tail": streams["stdout"]["tail"], "stderr_tail": streams["stderr"]["tail"], + "binary_integrity": {"initial": initial, "terminal": terminal}} + + +def _empty(role: str, status: str, reason: str | None = None) -> dict[str, Any]: + return {"status": status, "format_variant": None, + "evidence_classification": _classification(role), "reason": reason} + + +def _document( # pylint: disable=too-many-branches,too-many-arguments + # pylint: disable=too-many-positional-arguments + path: Path | None, + workspace: Path | None, + output: Path, + gpu: int, + reports: tuple[str, ...], + timeout: int, +) -> tuple[dict[str, Any], int]: + if path is None: + if workspace is None: + raise ValueError( + "--workspace is required when --native-verification is omitted" + ) + try: + auth = _authenticate_local(workspace, output, gpu, reports, timeout) + reason = None + except _Unavailable as exc: + auth, reason = None, str(exc) + elif not os.path.lexists(path.expanduser()): + reason = f"native verification is unavailable: {path}" + auth = None + else: + try: + auth = _authenticate(path, gpu, reports, timeout) + reason = None + except _Unavailable as exc: + auth, reason = None, str(exc) + aggregates = {role: _empty(role, "not_requested") for role in SAMPLES} + if auth is None: + for role in reports: + aggregates[role] = _empty(role, "unknown", reason) + native = {"initial_identity": None, "terminal_identity": None, + "kind": VERIFY_KIND, "schema_version": VERIFY_SCHEMA, + "ready": False, "status": "unknown"} + elif auth["source"] == "setup_verification": + for role in reports: + aggregates[role] = _report(role, auth["binaries"][role], gpu, timeout) + terminal_raw, terminal_identity, _ = _load(path, label="native verification") + env_raw, env_terminal, _ = _load( + Path(auth["environment_identity"]["path"]), label="environment") + if terminal_raw != auth["raw"] or env_raw != auth["environment_raw"]: + raise ValueError("native verification or environment changed during reports") + native = {"initial_identity": auth["identity"], "terminal_identity": terminal_identity, + "canonical_sha256": hashlib.sha256( + _dumps(auth["data"], compact=True)).hexdigest(), + "kind": VERIFY_KIND, "schema_version": VERIFY_SCHEMA, "ready": True, + "status": "operation_verified", + "environment_identity": env_terminal, + "package": {"name": PACKAGE, "version": auth["version"]}, + "samples": { + role: { + "binary": str(auth["binaries"][role]["path"]), + "identity": auth["binaries"][role]["identity"], + } + for role in reports + }} + else: + for role in reports: + if role in auth["binaries"]: + aggregates[role] = _report(role, auth["binaries"][role], gpu, timeout) + else: + aggregates[role] = _empty( + role, + "unknown", + auth["build"]["target_failures"].get( + role, f"{SAMPLES[role][0]} report binary is unavailable" + ), + ) + terminal_package = _local_package(timeout) + if any( + terminal_package.get(field) != auth["package"].get(field) + for field in ("name", "version", "sdk_root", "owned_path_count", "source_tree") + ): + raise ValueError("native package identity changed during reports") + _verify_tool_identities(auth["tools"]) + native = { + "initial_identity": None, + "terminal_identity": None, + "kind": VERIFY_KIND, + "schema_version": VERIFY_SCHEMA, + "ready": False, + "status": "not_supplied", + } + counts = {name: sum(item["status"] == name for item in aggregates.values()) + for name in ("completed", "unknown", "not_requested")} + status = ("complete" if counts["completed"] == len(reports) else + "partial" if counts["completed"] else "unknown") + result = {"kind": KIND, "schema_version": SCHEMA, + "generated_at": datetime.now(timezone.utc).isoformat(), + "status": status, "authority": "native_official_samples", "gpu": gpu, + "requested_reports": list(reports), "native_verification": native, + "aggregate_encoder_capabilities": aggregates["encoder"], + "aggregate_decoder_capabilities": aggregates["decoder"], "summary": counts} + if auth is not None and auth["source"] == "local_package_build": + result["local_authentication"] = { + "status": "authenticated", + "package": auth["package"], + "build": auth["build"], + "samples": { + role: { + "binary": str(auth["binaries"][role]["path"]), + "identity": auth["binaries"][role]["identity"], + "linkage": auth["binaries"][role]["linkage"], + } + for role in reports + if role in auth["binaries"] + }, + } + if auth["build"]["target_failures"]: + add_setup_remediation( + result, + surface="native", + reason="; ".join( + auth["build"]["target_failures"][role] + for role in reports + if role in auth["build"]["target_failures"] + ), + ) + elif auth is None: + add_setup_remediation(result, surface="native", reason=reason) + return result, 0 if status == "complete" else 2 + + +class _Parser(argparse.ArgumentParser): + def error(self, message: str) -> Any: + self.print_usage(sys.stderr) + print(f"{self.prog}: error: {message}", file=sys.stderr) + raise SystemExit(3) + + +def main() -> int: + parser = _Parser(description=__doc__) + parser.add_argument( + "--native-verification", + type=Path, + help=( + "Optional setup-produced schema-1.5 verification; when omitted, " + "authenticate the installed package and build requested report targets" + ), + ) + parser.add_argument( + "--workspace", + type=Path, + help="Fresh explicit build workspace required without --native-verification", + ) + parser.add_argument("--gpu", type=int, default=0) + parser.add_argument("--report", action="append", choices=tuple(SAMPLES)) + parser.add_argument("--timeout", type=int, default=300) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + try: + if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise ValueError("invoke this producer with isolated Python: python3 -I") + if args.gpu < 0 or args.timeout <= 0: + raise ValueError("--gpu must be non-negative and --timeout must be positive") + if os.path.lexists(args.output.expanduser()): + raise FileExistsError(f"output already exists: {args.output}") + reports = tuple(dict.fromkeys(args.report or SAMPLES)) + if (args.native_verification is None) == (args.workspace is None): + raise ValueError( + "provide --native-verification or --workspace, but not both" + ) + result, code = _document( + args.native_verification, + args.workspace, + args.output, + args.gpu, + reports, + args.timeout, + ) + rendered = _dumps(result) + exclusive_write_bytes(args.output, rendered) + print(rendered.decode("utf-8"), end="") + return code + except Exception as exc: # pylint: disable=broad-exception-caught + result = {"kind": KIND + "-error", "schema_version": SCHEMA, "status": "error", + "generated_at": datetime.now(timezone.utc).isoformat(), "error": str(exc)} + rendered = _dumps(result) + print(rendered.decode("utf-8"), end="") + if not os.path.lexists(args.output.expanduser()): + try: + exclusive_write_bytes(args.output, rendered) + except (OSError, ValueError): + pass + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-capability/scripts/validate_appenc_av1_ivf.py b/skills/jetson-video-capability/scripts/validate_appenc_av1_ivf.py new file mode 100644 index 00000000..e9c0878f --- /dev/null +++ b/skills/jetson-video-capability/scripts/validate_appenc_av1_ivf.py @@ -0,0 +1,356 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate an AppEncCuda AV1 IVF artifact without claiming codec operation success.""" + +# pylint: disable=missing-function-docstring,too-many-locals + +from __future__ import annotations + +import argparse +import hashlib +import os +import stat +import struct +import sys +from pathlib import Path +from typing import BinaryIO, Any + +_SCRIPT_DIR = Path(__file__).resolve().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position +from _capability_contract import ( # noqa: E402 + exclusive_write_bytes, + strict_json_dumps, +) + +SCHEMA_VERSION = "1.0" +IVF_FILE_HEADER = struct.Struct("<4sHH4sHHIIII") +IVF_FRAME_HEADER = struct.Struct(" int: + try: + parsed = int(value, 10) + except ValueError as exc: + raise argparse.ArgumentTypeError("must be an integer") from exc + if parsed <= 0: + raise argparse.ArgumentTypeError("must be positive") + return parsed + + +def _validated_expected(width: int, height: int, frames: int) -> dict[str, int]: + for label, value, maximum in ( + ("width", width, 0xFFFF), + ("height", height, 0xFFFF), + ("expected frame count", frames, 0xFFFFFFFF), + ): + if ( + not isinstance(value, int) + or isinstance(value, bool) + or value <= 0 + or value > maximum + ): + raise ValueError(f"{label} must be an integer in 1..{maximum}") + return {"width": width, "height": height, "frame_count": frames} + + +def _read_exact(handle: BinaryIO, size: int, label: str) -> bytes: + content = handle.read(size) + if len(content) != size: + raise IvfValidationError( + f"{label} is truncated: expected {size} bytes, found {len(content)}" + ) + return content + + +def _discard_exact(handle: BinaryIO, size: int, label: str) -> None: + remaining = size + while remaining: + chunk = handle.read(min(remaining, 1024 * 1024)) + if not chunk: + raise IvfValidationError( + f"{label} is truncated: expected {size} bytes, found {size - remaining}" + ) + remaining -= len(chunk) + + +def _file_identity_and_handle(path: Path) -> tuple[dict[str, Any], BinaryIO, os.stat_result]: + requested = path.expanduser().absolute() + if requested.is_symlink(): + raise IvfValidationError(f"input must not be a symlink: {requested}") + resolved = requested.resolve(strict=True) + flags = os.O_RDONLY + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(resolved, flags) + try: + initial = os.fstat(descriptor) + if not stat.S_ISREG(initial.st_mode): + raise IvfValidationError(f"input must be a regular file: {resolved}") + handle = os.fdopen(descriptor, "rb") + descriptor = -1 + finally: + if descriptor >= 0: + os.close(descriptor) + + try: + digest = hashlib.sha256() + observed = 0 + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + observed += len(chunk) + digest.update(chunk) + if observed != initial.st_size: + raise IvfValidationError("input size changed while hashing") + handle.seek(0) + except Exception: + handle.close() + raise + identity = { + "path": str(resolved), + "device": initial.st_dev, + "inode": initial.st_ino, + "size_bytes": initial.st_size, + "sha256": digest.hexdigest(), + } + return identity, handle, initial + + +def _parse_ivf( + handle: BinaryIO, *, width: int, height: int, expected_frames: int +) -> dict[str, Any]: + raw_header = _read_exact(handle, IVF_FILE_HEADER.size, "IVF file header") + ( + signature, + version, + header_size, + fourcc, + observed_width, + observed_height, + rate, + scale, + header_frame_count, + unused, + ) = IVF_FILE_HEADER.unpack(raw_header) + if signature != b"DKIF": + raise IvfValidationError("IVF signature is not DKIF") + if version != 0: + raise IvfValidationError(f"IVF version {version} is not 0") + if header_size != IVF_FILE_HEADER.size: + raise IvfValidationError( + f"IVF header size {header_size} is not {IVF_FILE_HEADER.size}" + ) + if fourcc != b"AV01": + raise IvfValidationError("IVF fourcc is not AV01") + if (observed_width, observed_height) != (width, height): + raise IvfValidationError( + "IVF geometry " + f"{observed_width}x{observed_height} does not match {width}x{height}" + ) + allowed_header_counts = {expected_frames, APPENC_CUDA_FRAME_COUNT_SENTINEL} + if header_frame_count not in allowed_header_counts: + raise IvfValidationError( + "IVF header frame count is neither the expected count nor the " + "AppEncCuda 0xFFFF sentinel" + ) + + payload_total = 0 + minimum_frame_size: int | None = None + maximum_frame_size: int | None = None + first_timestamp: int | None = None + last_timestamp: int | None = None + for index in range(expected_frames): + raw_frame_header = _read_exact( + handle, IVF_FRAME_HEADER.size, f"IVF frame {index} header" + ) + frame_size, timestamp = IVF_FRAME_HEADER.unpack(raw_frame_header) + if frame_size <= 0: + raise IvfValidationError(f"IVF frame {index} has an empty payload") + _discard_exact(handle, frame_size, f"IVF frame {index} payload") + payload_total += frame_size + minimum_frame_size = ( + frame_size + if minimum_frame_size is None + else min(minimum_frame_size, frame_size) + ) + maximum_frame_size = ( + frame_size + if maximum_frame_size is None + else max(maximum_frame_size, frame_size) + ) + if first_timestamp is None: + first_timestamp = timestamp + last_timestamp = timestamp + + if handle.read(1): + raise IvfValidationError( + "IVF contains trailing bytes or more frame records than expected" + ) + return { + "signature": signature.decode("ascii"), + "version": version, + "header_size_bytes": header_size, + "fourcc": fourcc.decode("ascii"), + "width": observed_width, + "height": observed_height, + "rate": rate, + "scale": scale, + "header_frame_count": header_frame_count, + "header_frame_count_kind": ( + "appenc_cuda_sentinel" + if header_frame_count == APPENC_CUDA_FRAME_COUNT_SENTINEL + else "exact" + ), + "unused": unused, + "record_count": expected_frames, + "payload_size_bytes": payload_total, + "minimum_frame_size_bytes": minimum_frame_size, + "maximum_frame_size_bytes": maximum_frame_size, + "first_timestamp": first_timestamp, + "last_timestamp": last_timestamp, + } + + +def validate_appenc_av1_ivf( + path: Path, *, width: int, height: int, expected_frames: int +) -> tuple[dict[str, Any], int]: + """Return compact structural evidence and a fail-closed process status.""" + expected = _validated_expected(width, height, expected_frames) + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-appenc-av1-ivf-structure-validation", + "status": "invalid", + "scope": "container_structure_only", + "operation_verified": False, + "authenticated_decode_required": True, + "expected": expected, + } + handle: BinaryIO | None = None + try: + identity, handle, initial = _file_identity_and_handle(path) + result["input"] = identity + parsed = _parse_ivf( + handle, + width=width, + height=height, + expected_frames=expected_frames, + ) + terminal = os.fstat(handle.fileno()) + initial_identity = ( + initial.st_dev, + initial.st_ino, + initial.st_size, + initial.st_mtime_ns, + initial.st_ctime_ns, + ) + terminal_identity = ( + terminal.st_dev, + terminal.st_ino, + terminal.st_size, + terminal.st_mtime_ns, + terminal.st_ctime_ns, + ) + if terminal_identity != initial_identity: + raise IvfValidationError("input identity changed while validating") + except (OSError, IvfValidationError) as exc: + result["reason"] = str(exc) + return result, 2 + finally: + if handle is not None: + handle.close() + result.update({"status": "structure_verified", "parsed": parsed}) + return result, 0 + + +class _ExitThreeParser(argparse.ArgumentParser): + """Convert malformed argv into the producer's strict JSON error contract.""" + + def error(self, message: str) -> Any: + raise ValueError(f"{self.prog}: {message}") + + +def main() -> int: + parser = _ExitThreeParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True, help="Fresh AppEncCuda AV1 output") + parser.add_argument("--width", type=_positive_int, required=True) + parser.add_argument("--height", type=_positive_int, required=True) + parser.add_argument("--expected-frames", type=_positive_int, required=True) + parser.add_argument( + "--output", + type=Path, + help="Optional fresh JSON evidence path; existing paths are never overwritten", + ) + try: + args = parser.parse_args() + except ValueError as exc: + print(strict_json_dumps({ + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-appenc-av1-ivf-structure-validation-error", + "status": "error", + "scope": "container_structure_only", + "operation_verified": False, + "authenticated_decode_required": True, + "error": str(exc), + }, sort_keys=True, separators=(",", ":"))) + return 3 + try: + result, code = validate_appenc_av1_ivf( + args.input, + width=args.width, + height=args.height, + expected_frames=args.expected_frames, + ) + except ValueError as exc: + result = { + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-appenc-av1-ivf-structure-validation", + "status": "invalid", + "scope": "container_structure_only", + "operation_verified": False, + "authenticated_decode_required": True, + "reason": str(exc), + } + code = 2 + except Exception as exc: # pylint: disable=broad-exception-caught + result = { + "schema_version": SCHEMA_VERSION, + "kind": "nvcodec-appenc-av1-ivf-structure-validation-error", + "status": "error", + "scope": "container_structure_only", + "operation_verified": False, + "authenticated_decode_required": True, + "error": str(exc), + } + code = 3 + rendered = strict_json_dumps(result, sort_keys=True, separators=(",", ":")) + "\n" + if args.output is not None: + try: + exclusive_write_bytes(args.output, rendered.encode("utf-8")) + except (OSError, ValueError) as exc: + failed = dict(result) + failed.update( + { + "validation_status": result["status"], + "status": "evidence_write_failed", + "evidence_output": str(args.output.expanduser().absolute()), + "reason": str(exc), + } + ) + print( + strict_json_dumps(failed, sort_keys=True, separators=(",", ":")) + ) + return 3 + print(rendered, end="") + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-capability/skill-card.md b/skills/jetson-video-capability/skill-card.md new file mode 100644 index 00000000..6f7481b1 --- /dev/null +++ b/skills/jetson-video-capability/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Use when Jetson codec, profile, chroma, bit-depth, dimension, engine-count, or operational support must be reconciled using live SDK APIs, authenticated NVIDIA samples, and NVIDIA documentation.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0
+## Use Case:
+Developers and engineers working with NVIDIA Jetson devices who need to determine video codec support, profile availability, and operational capability using live SDK APIs and NVIDIA documentation.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Capability Queries](references/capability-queries.md)
+- [Surface Selection Contract](references/surface-selection-contract.md)
+- [Agent Skills](https://agentskills.io/)
+ + +## Skill Output:
+**Output Type(s):** [Analysis, Shell commands]
+**Output Format:** [Markdown with inline bash code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+4 evaluation tasks (4 positive), each in an isolated sandbox pod.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Whether the skill avoids unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the final answer is correct against the reference answer.
+- Discoverability: Whether the expected skill was found and executed when needed.
+- Effectiveness: Whether the skill helped complete the user's goal and followed expected workflow behavior.
+- Efficiency: Whether the skill avoided wasted tool or skill usage and maintained routing quality.
+ +Underlying evaluation signals used in this run:
+- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 59% → 93% (+33 points) | 68% → 84% (+16 points) | +| Security | 100% → 100% (±0 points) | 100% → 75% (-25 points) | +| Correctness | 80% → 100% (+20 points) | 100% → 100% (±0 points) | +| Discoverability | 37% → 98% (+62 points) | 50% → 91% (+41 points) | +| Effectiveness | 53% → 74% (+20 points) | 75% → 58% (-17 points) | +| Efficiency | 27% → 91% (+64 points) | 18% → 98% (+80 points) | + +## Skill Version(s):
+e61c045 (source: git SHA, committed 2026-08-10)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/jetson-video-capability/skill.oms.sig b/skills/jetson-video-capability/skill.oms.sig new file mode 100644 index 00000000..c10dd85e --- /dev/null +++ b/skills/jetson-video-capability/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiamV0c29uLXZpZGVvLWNhcGFiaWxpdHkiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYmUyOGRjYzMzMDZiMDgyMzNkMDE0MzAxNTY1ZmExZDhjZTdjNDlkOGUyYzkxYjAwMmMwZDRlYThiODVkZTI0NiIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjhkOTBkZmQ3YWZjZDQ3MjFmMTBmMjcxYWRmOGMwYWVhNmE2MTU2ZGY3ODVmMDBmNjEzNDgxYTZlMzkxMmMyMzUiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjIyZmUwODM0YWQzNzI3Nzc4ZTU1MWYyMGE0NzkzOTQ2ODEzNWFjMDRiYWNjZGYzNDhlYjhkMzk5OWI1ZmIwNDkiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMmQ4NjlmYWE0ZmQ1MjhkMjNiNjFmMWRiNzc4YjUzYzAyZTYyMjNkMGNlOGMxNzQ5NWJkYTNmZjY5N2RjODcxYiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjNjNWEwNzNkNjA3M2I5Zjk5NjAxMmE2YzQ0ZGMwYWRkYTc4NTZmM2FkYjFjYjRlMjhlYzRlM2Y1ODNiNGI4MzIiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvY2FwYWJpbGl0eS1xdWVyaWVzLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTQ3M2JjMGUyZDAxNWE5MTUzODJlZDZhOGUzZTUxN2RhMDU4MDhiMmZmMmFjYmQ1ZmNkMGYyZjU2NzUyODI2ZCIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9zdXJmYWNlLXNlbGVjdGlvbi1jb250cmFjdC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImQ5MTQxM2Q2MWZkNWJlMjBmMDJhNDcwZjAxZGVjNjc0Mzc0YzNlNmUwYTc0MWZkYWNlM2ZkMmVjOWFhMjk3MmYiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvX2NhcGFiaWxpdHlfY29udHJhY3QucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlOTc3NDQ5YTg5MmRjNjcwNWU1YzE3YjA1MjIwNDJjNGVlMzljOGIzOTAyYTg4OGZlZjZlMTdiNjU2ZjkyMzQxIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3F1ZXJ5X2RlY29kZXJfY2Fwcy5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImUzMzhjMTYzNTkxZGFiOGFlMTcxOTJiYjk2YTFiYjczNDU5NjgxOGRiYjNkNDk0ZjE4ZmFhODdhMmI5ZmE3ZWQiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcXVlcnlfZW5jb2Rlcl9jYXBzLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMzkwZjIzNjhhNTU1OTgzNzgzYmNlOWYwYzgyZDMzNDc0YTI5ZmQ1ZWVlMjc5OTE2MTQ0MjA4MTFkNjIyNTVlNCIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9xdWVyeV9uYXRpdmVfc2FtcGxlX3JlcG9ydHMucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJhNjc5YTgyNGM4OTYwYTViYjg1NjE2YzJjNGQ0NjgxOWQ0NzIxN2ZiZTkxN2ZmYWNjNTdjNTU1NDQ1NzM0YzU3IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3ZhbGlkYXRlX2FwcGVuY19hdjFfaXZmLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMDU4NTNkYzk4MWY5NTA1ZDdlYmRiNWY4ZTkyOGU2ZWI5MWY4MmZjZTMwMGMxMDM1MDk1MmI1ZmFlMjk1OTNlOCIsCiAgICAgICAgIm5hbWUiOiAic2tpbGwtY2FyZC5tZCIKICAgICAgfQogICAgXSwKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgIm1ldGhvZCI6ICJmaWxlcyIKICAgIH0KICB9Cn0=","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQCexyZH9WsfrfnL8XFH6zidEmoepHNpXgG3r2jRE5UJUYlVqXiD9U1hhO+/C06AV3ICMQCa4bsXbR7bouCwHaV9Qkd/23uyTGW3Kzhta1rfvHmEIDjdXJbqB7qJkhiw7436Sok=","keyid":""}]}} \ No newline at end of file diff --git a/skills/jetson-video-pipeline/BENCHMARK.md b/skills/jetson-video-pipeline/BENCHMARK.md new file mode 100644 index 00000000..54f71116 --- /dev/null +++ b/skills/jetson-video-pipeline/BENCHMARK.md @@ -0,0 +1,98 @@ +# Skill Benchmark: jetson-video-pipeline + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `jetson-video-pipeline` +- Evaluation date: 2026-08-10 +- Evaluator version: `1.1.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 2 evaluation tasks (2 positive) +- Dataset digest: `sha256:2954de192ec504a13b63de5d546a514eafdc11544903c203f84432c93bb7a5be` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 50% → 100% (+50 points) | 35% → 71% (+36 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 40% → 100% (+60 points) | 40% → 30% (-10 points) | +| Discoverability | 50% → 100% (+50 points) | 22% → 94% (+72 points) | +| Effectiveness | 30% → 100% (+70 points) | 0% → 30% (+30 points) | +| Efficiency | 31% → 100% (+69 points) | 11% → 100% (+89 points) | + +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. + +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 1 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 2 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/jetson-video-pipeline/SKILL.md`) + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/jetson-video-pipeline/SKILL.md b/skills/jetson-video-pipeline/SKILL.md new file mode 100644 index 00000000..0db79358 --- /dev/null +++ b/skills/jetson-video-pipeline/SKILL.md @@ -0,0 +1,243 @@ +--- +name: jetson-video-pipeline +license: "Apache-2.0" +description: >- + Use when executing and verifying Jetson Video Codec SDK or PyNvVideoCodec + encode/decode, transcode, segmentation, container decode, AV1, or acceptance + workflows with exact artifact handoffs. +metadata: + author: "Vinit Bansal " + tags: [jetson, video-codec-sdk, pynvvideocodec, pipeline, nvenc, nvdec] + languages: [python] + data-classification: public +--- + +# Jetson Video Pipeline + +## Purpose + +Execute official-sample codec stages and prove that every consumer used the +exact artifact produced by the preceding stage. Use this skill for +encode-then-decode verification, native H.264-to-HEVC transcode, PyNvVideoCodec +segments, container decode triage, AV1 operation verification, or a compact +customer acceptance package. + +## Prerequisites + +- Run execution on the target Jetson with direct GPU access. A fresh validated + schema-1.2 `nvcodec-environment` identity from `jetson-video-setup` is + optional. When supplied it is authoritative, and invalid or stale evidence + fails closed without local fallback. The agent may obtain it from setup's + public read-only probe; it need not be supplied in the customer's prompt. +- Without setup evidence, authenticate only the selected installed surface. + Native routes inspect the fixed dpkg package, package-owned official sample + sources, build tools, and non-stub linkage. PyNvVideoCodec routes require an + authenticated setup environment or the caller's exact absolute + `pynvc_interpreter`; never scan for a venv. Before asking for that path, + invoke setup's public probe when that skill is installed and inspect its + typed result. + These read-only checks install, repair, register, and smoke-test nothing. +- Recipe-bearing routes require sibling `jetson-video-recipe` and one of its + validated schema-2 recipes. If its canonical public CLI is present, invoke + it; if absent, preserve `dependency_required`, name that skill, and tell the + user to install it and retry the stage. Recipe-free decode/segmentation + routes do not acquire that dependency. +- `capability_report` is an optional encode-request member, never a required + one. The established authority for PyNvVideoCodec encoder capabilities is the + `capabilities` block of the schema-1.2 `nvcodec-environment` artifact; an + encode request that omits `capability_report` is fully supported and reads + that block. When capability-owned freshness is wanted, a request may + additionally carry a schema-1.0 `nvcodec-encoder-capability-report` produced + by `jetson-video-capability` from the *same* environment artifact. When + supplied, that report becomes the selected Py encoder API evidence for the + check; it does not replace the environment artifact or its readiness facts. + Do not add the member to an independently constructed request merely because + the `pynvc` surface may be selected. It is optional on either surface, affects + Py capability classification only, and should be omitted for native; absence + never fails. +- When setup is installed, read its shared + [video content policy](../jetson-video-setup/references/video-content.md) + and apply its input gate before any normal pipeline dry run or execution. + Setup is not required solely for this policy: without it, require one exact + user-selected path or URL, never substitute catalog or synthetic media, and + preserve source URL, license, attribution, path, size, and SHA-256. +- PyNvVideoCodec routes that encode or invoke `advanced/decode.py`, including + encode/decode, segmentation, and Py container triage, require a separately + validated `full-samples` venv. The default `pynvc-smoke` environment is a + setup-readiness proof and must block these routes before workspace creation; + return a structured `jetson-video-setup` dependency and provision a new + full-samples venv rather than upgrading it in place. If that skill is absent, + tell the user to install it before retrying. +- The direct `encode` controller has one narrower consumer exception: + `jetson-video-capability` may bind setup's deterministic one-frame raw + fixture for an exact bounded capability smoke operation. That result is + operation evidence only, never representative pipeline or performance proof. + +## Compose requested sibling stages + +Recipe-free decode and segmentation routes require no sibling when the selected +SDK prerequisites already exist. Add `jetson-video-recipe` only for a +recipe-bearing stage, `jetson-video-benchmark` only for requested performance, +`jetson-video-setup` only for installation, repair, or one read-only handoff +when registered Python authority is required, and +`jetson-video-capability` only for a requested support verdict or fresh +acceptance capability artifact. Use the agent runtime's installed-skill catalog +before each stage; do not scan arbitrary directories. If the sibling is +present, read its `SKILL.md` and invoke its +documented public entry point; pass artifacts as data and never import sibling +code. If it is absent, preserve completed stages and artifacts and say, using +the actual names: `I can run , but it requires , which is not +installed. Install and retry this stage.` Never promote a partial +workflow to complete or require an optional sibling. + +## Instructions + +1. Apply the scope boundary. For a request + solely for objective quality metrics, including PSNR or SSIM, state only + that this skill does not provide them and that a separately authorized + quality workflow is required, then stop. Do not name or recommend an + external tool, and do not offer to configure or run the comparison. For a + request limited to capture, transport, AI, display, or glass-to-glass + latency, state that those stages are outside this codec skill and stop + without naming, recommending, or offering another tool or workflow. + Otherwise proceed immediately to the media gate in step 2; choose + `encode_decode`, `native_transcode`, `pynvc_segments`, `container_triage`, + `av1_verify`, or `acceptance` only after that gate clears. +2. For every remaining request to plan, dry-run, or execute a pipeline route, + including “plan only” or “do not run”, apply this gate + before route selection and before prerequisite, sibling, reference, or + script inspection. Do not decompose a media-gated pipeline request into a + media-free recipe subtask. If media is missing, return `input_required` and + stop before target probing, browsing, retrieval, authentication, dry run, or + operation launch. Ask only for the missing media at this gate; do not also + inspect controller help, describe or plan the route, list future stages or + handoffs, or request an interpreter, environment, recipe, or later-stage + field. The complete response at this terminal gate consists only of + `input_required` and one request for an exact target-local media path or + user-supplied HTTP(S) URL. Never choose substitute media. The + capability-smoke exception above applies only to the direct `encode` + controller and must not be promoted to pipeline completion. +3. Canonicalize and hash an exact local input. For URL input, preserve the + exact user-supplied URL, then retrieve, canonicalize, and hash it only after + target eligibility, authorization, and runtime-authority gates pass. +4. Preserve explicit `native`, `pynvc`, or `both`. Treat “whichever”, “best + available”, “choose for me”, and other unspecified-surface wording as + `auto`, never as `both`. Reserve `both` for an explicit request to run or + compare both surfaces. +5. After the input gate and surface classification, select exactly one runtime + authority for each selected surface. If the caller supplies a setup + environment identity, validate and bind that exact artifact to dry-run and + execute; never ignore it or substitute a local fallback. If PyNvVideoCodec + may participate and neither an environment nor exact interpreter was + supplied, invoke installed `jetson-video-setup` through its public read-only + `probe_nvcodec.py`: use `--runtime pynvc` for explicit Python or `--runtime + both` for `both`/`auto`, a fresh `--output`, and never + `--setup-candidate`. Inspect the fresh artifact; only a live artifact whose + selected Py surface is installed and whose `pynvc.identity.status` is + `verified` is usable. Snapshot that exact file as the controller's portable + `environment` identity with exactly `schema_version`, `kind`, canonical + absolute `path`, `size_bytes`, and lowercase `sha256`; do not import sibling + code or pass a blocked probe as authority. If setup is absent or reports any + not-ready, unreadable, stale, binding, or launch failure, ask for and supply + the exact `pynvc_interpreter` only for explicit `pynvc`/`both`. For `auto`, + keep Py `not_evaluated` and continue only an eligible native surface. The + controller derives a private local binding, never accepts that binding from + a request, and revalidates it before launch. If local authentication fails, + use setup for only that exact surface when installed. +6. Apply the `auto` gate using only the selected runtime authority: zero + eligible surfaces block, one runs, and two return `selection_required`; + never rank the surfaces in this gate. With two eligible surfaces, this gate + is unconditional: do not search old results or benchmark to make the choice. + Ask for exactly `native`, `pynvc`, or `both`, then stop + before dry run or launch. Never trust a prompt's statement that a surface is + ready: establish eligibility from the supplied or freshly probed authority. + Without setup evidence or an exact local `pynvc_interpreter`, record + PyNvVideoCodec as `not_evaluated` with the retry action; do not let that + optional peer block an otherwise eligible native `auto` route. Explicit + `pynvc` or `both` still requires one of those two authorities. +7. Before any codec launch, authenticate each selected executable from the + installed Video Codec SDK package or each Python sample from the selected + wheel and interpreter. Use only those authenticated NVIDIA sample routes; + if none can satisfy a stage, report that stage blocked. +8. For a multi-stage `pipeline` request, compose only the required siblings. + If a requested performance stage needs `jetson-video-benchmark`, invoke its + installed public controller; if absent, preserve completed pipeline stages + and report that the benchmark stage is `dependency_required` with an + install-and-retry action. Run pipeline `dry_run`, review the + complete recipe and sample arguments, then run `execute` with fresh result + and workspace paths. Invoke this skill's public controller directly: + + ```bash + python3 -I {baseDir}/scripts/pipeline_controller.py \ + --request request.json --workspace fresh-workspace \ + --output result.json + ``` + + A single encode-then-independent-decode request invokes + `scripts/encode_controller.py` with the same three arguments. That + controller is execution-only: validate and review its recipe and request + envelope first, then invoke it once with fresh output and workspace paths; + do not claim it performed an internal dry run. +9. Require exact positive markers and counts, no explicit failure marker, and + fresh nonempty outputs. Reopen and rehash every original handoff. An + independent decoder must consume the exact producer path, size, and + SHA-256 and produce the expected frames. +10. For native transcode, accept exactly one authenticated AppTrans completion + marker in either released form: legacy `(#totFrames=N)` or current + `Total frame transcoded: N`. Reject missing, duplicate, or mixed markers. +11. Preserve each segment or surface result independently. A failed peer yields + an honest partial result rather than summary-level completion. +12. For acceptance, let the controller validate and write its nine physical + pre-seal files and return `seal_pending: true`; those are distinct from the + reference keys and stage rows. Keep large media/build artifacts external, + then have the agent create the checksum manifest last—the controller does + not create it. + +## References + +- [Pipeline workflow](references/pipeline-workflow.md) defines routes, + acceptance, sealing, and the pipeline-only compact content-evidence and + validator contract. +- Setup's shared + [video content policy](../jetson-video-setup/references/video-content.md) + defines the exact user-input gate and common content evidence. +- [Official sample contract](references/official-sample-contract.md) defines + allowed routes and operation proof. + +## Available Scripts + +Invoke each public script directly in isolated mode: + +```bash +python3 -I {baseDir}/scripts/encode_controller.py --help +python3 -I {baseDir}/scripts/pipeline_controller.py --help +python3 -I {baseDir}/scripts/validate_representative_content_summary.py --help +``` + +| Script | Purpose | Arguments | +|---|---|---| +| `scripts/encode_controller.py` | Execute one recipe-bound encode followed by independent decode. | `--request`, `--workspace`, and `--output`. | +| `scripts/pipeline_controller.py` | Dry-run or execute the six multi-stage pipeline routes. | `--request`, `--workspace`, and `--output`. | +| `scripts/validate_representative_content_summary.py` | Rehash external media and validate compact content metadata without modifying it. | Inspect `--help` for summary/input arguments. | + +## Limitations + +- This skill covers NVIDIA codec stages and their artifact handoffs, not + capture, network transport, AI inference, display, or glass-to-glass latency. +- It does not implement PSNR or SSIM quality measurement. +- A capability query, exit zero, or output-file creation is never operation + proof. +- Container demux is allowed only through libavformat embedded in an + authenticated released NVIDIA sample. + +## Troubleshooting + +- Return the exact failed gate, producer, consumer, artifact path, and reason. +- Preserve `input_required`, `selection_required`, `blocked`, `partial`, and + `failed` rather than claiming a complete pipeline. +- Reject stale outputs, symlinks where forbidden, path/size/SHA drift, + malformed request or evidence JSON, wrong frame counts, and duplicate + completion markers. +- Retry at most once and only after evidence identifies a changed condition, + such as a repaired dependency, a newly supplied artifact, or a changed + path/size/SHA-256 binding. Repeating an unchanged failed command is forbidden. diff --git a/skills/jetson-video-pipeline/evals/evals.json b/skills/jetson-video-pipeline/evals/evals.json new file mode 100644 index 00000000..8b22226c --- /dev/null +++ b/skills/jetson-video-pipeline/evals/evals.json @@ -0,0 +1,29 @@ +{ + "skill_name": "jetson-video-pipeline", + "evals": [ + { + "id": "pipeline-dual-surface-encode-decode", + "prompt": "Plan equivalent native Video Codec SDK and PyNvVideoCodec H.264 encode-to-decode pipelines with exact artifact handoffs. Do not run them.", + "expected_output": "An input-required response asking only for one exact media path or URL before either pipeline is planned.", + "assertions": [ + "Returns input_required and asks only for one exact target-local path or user-supplied HTTP(S) URL", + "Stops before probing, authentication, dry-run planning, or operation launch", + "Does not choose catalog or synthetic media and makes no execution or pipeline-completion claim" + ], + "expected_skill": "jetson-video-pipeline", + "expected_workflow": "pipeline" + }, + { + "id": "pipeline-native-transcode", + "prompt": "Plan a native H.264-to-HEVC transcode with an independent decode check. Do not run it.", + "expected_output": "An input-required response asking only for one exact H.264 media path or URL before planning the transcode.", + "assertions": [ + "Returns input_required and asks only for one exact target-local H.264 path or user-supplied HTTP(S) URL", + "Stops before probing, authentication, dry-run planning, AppTrans, or AppDec", + "Does not choose catalog or synthetic media and makes no transcode or verification claim" + ], + "expected_skill": "jetson-video-pipeline", + "expected_workflow": "pipeline" + } + ] +} diff --git a/skills/jetson-video-pipeline/references/official-sample-contract.md b/skills/jetson-video-pipeline/references/official-sample-contract.md new file mode 100644 index 00000000..07e4cb6b --- /dev/null +++ b/skills/jetson-video-pipeline/references/official-sample-contract.md @@ -0,0 +1,61 @@ +# Official sample routes and provenance + +Only released NVIDIA samples may perform codec work. This skill keeps its exact +pipeline subset beside its authenticator: + +- `scripts/_pipeline_provenance.py:NATIVE_ROUTES` owns the native package + sample paths, source/CMake paths, and required runtime libraries used here. +- `scripts/_pipeline_provenance.py:PYNVC_ROUTES` owns the PyNvVideoCodec 2.1 + wheel sample paths and required `RECORD` members used here. + +The allowlists are static routing facts, not readiness or operation proof. Every selected +file is still authenticated from the current installed package or wheel before launch, +and its path, size, and SHA-256 are rechecked immediately before execution. +Authenticating a route never selects or acquires non-smoke media. Recipe execution, +performance/benchmarking, content-sensitive comparison, and pipeline routes require an absolute +target-local path or exact user-supplied HTTP(S) URL, never catalog or synthetic substitution, and +preserved provenance and identity. When setup is installed, also apply its shared +[video content policy](../../jetson-video-setup/references/video-content.md). +Missing input is `input_required` and pauses before route authentication or +launch. Only setup/readiness and minimal capability-operation smoke tests may +use the documented deterministic synthetic fixture. + +## Active routes + +Pipeline-native routes are `AppEncCuda`, `AppDec`, and `AppTrans`. + +Pipeline PyNvVideoCodec routes are `samples/basic/encode.py`, +`samples/advanced/decode.py`, and `samples/basic/create_video_segments.py`. + +The current public controllers intentionally use smaller route subsets: + +- pipeline encode: `AppEncCuda` + `AppDec`, or basic encode + advanced decode; +- multi-stage pipeline: `AppEncCuda`, `AppDec`, `AppTrans`, basic segments, and + advanced decode. + +Do not route a sample merely because it appears in an SDK source tree. It must be present +in the installed package or selected wheel and implemented by the chosen controller. + +## Evidence classifications + +Keep these observations separate: + +- `api_query_helper`: API fields only; never operation proof or live availability; +- `official_sample_report`: an authenticated official sample report mode; +- `official_sample_operation`: an authenticated sample actually executed; +- `documentation_reference`: product-support authority for the exact tuple, but never live + readiness, availability, or operation proof; +- `absent_release_sample`: not shipped by the authenticated release payload; +- `source_tree_candidate`: visible in source, but not yet an installed runnable route. + +An operation is verified only when its exact positive markers and counts are present, no +failure marker appears, the requested output is fresh and nonempty, and the independent +consumer verifies the same path and SHA-256. Exit zero, import success, or output creation +alone is insufficient. + +## Updating a route + +Changing either allowlist requires matching controller support and focused tests for its +package/wheel identity, exact argv, help-advertised options, markers, fresh outputs, and +producer-to-consumer handoff. Never add a compatibility alias or fallback that silently +changes the requested codec, surface, format, or operation. diff --git a/skills/jetson-video-pipeline/references/pipeline-workflow.md b/skills/jetson-video-pipeline/references/pipeline-workflow.md new file mode 100644 index 00000000..d9fa2dfc --- /dev/null +++ b/skills/jetson-video-pipeline/references/pipeline-workflow.md @@ -0,0 +1,224 @@ +# Pipeline workflow + +## Contents + +- [Owner and exact CLI](#owner-and-exact-cli) +- [Pipeline request schema](#pipeline-request-schema) +- [Workflow](#workflow) +- [Acceptance content evidence](#acceptance-content-evidence) + +Run compact, authenticated multi-stage Video SDK routes and verify every +producer-to-consumer handoff. This workflow owns encode/decode chains, native +transcode, PyNvVideoCodec segmentation, container decode triage, AV1 verification, +and the customer acceptance package. Each stage uses only released samples; libavformat +embedded in released NVIDIA samples is allowed only for container demux. + +## Owner and exact CLI + +One controller owns this domain: +**`jetson-video-pipeline/scripts/pipeline_controller.py`**. Launch it directly +in isolated mode: + +```text +python3 -I scripts/pipeline_controller.py \ + --request nvcodec-pipeline-request.json \ + --workspace fresh-workspace-dir \ + --output nvcodec-pipeline-result.json +``` + +All three flags are required (`--help` supported). Exit `0` when `status` is `complete` +or `planned`, `2` on a safe block or stage failure, `3` on malformed input or an +internal error. + +## Pipeline request schema + +`--request` is a `schema_version: "1.0"`, `kind: "nvcodec-pipeline-request"` object. +`mode` is exactly `dry_run` (plan only) or `execute` (default `execute`). `route` is +exactly one of: + +| `route` | Purpose | +|---|---| +| `encode_decode` | Encode then independently decode/verify the same artifact (native, or the same request through native and Python for a dual-surface comparison) | +| `native_transcode` | Native H.264 → HEVC transcode with usable-output proof | +| `pynvc_segments` | Split a clip into independently usable PyNvVideoCodec segments and validate each | +| `container_triage` | Demux a user-supplied local container, or retrieve the exact user-supplied HTTP(S) URL, then decode and report hardware-decode support (`surface` must be `native` or `pynvc`) | +| `av1_verify` | Verify AV1 encode from an AV1 recipe with an exact native projection and positive `frame_count` | +| `acceptance` | Aggregate setup, capabilities, P4/P5 encode/decode, and throughput into one acceptance package | + +`container_triage` also requires `target_eligibility` with exactly `eligible` +(boolean) and `reasons` (a list of nonempty strings). Use +`{"eligible": true, "reasons": []}` only after identifying the current target +as an eligible released Jetson route; otherwise set `eligible` false and record +the observed reasons. This gate is request planning context, not an SDK setup +artifact or a codec-support verdict. + +For `encode_decode`, the controller consumes one authenticated encode-request artifact +and calls the landed recipe/encode owner. Other routes authenticate their exact sample +set directly. A completed `container_triage` result retains its workspace-relative +decoded-frame evidence and also returns `decode.raw_video` as the exact portable absolute +identity that the encode controller accepts directly. `av1_verify` owns the complete IVF +frame walk and independent AppDec proof; +setup capability queries remain separate context and are never treated as that proof. + +`environment` is an optional portable identity for a fresh schema-1.2 setup +artifact. Setup emits raw JSON; the request carries an exact identity with only +`schema_version`, `kind`, canonical absolute `path`, `size_bytes`, and lowercase +`sha256`. When supplied it is authoritative and must validate. When omitted, +the controller authenticates the selected installed surface locally; +PyNvVideoCodec routes additionally require the exact absolute +`pynvc_interpreter`. A controller-produced local binding is private evidence +and is never accepted as a request member. + +After the media gate, an agent that needs `pynvc` or `both` and has neither a +supplied environment nor exact interpreter first checks the installed skill +catalog. If `jetson-video-setup` is present, invoke its public +`probe_nvcodec.py` with a fresh output, `--runtime pynvc` for Python-only or +`--runtime both` for `both`/`auto`, and no `--setup-candidate`. Setup alone +resolves and reauthenticates its registry. Inspect the fresh JSON before +constructing the identity: require `mode=live`, the requested GPU, +`pynvc.installed=true`, and `pynvc.identity.status=verified`. Derive the +identity's path, size, and SHA-256 from that exact file and let this controller +authenticate it again. For explicit `pynvc`/`both`, ask for an exact interpreter +only when setup is unavailable or returns any absent, stale, unreadable, +invalid-binding, or launch-failure result. For `auto`, keep Py `not_evaluated` +and continue only an eligible native surface. Never pass a blocked probe as +authority. + +## Workflow + +1. Apply the media gate before classifying the request into exactly one route + above. Require its media input to originate from an absolute target-local + path or one exact user-supplied HTTP(S) URL; never substitute catalog or + synthetic media, and preserve provenance and identity. When setup is installed, also apply its + shared [video content policy](../../jetson-video-setup/references/video-content.md). + A route may consume a prior artifact only when that artifact remains bound + to the same user-selected source. If no input is supplied, return + `input_required`, ask for one, and pause before probing, retrieval, + conversion, authority selection, dry run, or pipeline execution. Ask only + for media at that gate; never select catalog media or use the synthetic + setup fixture. +2. Honor an explicit `native`/`pynvc`/`both` surface. Map “whichever”, “best + available”, “choose for me”, and any other unspecified-surface wording to + `auto`; never broaden it to `both`. Only an explicit dual-surface request + selects `both`. +3. After the input gate and surface classification, use a supplied setup + environment identity, obtain a fresh one through setup's public probe as + described above, or use local selected-surface authentication. Supplied + evidence must validate and never falls back. Without it, native derives raw + dpkg/package-source/toolchain facts; Python derives raw wheel/import facts + from the exact `pynvc_interpreter`. The consumer stages those facts as + private evidence, derives its normalized surface view internally, and + revalidates the binding before launch. It never accepts a serialized local + binding from the caller and never scans for a Python environment. A missing + or invalid SDK routes only that surface to `jetson-video-setup`. +4. Apply the `auto` gate using only that authority: zero + eligible surfaces block, one runs, and two + return `selection_required` before searching prior results, dry run, or + codec launch. Eligibility authentication happens first. This gate is + unconditional when two surfaces are + eligible: ask for exactly `native`, `pynvc`, or `both`. For `both`, preserve + every eligible branch and each blocked peer. One surface never authorizes or + blocks the other. If local `auto` has no exact `pynvc_interpreter`, disclose + PyNvVideoCodec as `not_evaluated` with the retry action and continue only an + eligible native branch; explicit `pynvc` and `both` still require setup + evidence or the exact interpreter. A prompt's claim that a surface is ready + is never authority. Eligibility is decided at planning time: each surface is + structurally validated against its own required subset of the schema-1.2 + environment — native `installed`/`package`/`sdk_root`/`cuda`/`tools` versions, + pynvc `installed`/`version`/`interpreter`/`interpreter_identity`/`sys_prefix`/ + `extension`/`module` — ignoring additive keys. A malformed surface is blocked + during planning, never deferred to execute-time authentication, so a broken + surface never consumes a run slot or couples the peer through late failure. +5. Run `dry_run` to review the planned stages and handoffs, then `execute` with a fresh + output path and bounded per-stage timeouts. +6. Bind each handoff to its original canonical path, size, and SHA-256; reopen and + rehash the original artifact — never trust copied status prose. A stage is complete + only when its exact positive marker, count, and fresh nonempty output are proven. + +For `acceptance`, the controller writes exactly nine compact pre-seal files and reports +`seal_pending: true`; it intentionally does not create its own checksum manifest or an +inventory. Its `references` object must name `readiness`, `capabilities`, `content`, +`p4_recipe`, `p5_recipe`, `p4_encode_decode`, `p5_encode_decode`, `p4_benchmark`, +`p5_benchmark`, `handoffs`, `commands`, `task_results`, +and `timing`. The task-results JSONL contains exactly one +`{task_id, status: "complete", evidence}` row for each of the ten stage names; +`evidence` is a nonempty list of names from `references`. Recipes must validate as P4/P5, +each encode/decode stage must consume its accepted recipe and the same exact raw input, +and each benchmark variant must retain that exact recipe identity and input identity. +Encode/decode artifacts must be complete, benchmarks must retain a warmup and at least +three measurements; the handoff list must be nonempty and every entry must be verified. +Each accepted benchmark branch retains the successful `warmup` command and +sequential measured repetitions whose command phase is `measure`; every command argv must +equal the authenticated launcher plus `benchmark_arguments`, exit zero, and not time out. +FPS and megapixels/second must be positive finite values, and their repetition count, mean, +minimum, and maximum must agree with the retained measurements. + +The acceptance assembler copies the six verified media metadata fields from the `content` +evidence into `evidence/summary.json.representative_content`: exactly +`source_url`, `license`, `attribution`, `path`, `size_bytes`, and `sha256`. +`source_url` is the exact user-supplied HTTP(S) URL for retrieved media and JSON +`null` for target-local media. `license` and `attribution` are nonempty honest +strings; either may be the literal `unknown`. It writes the corresponding exact +direct source record—not a generic reference table—to +`evidence/representative-content.json` with exactly `kind`, those six metadata +fields, `expected_size_bytes`, `expected_sha256`, and `verified`. Require +matching expected size/SHA-256 and `verified: true`. Both files therefore +satisfy the six-field compact metadata contract in +[setup's shared video content policy](../../jetson-video-setup/references/video-content.md) +and can be passed directly to the `content-summary` controller while the +external media remains at its bound canonical path. Retrieval-command or local +user-selection evidence supports acquisition claims separately and is not part +of this six-field validator contract. + +### Acceptance content evidence + +Keep the source record and fresh validation result in the compact package and +bind both in its manifest. Exclude the media itself from the package, retaining +its canonical path, size, and SHA-256 in the summary. Run +`scripts/validate_representative_content_summary.py` directly before cleaning +up excluded media: + +```bash +python3 -I scripts/validate_representative_content_summary.py \ + --summary /absolute/path/to/evidence/summary.json \ + --source-artifact /absolute/path/to/evidence/representative-content.json \ + --output /absolute/path/to/evidence/representative-content-summary-validation.json +``` + +The validator reads the two JSON files directly, requires strict shapes and +values, and freshly rehashes the current content file without modifying it. It +proves current byte identity and exact cross-record metadata equality only. It +does not prove URL origin, publisher-supplied expectations, license, +attribution, representativeness, or nonuniformity. Preserve a retrieval command +record for URL media or user-selection evidence for local media when making an +acquisition claim; these are separate from the six-field metadata consistency +check and need not be copied into the compact package. Claim content +characteristics only when separate observed evidence supports them. + +Exit `0` means this limited consistency check is verified, exit `2` means +invalid or unauthenticated input, and exit `3` means validator or fresh-output +failure. The compact package cannot rerun the check after excluded media is +removed unless that external media is restored at its bound path. Do not pipe a +log or stdout record into this check, and do not use command substitution in +place of either file. + +Set `validation_result_filename` to one fresh plain JSON filename and pass the matching +workspace-relative path to the CLI, for example +`--output evidence/customer-validation.json`. The controller writes the other eight files; +the mandatory `--output` is the ninth validation-result file. It does not contain its own +artifact identity. After the result is complete and every JSON/JSONL file parses, seal the +final small package from its root: + +```bash +( + cd "$PACKAGE_ROOT" + find . -type f ! -path './evidence/manifest-sha256.txt' -print0 | + LC_ALL=C sort -z | xargs -0 sha256sum +) > "$PACKAGE_ROOT/evidence/manifest-sha256.txt" +``` + +Do not place media, raw frames, streams, SDK trees, build directories, virtual +environments, or caches in the package; record their external path, size, and SHA-256. + +The allowed operation routes are defined by +[official-sample-contract.md](official-sample-contract.md). diff --git a/skills/jetson-video-pipeline/scripts/_pipeline_provenance.py b/skills/jetson-video-pipeline/scripts/_pipeline_provenance.py new file mode 100644 index 00000000..41046043 --- /dev/null +++ b/skills/jetson-video-pipeline/scripts/_pipeline_provenance.py @@ -0,0 +1,1560 @@ +#!/usr/bin/env python3 +"""Private official-sample authentication for pipeline operations.""" +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations +import base64 +import csv +import hashlib +import io +import json +import re +import stat +import subprocess +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Mapping, Sequence +import _pipeline_runtime as artifact_io +# The two authenticators intentionally validate complete provenance +# conjunctions in one owner. +# pylint: disable=too-many-arguments,too-many-boolean-expressions +# pylint: disable=too-many-branches,too-many-instance-attributes,too-many-locals +# pylint: disable=too-many-lines +command_runner = artifact_io +_COMMAND_ENV = { + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +} +_DEBIAN_VERSION = re.compile( + r"^(?:(?P[0-9]+):)?(?P13\.0(?:\.[0-9]+)*)" + r"(?:\+[0-9A-Za-z.]+)?(?:-[0-9A-Za-z.+]+)?$" +) +# The normalized native toolchain names exactly these five build tools; the +# generator record adds its CMake generator name. +_BUILD_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +_CMAKE_GENERATORS = frozenset({"Ninja", "Unix Makefiles"}) +_PKG_CONFIG_MODULES = ("libavcodec", "libavformat", "libavutil", "libswresample") +_SYSTEM_TOOLS = { + "cmake": Path("/usr/bin/cmake"), + "g++": Path("/usr/bin/g++"), + "pkg-config": Path("/usr/bin/pkg-config"), + "ninja": Path("/usr/bin/ninja"), + "make": Path("/usr/bin/make"), +} +_DIRECT_OUTPUT_LIMIT = 1024 * 1024 +OFFICIAL_SAMPLE_FAILURE_PATTERN = re.compile( + r"""(?imx)^\s*(?: + traceback\ \(most\ recent\ call\ last\) + | error\b + | an\ (?:unexpected\ )?error\ occurred(?:\ with\ [^:\r\n]+)?: + | operation\ or\ configuration\ not\ supported: + | decode\ error\ occurred\ for\ picture\b + | create(?:encoder|decoder)\ failure: + | (?:worker\ [^:\r\n]+:\s*)?(?:an\ unexpected\ error\ occurred|setup\ failed): + | (?:thread|process)\ [^:\r\n]+\ error: + | encoding\ aborted:\ setup\ failed\b + | nvdecoder::~nvdecoder\(\):\ exception\ during\ cleanup\ \(suppressed\) + | (?:[a-z_][a-z0-9_:~]*\s*:\s*)?[a-z_][a-z0-9_.]* + (?:\([^\r\n]*\))?\s+returned\ error\b + | (?:cuda(?:\ (?:driver|runtime)\ api)?|nvenc|hresult|glenum|general|ffmpeg) + \ error\b + | cudacheckerror\(\)\ failed\b + | \[(?:error|fatal)\s*\](?:\[[^]\r\n]+\])? + | failed\b + | failure\b + | (?:operation|command|sample|encode|decode|encoding|decoding|encoder| + decoder|setup|test)\b[^\r\n]*\b(?:failed|failure)\b + )""" +) +NATIVE_ROUTES = { + "AppEncCuda": ( + "AppEncode/AppEncCuda/AppEncCuda", + "Samples/AppEncode/AppEncCuda/AppEncCuda.cpp", + "Samples/AppEncode/AppEncCuda/CMakeLists.txt", + ("libcuda.so.1", "libnvidia-encode.so.1"), + ), + "AppDec": ( + "AppDecode/AppDec/AppDec", + "Samples/AppDecode/AppDec/AppDec.cpp", + "Samples/AppDecode/AppDec/CMakeLists.txt", + ("libcuda.so.1", "libnvcuvid.so.1"), + ), + "AppTrans": ( + "AppTranscode/AppTrans/AppTrans", + "Samples/AppTranscode/AppTrans/AppTrans.cpp", + "Samples/AppTranscode/AppTrans/CMakeLists.txt", + ("libcuda.so.1", "libnvcuvid.so.1", "libnvidia-encode.so.1"), + ), +} +PYNVC_ROUTES = { + "samples/basic/encode.py": ( + "samples/utils/__init__.py", + "samples/utils/Utils.py", + "samples/utils/encode_parser.py", + "samples/utils/frame_utils.py", + ), + "samples/advanced/decode.py": ( + "samples/utils/__init__.py", + "samples/utils/Utils.py", + "samples/utils/decode_parser.py", + ), + "samples/basic/create_video_segments.py": ( + "samples/utils/__init__.py", + "samples/utils/transcode_parser.py", + "samples/basic/segments.txt", + "samples/basic/transcode_config.json", + ), +} + + +class ProvenanceError(ValueError): + """Official sample authority cannot be established.""" + + +PynvcProvenanceError = ProvenanceError + + +@dataclass(frozen=True) +class FileSeal: + """One protected external file identity.""" + identity: Mapping[str, Any] + + +@dataclass(frozen=True) +class AuthenticatedSample: + """Frozen authority for one exact sample launcher.""" + surface: str + sample: str + launcher: tuple[str, ...] + workspace: str + protected_files: tuple[FileSeal, ...] + runtime_libraries: tuple[FileSeal, ...] + environment_workspace: str + environment_identity: Mapping[str, Any] + + +def _select( + samples: Sequence[AuthenticatedSample], sample: str, surface: str +) -> AuthenticatedSample: + matches = [item for item in samples if item.sample == sample] + if len(matches) != 1: + raise ProvenanceError(f"{surface} sample is not authenticated: {sample!r}") + return matches[0] + + +@dataclass(frozen=True) +class NativeAuthentication: + """Native sample records and provenance report.""" + samples: tuple[AuthenticatedSample, ...] + report_identity: Mapping[str, Any] + + def token(self, sample: str) -> AuthenticatedSample: + """Return the unique authenticated native sample.""" + return _select(self.samples, sample, "native") + + +@dataclass(frozen=True) +class PynvcAuthentication: + """PyNvVideoCodec sample records and provenance report.""" + samples: tuple[AuthenticatedSample, ...] + report_identity: Mapping[str, Any] + + def token(self, sample: str) -> AuthenticatedSample: + """Return the unique authenticated Python sample.""" + return _select(self.samples, sample, "PyNvVideoCodec") + + +def _identity(path: Path, kind: str) -> dict[str, Any]: + return artifact_io.snapshot_external_artifact( + path, schema_version="1", kind=kind + ) + + +def _sample_record( + *, + surface: str, + sample: str, + launcher: tuple[str, ...], + workspace: Path, + protected: Sequence[Mapping[str, Any]], + runtime: Sequence[Mapping[str, Any]], + environment_workspace: Path, + environment_identity: Mapping[str, Any], +) -> AuthenticatedSample: + return AuthenticatedSample( + surface=surface, + sample=sample, + launcher=launcher, + workspace=str(workspace), + protected_files=tuple(FileSeal(dict(item)) for item in protected), + runtime_libraries=tuple(FileSeal(dict(item)) for item in runtime), + environment_workspace=str(environment_workspace), + environment_identity=dict(environment_identity), + ) + + +def _direct_read_only(argv: Sequence[str], *, timeout_seconds: float) -> str: + """Run one fixed read-only probe without creating an evidence workspace.""" + try: + completed = subprocess.run( + list(argv), + cwd="/", + env=_COMMAND_ENV, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_seconds, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise ProvenanceError(f"read-only local probe failed: {argv[0]}: {exc}") from exc + if ( + completed.returncode != 0 + or len(completed.stdout) > _DIRECT_OUTPUT_LIMIT + or len(completed.stderr) > _DIRECT_OUTPUT_LIMIT + ): + detail = completed.stderr[-500:].decode("utf-8", errors="replace") + raise ProvenanceError( + f"read-only local probe failed: {list(argv)!r}: " + f"exit={completed.returncode}: {detail}" + ) + return completed.stdout.decode("utf-8", errors="strict") + + +def _plain_identity(path: Path) -> dict[str, Any]: + identity = _identity(path, "local-runtime-file") + return { + field: identity[field] for field in ("path", "size_bytes", "sha256") + } + + +def _system_tool(name: str) -> Path: + candidate = _SYSTEM_TOOLS.get(name) + if candidate is None: + raise ProvenanceError(f"native build tool is not allowlisted: {name}") + try: + resolved = candidate.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ProvenanceError( + f"required native build tool is missing: {name}" + ) from exc + if ( + not resolved.is_relative_to(Path("/usr/bin")) + or not resolved.is_file() + or resolved.stat().st_mode & 0o111 == 0 + ): + raise ProvenanceError(f"required native build tool is not executable: {name}") + return resolved + + +def _local_cuda() -> tuple[dict[str, Any], Path]: + candidates = [Path("/usr/local/cuda")] + candidates.extend( + sorted(Path("/usr/local").glob("cuda-*"), reverse=True) + ) + roots: list[Path] = [] + for candidate in candidates: + try: + root = candidate.resolve(strict=True) + except (OSError, RuntimeError): + continue + if root not in roots and (root / "bin" / "nvcc").is_file(): + roots.append(root) + if not roots: + raise ProvenanceError("no installed CUDA toolkit with nvcc was found") + root = roots[0] + output = _direct_read_only( + [str(root / "bin" / "nvcc"), "--version"], timeout_seconds=15 + ) + match = re.search(r"\brelease\s+(13\.[0-9]+(?:\.[0-9]+)*)", output) + if match is None: + raise ProvenanceError("selected nvcc does not report a CUDA 13.x release") + return { + "status": "available", + "version": match.group(1), + "root": str(root), + "nvcc_discovery": {"root": str(root), "nvcc": str(root / "bin" / "nvcc")}, + }, root + + +def _local_build_tools(cuda_root: Path) -> dict[str, Any]: + tools = { + "cmake": _plain_identity(_system_tool("cmake")), + "g++": _plain_identity(_system_tool("g++")), + "nvcc": _plain_identity((cuda_root / "bin" / "nvcc").resolve(strict=True)), + "pkg-config": _plain_identity(_system_tool("pkg-config")), + } + for executable, generator in (("ninja", "Ninja"), ("make", "Unix Makefiles")): + try: + candidate = _system_tool(executable) + except ProvenanceError: + continue + tools[executable] = { + **_plain_identity(candidate), + "name": generator, + "executable": executable, + } + break + else: + raise ProvenanceError("neither Ninja nor Make is installed") + return tools + + +def _local_build_prerequisites(pkg_config: str) -> dict[str, Any]: + modules: dict[str, Any] = {} + unresolved: list[str] = [] + for module in _PKG_CONFIG_MODULES: + try: + version = _direct_read_only( + [pkg_config, "--modversion", module], timeout_seconds=15 + ).strip() + except ProvenanceError: + version = "" + ready = bool(version) + modules[module] = { + "status": "available" if ready else "missing", + "version": version or None, + } + if not ready: + unresolved.append(module) + return { + "status": "complete" if not unresolved else "incomplete", + "unresolved_modules": unresolved, + "consumer": "Samples/AppDecode/AppDec", + "required_modules": list(_PKG_CONFIG_MODULES), + "modules": modules, + } + + +def _local_native_evidence( + *, timeout_seconds: float +) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], dict[str, Any]]: + """Authenticate the installed native package enough to plan its own routes.""" + package_output = _direct_read_only( + [ + "/usr/bin/dpkg-query", + "-W", + "-f=${db:Status-Abbrev}\\t${Version}\\t${binary:Package}\\n", + "nvidia-video-codec-sdk", + ], + timeout_seconds=timeout_seconds, + ) + fields = package_output.rstrip("\n").split("\t") + if len(fields) != 3 or fields[0] != "ii " or fields[2] != "nvidia-video-codec-sdk": + raise ProvenanceError("native Video Codec SDK package is not installed by dpkg") + _public_version(fields[1]) + verified = _direct_read_only( + ["/usr/bin/dpkg", "--verify", "nvidia-video-codec-sdk"], + timeout_seconds=timeout_seconds, + ) + if verified.strip(): + raise ProvenanceError("dpkg --verify reported modified native SDK files") + listing = _direct_read_only( + ["/usr/bin/dpkg-query", "-L", "nvidia-video-codec-sdk"], + timeout_seconds=timeout_seconds, + ) + roots = sorted( + { + str(Path(line).parent.parent.resolve(strict=True)) + for line in listing.splitlines() + if line.startswith("/") and line.endswith("/Samples/CMakeLists.txt") + } + ) + if len(roots) != 1: + raise ProvenanceError( + "native package does not expose exactly one canonical SDK sample root" + ) + cuda, cuda_root = _local_cuda() + tools = _local_build_tools(cuda_root) + prerequisites = _local_build_prerequisites(tools["pkg-config"]["path"]) + native = { + "status": "installed", + "package": { + "name": "nvidia-video-codec-sdk", + "status": "installed", + "version": fields[1], + }, + "complete_roots": roots, + } + return native, cuda, tools, prerequisites + + +_LOCAL_PYNVC_PROBE = r""" +import base64 +import csv +import hashlib +import importlib.metadata +import io +import json +import pathlib +import re +import sys + +distribution = importlib.metadata.distribution("PyNvVideoCodec") +module = __import__("PyNvVideoCodec") +dist_version = str(distribution.version) +module_version = str(getattr(module, "__version__", "") or "") +if dist_version != "2.1.0" or module_version != dist_version: + raise RuntimeError("distribution and imported module must both be PyNvVideoCodec 2.1.0") +prefix = pathlib.Path(sys.prefix).resolve(strict=True) +launcher = pathlib.Path(sys.executable) +if launcher.parent.parent.resolve(strict=True) != prefix: + raise RuntimeError("selected interpreter does not belong to its reported venv prefix") +files = [str(item).replace("\\", "/") for item in (distribution.files or [])] +paths = { + item: pathlib.Path(distribution.locate_file(pathlib.Path(item))).resolve(strict=True) + for item in files +} +records = [item for item in files if item.endswith(".dist-info/RECORD")] +if len(records) != 1: + raise RuntimeError("expected exactly one PyNvVideoCodec RECORD") +record_path = paths[records[0]] +rows = {} +for row in csv.reader(io.StringIO(record_path.read_text(encoding="utf-8"))): + if len(row) != 3 or not row[0] or row[0] in rows: + raise RuntimeError("wheel RECORD is not canonical") + rows[row[0]] = (row[1], row[2]) + +native_extension_member = re.compile( + r"PyNvVideoCodec/_?PyNvVideoCodec(?:_(?:121|130))?\.[^/]*\.so" +) + +def owned_identity(value, label, allow_native_stale=False): + lexical = pathlib.Path(value) + resolved = lexical.resolve(strict=True) + if not resolved.is_relative_to(prefix): + raise RuntimeError(f"{label} is outside selected venv") + matches = [name for name, path in paths.items() if path == resolved] + if len(matches) != 1 or matches[0] not in rows: + raise RuntimeError(f"{label} is not one unique wheel member") + encoded, size_text = rows[matches[0]] + payload = resolved.read_bytes() + actual = base64.urlsafe_b64encode(hashlib.sha256(payload).digest()).rstrip(b"=").decode() + matches_record = ( + encoded.startswith("sha256=") + and encoded[7:] == actual + and size_text == str(len(payload)) + ) + known_stale = ( + allow_native_stale + and dist_version == "2.1.0" + and native_extension_member.fullmatch(matches[0]) is not None + and encoded.startswith("sha256=") + and size_text.isdecimal() + ) + if not matches_record and not known_stale: + raise RuntimeError(f"{label} differs from wheel RECORD") + return { + "path": str(resolved), "loaded_path": str(resolved), + "size_bytes": len(payload), "sha256": hashlib.sha256(payload).hexdigest(), + "record_member": matches[0], + "record_consistency": ( + "matches_record" if matches_record else "known_upstream_stale_record" + ), + } + +module_path = getattr(module, "__file__", None) +extension_path = getattr(getattr(module, "_PyNvVideoCodec", None), "__file__", None) +if not module_path or not extension_path: + raise RuntimeError("imported module or executed extension path is unavailable") +module_identity = owned_identity(module_path, "imported module") +module_identity["version"] = module_version +extension_identity = owned_identity( + extension_path, "executed extension", allow_native_stale=True +) +suffix = str(getattr(module, "module_suffix", "") or "") +if suffix not in {"_121", "_130"}: + raise RuntimeError("executed extension API suffix is not recognized") +extension_identity["module_suffix"] = suffix +packages = {} +for name in ("numpy", "pycuda", "torch"): + try: + version = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + version = None + ready = version is not None + packages[name] = { + "status": "installed" if ready else "missing", + "version": version, + "requirement_satisfied": ready, + } +if packages["torch"]["status"] == "installed": + try: + import torch + cuda_build = getattr(torch.version, "cuda", None) + cuda_available = bool(torch.cuda.is_available()) + except Exception: + cuda_build = None + cuda_available = False + torch_ready = ( + packages["torch"]["version"] == "2.9.1+cu130" + and cuda_build == "13.0" + and cuda_available + ) + packages["torch"].update({ + "cuda_build": cuda_build, + "cuda_available": cuda_available, + "requirement_satisfied": torch_ready, + }) +target = launcher.resolve(strict=True) +target_payload = target.read_bytes() +interpreter_identity = { + "path": str(target), "size_bytes": len(target_payload), + "sha256": hashlib.sha256(target_payload).hexdigest(), +} +print(json.dumps({ + "pynvc": { + "imported": True, + "distribution_version": dist_version, + "identity": { + "status": "verified", + "interpreter": str(launcher), + "sys_prefix": str(prefix), + "module": module_identity, + "extension": extension_identity, + }, + }, + "python": { + "executable": str(launcher), + "sys_prefix": str(prefix), + "interpreter_identities": {"running": interpreter_identity}, + "packages": packages, + }, +}, sort_keys=True)) +""".strip() + + +def _local_pynvc_evidence( + interpreter: str, *, timeout_seconds: float +) -> tuple[dict[str, Any], dict[str, Any]]: + if ( + not isinstance(interpreter, str) + or not interpreter + or interpreter != interpreter.strip() + or not Path(interpreter).is_absolute() + or str(Path(interpreter)) != interpreter + ): + raise ProvenanceError( + "pynvc_interpreter must be one canonical absolute path" + ) + launcher = Path(interpreter) + try: + target = launcher.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ProvenanceError("pynvc_interpreter is unavailable") from exc + if not launcher.is_file() or target.stat().st_mode & 0o111 == 0: + raise ProvenanceError("pynvc_interpreter is not executable") + output = _direct_read_only( + [str(launcher), "-I", "-c", _LOCAL_PYNVC_PROBE], + timeout_seconds=timeout_seconds, + ) + try: + value = json.loads(output) + except (UnicodeError, json.JSONDecodeError) as exc: + raise ProvenanceError("local PyNvVideoCodec probe returned invalid JSON") from exc + if ( + not isinstance(value, dict) + or set(value) != {"pynvc", "python"} + or not isinstance(value.get("pynvc"), dict) + or not isinstance(value.get("python"), dict) + ): + raise ProvenanceError("local PyNvVideoCodec probe returned an invalid shape") + return dict(value["pynvc"]), dict(value["python"]) + + +def build_local_runtime_binding( + required_surfaces: Sequence[str], + *, + pynvc_interpreter: str | None, + gpu: int, + timeout_seconds: float, +) -> tuple[dict[str, Any], dict[str, str]]: + """Build an in-process-only read-only authority for requested surfaces. + + The returned document is private evidence staged by the consumer. It is + never accepted from a request and never substitutes for a supplied setup + artifact. Each requested surface is probed independently. + """ + selected = tuple(required_surfaces) + if ( + not selected + or len(selected) != len(set(selected)) + or any(surface not in artifact_io.SURFACE_ORDER for surface in selected) + ): + raise ProvenanceError("local binding surfaces are invalid") + if isinstance(gpu, bool) or not isinstance(gpu, int) or gpu < 0: + raise ProvenanceError("local binding gpu must be a non-negative integer") + errors: dict[str, str] = {} + native: dict[str, Any] = { + "status": "missing", + "package": { + "name": "nvidia-video-codec-sdk", + "status": "missing", + "version": None, + }, + "complete_roots": [], + } + cuda: dict[str, Any] = { + "status": "absent", + "version": None, + "nvcc_discovery": {"root": None, "nvcc": None}, + } + tools: dict[str, Any] = {} + prerequisites: dict[str, Any] = { + "status": "incomplete", + "unresolved_modules": list(_PKG_CONFIG_MODULES), + } + pynvc: dict[str, Any] = { + "imported": False, + "distribution_version": None, + "identity": {"status": "not_ready"}, + } + python: dict[str, Any] = { + "interpreter_identities": {}, + "packages": {}, + } + if "native" in selected: + try: + native, cuda, tools, prerequisites = _local_native_evidence( + timeout_seconds=timeout_seconds + ) + except (OSError, ValueError) as exc: + errors["native"] = str(exc) + if "pynvc" in selected: + if pynvc_interpreter is None: + errors["pynvc"] = ( + "PyNvVideoCodec was not evaluated because no exact absolute " + "pynvc_interpreter was supplied" + ) + else: + try: + pynvc, python = _local_pynvc_evidence( + pynvc_interpreter, timeout_seconds=timeout_seconds + ) + except (OSError, ValueError) as exc: + errors["pynvc"] = str(exc) + runtime = ( + "both" if len(selected) == 2 else selected[0] + ) + binding = { + "schema_version": artifact_io.LOCAL_RUNTIME_BINDING_SCHEMA_VERSION, + "kind": artifact_io.LOCAL_RUNTIME_BINDING_KIND, + "mode": "live", + "source": "pipeline-local-discovery", + "requested_runtime": runtime, + "selected_gpu": gpu, + "installation": { + "native_sdk": native, + "cuda_toolkit": cuda, + "native_build_prerequisites": prerequisites, + "build_tool_identities": tools, + "python": python, + }, + "pynvc": pynvc, + } + artifact_io.validate_local_runtime_binding( + binding, + label="local runtime", + error_type=ProvenanceError, + required_surfaces=selected, + ) + return binding, errors + + +def _environment_value( + workspace: Path, identity: Mapping[str, Any], label: str +) -> dict[str, Any]: + required = ("pynvc",) if label.lower().startswith("py") else ("native",) + return artifact_io.read_live_environment( + workspace, + identity, + label=label, + error_type=ProvenanceError, + required_surfaces=required, + ) + + +def _run_prepare( + service: Any, + commands: list[dict[str, Any]], + argv: Sequence[str], + *, + cwd: Path, + timeout_seconds: float, + stage: str, +) -> tuple[dict[str, Any], str, str]: + return artifact_io.run_prepare_command( + service, + commands, + argv, + command_environment=_COMMAND_ENV, + error_type=ProvenanceError, + cwd=cwd, + timeout_seconds=timeout_seconds, + stage=stage, + ) + + +def _public_version(value: Any) -> str: + match = _DEBIAN_VERSION.fullmatch( + value.strip() if isinstance(value, str) else "" + ) + if match is None or "~" in str(value) or "really" in str(value).lower(): + raise ProvenanceError( + "native SDK package is not a stable public 13.0.x release" + ) + return match.group("upstream") + + +def _native_surface(environment: Mapping[str, Any]) -> tuple[str, str]: + """Read the required native subset of one schema-1.2 environment. + + The normalized surface carries one canonical ``sdk_root`` string, taken + from the single complete root 1.2 publishes. Ownership of that root is + re-authenticated live in ``_authenticated_sdk_root`` against ``dpkg`` + rather than replayed from the artifact's inventory. + """ + native = artifact_io.environment_surface(environment, "native") + package = native.get("package") if native is not None else None + sdk_root = native.get("sdk_root") if native is not None else None + if ( + native is None + or not isinstance(package, Mapping) + or package.get("name") != "nvidia-video-codec-sdk" + or package.get("status") != "installed" + or not isinstance(sdk_root, str) + or not Path(sdk_root).is_absolute() + ): + raise ProvenanceError( + "environment does not publish one installed native SDK package and root" + ) + _public_version(package.get("version")) + return sdk_root, str(package["version"]) + + +def _tool_identity( + tools: Mapping[str, Any], name: str +) -> dict[str, Any]: + record = tools.get(name) + if not isinstance(record, Mapping): + raise ProvenanceError(f"environment did not publish required build tool {name}") + path = record.get("path") + sha256 = record.get("sha256") + if not isinstance(path, str) or not isinstance(sha256, str): + raise ProvenanceError( + f"environment has no canonical path and SHA-256 for build tool {name}" + ) + # The returned value stays an exact artifact identity: it is sealed and + # re-verified before every launch, so it carries no extra keys. + observed = _identity(Path(path), f"native-build-tool-{name}") + # Schema 1.2 binds a build tool by path, size and content hash; every field + # it published must still match, and it publishes no tool version. + if any( + record.get(field) not in (None, observed[field]) + for field in ("path", "size_bytes", "sha256") + ): + raise ProvenanceError(f"build tool {name} changed after the environment probe") + return observed + + +def _native_toolchain( + environment: Mapping[str, Any], +) -> tuple[dict[str, dict[str, Any]], str, Path, Path]: + """Re-hash the published build toolchain and resolve the CUDA root live.""" + native = artifact_io.environment_surface(environment, "native") or {} + records = native.get("tools") + if not isinstance(records, Mapping): + raise ProvenanceError("environment publishes no native build tools") + generator_record = records.get("generator") + generator = ( + generator_record.get("name") if isinstance(generator_record, Mapping) else None + ) + if generator not in _CMAKE_GENERATORS: + raise ProvenanceError( + f"environment generator must be one of {sorted(_CMAKE_GENERATORS)}" + ) + tools = {name: _tool_identity(records, name) for name in _BUILD_TOOLS} + cuda = native.get("cuda") + if not isinstance(cuda, Mapping) or cuda.get("status") != "installed": + raise ProvenanceError("environment does not report an installed CUDA toolkit") + cuda_version = cuda.get("version") + if ( + not isinstance(cuda_version, str) + or not cuda_version + or not cuda_version.startswith("13.") + ): + raise ProvenanceError("environment does not report a required CUDA 13.x version") + cuda_root_value = cuda.get("root") + if not isinstance(cuda_root_value, str): + raise ProvenanceError("environment did not record the selected CUDA root") + cuda_root = Path(cuda_root_value).resolve(strict=True) + if not Path(tools["nvcc"]["path"]).is_relative_to(cuda_root): + raise ProvenanceError("environment nvcc is outside the selected CUDA root") + marker = next( + ( + path + for path in (cuda_root / "version.json", cuda_root / "version.txt") + if path.is_file() + ), + None, + ) + if marker is None: + raise ProvenanceError("selected CUDA root has no version marker") + return tools, str(generator), cuda_root, marker + + +def _package_authority( + service: Any, + commands: list[dict[str, Any]], + *, + cwd: Path, + timeout_seconds: float, + expected_version: str, +) -> tuple[set[Path], set[Path]]: + """Re-authenticate live package ownership and payload integrity.""" + _, output, _ = _run_prepare( + service, + commands, + [ + "/usr/bin/dpkg-query", + "-W", + "-f=${db:Status-Abbrev}\\t${Version}\\t${binary:Package}\\n", + "nvidia-video-codec-sdk", + ], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-version", + ) + if output.rstrip("\n").split("\t") != [ + "ii ", + expected_version, + "nvidia-video-codec-sdk", + ]: + raise ProvenanceError("live dpkg package identity differs from the environment") + _, listing, _ = _run_prepare( + service, + commands, + ["/usr/bin/dpkg-query", "-L", "nvidia-video-codec-sdk"], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-files", + ) + owned: set[Path] = set() + directories: set[Path] = set() + for line in listing.splitlines(): + if not line.startswith("/"): + continue + entry = Path(line) + if entry.is_dir(): + directories.add(entry.resolve(strict=True)) + elif entry.is_file(): + owned.add(entry.resolve(strict=True)) + _, verified, _ = _run_prepare( + service, + commands, + ["/usr/bin/dpkg", "--verify", "nvidia-video-codec-sdk"], + cwd=cwd, + timeout_seconds=timeout_seconds, + stage="native-package-verify", + ) + if verified.strip(): + raise ProvenanceError("dpkg --verify reported modified native SDK files") + return owned, directories + + +def _authenticated_sdk_root(declared: str, directories: set[Path]) -> Path: + """Prove the published SDK root is a live package-owned directory.""" + try: + root = Path(declared).resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ProvenanceError( + f"published native SDK root is unavailable: {declared}" + ) from exc + if not root.is_dir() or root not in directories: + raise ProvenanceError( + "published native SDK root is not a live package-owned directory" + ) + return root + + +def _cache_values(path: Path) -> dict[str, str]: + values: dict[str, str] = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if line.startswith(("//", "#")) or "=" not in line or ":" not in line: + continue + name_type, value = line.split("=", 1) + name, _field_type = name_type.split(":", 1) + values[name] = value + return values + + +def _compile_sources(path: Path) -> set[Path]: + value = artifact_io.strict_json_loads(path.read_bytes()) + if not isinstance(value, list) or not value: + raise ProvenanceError("CMake compile_commands.json must be a non-empty array") + if any( + not isinstance(entry, dict) or not isinstance(entry.get("file"), str) + for entry in value + ): + raise ProvenanceError("compile_commands.json entry has no source file") + return {Path(entry["file"]).resolve(strict=True) for entry in value} + + +def _runtime_libraries( + output: str, required: Sequence[str] +) -> list[dict[str, Any]]: + if re.search(r"(?m)=>\s+not found\s*$", output): + raise ProvenanceError("ldd reported an unresolved runtime library") + resolved: dict[str, Path] = {} + for line in output.splitlines(): + match = re.match(r"\s*(\S+)\s+=>\s+(\/\S+)\s+\(", line) + if match: + library, path = match.groups() + candidate = Path(path).resolve(strict=True) + if "stubs" in candidate.parts: + raise ProvenanceError(f"ldd resolved a CUDA stub library: {candidate}") + resolved[library] = candidate + missing = [name for name in required if name not in resolved] + if missing: + raise ProvenanceError( + f"ldd did not resolve required codec libraries: {missing}" + ) + return [ + _identity(resolved[name], f"native-runtime-library-{name}") + for name in required + ] + + +def _prepare_native( # pylint: disable=too-many-arguments,too-many-locals + sample: str, + *, + sdk_root: Path, + owned: set[Path], + environment: dict[str, Any], + environment_identity: Mapping[str, Any], + environment_workspace: Path, + workspace: Path, + service: Any, + commands: list[dict[str, Any]], + timeout_seconds: float, +) -> tuple[AuthenticatedSample, dict[str, Any]]: + binary_suffix, source_suffix, cmake_suffix, libraries = NATIVE_ROUTES[sample] + source = (sdk_root / source_suffix).resolve(strict=True) + cmake_file = (sdk_root / cmake_suffix).resolve(strict=True) + top_cmake = (sdk_root / "Samples/CMakeLists.txt").resolve(strict=True) + if not {source, cmake_file, top_cmake} <= owned: + raise ProvenanceError(f"{sample} source/CMake files are not package-owned") + tools, generator, cuda_root, marker = _native_toolchain(environment) + build = artifact_io.create_private_workspace( + workspace / f"native-build-{sample}" + ) + configure = [ + tools["cmake"]["path"], + "-S", + str(sdk_root / "Samples"), + "-B", + str(build), + "-G", + generator, + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", + f"-DCMAKE_CXX_COMPILER={tools['cxx']['path']}", + f"-DCMAKE_CUDA_COMPILER={tools['nvcc']['path']}", + f"-DCMAKE_MAKE_PROGRAM={tools['generator']['path']}", + f"-DCMAKE_PREFIX_PATH={cuda_root}", + ] + _run_prepare( + service, + commands, + configure, + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"configure-{sample.lower()}", + ) + _run_prepare( + service, + commands, + [ + tools["cmake"]["path"], + "--build", + str(build), + "--target", + sample, + "--parallel", + "1", + ], + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"build-{sample.lower()}", + ) + cache_path = build / "CMakeCache.txt" + compile_path = build / "compile_commands.json" + expected_cache = { + "CMAKE_HOME_DIRECTORY": str(sdk_root / "Samples"), + "CMAKE_CXX_COMPILER": tools["cxx"]["path"], + "CMAKE_CUDA_COMPILER": tools["nvcc"]["path"], + "CMAKE_MAKE_PROGRAM": tools["generator"]["path"], + "CMAKE_GENERATOR": generator, + } + cache = _cache_values(cache_path) + for name, expected in expected_cache.items(): + observed = cache.get(name) + differs = ( + observed != expected + if name == "CMAKE_GENERATOR" + else not observed or Path(observed).resolve() != Path(expected).resolve() + ) + if differs: + raise ProvenanceError(f"CMake cache disagrees for {name}") + compiled = _compile_sources(compile_path) + if source not in compiled or not compiled <= owned: + raise ProvenanceError( + f"{sample} compile target is not bound to package-owned sources" + ) + executable = (build / binary_suffix).resolve(strict=True) + if not stat.S_ISREG(executable.stat().st_mode) or executable.stat().st_mode & 0o111 == 0: + raise ProvenanceError(f"built {sample} output is not executable") + _, linkage, _ = _run_prepare( + service, + commands, + ["/usr/bin/ldd", str(executable)], + cwd=workspace, + timeout_seconds=timeout_seconds, + stage=f"linkage-{sample.lower()}", + ) + runtime = _runtime_libraries(linkage, libraries) + protected = [ + _identity(executable, "native-official-sample-executable"), + *(_identity(path, "native-package-source") for path in sorted(compiled)), + _identity(cache_path, "native-cmake-cache"), + _identity(compile_path, "native-compile-commands"), + _identity(marker, "cuda-root-version-marker"), + *tools.values(), + ] + record = _sample_record( + surface="native", + sample=sample, + launcher=(str(executable),), + workspace=workspace, + protected=protected, + runtime=runtime, + environment_workspace=environment_workspace, + environment_identity=environment_identity, + ) + return record, { + "sample": sample, + "launcher": list(record.launcher), + "source": str(source), + "configure_argv": configure, + "compiled_sources": [str(path) for path in sorted(compiled)], + "protected_files": protected, + "runtime_libraries": runtime, + } + + +def authenticate_native_samples( # pylint: disable=too-many-arguments,too-many-locals + *, + environment_workspace: Path, + environment_identity: Mapping[str, Any], + workspace: Path, + samples: Sequence[str], + runner: Any = None, + timeout_seconds: float = 600, + report_path: Path | None = None, +) -> NativeAuthentication: + """Build and authenticate exact pipeline-owned native sample routes.""" + root = artifact_io.resolve_private_workspace(workspace) + env_root = artifact_io.resolve_private_workspace(environment_workspace) + requested = tuple(samples) + if not requested or len(set(requested)) != len(requested): + raise ProvenanceError("native sample list must be non-empty and unique") + unknown = sorted(set(requested) - set(NATIVE_ROUTES)) + if unknown: + raise ProvenanceError(f"native sample is not allowlisted: {unknown}") + destination = report_path or root / "native-sample-provenance.json" + commands: list[dict[str, Any]] = [] + evidence: list[dict[str, Any]] = [] + records: list[AuthenticatedSample] = [] + service = runner or command_runner.CommandRunner(root) + report: dict[str, Any] = { + "schema_version": "1", + "kind": "native-sample-provenance", + "status": "failed", + "environment": dict(environment_identity), + "environment_workspace": str(env_root), + "requested_samples": list(requested), + "samples": evidence, + "commands": commands, + } + try: + environment = _environment_value(env_root, environment_identity, "native") + declared_root, package_version = _native_surface(environment) + owned, directories = _package_authority( + service, + commands, + cwd=root, + timeout_seconds=timeout_seconds, + expected_version=package_version, + ) + sdk_root = _authenticated_sdk_root(declared_root, directories) + for sample in requested: + record, item = _prepare_native( + sample, + sdk_root=sdk_root, + owned=owned, + environment=environment, + environment_identity=environment_identity, + environment_workspace=env_root, + workspace=root, + service=service, + commands=commands, + timeout_seconds=timeout_seconds, + ) + records.append(record) + evidence.append(item) + report["status"] = "authenticated" + report["package"] = { + "name": "nvidia-video-codec-sdk", + "version": package_version, + "root": str(sdk_root), + "root_authentication": { + "declared_root": declared_root, + "method": "live-dpkg-ownership", + "package_owned_directory": True, + "payload_verified": True, + }, + } + except Exception as exc: + report["failure"] = f"{type(exc).__name__}: {exc}" + artifact_io.write_fresh_json(root, destination, report) + raise + return NativeAuthentication( + tuple(records), artifact_io.write_fresh_json(root, destination, report) + ) + + +_PROBE_PROGRAM = r""" +import importlib.metadata +import json +import pathlib +import re +import sys +dist = importlib.metadata.distribution("PyNvVideoCodec") +name = str(dist.metadata.get("Name", "")) +files = [str(item).replace("\\", "/") for item in (dist.files or [])] +records = [item for item in files if item.endswith(".dist-info/RECORD")] +if len(records) != 1: + raise RuntimeError("expected exactly one PyNvVideoCodec RECORD entry") +root = pathlib.Path(dist.locate_file(pathlib.Path("."))).resolve() +paths = { + item: str(pathlib.Path(dist.locate_file(pathlib.Path(item))).resolve()) + for item in files +} +print(json.dumps({ + "schema_version": "1", "kind": "pynvc-record-probe", + "interpreter": str(pathlib.Path(sys.executable).resolve()), + "sys_prefix": str(pathlib.Path(sys.prefix).resolve()), "name": name, + "normalized_name": re.sub(r"[-_.]+", "-", name).lower(), + "version": str(dist.version), "record_entry": records[0], + "distribution_root": str(root), "paths": paths, +}, sort_keys=True)) +""".strip() + + +def _probe_record( + *, + python: Path, + workspace: Path, + runner: Any, + timeout_seconds: float, +) -> dict[str, Any]: + root = artifact_io.resolve_private_workspace(workspace) + launcher = Path(python) + if not launcher.is_absolute(): + raise ProvenanceError("selected virtual-environment Python must be absolute") + target = launcher.resolve(strict=True) + if not target.is_file() or target.stat().st_mode & 0o111 == 0: + raise ProvenanceError("selected virtual-environment Python is not executable") + result = runner.run( + [str(launcher), "-I", "-c", _PROBE_PROGRAM], + cwd=root, + env=_COMMAND_ENV, + timeout_seconds=timeout_seconds, + stage="pynvc-record-probe", + phase="probe", + ) + runner_root = artifact_io.resolve_private_workspace(Path(runner.workspace)) + stderr = artifact_io.read_verified_text(runner_root, result["stderr"]) + if ( + result.get("timed_out") + or result.get("launch_error") + or result.get("exit_code") != 0 + ): + raise ProvenanceError( + f"isolated PyNvVideoCodec RECORD probe failed: {stderr[-500:]}" + ) + value = artifact_io.strict_json_loads( + artifact_io.read_verified_text(runner_root, result["stdout"]) + ) + if ( + not isinstance(value, dict) + or value.get("schema_version") != "1" + or value.get("kind") != "pynvc-record-probe" + or value.get("interpreter") != str(target) + or re.sub(r"[-_.]+", "-", str(value.get("name", ""))).lower() + != "pynvvideocodec" + or value.get("normalized_name") != "pynvvideocodec" + or value.get("version") != "2.1.0" + or not isinstance(value.get("paths"), dict) + ): + raise ProvenanceError( + "isolated probe did not select exact PyNvVideoCodec 2.1.0" + ) + prefix = Path(value.get("sys_prefix", "")).resolve(strict=True) + if launcher.parent.parent.resolve(strict=True) != prefix: + raise ProvenanceError("isolated probe sys.prefix differs from selected venv") + for member, path_value in value["paths"].items(): + if ( + not isinstance(member, str) + or member != member.replace("\\", "/") + or not isinstance(path_value, str) + or not Path(path_value).resolve(strict=True).is_relative_to(prefix) + ): + raise ProvenanceError("wheel member resolved outside selected venv") + value["command"] = result + return value + + +def _within_prefix(probe: Mapping[str, Any], path_value: Any, label: str) -> Path: + prefix_value = probe.get("sys_prefix") + if ( + not isinstance(prefix_value, str) + or not Path(prefix_value).is_absolute() + or not isinstance(path_value, str) + or not Path(path_value).is_absolute() + ): + raise ProvenanceError(f"{label} path is invalid") + prefix = Path(prefix_value).resolve(strict=True) + candidate = Path(path_value).resolve(strict=True) + if not candidate.is_relative_to(prefix): + raise ProvenanceError(f"{label} resolved outside selected venv") + return candidate + + +def _record_rows( + probe: Mapping[str, Any], +) -> tuple[dict[str, tuple[str, str]], dict[str, Any]]: + paths = probe.get("paths") + member = probe.get("record_entry") + if not isinstance(paths, dict) or not isinstance(member, str): + raise ProvenanceError("record probe shape is invalid") + record_path = _within_prefix(probe, paths.get(member), "wheel RECORD") + try: + rows = list( + csv.reader( + io.StringIO(record_path.read_bytes().decode("utf-8", errors="strict")) + ) + ) + except (UnicodeDecodeError, csv.Error) as exc: + raise ProvenanceError(f"wheel RECORD is invalid: {exc}") from exc + parsed: dict[str, tuple[str, str]] = {} + for row in rows: + if ( + len(row) != 3 + or not row[0] + or row[0] != row[0].replace("\\", "/") + or row[0] in parsed + ): + raise ProvenanceError("wheel RECORD row is not unique and canonical") + parsed[row[0]] = (row[1], row[2]) + return parsed, _identity(record_path, "pynvc-wheel-record") + + +def _record_members( + probe: Mapping[str, Any], members: Sequence[str] +) -> tuple[dict[str, Any], ...]: + rows, _record_identity = _record_rows(probe) + paths = probe["paths"] + result = [] + for member in members: + if member not in rows or member not in paths: + raise ProvenanceError(f"wheel RECORD member is missing: {member!r}") + encoded_hash, size_text = rows[member] + if not encoded_hash.startswith("sha256=") or not size_text.isdecimal(): + raise ProvenanceError(f"wheel RECORD member has no SHA-256/size: {member}") + path = _within_prefix(probe, paths[member], "wheel member") + payload = path.read_bytes() + actual = ( + base64.urlsafe_b64encode(hashlib.sha256(payload).digest()) + .rstrip(b"=") + .decode() + ) + if encoded_hash.removeprefix("sha256=") != actual or int(size_text) != len(payload): + raise ProvenanceError(f"installed wheel member differs from RECORD: {member}") + result.append( + { + "member": member, + "record_hash": encoded_hash, + "identity": _identity(path, "pynvc-wheel-member"), + } + ) + return tuple(result) + + +def _pynvc_environment( + workspace: Path, identity: Mapping[str, Any] +) -> tuple[dict[str, Any], Path, Path, dict[str, Any]]: + """Resolve the schema-1.2 Py surface identity into a launchable venv. + + The normalized surface carries identity only: the exact lexical + ``interpreter``, the resolved ``interpreter_identity``, ``sys_prefix``, the + PyNv ``version`` and the ``extension``. Everything the artifact asserts is + re-authenticated live from the isolated RECORD probe. + """ + value = _environment_value(workspace, identity, "PyNVC") + surface = artifact_io.environment_surface(value, "pynvc") + if surface is None or surface.get("version") != "2.1.0": + raise ProvenanceError( + "environment lacks one installed PyNvVideoCodec 2.1.0 surface" + ) + launcher_value = surface.get("interpreter") + prefix = surface.get("sys_prefix") + resolved = surface.get("interpreter_identity") + interpreter = resolved.get("path") if isinstance(resolved, Mapping) else None + if ( + not isinstance(launcher_value, str) + or not Path(launcher_value).is_absolute() + or not isinstance(prefix, str) + or not isinstance(interpreter, str) + ): + raise ProvenanceError("environment lacks selected PyNvVideoCodec venv") + launcher = Path(launcher_value) + target = launcher.resolve(strict=True) + if ( + not launcher.is_file() + or launcher.stat().st_mode & 0o111 == 0 + or target != Path(interpreter).resolve(strict=True) + or launcher.parent.parent.resolve(strict=True) != Path(prefix).resolve(strict=True) + ): + raise ProvenanceError("selected venv launcher or prefix is invalid") + return value, launcher, target, surface + + +def _metadata_members(probe: Mapping[str, Any]) -> tuple[str, str, str]: + record = probe.get("record_entry") + if not isinstance(record, str) or not record.endswith(".dist-info/RECORD"): + raise ProvenanceError("wheel RECORD member is invalid") + root = record.removesuffix("RECORD") + metadata, wheel = root + "METADATA", root + "WHEEL" + paths = probe.get("paths") + if not isinstance(paths, dict) or metadata not in paths or wheel not in paths: + raise ProvenanceError("wheel METADATA/WHEEL members are missing") + return metadata, wheel, record + + +def _module_member(surface: Mapping[str, Any], probe: Mapping[str, Any]) -> str: + """Bind the imported module to the exact wheel RECORD member. + + The imported ``module.version`` must equal the distribution version, so an + interpreter that imported some other build fails closed here rather than + being inferred as correct from metadata alone. + """ + module = surface.get("module") + paths = probe.get("paths") + if ( + not isinstance(module, Mapping) + or not isinstance(paths, Mapping) + or not isinstance(module.get("path"), str) + ): + raise ProvenanceError("environment module identity is invalid") + if module.get("version") != surface.get("version"): + raise ProvenanceError( + "imported module version differs from the distribution version" + ) + if module["version"] != probe.get("version"): + raise ProvenanceError( + "imported module version differs from the live distribution probe" + ) + matches = [name for name, path in paths.items() if path == module["path"]] + if matches != ["PyNvVideoCodec/__init__.py"]: + raise ProvenanceError("package module is not an exact wheel RECORD member") + return matches[0] + + +def _extension_identity( + surface: Mapping[str, Any], probe: Mapping[str, Any] +) -> list[dict[str, Any]]: + """Bind the authenticated extension to live bytes inside the venv prefix. + + Hashing a ``.so`` found through distribution metadata proves only that a + matching file exists on disk, so the authenticated path is resolved inside + the probe's own prefix and re-hashed here. ``loaded_path`` is additive: 1.2 + publishes none, and an environment that does carry one must name the very + same file rather than a second build. + """ + extension = surface.get("extension") + if not isinstance(extension, Mapping): + raise ProvenanceError( + "environment publishes no PyNvVideoCodec extension identity" + ) + path_value = extension.get("path") + sha256 = extension.get("sha256") + loaded_path = extension.get("loaded_path") + if loaded_path is None: + loaded_path = path_value + if ( + not isinstance(path_value, str) + or not isinstance(sha256, str) + or not isinstance(loaded_path, str) + or not path_value.endswith(".so") + ): + raise ProvenanceError( + "environment publishes no PyNvVideoCodec extension identity" + ) + if loaded_path != path_value: + raise ProvenanceError( + "the interpreter loaded a different extension than the authenticated one" + ) + candidate = _within_prefix(probe, path_value, "PyNvVideoCodec extension") + identity = _identity(candidate, "pynvc-extension") + if identity["sha256"] != sha256: + raise ProvenanceError( + "installed PyNvVideoCodec extension differs from the environment" + ) + return [identity] + + +def authenticate_pynvc_samples( # pylint: disable=too-many-arguments,too-many-locals + *, + environment_workspace: Path, + environment_identity: Mapping[str, Any], + workspace: Path, + samples: Sequence[str], + runner: Any = None, + timeout_seconds: float = 600, + report_path: Path | None = None, +) -> PynvcAuthentication: + """Authenticate exact pipeline-owned PyNvVideoCodec wheel samples.""" + root = artifact_io.resolve_private_workspace(workspace) + env_root = artifact_io.resolve_private_workspace(environment_workspace) + requested = tuple(samples) + if not requested or len(set(requested)) != len(requested): + raise ProvenanceError("PyNvVideoCodec sample list must be non-empty and unique") + unknown = sorted(set(requested) - set(PYNVC_ROUTES)) + if unknown: + raise ProvenanceError(f"PyNvVideoCodec sample is not allowlisted: {unknown}") + destination = report_path or root / "pynvc-sample-provenance.json" + report: dict[str, Any] = { + "schema_version": "1", + "kind": "pynvc-sample-provenance", + "status": "failed", + "environment": dict(environment_identity), + "requested_samples": list(requested), + "samples": [], + } + try: + _value, launcher, target, surface = _pynvc_environment( + env_root, environment_identity + ) + service = runner or command_runner.CommandRunner(root) + probe = _probe_record( + python=launcher, + workspace=root, + runner=service, + timeout_seconds=timeout_seconds, + ) + if ( + Path(str(probe.get("sys_prefix"))).resolve(strict=True) + != Path(str(surface.get("sys_prefix"))).resolve(strict=True) + ): + raise ProvenanceError( + "isolated RECORD probe differs from environment identity" + ) + metadata, wheel, record = _metadata_members(probe) + module = _module_member(surface, probe) + common = _record_members(probe, (module, metadata, wheel)) + _rows, record_identity = _record_rows(probe) + extension_protected = _extension_identity(surface, probe) + interpreter_identity = _identity(target, "pynvc-venv-python") + published = surface.get("interpreter_identity") + if ( + not isinstance(published, Mapping) + or published.get("sha256") != interpreter_identity["sha256"] + ): + raise ProvenanceError( + "selected venv interpreter differs from the environment identity" + ) + records: list[AuthenticatedSample] = [] + for sample in requested: + members = (sample, *PYNVC_ROUTES[sample]) + owned = _record_members(probe, members) + protected = [ + interpreter_identity, + record_identity, + *(item["identity"] for item in common), + *(item["identity"] for item in owned), + *extension_protected, + ] + record_value = _sample_record( + surface="pynvc", + sample=sample, + launcher=(str(launcher), "-I", probe["paths"][sample]), + workspace=root, + protected=protected, + runtime=(), + environment_workspace=env_root, + environment_identity=environment_identity, + ) + records.append(record_value) + report["samples"].append( + { + "sample": sample, + "launcher": list(record_value.launcher), + "record_members": list(members), + "protected_files": protected, + } + ) + report["status"] = "authenticated" + report["distribution"] = { + "reported_name": probe["name"], + "normalized_name": probe["normalized_name"], + "version": probe["version"], + } + report["record_probe_command"] = probe["command"] + report["record"] = record_identity + report["record_member"] = record + except Exception as exc: + report["failure"] = f"{type(exc).__name__}: {exc}" + artifact_io.write_fresh_json(root, destination, report) + raise + return PynvcAuthentication( + tuple(records), artifact_io.write_fresh_json(root, destination, report) + ) + + +def run_authenticated( # pylint: disable=too-many-arguments + token: AuthenticatedSample, + arguments: Sequence[str], + *, + workspace: Path, + runner: Any, + cwd: Path, + timeout_seconds: float, + stage: str, + phase: str, +) -> dict[str, Any]: + """Rehash every seal immediately before launching one exact sample.""" + return artifact_io.run_authenticated_sample( + token, + arguments, + token_type=AuthenticatedSample, + error_type=ProvenanceError, + command_environment=_COMMAND_ENV, + workspace=workspace, + runner=runner, + cwd=cwd, + timeout_seconds=timeout_seconds, + stage=stage, + phase=phase, + ) diff --git a/skills/jetson-video-pipeline/scripts/_pipeline_runtime.py b/skills/jetson-video-pipeline/scripts/_pipeline_runtime.py new file mode 100644 index 00000000..093ae1a5 --- /dev/null +++ b/skills/jetson-video-pipeline/scripts/_pipeline_runtime.py @@ -0,0 +1,1561 @@ +#!/usr/bin/env python3 +"""Private artifact, command, routing, and recipe runtime for this skill.""" + +# This module is the skill's single private runtime owner: artifact identity, +# command execution, surface routing, environment validation and the recipe +# bridge each keep their complete contract local and reviewable in one file. +# pylint: disable=too-many-lines + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import hashlib +import json +import math +import os +import re +import signal +import stat +import subprocess +import sys +import time +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any, Mapping, Sequence + +IDENTITY_KEYS = frozenset( + {"schema_version", "kind", "path", "size_bytes", "sha256"} +) +MAX_JSON_BYTES = 64 * 1024 * 1024 +MAX_ARTIFACT_BYTES = 64 * 1024 * 1024 * 1024 +_HASH_CHUNK_BYTES = 1024 * 1024 +_TAIL_BYTES = 4096 +_STAGE_PATTERN = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,63}\Z") +COMMAND_PHASES = ( + "probe", + "plan", + "prepare", + "install", + "warmup", + "measure", + "verify", + "execute", +) +SURFACE_ORDER = ("native", "pynvc") +REQUESTED_SURFACES = ("native", "pynvc", "auto", "both") +ENVIRONMENT_KIND = "nvcodec-environment" +# The frozen `nvcodec-environment` 1.2 contract. Consumers pin this exact +# version and validate its required subset only; additive keys are tolerated. +ENVIRONMENT_SCHEMA_VERSION = "1.2" +ENVIRONMENT_RUNTIMES = ("pynvc", "native", "both") +LOCAL_RUNTIME_BINDING_KIND = "nvcodec-local-runtime-binding" +LOCAL_RUNTIME_BINDING_SCHEMA_VERSION = "1.0" +CAPABILITY_REPORT_KIND = "nvcodec-encoder-capability-report" +CAPABILITY_REPORT_SCHEMA_VERSION = "1.0" +NATIVE_PACKAGE_NAME = "nvidia-video-codec-sdk" +REQUIRED_BUILD_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +# 1.2 publishes `installation.build_tool_identities` under the executable's own +# name; these are the internal names the toolchain is consumed under. +_PUBLISHED_BUILD_TOOLS = ( + ("cmake", "cmake"), + ("cxx", "g++"), + ("nvcc", "nvcc"), + ("pkg_config", "pkg-config"), +) +# CMake generator preference, highest first, with its exact generator name. +_PUBLISHED_GENERATORS = (("ninja", "Ninja"), ("make", "Unix Makefiles")) +_ELIGIBILITY_KEYS = frozenset({"eligible", "reasons"}) +_RECIPE_OUTPUT_LIMIT = 1024 * 1024 + + +class ArtifactError(ValueError): + """An artifact violates the portable content contract.""" + + +class CommandError(ValueError): + """A command violates the deterministic execution contract.""" + + +class RecipeBridgeError(ValueError): + """The installed recipe skill rejected a recipe operation.""" + + +class SkillDependencyError(RecipeBridgeError): + """A route cannot continue because one required sibling skill is absent.""" + + def __init__(self, skill: str, needed_for: str): + self.skill = skill + self.needed_for = needed_for + super().__init__( + f"I can run {needed_for}, but it requires the {skill} skill, " + f"which is not installed. Install {skill} and retry this stage." + ) + + +def _reject_constant(value: str) -> None: + raise ValueError(f"non-finite JSON number is not allowed: {value}") + + +def _object_from_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + value: dict[str, Any] = {} + for key, item in pairs: + if key in value: + raise ValueError(f"duplicate JSON object name: {key!r}") + value[key] = item + return value + + +def strict_json_loads(payload: bytes | str) -> Any: + """Decode strict UTF-8 JSON without duplicate names or non-finites.""" + if isinstance(payload, bytes): + try: + text = payload.decode("utf-8", errors="strict") + except UnicodeDecodeError as exc: + raise ArtifactError(f"JSON is not UTF-8: {exc}") from exc + elif isinstance(payload, str): + text = payload + else: + raise ArtifactError("JSON payload must be bytes or text") + try: + return json.loads( + text, + object_pairs_hook=_object_from_pairs, + parse_constant=_reject_constant, + ) + except (json.JSONDecodeError, ValueError) as exc: + raise ArtifactError(f"invalid strict JSON: {exc}") from exc + + +def canonical_json_bytes(value: Any) -> bytes: + """Encode deterministic strict JSON as UTF-8 with one trailing newline.""" + try: + rendered = json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + indent=2, + sort_keys=True, + ) + except (TypeError, ValueError) as exc: + raise ArtifactError( + f"value cannot be represented as strict JSON: {exc}" + ) from exc + return (rendered + "\n").encode("utf-8") + + +def _contract_string(value: Any, label: str) -> str: + if not isinstance(value, str) or not value or value != value.strip(): + raise ArtifactError(f"{label} must be a non-empty canonical string") + return value + + +def _contract_path(value: Any) -> str: + locator = _contract_string(value, "path") + if any(ord(character) < 32 or ord(character) == 127 for character in locator): + raise ArtifactError("path must not contain control characters") + return locator + + +def _bound(value: Any, label: str, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or value < 0: + raise ArtifactError(f"{label} must be a non-negative integer") + if value > maximum: + raise ArtifactError(f"{label} exceeds the {maximum}-byte bound") + return value + + +def _existing_regular(path: Path) -> Path: + requested = Path(_contract_path(str(path))) + try: + candidate = requested.expanduser().resolve(strict=True) + details = candidate.stat() + except (OSError, RuntimeError) as exc: + raise ArtifactError(f"artifact is not an accessible file: {path}") from exc + if not stat.S_ISREG(details.st_mode): + raise ArtifactError(f"artifact is not a regular file: {candidate}") + return candidate + + +def create_private_workspace(path: Path) -> Path: + """Create one fresh canonical managed workspace with exact mode 0700.""" + requested = Path(_contract_path(str(path))).expanduser() + try: + parent = requested.parent.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ArtifactError( + f"workspace parent is unavailable: {requested.parent}" + ) from exc + candidate = parent / requested.name + try: + os.mkdir(candidate, mode=0o700) + os.chmod(candidate, 0o700) + except OSError as exc: + raise ArtifactError( + f"cannot create fresh workspace {candidate}: {exc}" + ) from exc + return resolve_private_workspace(candidate) + + +def resolve_private_workspace(path: Path) -> Path: + """Resolve an existing managed workspace and require exact mode 0700.""" + requested = Path(_contract_path(str(path))) + try: + candidate = requested.expanduser().resolve(strict=True) + details = candidate.stat() + except (OSError, RuntimeError) as exc: + raise ArtifactError(f"workspace is unavailable: {path}") from exc + if not stat.S_ISDIR(details.st_mode): + raise ArtifactError(f"workspace is not a directory: {candidate}") + if stat.S_IMODE(details.st_mode) != 0o700: + raise ArtifactError(f"workspace mode must be exactly 0700: {candidate}") + return candidate + + +def preflight_fresh_output(workspace: Path, path: Path) -> Path: + """Validate a fresh result destination without creating the workspace.""" + requested_root = Path(_contract_path(str(workspace))).expanduser() + if requested_root.exists(): + root = resolve_private_workspace(requested_root) + else: + if requested_root.is_symlink(): + raise ArtifactError(f"workspace must not be a symlink: {requested_root}") + try: + parent = requested_root.parent.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ArtifactError( + f"workspace parent is unavailable: {requested_root.parent}" + ) from exc + root = parent / requested_root.name + requested = Path(_contract_path(str(path))).expanduser() + candidate = requested if requested.is_absolute() else root / requested + try: + candidate = candidate.resolve(strict=False) + candidate.relative_to(root) + except (OSError, RuntimeError, ValueError) as exc: + raise ArtifactError(f"artifact is outside managed workspace: {candidate}") from exc + if candidate == root or candidate.exists() or candidate.is_symlink(): + raise ArtifactError(f"artifact output must be a fresh file: {candidate}") + return candidate + + +def _workspace_path(workspace: Path, path: Path, *, must_exist: bool) -> Path: + root = resolve_private_workspace(workspace) + requested = Path(_contract_path(str(path))) + candidate = requested if requested.is_absolute() else root / requested + try: + resolved = candidate.resolve(strict=must_exist) + except (OSError, RuntimeError) as exc: + raise ArtifactError( + f"workspace artifact is unavailable: {candidate}" + ) from exc + try: + resolved.relative_to(root) + except ValueError as exc: + raise ArtifactError( + f"artifact is outside managed workspace: {resolved}" + ) from exc + if must_exist: + return _existing_regular(resolved) + try: + parent = resolved.parent.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise ArtifactError( + f"artifact output parent is unavailable: {resolved.parent}" + ) from exc + if not parent.is_dir(): + raise ArtifactError(f"artifact output parent is not a directory: {parent}") + return parent / resolved.name + + +def _hash(path: Path, maximum: int = MAX_ARTIFACT_BYTES) -> tuple[int, str]: + digest = hashlib.sha256() + total = 0 + try: + with path.open("rb") as stream: + while True: + chunk = stream.read(_HASH_CHUNK_BYTES) + if not chunk: + break + total += len(chunk) + if total > maximum: + raise ArtifactError( + f"artifact exceeds the {maximum}-byte bound: {path}" + ) + digest.update(chunk) + except OSError as exc: + raise ArtifactError(f"cannot read artifact {path}: {exc}") from exc + return total, digest.hexdigest() + + +def _identity( + path_locator: str, + size_bytes: int, + sha256: str, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + return { + "schema_version": _contract_string(schema_version, "schema_version"), + "kind": _contract_string(kind, "kind"), + "path": _contract_path(path_locator), + "size_bytes": size_bytes, + "sha256": sha256, + } + + +def _validate_identity(identity: Mapping[str, Any]) -> tuple[str, int, str]: + if not isinstance(identity, Mapping) or set(identity) != IDENTITY_KEYS: + raise ArtifactError( + f"artifact identity fields must be exactly {sorted(IDENTITY_KEYS)}" + ) + _contract_string(identity["schema_version"], "schema_version") + _contract_string(identity["kind"], "kind") + locator = _contract_path(identity["path"]) + size_bytes = _bound(identity["size_bytes"], "size_bytes", MAX_ARTIFACT_BYTES) + sha256 = identity["sha256"] + if ( + not isinstance(sha256, str) + or len(sha256) != 64 + or any(character not in "0123456789abcdef" for character in sha256) + ): + raise ArtifactError("sha256 must be 64 lowercase hexadecimal characters") + return locator, size_bytes, sha256 + + +def snapshot_external_artifact( + path: Path, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + """Snapshot an external file with a canonical absolute path locator.""" + candidate = _existing_regular(Path(path)) + size_bytes, sha256 = _hash(candidate) + return _identity( + str(candidate), + size_bytes, + sha256, + schema_version=schema_version, + kind=kind, + ) + + +def verify_external_artifact(identity: Mapping[str, Any]) -> Path: + """Verify one external identity and return its canonical current path.""" + locator, expected_size, expected_sha = _validate_identity(identity) + if not Path(locator).is_absolute(): + raise ArtifactError("external artifact path must be absolute") + candidate = _existing_regular(Path(locator)) + if str(candidate) != locator: + raise ArtifactError("external artifact path must be canonical") + if _hash(candidate) != (expected_size, expected_sha): + raise ArtifactError(f"external artifact changed: {candidate}") + return candidate + + +def snapshot_artifact( + workspace: Path, + path: Path, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + """Snapshot a workspace file with a portable relative path locator.""" + root = resolve_private_workspace(workspace) + candidate = _workspace_path(root, path, must_exist=True) + size_bytes, sha256 = _hash(candidate) + return _identity( + candidate.relative_to(root).as_posix(), + size_bytes, + sha256, + schema_version=schema_version, + kind=kind, + ) + + +def _workspace_identity_path( + workspace: Path, identity: Mapping[str, Any] +) -> Path: + locator, _size_bytes, _sha256 = _validate_identity(identity) + pure = PurePosixPath(locator) + if ( + pure.is_absolute() + or not pure.parts + or any(part in {"", ".", ".."} for part in pure.parts) + or pure.as_posix() != locator + ): + raise ArtifactError( + "workspace artifact path must be a canonical relative locator" + ) + return _workspace_path(workspace, Path(*pure.parts), must_exist=True) + + +def verify_artifact(workspace: Path, identity: Mapping[str, Any]) -> Path: + """Verify one workspace identity and return its canonical current path.""" + candidate = _workspace_identity_path(workspace, identity) + _locator, expected_size, expected_sha = _validate_identity(identity) + if _hash(candidate) != (expected_size, expected_sha): + raise ArtifactError(f"workspace artifact changed: {candidate}") + return candidate + + +def write_fresh_bytes( + workspace: Path, + path: Path, + payload: bytes, + *, + schema_version: str, + kind: str, +) -> dict[str, Any]: + """Write one new 0600 workspace artifact and return its identity.""" + if not isinstance(payload, bytes): + raise ArtifactError("artifact payload must be bytes") + _bound(len(payload), "artifact payload", MAX_ARTIFACT_BYTES) + root = resolve_private_workspace(workspace) + candidate = _workspace_path(root, path, must_exist=False) + try: + descriptor = os.open(candidate, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + except OSError as exc: + raise ArtifactError(f"cannot create fresh artifact {candidate}: {exc}") from exc + try: + os.fchmod(descriptor, 0o600) + with os.fdopen(descriptor, "wb") as stream: + descriptor = -1 + stream.write(payload) + stream.flush() + os.fsync(stream.fileno()) + except OSError as exc: + if descriptor >= 0: + os.close(descriptor) + try: + candidate.unlink(missing_ok=True) + except OSError: + pass + raise ArtifactError(f"cannot write fresh artifact {candidate}: {exc}") from exc + return snapshot_artifact( + root, + candidate, + schema_version=schema_version, + kind=kind, + ) + + +def write_fresh_json(workspace: Path, path: Path, value: Any) -> dict[str, Any]: + """Write strict JSON, taking schema version and kind from its envelope.""" + if not isinstance(value, dict): + raise ArtifactError("JSON artifact must be an object") + return write_fresh_bytes( + workspace, + path, + canonical_json_bytes(value), + schema_version=_contract_string(value.get("schema_version"), "schema_version"), + kind=_contract_string(value.get("kind"), "kind"), + ) + + +def _read_bounded_exact_bytes( + path: Path, + *, + expected_size: int, + expected_sha256: str, + max_bytes: int, +) -> bytes: + """Read no more than one byte past an artifact's expected bounded size. + + Requesting exactly ``size_bytes + 1`` means an artifact that grew after its + identity was recorded is never fully materialised: the extra byte only + proves that it grew. Length and SHA-256 are both re-verified after the read, + so the returned bytes are exactly the ones the identity names. + """ + bound = _bound(max_bytes, "max_bytes", MAX_ARTIFACT_BYTES) + size_bytes = _bound(expected_size, "size_bytes", bound) + try: + with path.open("rb") as stream: + payload = stream.read(size_bytes + 1) + except OSError as exc: + raise ArtifactError(f"cannot read artifact {path}: {exc}") from exc + if len(payload) > bound: + raise ArtifactError(f"artifact exceeds the {bound}-byte read bound: {path}") + if len(payload) != size_bytes: + raise ArtifactError(f"artifact changed while it was read: {path}") + if hashlib.sha256(payload).hexdigest() != expected_sha256: + raise ArtifactError(f"artifact changed while it was read: {path}") + return payload + + +def _verified_payload( + workspace: Path, + identity: Mapping[str, Any], + maximum: int, +) -> bytes: + bound = _bound(maximum, "max_bytes", MAX_ARTIFACT_BYTES) + _locator, size_bytes, sha256 = _validate_identity(identity) + _bound(size_bytes, "size_bytes", bound) + path = verify_artifact(workspace, identity) + return _read_bounded_exact_bytes( + path, + expected_size=size_bytes, + expected_sha256=sha256, + max_bytes=bound, + ) + + +def read_verified_bytes( + workspace: Path, + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> bytes: + """Verify and bounded-read one workspace byte artifact.""" + return _verified_payload(workspace, identity, max_bytes) + + +def read_verified_text( + workspace: Path, + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> str: + """Verify and bounded-read one UTF-8 workspace text artifact.""" + try: + return _verified_payload(workspace, identity, max_bytes).decode( + "utf-8", errors="strict" + ) + except UnicodeDecodeError as exc: + raise ArtifactError(f"verified artifact is not UTF-8: {exc}") from exc + + +def read_verified_json( + workspace: Path, + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> Any: + """Verify, bounded-read, and strictly parse one workspace JSON artifact.""" + return strict_json_loads(_verified_payload(workspace, identity, max_bytes)) + + +def read_verified_external_json( + identity: Mapping[str, Any], + *, + max_bytes: int = MAX_JSON_BYTES, +) -> Any: + """Verify, bounded-read, and strictly parse one external JSON artifact.""" + bound = _bound(max_bytes, "max_bytes", MAX_ARTIFACT_BYTES) + _locator, size_bytes, sha256 = _validate_identity(identity) + _bound(size_bytes, "size_bytes", bound) + path = verify_external_artifact(identity) + return strict_json_loads( + _read_bounded_exact_bytes( + path, + expected_size=size_bytes, + expected_sha256=sha256, + max_bytes=bound, + ) + ) + + +def _command_argv(value: Sequence[str]) -> list[str]: + if isinstance(value, (str, bytes)) or not isinstance(value, Sequence): + raise CommandError("argv must be a non-empty sequence of strings") + copied = list(value) + if not copied or not copied[0]: + raise CommandError("argv[0] must be a non-empty executable name") + for index, token in enumerate(copied): + if not isinstance(token, str) or "\x00" in token: + raise CommandError(f"argv[{index}] must be a NUL-free string") + return copied + + +def _command_cwd(value: str | os.PathLike[str]) -> Path: + try: + candidate = Path(value).expanduser().resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise CommandError(f"cwd is unavailable: {value}") from exc + if not candidate.is_dir(): + raise CommandError(f"cwd is not a directory: {candidate}") + return candidate + + +def _command_environment(value: Mapping[str, str]) -> dict[str, str]: + if not isinstance(value, Mapping): + raise CommandError("env must be an object of string names and values") + copied: dict[str, str] = {} + for key, item in value.items(): + invalid_key = ( + not isinstance(key, str) + or not key + or "=" in key + or "\x00" in key + ) + invalid_value = not isinstance(item, str) or "\x00" in item + if invalid_key or invalid_value: + raise CommandError("environment names and values must be valid strings") + copied[key] = item + return dict(sorted(copied.items())) + + +def _positive_seconds(value: Any, label: str) -> float: + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(float(value)) + or float(value) <= 0 + ): + raise CommandError(f"{label} must be a positive finite number") + return float(value) + + +class CommandRunner: + """Run no-shell commands in process groups with complete log evidence.""" + + def __init__( + self, + workspace: str | os.PathLike[str], + *, + termination_grace_seconds: float = 0.25, + ) -> None: + self.workspace = resolve_private_workspace(Path(workspace)) + self.log_directory = create_private_workspace( + self.workspace / "command-logs" + ) + self.termination_grace_seconds = _positive_seconds( + termination_grace_seconds, "termination_grace_seconds" + ) + self._sequence = 0 + + def _paths(self, stage: str, phase: str) -> tuple[str, Path, Path]: + self._sequence += 1 + stem = f"{self._sequence:06d}-{stage}-{phase}" + return ( + stem, + self.log_directory / f"{stem}.stdout.log", + self.log_directory / f"{stem}.stderr.log", + ) + + @staticmethod + def _tail(path: Path) -> str: + try: + with path.open("rb") as stream: + stream.seek(0, os.SEEK_END) + size = stream.tell() + stream.seek(max(0, size - _TAIL_BYTES), os.SEEK_SET) + payload = stream.read(_TAIL_BYTES) + except OSError as exc: + raise CommandError(f"cannot read command log tail {path}: {exc}") from exc + return payload.decode("utf-8", errors="replace") + + @staticmethod + def _terminate_group(process: subprocess.Popen[Any], grace: float) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + try: + process.wait(timeout=grace) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + # pylint: disable=too-many-arguments,too-many-locals + def run( + self, + argv: Sequence[str], + *, + cwd: str | os.PathLike[str], + env: Mapping[str, str], + phase: str, + stage: str, + timeout_seconds: float, + ) -> dict[str, Any]: + """Execute one exact argv and return timing, exit, and log identities.""" + command = _command_argv(argv) + command_cwd = _command_cwd(cwd) + command_env = _command_environment(env) + if phase not in COMMAND_PHASES: + raise CommandError(f"phase must be exactly one of {list(COMMAND_PHASES)}") + if not isinstance(stage, str) or _STAGE_PATTERN.fullmatch(stage) is None: + raise CommandError("stage must be 1-64 safe ASCII characters") + timeout = _positive_seconds(timeout_seconds, "timeout_seconds") + command_id, stdout_path, stderr_path = self._paths(stage, phase) + started_at = datetime.now(timezone.utc).isoformat() + started_ns = time.monotonic_ns() + process: subprocess.Popen[Any] | None = None + timed_out = False + launch_error: str | None = None + exit_code: int | None = None + write_fresh_bytes( + self.log_directory, + stdout_path, + b"", + schema_version="1", + kind="command-stdout-log", + ) + write_fresh_bytes( + self.log_directory, + stderr_path, + b"", + schema_version="1", + kind="command-stderr-log", + ) + with stdout_path.open("wb") as stdout_stream, stderr_path.open( + "wb" + ) as stderr_stream: + try: + process = subprocess.Popen( # pylint: disable=consider-using-with + command, + cwd=str(command_cwd), + env=command_env, + stdin=subprocess.DEVNULL, + stdout=stdout_stream, + stderr=stderr_stream, + shell=False, + start_new_session=True, + ) + try: + exit_code = process.wait(timeout=timeout) + except subprocess.TimeoutExpired: + timed_out = True + self._terminate_group(process, self.termination_grace_seconds) + exit_code = process.returncode + except OSError as exc: + launch_error = f"{type(exc).__name__}: {exc}" + if process is not None and process.poll() is None: + self._terminate_group(process, self.termination_grace_seconds) + exit_code = process.returncode + ended_ns = time.monotonic_ns() + stdout_identity = snapshot_artifact( + self.workspace, + stdout_path, + schema_version="1", + kind="command-stdout-log", + ) + stderr_identity = snapshot_artifact( + self.workspace, + stderr_path, + schema_version="1", + kind="command-stderr-log", + ) + return { + "schema_version": "1", + "kind": "command-result", + "command_id": command_id, + "stage": stage, + "phase": phase, + "argv": command, + "cwd": str(command_cwd), + "environment_keys": sorted(command_env), + "started_at": started_at, + "ended_at": datetime.now(timezone.utc).isoformat(), + "duration_seconds": (ended_ns - started_ns) / 1_000_000_000, + "approval_wait_seconds": 0.0, + "timeout_seconds": timeout, + "exit_code": exit_code, + "timed_out": timed_out, + "launch_error": launch_error, + "stdout": stdout_identity, + "stderr": stderr_identity, + "stdout_tail": self._tail(stdout_path), + "stderr_tail": self._tail(stderr_path), + } + + +def run_authenticated_sample( # pylint: disable=too-many-arguments,too-many-locals + token: Any, + arguments: Sequence[str], + *, + token_type: type, + error_type: type[ValueError], + command_environment: Mapping[str, str], + workspace: Path, + runner: Any, + cwd: Path, + timeout_seconds: float, + stage: str, + phase: str, +) -> dict[str, Any]: + """Rehash every authority seal immediately before an exact launch.""" + if not isinstance(token, token_type): + raise error_type("official sample has no authenticated record") + root = resolve_private_workspace(workspace) + if root != Path(token.workspace).resolve(strict=True): + raise error_type("launch workspace differs from authenticated workspace") + command_cwd = Path(cwd).resolve(strict=True) + try: + command_cwd.relative_to(root) + except ValueError as exc: + raise error_type("official sample cwd must be inside workspace") from exc + suffix = tuple(arguments) + if any(not isinstance(item, str) or "\0" in item for item in suffix): + raise error_type("official sample arguments must be NUL-free strings") + verify_artifact(Path(token.environment_workspace), token.environment_identity) + for item in (*token.protected_files, *token.runtime_libraries): + verify_external_artifact(item.identity) + executable = Path(token.launcher[0]) + if executable.stat().st_mode & 0o111 == 0: + raise error_type("authenticated launcher is no longer executable") + expected_kind = { + "native": "native-official-sample-executable", + "pynvc": "pynvc-venv-python", + }.get(token.surface) + launcher_seals = [ + seal + for seal in token.protected_files + if seal.identity.get("kind") == expected_kind + ] + if expected_kind is None or len(launcher_seals) != 1: + raise error_type("authenticated launcher seal is absent or duplicated") + if executable.resolve(strict=True) != Path(launcher_seals[0].identity["path"]): + raise error_type("authenticated launcher resolves outside sealed target") + argv = [*token.launcher, *suffix] + result = runner.run( + argv, + cwd=command_cwd, + env=command_environment, + timeout_seconds=timeout_seconds, + stage=stage, + phase=phase, + ) + expected = { + "argv": argv, + "cwd": str(command_cwd), + "stage": stage, + "phase": phase, + "timeout_seconds": float(timeout_seconds), + } + if not isinstance(result, dict) or any( + result.get(key) != value for key, value in expected.items() + ): + raise error_type("command evidence does not match authenticated launch") + return result + + +def validate_environment_envelope( + value: Any, + *, + label: str, + error_type: type[ValueError], + required_surfaces: Sequence[str] = SURFACE_ORDER, +) -> dict[str, Any]: + """Accept one live environment envelope by required subset, ignoring extras. + + Additive optional keys are permitted by the schema, so this validates the + required subset only and never compares the document's key set for + equality. Any other version is an unknown version and is rejected. + """ + selected = tuple(required_surfaces) + if ( + not selected + or len(selected) != len(set(selected)) + or any(surface not in SURFACE_ORDER for surface in selected) + ): + raise error_type( + f"{label} required surfaces must be a non-empty subset of " + f"{list(SURFACE_ORDER)}" + ) + if not isinstance(value, dict): + raise error_type(f"{label} environment must be a JSON object") + if value.get("kind") != ENVIRONMENT_KIND: + raise error_type(f"{label} artifact is not one {ENVIRONMENT_KIND} document") + if value.get("schema_version") != ENVIRONMENT_SCHEMA_VERSION: + raise error_type( + f"{label} requires {ENVIRONMENT_KIND} schema " + f"{ENVIRONMENT_SCHEMA_VERSION}, not {value.get('schema_version')!r}" + ) + if value.get("mode") != "live": + raise error_type(f"{label} requires one live environment artifact") + if value.get("requested_runtime") not in ENVIRONMENT_RUNTIMES: + raise error_type( + f"{label} environment requested_runtime must be exactly one of " + f"{list(ENVIRONMENT_RUNTIMES)}" + ) + if not isinstance(value.get("installation"), Mapping): + raise error_type(f"{label} environment has no installation facts") + if "pynvc" in selected and not isinstance(value.get("pynvc"), Mapping): + raise error_type(f"{label} environment has no selected pynvc facts") + selected_gpu = value.get("selected_gpu") + if ( + isinstance(selected_gpu, bool) + or not isinstance(selected_gpu, int) + or selected_gpu < 0 + ): + raise error_type(f"{label} environment selected_gpu is malformed") + return value + + +def validate_local_runtime_binding( + value: Any, + *, + label: str, + error_type: type[ValueError], + required_surfaces: Sequence[str], +) -> dict[str, Any]: + """Validate one consumer-owned read-only binding for selected surfaces.""" + selected = tuple(required_surfaces) + if ( + not selected + or len(selected) != len(set(selected)) + or any(surface not in SURFACE_ORDER for surface in selected) + ): + raise error_type( + f"{label} required surfaces must be a non-empty subset of " + f"{list(SURFACE_ORDER)}" + ) + if not isinstance(value, dict): + raise error_type(f"{label} local runtime binding must be a JSON object") + expected = { + "schema_version", + "kind", + "mode", + "source", + "requested_runtime", + "selected_gpu", + "installation", + "pynvc", + } + if set(value) != expected: + raise error_type( + f"{label} local runtime binding fields must be exactly {sorted(expected)}" + ) + if ( + value.get("schema_version") != LOCAL_RUNTIME_BINDING_SCHEMA_VERSION + or value.get("kind") != LOCAL_RUNTIME_BINDING_KIND + or value.get("mode") != "live" + ): + raise error_type( + f"{label} requires live {LOCAL_RUNTIME_BINDING_KIND} " + f"{LOCAL_RUNTIME_BINDING_SCHEMA_VERSION}" + ) + if value.get("source") != "pipeline-local-discovery": + raise error_type(f"{label} local binding source is invalid") + selected_gpu = value.get("selected_gpu") + if ( + isinstance(selected_gpu, bool) + or not isinstance(selected_gpu, int) + or selected_gpu < 0 + ): + raise error_type(f"{label} local binding selected_gpu is malformed") + if value.get("requested_runtime") not in ENVIRONMENT_RUNTIMES: + raise error_type( + f"{label} local binding requested_runtime must be exactly one of " + f"{list(ENVIRONMENT_RUNTIMES)}" + ) + if not isinstance(value.get("installation"), Mapping): + raise error_type(f"{label} local binding has no installation facts") + if "pynvc" in selected and not isinstance(value.get("pynvc"), Mapping): + raise error_type(f"{label} local binding has no selected pynvc facts") + return value + + +def _member(value: Any, key: str) -> Mapping[str, Any]: + """Return one nested mapping member, or an empty mapping when it is absent.""" + member = value.get(key) if isinstance(value, Mapping) else None + return member if isinstance(member, Mapping) else {} + + +def _normalized_prerequisites(record: Any) -> dict[str, Any] | None: + """Recover the internal unresolved-module list from the 1.2 AppDec record. + + Schema 1.2 publishes ``missing_modules``/``unknown_modules`` and sets + ``status`` to ``complete`` exactly when neither is populated and pkg-config + itself was available, so the internal list is recoverable without loss. An + incomplete record that itemizes nothing leaves every required module + unproven. + """ + if not isinstance(record, Mapping): + return None + if isinstance(record.get("unresolved_modules"), list): + return dict(record) + unresolved = sorted( + { + str(name) + for key in ("missing_modules", "unknown_modules") + for name in (record.get(key) or []) + } + ) + if not unresolved and record.get("status") != "complete": + unresolved = [str(name) for name in (record.get("required_modules") or ["pkg-config"])] + return {**record, "unresolved_modules": unresolved} + + +def _normalized_native(environment: Mapping[str, Any]) -> dict[str, Any]: + """Normalize the 1.2 ``installation`` tree into the internal native view.""" + installation = _member(environment, "installation") + native = _member(installation, "native_sdk") + cuda = _member(installation, "cuda_toolkit") + identities = _member(installation, "build_tool_identities") + roots = native.get("complete_roots") + tools = { + internal: dict(_member(identities, published)) + for internal, published in _PUBLISHED_BUILD_TOOLS + if isinstance(identities.get(published), Mapping) + } + for published, generator in _PUBLISHED_GENERATORS: + record = identities.get(published) + if isinstance(record, Mapping) and _absolute(record.get("path")): + tools["generator"] = {**record, "name": generator} + break + return { + "installed": native.get("status") == "installed", + "package": native.get("package"), + # 1.2 records a root inventory; exactly one complete root is the + # canonical SDK root, and an ambiguous inventory selects none. + "sdk_root": roots[0] if isinstance(roots, list) and len(roots) == 1 else None, + "build_prerequisites": _normalized_prerequisites( + installation.get("native_build_prerequisites") + ), + "cuda": { + "status": "installed" if cuda.get("status") == "available" else cuda.get("status"), + "version": cuda.get("version"), + "root": _member(cuda, "nvcc_discovery").get("root"), + }, + "tools": tools, + } + + +def _normalized_pynvc(environment: Mapping[str, Any]) -> dict[str, Any]: + """Normalize the 1.2 ``pynvc``/``installation.python`` trees into one view.""" + pynvc = _member(environment, "pynvc") + identity = _member(pynvc, "identity") + python = _member(_member(environment, "installation"), "python") + packages = _member(python, "packages") + return { + "installed": ( + pynvc.get("imported") is True and identity.get("status") == "verified" + ), + "version": pynvc.get("distribution_version"), + "interpreter": identity.get("interpreter"), + # 1.2 requires the verified wheel interpreter to be the probe + # interpreter, so its running identity is this surface's identity. + "interpreter_identity": _member(python, "interpreter_identities").get("running"), + "sys_prefix": identity.get("sys_prefix"), + "extension": identity.get("extension"), + "module": identity.get("module"), + "dependencies": { + name: {**record, "ready": record.get("requirement_satisfied") is True} + for name, record in packages.items() + if isinstance(record, Mapping) + }, + } + + +def environment_surface( + environment: Mapping[str, Any], name: str +) -> dict[str, Any] | None: + """Return one installed independent surface, or None when it is absent. + + Schema 1.2 publishes ``installation`` and ``pynvc``, never a ``surfaces`` + key, so the per-surface view stays an internal normalized representation + derived here. A document that additively carries an already normalized + ``surfaces`` mapping is used as supplied. Surfaces stay structurally + independent: either may be absent or report ``installed: false`` with no + effect on the other. + """ + if not isinstance(environment, Mapping): + return None + recognized_kind = environment.get("kind") in { + ENVIRONMENT_KIND, + LOCAL_RUNTIME_BINDING_KIND, + } + if recognized_kind and name == "native": + surface = _normalized_native(environment) + elif recognized_kind and name == "pynvc": + surface = _normalized_pynvc(environment) + else: + surface = None + if not isinstance(surface, Mapping) or surface.get("installed") is not True: + return None + return dict(surface) + + +def read_live_environment( + workspace: Path, + identity: Mapping[str, Any], + *, + label: str, + error_type: type[ValueError], + required_surfaces: Sequence[str] = SURFACE_ORDER, +) -> dict[str, Any]: + """Read one supplied setup envelope or consumer-owned local binding.""" + value = read_verified_json(workspace, identity) + if isinstance(value, Mapping) and value.get("kind") == LOCAL_RUNTIME_BINDING_KIND: + return validate_local_runtime_binding( + value, + label=f"{label} authentication", + error_type=error_type, + required_surfaces=required_surfaces, + ) + return validate_environment_envelope( + value, + label=f"{label} authentication", + error_type=error_type, + required_surfaces=required_surfaces, + ) + + +def _text(value: Any) -> bool: + """Return whether one value is a non-empty canonical string.""" + return isinstance(value, str) and bool(value) and value == value.strip() + + +def _absolute(value: Any) -> bool: + return _text(value) and PurePosixPath(str(value)).is_absolute() + + +def _sha256(value: Any) -> bool: + return isinstance(value, str) and re.fullmatch(r"[0-9a-f]{64}", value) is not None + + +def _extension_defects(surface: Mapping[str, Any]) -> list[str]: + """Require the authenticated extension identity, and honour a loaded path. + + Schema 1.2 authenticates the extension by wheel-member identity and does + not publish which file the interpreter actually loaded, so ``loaded_path`` + is additive: when a document does carry it, it must name the very same + file, and it is never inferred when absent. The live isolated RECORD probe + remains the authority for what was loaded. + """ + extension = surface.get("extension") + if not isinstance(extension, Mapping): + return ["pynvc extension identity is missing"] + if not (_absolute(extension.get("path")) and _sha256(extension.get("sha256"))): + return ["pynvc extension path or sha256 is missing or malformed"] + loaded = extension.get("loaded_path") + if loaded is None: + return [] + if not _absolute(loaded): + return ["pynvc extension loaded_path is malformed"] + if loaded != extension["path"]: + return [ + "the interpreter loaded a different extension than the authenticated one: " + f"{loaded!r} is not {extension['path']!r}" + ] + return [] + + +def _module_defects(surface: Mapping[str, Any]) -> list[str]: + """Require the imported module to agree with the distribution version.""" + module = surface.get("module") + if not isinstance(module, Mapping): + return ["pynvc imported module identity is missing"] + if not (_text(module.get("version")) and _absolute(module.get("path"))): + return ["pynvc module version or path is missing or malformed"] + if module["version"] != surface.get("version"): + return [ + "imported module version differs from the distribution version: " + f"{module['version']!r} is not {surface.get('version')!r}" + ] + return [] + + +def native_surface_defects(environment: Mapping[str, Any]) -> list[str]: + """Structurally validate the native surface's required subset at planning. + + Additive keys are ignored and no key set is compared for equality. A + malformed native surface is blocked here rather than at execute-time + authentication, so the Py surface stays independently actionable. + """ + surface = environment_surface(environment, "native") + if surface is None: + return ["native surface is absent or does not report installed"] + defects: list[str] = [] + package = surface.get("package") + if ( + not isinstance(package, Mapping) + or package.get("name") != NATIVE_PACKAGE_NAME + or package.get("status") != "installed" + or not _text(package.get("version")) + ): + defects.append("native package name, status, or version is missing or malformed") + if not _absolute(surface.get("sdk_root")): + defects.append("native sdk_root is missing or is not an absolute path") + prerequisites = surface.get("build_prerequisites") + if ( + not isinstance(prerequisites, Mapping) + or prerequisites.get("status") != "complete" + or prerequisites.get("unresolved_modules") != [] + ): + defects.append("native build prerequisites are incomplete or malformed") + cuda = surface.get("cuda") + if ( + not isinstance(cuda, Mapping) + or cuda.get("status") != "installed" + or not _text(cuda.get("version")) + ): + defects.append("native CUDA status or version is missing or malformed") + tools = surface.get("tools") + if not isinstance(tools, Mapping): + defects.append("native build tools are missing") + else: + # 1.2 identifies each build tool by path and content hash and publishes + # no tool version, so a version is honoured when present and never + # required. The content hash is what the launch is bound to. + incomplete = [ + name + for name in REQUIRED_BUILD_TOOLS + if not isinstance(tools.get(name), Mapping) + or not _absolute(tools[name].get("path")) + or not _sha256(tools[name].get("sha256")) + or (tools[name].get("version") is not None + and not _text(tools[name].get("version"))) + or ( + name == "generator" + and not _text(tools[name].get("name")) + ) + ] + if incomplete: + defects.append( + "native build tools are missing a required path or SHA-256: " + f"{incomplete}" + ) + return defects + + +def pynvc_surface_defects(environment: Mapping[str, Any]) -> list[str]: + """Structurally validate the Py surface's required subset at planning.""" + surface = environment_surface(environment, "pynvc") + if surface is None: + return ["pynvc surface is absent or does not report installed"] + defects: list[str] = [] + if not _text(surface.get("version")): + defects.append("pynvc version is missing or malformed") + if not _absolute(surface.get("interpreter")): + defects.append("pynvc lexical interpreter is missing or is not an absolute path") + identity = surface.get("interpreter_identity") + if ( + not isinstance(identity, Mapping) + or not _absolute(identity.get("path")) + or not _sha256(identity.get("sha256")) + ): + defects.append("pynvc interpreter_identity path or sha256 is missing or malformed") + if not _absolute(surface.get("sys_prefix")): + defects.append("pynvc sys_prefix is missing or is not an absolute path") + defects.extend(_extension_defects(surface)) + defects.extend(_module_defects(surface)) + return defects + + +def pynvc_full_sample_defects(environment: Mapping[str, Any]) -> list[str]: + """Require the verified dependency closure used by full PyNv sample routes.""" + defects = pynvc_surface_defects(environment) + surface = environment_surface(environment, "pynvc") + if surface is None: + return defects + dependencies = surface.get("dependencies") + dependencies = dependencies if isinstance(dependencies, Mapping) else {} + incomplete = [ + name + for name in ("numpy", "pycuda") + if not isinstance(dependencies.get(name), Mapping) + or dependencies[name].get("status") != "installed" + or dependencies[name].get("ready") is not True + ] + torch = dependencies.get("torch") + if not ( + isinstance(torch, Mapping) + and torch.get("status") == "installed" + and torch.get("version") == "2.9.1+cu130" + and torch.get("cuda_build") == "13.0" + and torch.get("cuda_available") is True + and torch.get("ready") is True + ): + incomplete.append("torch") + if incomplete: + defects.append( + "authenticated Py full-samples routes require ready NumPy and PyCUDA plus exact " + "CUDA-enabled Torch 2.9.1+cu130; incomplete dependencies: " + + ", ".join(sorted(set(incomplete))) + + ". A jetson-video-setup pynvc-smoke environment deliberately omits Torch. " + "Provision a NEW full-samples PyNvVideoCodec venv with plan_install.py " + "--profile full-samples --venv NEW_PATH; never upgrade the smoke venv in place." + ) + return defects + + +def run_prepare_command( # pylint: disable=too-many-arguments + service: Any, + commands: list[dict[str, Any]], + argv: Sequence[str], + *, + command_environment: Mapping[str, str], + error_type: type[ValueError], + cwd: Path, + timeout_seconds: float, + stage: str, +) -> tuple[dict[str, Any], str, str]: + """Run one authenticated preparation command and consume its evidence.""" + result = service.run( + list(argv), + cwd=cwd, + env=command_environment, + timeout_seconds=timeout_seconds, + stage=stage, + phase="prepare", + ) + if not isinstance(result, dict): + raise error_type("command runner returned a non-object result") + commands.append(result) + stdout = read_verified_text(service.workspace, result["stdout"]) + stderr = read_verified_text(service.workspace, result["stderr"]) + if ( + result.get("timed_out") + or result.get("launch_error") + or result.get("exit_code") != 0 + ): + raise error_type( + f"command failed during {stage}: exit={result.get('exit_code')} " + f"timeout={result.get('timed_out')}: {stderr[-500:]}" + ) + return result, stdout, stderr + + +def _candidate(value: Any, surface: str) -> tuple[bool, list[str]]: + if isinstance(value, bool): + return value, [] if value else [f"{surface} is not eligible"] + if not isinstance(value, Mapping) or set(value) != _ELIGIBILITY_KEYS: + raise ValueError( + f"eligibility.{surface} must contain exactly eligible and reasons" + ) + eligible = value["eligible"] + reasons = value["reasons"] + if not isinstance(eligible, bool) or not isinstance(reasons, list): + raise ValueError(f"eligibility.{surface} has invalid types") + if any( + not isinstance(reason, str) + or not reason + or reason != reason.strip() + for reason in reasons + ): + raise ValueError(f"eligibility.{surface}.reasons are invalid") + if len(set(reasons)) != len(reasons): + raise ValueError(f"eligibility.{surface}.reasons contain duplicates") + if eligible and reasons: + raise ValueError(f"eligible surface {surface} must not contain reasons") + return eligible, reasons or ([] if eligible else [f"{surface} is not eligible"]) + + +def build_surface_plan( + requested_surface: str, + eligibility: Mapping[str, Any], +) -> dict[str, Any]: + """Return the exact native/Py surface-selection truth table.""" + if requested_surface not in REQUESTED_SURFACES: + raise ValueError( + f"requested_surface must be exactly one of {list(REQUESTED_SURFACES)}" + ) + if not isinstance(eligibility, Mapping) or set(eligibility) != set(SURFACE_ORDER): + raise ValueError(f"eligibility fields must be exactly {list(SURFACE_ORDER)}") + eligible: list[str] = [] + surface_reasons: dict[str, list[str]] = {} + for surface in SURFACE_ORDER: + ready, reasons = _candidate(eligibility[surface], surface) + if ready: + eligible.append(surface) + surface_reasons[surface] = reasons + selected: list[str] = [] + reasons: list[str] = [] + if requested_surface in SURFACE_ORDER: + if requested_surface in eligible: + classification = "ready" + selected = [requested_surface] + else: + classification = "blocked" + reasons = list(surface_reasons[requested_surface]) + elif requested_surface == "auto": + if len(eligible) == 1: + classification = "ready" + selected = list(eligible) + elif len(eligible) == 2: + classification = "selection_required" + reasons = [ + "auto requires an explicit surface because native and pynvc are eligible" + ] + else: + classification = "blocked" + reasons = [ + reason + for surface in SURFACE_ORDER + for reason in surface_reasons[surface] + ] + else: + selected = list(eligible) + classification = "ready" if len(eligible) == 2 else "blocked" + reasons = [ + reason + for surface in SURFACE_ORDER + if surface not in eligible + for reason in surface_reasons[surface] + ] + return { + "requested_surface": requested_surface, + "classification": classification, + "selected_surfaces": selected, + "eligible_surfaces": eligible, + "reasons": reasons, + } + + +def _recipe_cli() -> Path: + """Resolve the fixed sibling recipe public CLI without importing it.""" + skill_root = ( + Path(__file__).absolute().parents[2] + / "jetson-video-recipe" + ) + candidate = ( + skill_root + / "scripts" + / "recipes" + / "recipe_model.py" + ) + try: + resolved_root = skill_root.resolve(strict=True) + details = candidate.lstat() + resolved = candidate.resolve(strict=True) + except OSError as exc: + raise SkillDependencyError( + "jetson-video-recipe", "this recipe-bearing pipeline stage" + ) from exc + if stat.S_ISLNK(details.st_mode) or not stat.S_ISREG(details.st_mode): + raise RecipeBridgeError("the recipe public CLI must be a regular file") + if resolved != resolved_root / "scripts" / "recipes" / "recipe_model.py": + raise RecipeBridgeError("the recipe public CLI path is not canonical") + return resolved + + +def _recipe_command( + arguments: Sequence[str], + timeout_seconds: float = 60, + *, + accepted_returncodes: Sequence[int] = (0,), +) -> Any: + """Execute and strictly consume one size-checked recipe CLI JSON response.""" + argv = [sys.executable, "-I", str(_recipe_cli()), *arguments] + try: + completed = subprocess.run( + argv, + cwd=str(Path(__file__).resolve().parent), + env={ + "LANG": "C.UTF-8", + "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, + stdin=subprocess.DEVNULL, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + timeout=timeout_seconds, + check=False, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + raise RecipeBridgeError(f"recipe public CLI launch failed: {exc}") from exc + if ( + len(completed.stdout) > _RECIPE_OUTPUT_LIMIT + or len(completed.stderr) > _RECIPE_OUTPUT_LIMIT + ): + raise RecipeBridgeError("recipe public CLI output exceeded its bound") + try: + value = strict_json_loads(completed.stdout) + except ValueError as exc: + raise RecipeBridgeError( + "recipe public CLI did not return one strict JSON response" + ) from exc + if completed.returncode not in accepted_returncodes: + detail = value.get("error") if isinstance(value, dict) else None + raise RecipeBridgeError( + f"recipe public CLI rejected the request: {detail or completed.returncode}" + ) + return value + + +def validate_recipe_identity(identity: Mapping[str, Any]) -> None: + """Replay one current recipe through its owning public CLI.""" + recipe = verify_external_artifact(identity) + value = _recipe_command(("validate", "--recipe", str(recipe))) + if value != {"valid": True, "errors": []}: + raise RecipeBridgeError("recipe public CLI returned an invalid validation envelope") + + +def _bind_capability_report( + capability_identity: Mapping[str, Any], + environment_identity: Mapping[str, Any], +) -> Path: + """Require the report to be the one produced from this exact environment. + + The report records the environment artifact's ``{path, size_bytes, + sha256}``. Recipe fails closed on a mismatch; the pairing is also refused + here so a cached report is never sent alongside a fresh environment. + """ + capability = verify_external_artifact(capability_identity) + report = read_verified_external_json(capability_identity) + if ( + not isinstance(report, dict) + or report.get("kind") != CAPABILITY_REPORT_KIND + or report.get("schema_version") != CAPABILITY_REPORT_SCHEMA_VERSION + ): + raise RecipeBridgeError( + f"capability report must be schema {CAPABILITY_REPORT_SCHEMA_VERSION} " + f"kind {CAPABILITY_REPORT_KIND}" + ) + bound = report.get("environment") + if not isinstance(bound, Mapping): + raise RecipeBridgeError("capability report records no bound environment") + if any( + bound.get(field) != environment_identity.get(field) + for field in ("path", "size_bytes", "sha256") + ): + raise RecipeBridgeError( + "capability report was produced from a different environment artifact" + ) + return capability + + +def check_live_recipe( + recipe_identity: Mapping[str, Any], + environment_identity: Mapping[str, Any] | None, + surface: str, + *, + buffer_mode: str, + capability_identity: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Classify one projection through the recipe skill's public CLI. + + The recipe is required. A supplied setup environment is an optional but + authoritative fast path; when absent, the recipe owner performs only its + own media-free projection check. A capability report can be accepted only + with the exact environment to which it is bound. + """ + recipe = verify_external_artifact(recipe_identity) + arguments = [ + "check-live", + "--recipe", + str(recipe), + "--surface", + surface, + "--buffer-mode", + buffer_mode, + ] + if environment_identity is not None: + environment = verify_external_artifact(environment_identity) + arguments.extend(["--environment", str(environment)]) + if capability_identity is not None: + if environment_identity is None: + raise RecipeBridgeError( + "a capability report requires its exact supplied environment identity" + ) + capability = _bind_capability_report(capability_identity, environment_identity) + arguments.extend(["--capability-report", str(capability)]) + value = _recipe_command(tuple(arguments), accepted_returncodes=(0, 2)) + if ( + not isinstance(value, dict) + or value.get("classification") not in {"compatible", "unknown", "unsupported"} + or not isinstance(value.get("reasons"), list) + ): + raise RecipeBridgeError("recipe public CLI returned an invalid live envelope") + return value diff --git a/skills/jetson-video-pipeline/scripts/encode_controller.py b/skills/jetson-video-pipeline/scripts/encode_controller.py new file mode 100644 index 00000000..344da6ad --- /dev/null +++ b/skills/jetson-video-pipeline/scripts/encode_controller.py @@ -0,0 +1,1153 @@ +#!/usr/bin/env python3 +"""Execute one authenticated recipe through official encode and decode samples.""" + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import json +import math +import os +import re +import stat +import sys +from pathlib import Path +from typing import Any, Mapping + +if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise SystemExit("invoke this producer with isolated Python: python3 -I") + +# Isolated mode excludes the script directory. Add only this skill's fixed +# private module directory; no sibling skill is imported as Python code. +_SCRIPT_DIR = Path(__file__).absolute().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position,import-error +import _pipeline_provenance as sample_provenance # noqa: E402 +import _pipeline_runtime as artifact_io # noqa: E402 +# pylint: enable=wrong-import-position,import-error + +# This controller intentionally keeps request planning and both operation +# branches together so their artifact and marker contracts cannot diverge. +# pylint: disable=too-many-lines + +command_runner = artifact_io +surface_router = artifact_io + +SCHEMA_VERSION = "1.0" +REQUEST_KIND = "nvcodec-encode-request" +RESULT_KIND = "nvcodec-encode-result" +IDENTITY_KEYS = {"schema_version", "kind", "path", "size_bytes", "sha256"} +COMMAND_KEYS = { + "schema_version", + "kind", + "command_id", + "stage", + "phase", + "argv", + "cwd", + "environment_keys", + "started_at", + "ended_at", + "duration_seconds", + "approval_wait_seconds", + "timeout_seconds", + "exit_code", + "timed_out", + "launch_error", + "stdout", + "stderr", + "stdout_tail", + "stderr_tail", +} +REQUEST_REQUIRED = { + "schema_version", + "kind", + "surface", + "recipe", + "input", + "timeout_seconds", +} +# capability_report stays optional and purely additive: it refines a live +# classification when a request supplies one, and no surface fails for its +# absence, so an independently constructed request needs no new field. +REQUEST_OPTIONAL = { + "buffer_mode", + "capability_report", + "environment", + "pynvc_interpreter", +} +NON_MEDIA_REQUEST_REQUIRED = REQUEST_REQUIRED - {"input"} +ACCEPTED_MEDIA_INPUTS = ("local_path", "http_url", "https_url") +SURFACES = ("native", "pynvc") +SAMPLES = { + "native": ("AppEncCuda", "AppDec"), + "pynvc": ("samples/basic/encode.py", "samples/advanced/decode.py"), +} +EXTENSIONS = {"h264": "h264", "hevc": "hevc", "av1": "av1"} +FAILURE_PATTERN = sample_provenance.OFFICIAL_SAMPLE_FAILURE_PATTERN +_provenance_providers: tuple[Any, Any, Any] | None = None # pylint: disable=invalid-name + + +class EncodeError(ValueError): + """The request or official operation failed its evidence contract.""" + + +def _providers() -> tuple[Any, Any, Any]: + global _provenance_providers # pylint: disable=global-statement + if _provenance_providers is None: + _provenance_providers = ( + sample_provenance.authenticate_native_samples, + sample_provenance.authenticate_pynvc_samples, + sample_provenance.run_authenticated, + ) + return _provenance_providers + + +def _artifact_identity(value: Any, label: str) -> dict[str, Any]: + """Accept one exact, currently valid portable artifact identity.""" + if not isinstance(value, Mapping) or set(value) != IDENTITY_KEYS: + raise EncodeError(f"{label} must be an exact portable artifact identity") + copied = dict(value) + try: + artifact_io.verify_external_artifact(copied) + except (OSError, ValueError) as exc: + raise EncodeError(f"{label} identity is not current: {exc}") from exc + return copied + + +def _identity(value: Any, *, schema: str, kind: str, label: str) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != IDENTITY_KEYS: + raise EncodeError(f"{label} must be an exact portable artifact identity") + if value.get("schema_version") != schema or value.get("kind") != kind: + raise EncodeError(f"{label} identity must be schema {schema} kind {kind}") + return _artifact_identity(value, label) + + +def _positive_timeout(value: Any, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise EncodeError("request timeout_seconds must be a positive finite number") + timeout = float(value) + if not math.isfinite(timeout) or timeout <= 0 or timeout > maximum: + raise EncodeError(f"request timeout_seconds must be in (0, {float(maximum)}]") + return timeout + + +def _workspace(path: Path) -> Path: + requested = Path(path).expanduser() + if requested.exists(): + root = artifact_io.resolve_private_workspace(requested) + if any(root.iterdir()): + raise EncodeError("encode workspace must be fresh and empty") + return root + return artifact_io.create_private_workspace(requested) + + +def _request_envelope(value: Any, timeout_ceiling: float) -> dict[str, Any]: + """Validate non-media request fields without opening artifact identities.""" + if not isinstance(value, Mapping): + raise EncodeError("request must be a JSON object") + if ( + set(value) - (REQUEST_REQUIRED | REQUEST_OPTIONAL) + or not NON_MEDIA_REQUEST_REQUIRED.issubset(value) + ): + raise EncodeError( + f"request fields must be required {sorted(REQUEST_REQUIRED)} plus optional " + f"{sorted(REQUEST_OPTIONAL)}" + ) + request = dict(value) + if request["schema_version"] != SCHEMA_VERSION or request["kind"] != REQUEST_KIND: + raise EncodeError(f"request identity must be schema {SCHEMA_VERSION} kind {REQUEST_KIND}") + if ( + not isinstance(request["surface"], str) + or request["surface"] not in {"native", "pynvc", "auto", "both"} + ): + raise EncodeError("surface must be native, pynvc, auto, or both") + request["buffer_mode"] = request.get("buffer_mode", "gpu") + if ( + not isinstance(request["buffer_mode"], str) + or request["buffer_mode"] not in {"cpu", "gpu"} + ): + raise EncodeError("buffer_mode must be cpu or gpu") + request["timeout_seconds"] = _positive_timeout(request["timeout_seconds"], timeout_ceiling) + return request + + +def _request(value: Any, timeout_ceiling: float) -> dict[str, Any]: + request = _request_envelope(value, timeout_ceiling) + request["recipe"] = _identity( + request["recipe"], schema="2.0", kind="nvcodec-recipe", label="recipe" + ) + if request.get("environment") is not None: + request["environment"] = _identity( + request["environment"], + schema=artifact_io.ENVIRONMENT_SCHEMA_VERSION, + kind=artifact_io.ENVIRONMENT_KIND, + label="environment", + ) + interpreter = request.get("pynvc_interpreter") + if interpreter is not None and ( + not isinstance(interpreter, str) + or not interpreter + or interpreter != interpreter.strip() + or not Path(interpreter).is_absolute() + ): + raise EncodeError("pynvc_interpreter must be one exact absolute path") + if request.get("capability_report") is not None: + request["capability_report"] = _identity( + request["capability_report"], + schema=artifact_io.CAPABILITY_REPORT_SCHEMA_VERSION, + kind=artifact_io.CAPABILITY_REPORT_KIND, + label="capability report", + ) + if request.get("input") is not None: + request["input"] = _identity( + request["input"], schema="1", kind="nvcodec-raw-video", label="input" + ) + return request + + +def _input_required_result() -> dict[str, Any]: + """Return the non-mutating terminal state for an absent user media input.""" + return { + "schema_version": SCHEMA_VERSION, + "kind": RESULT_KIND, + "status": "input_required", + "gate": "media_input", + "accepted_inputs": list(ACCEPTED_MEDIA_INPUTS), + "next_action": "provide_media_path_or_url", + "synthetic_input_allowed": False, + "selected_surfaces": [], + "operations": {}, + "output_artifacts": {}, + } + + +def _interpreter_required_result() -> dict[str, Any]: + """Ask for the one explicit Py interpreter without scanning environments.""" + return { + "schema_version": SCHEMA_VERSION, + "kind": RESULT_KIND, + "status": "input_required", + "gate": "surface_selector", + "surface": "pynvc", + "required_field": "pynvc_interpreter", + "reason": ( + "provide the exact absolute Python interpreter for the existing " + "PyNvVideoCodec environment; no environment scan is performed" + ), + "next_action": "provide_pynvc_interpreter", + "selected_surfaces": [], + "operations": {}, + "output_artifacts": {}, + } + + +def _setup_dependency(surface: str, reason: str) -> dict[str, Any]: + """Describe the exact setup handoff when local authentication fails.""" + candidate = Path(__file__).absolute().parents[2] / "jetson-video-setup" / "SKILL.md" + try: + details = candidate.lstat() + resolved_root = candidate.parent.resolve(strict=True) + resolved = candidate.resolve(strict=True) + except OSError: + installed = False + else: + installed = bool( + stat.S_ISREG(details.st_mode) + and not stat.S_ISLNK(details.st_mode) + and resolved_root.is_dir() + and resolved == resolved_root / "SKILL.md" + ) + action = ( + f"use jetson-video-setup for the {surface} surface and retry" + if installed + else "install jetson-video-setup, configure the selected surface, and retry" + ) + return { + "schema_version": SCHEMA_VERSION, + "kind": RESULT_KIND, + "status": "dependency_required", + "gate": "sdk_setup", + "surface": surface, + "reason": reason, + "dependency": { + "skill": "jetson-video-setup", + "installed": installed, + "needed_for": f"{surface} SDK installation or repair", + "next_action": action, + }, + "selected_surfaces": [], + "operations": {}, + "output_artifacts": {}, + } + + +def _skill_dependency(error: artifact_io.SkillDependencyError) -> dict[str, Any]: + """Preserve a missing sibling as a customer-actionable result envelope.""" + return { + "schema_version": SCHEMA_VERSION, + "kind": RESULT_KIND, + "status": "dependency_required", + "gate": "skill_dependency", + "dependency": { + "skill": error.skill, + "needed_for": error.needed_for, + "reason": str(error), + "next_action": f"install_{error.skill}_and_retry_stage", + }, + "selected_surfaces": [], + "operations": {}, + "output_artifacts": {}, + } + + +def _frame_bytes(intent: Mapping[str, Any]) -> int: + width = intent["width"] + height = intent["height"] + fmt = intent["format"] + if fmt in {"NV12", "YUV420", "P010"} and (width % 2 or height % 2): + raise EncodeError(f"{fmt} execution requires even width and height") + if fmt in {"NV16", "P210"} and width % 2: + raise EncodeError(f"{fmt} execution requires even width") + pixels = width * height + sizes = { + "NV12": pixels * 3 // 2, + "YUV420": pixels * 3 // 2, + "P010": pixels * 3, + "NV16": pixels * 2, + "P210": pixels * 4, + "YUV444": pixels * 3, + "YUV444_16BIT": pixels * 6, + "ARGB": pixels * 4, + "ABGR": pixels * 4, + } + try: + return sizes[fmt] + except KeyError as exc: + raise EncodeError(f"no exact raw frame-size contract for {fmt}") from exc + + +def _validate_raw(recipe: Mapping[str, Any], identity: Mapping[str, Any]) -> int: + intent = recipe["encoder_intent"] + count = intent.get("frame_count") + if isinstance(count, bool) or not isinstance(count, int) or count <= 0: + raise EncodeError("executable recipe requires an exact positive frame_count") + expected = _frame_bytes(intent) * count + if identity["size_bytes"] != expected: + raise EncodeError( + f"raw input size must equal exactly {count} complete {intent['format']} " + f"frames ({expected} bytes)" + ) + return count + + +def _native_readiness(environment: Mapping[str, Any]) -> list[str]: + """Block a malformed or unready native surface during planning.""" + reasons = artifact_io.native_surface_defects(environment) + surface = artifact_io.environment_surface(environment, "native") or {} + cuda = surface.get("cuda") + if isinstance(cuda, Mapping) and not ( + str(cuda.get("version", "")).startswith("13.") + and isinstance(cuda.get("root"), str) + and cuda["root"] + ): + reasons.append("live environment does not report an installed CUDA 13.x toolkit root") + return reasons + + +def _pynvc_readiness(environment: Mapping[str, Any]) -> list[str]: + """Block a malformed or unready Py surface during planning.""" + return artifact_io.pynvc_full_sample_defects(environment) + + +def _surface_plan( # pylint: disable=too-many-arguments,too-many-locals + requested_surface: str, + recipe: dict[str, Any], + environment: dict[str, Any], + buffer_mode: str, + *, + recipe_identity: Mapping[str, Any], + environment_identity: Mapping[str, Any] | None, + capability_identity: Mapping[str, Any] | None = None, + local_errors: Mapping[str, str] | None = None, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]]]: + """Decide eligibility per surface at planning time, never later. + + Each surface is judged on its own required subset and its own live + classification. One surface's defect never raises out of this function, so + a malformed surface is blocked here while its peer stays independently + actionable. + """ + candidates: dict[str, Any] = {} + details: dict[str, dict[str, Any]] = {} + evaluated_surfaces = ( + (requested_surface,) + if requested_surface in SURFACES + else SURFACES + ) + for surface in SURFACES: + if surface not in evaluated_surfaces: + reason = f"{surface} was not evaluated because it was not selected" + candidates[surface] = {"eligible": False, "reasons": [reason]} + details[surface] = { + "projection": "not_evaluated", + "live": {"classification": "not_evaluated", "reasons": [reason]}, + "eligible": False, + "reasons": [reason], + } + continue + projection = recipe["projections"][surface] + reasons = [] + try: + _decoded_layout(recipe["encoder_intent"]) + except EncodeError as exc: + reasons.append(str(exc)) + if projection["status"] != "exact": + reasons.append("recipe projection has blocking losses") + reasons.extend( + _native_readiness(environment) + if surface == "native" + else _pynvc_readiness(environment) + ) + if local_errors and surface in local_errors: + reasons.append(local_errors[surface]) + blocked = bool(reasons) + live = _live_classification( + surface, + blocked=blocked, + recipe_identity=recipe_identity, + environment_identity=environment_identity, + capability_identity=capability_identity, + buffer_mode=buffer_mode, + ) + if not blocked and surface == "pynvc" and live["classification"] != "compatible": + reasons.extend(live["reasons"]) + candidates[surface] = {"eligible": not reasons, "reasons": reasons} + details[surface] = { + "projection": projection["status"], + "live": live, + "eligible": not reasons, + "reasons": list(reasons), + } + local_reason = local_errors.get(surface) if local_errors else None + local_reason_text = str(local_reason) if isinstance(local_reason, str) else "" + if ( + surface == "pynvc" + and "not evaluated" in local_reason_text + and "pynvc_interpreter" in local_reason_text + ): + details[surface]["evaluation_status"] = "not_evaluated" + details[surface]["next_action"] = "provide_pynvc_interpreter_and_retry" + return surface_router.build_surface_plan(requested_surface, candidates), details + + +def _live_classification( # pylint: disable=too-many-arguments + surface: str, + *, + blocked: bool, + recipe_identity: Mapping[str, Any], + environment_identity: Mapping[str, Any] | None, + capability_identity: Mapping[str, Any] | None, + buffer_mode: str, +) -> dict[str, Any]: + """Classify one surface without letting its failure reach its peer.""" + if blocked: + return { + "classification": "not_evaluated", + "reasons": ["surface was blocked during planning validation"], + } + try: + return artifact_io.check_live_recipe( + recipe_identity, + environment_identity, + surface, + buffer_mode=buffer_mode, + capability_identity=capability_identity, + ) + except (OSError, ValueError) as exc: + return { + "classification": "unknown", + "reasons": [f"live recipe classification failed: {type(exc).__name__}: {exc}"], + } + + +def _stage_environment(root: Path, branch: Path, environment: dict[str, Any]) -> dict[str, Any]: + del root + return artifact_io.write_fresh_json(branch, branch / "environment.json", environment) + + +def _configuration( + branch: Path, surface: str, recipe: dict[str, Any] +) -> tuple[dict[str, Any], Path | None]: + if surface == "pynvc": + value = recipe["projections"]["pynvc"]["config"] + path = branch / "encode-config.json" + identity = artifact_io.write_fresh_bytes( + branch, + path, + artifact_io.canonical_json_bytes(value), + schema_version="1", + kind="pynvc-encode-config", + ) + return identity, path + value = { + "schema_version": "1", + "kind": "native-encode-projection", + "cli_options": recipe["projections"]["native"]["cli_options"], + } + path = branch / "encode-projection.json" + return artifact_io.write_fresh_json(branch, path, value), path + + +def _root_identity(root: Path, child: Path, identity: Mapping[str, Any]) -> dict[str, Any]: + path = artifact_io.verify_artifact(child, identity) + return artifact_io.snapshot_artifact( + root, path, schema_version=identity["schema_version"], kind=identity["kind"] + ) + + +def _command_texts(runner: Any, result: Mapping[str, Any]) -> tuple[str, str]: + logs = artifact_io.resolve_private_workspace(Path(runner.workspace)) + return ( + artifact_io.read_verified_text(logs, result["stdout"], max_bytes=16 * 1024 * 1024), + artifact_io.read_verified_text(logs, result["stderr"], max_bytes=16 * 1024 * 1024), + ) + + +def _command_evidence( # pylint: disable=too-many-arguments + result: Any, + *, + token: Any, + suffix: list[str], + branch: Path, + runner: Any, + timeout: float, + stage: str, + phase: str, +) -> tuple[dict[str, Any], str, str]: + expected = { + "schema_version": "1", + "kind": "command-result", + "argv": [*token.launcher, *suffix], + "cwd": str(branch), + "stage": stage, + "phase": phase, + "timeout_seconds": float(timeout), + } + if ( + not isinstance(result, dict) + or set(result) != COMMAND_KEYS + or any(result.get(key) != value for key, value in expected.items()) + ): + raise EncodeError("authenticated command result does not bind the exact launch") + if ( + result.get("exit_code") != 0 + or result.get("timed_out") is not False + or result.get("launch_error") is not None + ): + raise EncodeError("authenticated command did not exit successfully") + for key in ("duration_seconds", "approval_wait_seconds"): + value = result.get(key) + if ( + isinstance(value, bool) + or not isinstance(value, (int, float)) + or not math.isfinite(value) + or value < 0 + ): + raise EncodeError(f"command result {key} is invalid") + if result["approval_wait_seconds"] != 0: + raise EncodeError("non-interactive command cannot report approval wait") + if not isinstance(result["started_at"], str) or not isinstance(result["ended_at"], str): + raise EncodeError("command UTC timestamps are invalid") + if not isinstance(result["command_id"], str) or not result["command_id"]: + raise EncodeError("command ID is invalid") + if not isinstance(result["environment_keys"], list) or not all( + isinstance(item, str) for item in result["environment_keys"] + ): + raise EncodeError("command environment evidence is invalid") + if not isinstance(result["stdout_tail"], str) or not isinstance(result["stderr_tail"], str): + raise EncodeError("command log tails are invalid") + if ( + result["stdout"].get("kind") != "command-stdout-log" + or result["stderr"].get("kind") != "command-stderr-log" + ): + raise EncodeError("command log identities have invalid kinds") + stdout, stderr = _command_texts(runner, result) + if FAILURE_PATTERN.search(stdout) or FAILURE_PATTERN.search(stderr): + raise EncodeError("authenticated command output contains an explicit failure marker") + return dict(result), stdout, stderr + + +def _exact_marker(text: str, pattern: str, expected: int, label: str) -> None: + matches = re.findall(pattern, text, flags=re.MULTILINE) + if matches != [str(expected)]: + raise EncodeError(f"{label} must report exactly {expected} frames once") + + +def _structure( # pylint: disable=too-many-branches,too-many-locals + path: Path, codec: str, width: int, height: int, frame_count: int +) -> dict[str, Any]: + with path.open("rb") as stream: + prefix = stream.read(1024 * 1024) + if not prefix: + raise EncodeError("encoded output is empty") + if codec == "av1": + if len(prefix) < 32 or prefix[:4] != b"DKIF": + raise EncodeError("AV1 output is not an IVF AV01 stream") + version = int.from_bytes(prefix[4:6], "little") + header_size = int.from_bytes(prefix[6:8], "little") + if version != 0 or header_size != 32 or prefix[8:12] != b"AV01": + raise EncodeError("AV1 output has an invalid IVF version, header, or fourcc") + if ( + int.from_bytes(prefix[12:14], "little") != width + or int.from_bytes(prefix[14:16], "little") != height + ): + raise EncodeError("AV1 IVF dimensions differ from the recipe") + header_count = int.from_bytes(prefix[24:28], "little") + if header_count not in {frame_count, 0xFFFF}: + raise EncodeError( + "AV1 IVF header frame count is neither exact nor the " + "AppEncCuda 0xFFFF sentinel" + ) + size = path.stat().st_size + walked = 0 + with path.open("rb") as stream: + stream.seek(32) + position = 32 + while position < size: + frame_header = stream.read(12) + if len(frame_header) != 12: + raise EncodeError("AV1 IVF has a truncated frame header") + payload_size = int.from_bytes(frame_header[:4], "little") + if payload_size <= 0 or position + 12 + payload_size > size: + raise EncodeError("AV1 IVF has a zero or truncated frame payload") + stream.seek(payload_size, os.SEEK_CUR) + position += 12 + payload_size + walked += 1 + if position != size or stream.read(1): + raise EncodeError("AV1 IVF has trailing bytes") + if walked != frame_count: + raise EncodeError(f"AV1 IVF contains {walked} frames, expected {frame_count}") + return { + "format": "ivf-av01", + "header_frame_count": header_count, + "walked_frame_count": walked, + } + starts = [match.end() for match in re.finditer(b"\x00\x00\x01", prefix)] + if not starts: + raise EncodeError(f"{codec} output has no Annex-B start code") + nal_types = [] + for offset in starts: + if offset < len(prefix): + nal_types.append( + prefix[offset] & 0x1F if codec == "h264" else (prefix[offset] >> 1) & 0x3F + ) + required = 7 if codec == "h264" else 33 + if required not in nal_types: + raise EncodeError(f"{codec} output has no authenticated sequence-parameter NAL") + return {"format": f"annex-b-{codec}", "sequence_parameter_nal": required} + + +def _suffixes( # pylint: disable=too-many-arguments,too-many-positional-arguments + surface: str, + recipe: dict[str, Any], + raw: Path, + encoded: Path, + decoded: Path, + config: Path | None, + frame_count: int, + buffer_mode: str, +) -> tuple[list[str], list[str]]: + intent = recipe["encoder_intent"] + if surface == "native": + encode = [ + "-i", + str(raw), + "-o", + str(encoded), + *recipe["projections"]["native"]["cli_options"], + ] + decode = ["-i", str(encoded), "-o", str(decoded), "-gpu", str(intent["gpu"])] + else: + if config is None: + raise EncodeError("Py encode config was not staged") + encode = [ + "-i", + str(raw), + "-o", + str(encoded), + "-s", + f"{intent['width']}x{intent['height']}", + "-m", + buffer_mode, + "-if", + intent["format"], + "-f", + str(frame_count), + "-g", + str(intent["gpu"]), + "-c", + intent["codec"], + "-json", + str(config), + ] + decode = [ + "-i", + str(encoded), + "-o", + str(decoded), + "-g", + str(intent["gpu"]), + "-d", + "0", + "-f", + str(frame_count), + ] + return encode, decode + + +def _verify_encode_markers( + surface: str, + encode_stdout: str, + encoded: Path, + count: int, + buffer_mode: str, +) -> None: + if surface == "native": + _exact_marker(encode_stdout, r"^Total frames encoded: ([0-9]+)$", count, "native encode") + return + _exact_marker( + encode_stdout, + rf"^Completed encoding ([0-9]+) frames using {buffer_mode.upper()} buffers$", + count, + "Py encode", + ) + output_lines = re.findall(r"^Output file: (.+)$", encode_stdout, flags=re.MULTILINE) + if output_lines != [str(encoded), str(encoded)]: + raise EncodeError("Py encode must print the same requested output path exactly twice") + if len(re.findall(r"^ENCODING COMPLETE$", encode_stdout, flags=re.MULTILINE)) != 1: + raise EncodeError("Py encode completion marker is missing or duplicated") + + +def _verify_markers( + surface: str, + decode_stdout: str, + decoded: Path, + count: int, +) -> str: + if surface == "native": + _exact_marker(decode_stdout, r"^Total frame decoded: ([0-9]+)$", count, "native decode") + saved = re.findall( + rf"^Saved in file {re.escape(str(decoded))} in " + r"(NV12|P016|YUV444|YUV444P16|NV16|P216) format$", + decode_stdout, + flags=re.MULTILINE, + ) + if len(saved) != 1: + raise EncodeError("native decode must report one exact saved output layout") + return saved[0] + success = re.findall( + r"^Successfully decoded requested ([0-9]+) frames to (.+)$", + decode_stdout, + flags=re.MULTILINE, + ) + if success != [(str(count), str(decoded))]: + raise EncodeError("Py decode did not report the exact requested frame count and output") + return "derived" + + +def _decoded_layout(intent: Mapping[str, Any]) -> tuple[str, int]: + pixels = intent["width"] * intent["height"] + layouts = { + "NV12": ("NV12", pixels * 3 // 2), + "YUV420": ("NV12", pixels * 3 // 2), + "P010": ("P016", pixels * 3), + "NV16": ("NV16", pixels * 2), + "P210": ("P216", pixels * 4), + "YUV444": ("YUV444", pixels * 3), + "YUV444_16BIT": ("YUV444P16", pixels * 6), + } + try: + return layouts[intent["format"]] + except KeyError as exc: + raise EncodeError( + f"exact decoder output layout is not proven for {intent['format']}" + ) from exc + + +def _execute_branch( # pylint: disable=too-many-arguments,too-many-locals + *, + surface: str, + root: Path, + branch: Path, + authentication: Any, + recipe: dict[str, Any], + request: dict[str, Any], + runner: Any, + config_identity: dict[str, Any], + config_path: Path | None, + raw_path: Path, + frame_count: int, + launch: Any, +) -> dict[str, Any]: + codec = recipe["encoder_intent"]["codec"] + encoded = branch / f"encoded.{EXTENSIONS[codec]}" + decoded = branch / "decoded.yuv" + if encoded.exists() or decoded.exists(): + raise EncodeError("operation outputs must be fresh") + encode_suffix, decode_suffix = _suffixes( + surface, + recipe, + raw_path, + encoded, + decoded, + config_path, + frame_count, + request["buffer_mode"], + ) + encode_token = authentication.token(SAMPLES[surface][0]) + decode_token = authentication.token(SAMPLES[surface][1]) + timeout = request["timeout_seconds"] + encode_result = launch( + encode_token, + encode_suffix, + workspace=branch, + runner=runner, + cwd=branch, + timeout_seconds=timeout, + stage=f"{surface}-encode", + phase="execute", + ) + encode_command, encode_stdout, _ = _command_evidence( + encode_result, + token=encode_token, + suffix=encode_suffix, + branch=branch, + runner=runner, + timeout=timeout, + stage=f"{surface}-encode", + phase="execute", + ) + _verify_encode_markers( + surface, + encode_stdout, + encoded, + frame_count, + request["buffer_mode"], + ) + encoded_identity = artifact_io.snapshot_artifact( + root, encoded, schema_version="1", kind=f"nvcodec-{codec}-bitstream" + ) + if encoded_identity["size_bytes"] <= 0: + raise EncodeError("encoded output is empty") + structure = _structure( + encoded, + codec, + recipe["encoder_intent"]["width"], + recipe["encoder_intent"]["height"], + frame_count, + ) + decode_result = launch( + decode_token, + decode_suffix, + workspace=branch, + runner=runner, + cwd=branch, + timeout_seconds=timeout, + stage=f"{surface}-decode", + phase="verify", + ) + decode_command, decode_stdout, _ = _command_evidence( + decode_result, + token=decode_token, + suffix=decode_suffix, + branch=branch, + runner=runner, + timeout=timeout, + stage=f"{surface}-decode", + phase="verify", + ) + decoded_identity = artifact_io.snapshot_artifact( + root, decoded, schema_version="1", kind="nvcodec-decoded-frames" + ) + if decoded_identity["size_bytes"] <= 0: + raise EncodeError("independent decode produced no frame bytes") + observed_layout = _verify_markers( + surface, + decode_stdout, + decoded, + frame_count, + ) + expected_layout, bytes_per_frame = _decoded_layout(recipe["encoder_intent"]) + if surface == "native" and observed_layout != expected_layout: + raise EncodeError(f"native decoder reported {observed_layout}, expected {expected_layout}") + expected_decoded_size = bytes_per_frame * frame_count + if decoded_identity["size_bytes"] != expected_decoded_size: + raise EncodeError( + "independent decode byte count differs from the exact evidenced layout: " + f"expected {expected_decoded_size}, got {decoded_identity['size_bytes']}" + ) + artifact_io.verify_external_artifact(request["recipe"]) + if request.get("environment") is not None: + artifact_io.verify_external_artifact(request["environment"]) + artifact_io.verify_external_artifact(request["input"]) + artifact_io.verify_artifact(branch, config_identity) + artifact_io.verify_artifact(root, encoded_identity) + artifact_io.verify_artifact(root, decoded_identity) + return { + "surface": surface, + "status": "operation_verified", + "reasons": [], + "provenance": { + "report": _root_identity(root, branch, authentication.report_identity), + "samples": list(SAMPLES[surface]), + }, + "configuration": _root_identity(root, branch, config_identity), + "encode": { + "command": encode_command, + "frames": frame_count, + "output": encoded_identity, + "structure": structure, + }, + "decode": { + "command": decode_command, + "frames": frame_count, + "output": decoded_identity, + "independent": True, + "layout": expected_layout, + "expected_size_bytes": expected_decoded_size, + "consumed_bitstream_sha256": encoded_identity["sha256"], + }, + } + + +def run_encode_request( # pylint: disable=too-many-branches,too-many-locals,too-many-statements + request: Mapping[str, Any], + *, + workspace: Path, + runner: Any = None, + timeout_seconds: float = 600, +) -> dict[str, Any]: + """Validate, preauthenticate, encode, independently decode, and report.""" + ceiling = _positive_timeout(timeout_seconds, float("inf")) + envelope = _request_envelope(request, ceiling) + if envelope.get("input") is None: + return _input_required_result() + value = _request(request, ceiling) + if ( + value["surface"] in {"pynvc", "both"} + and value.get("environment") is None + and value.get("pynvc_interpreter") is None + ): + return _interpreter_required_result() + try: + artifact_io.validate_recipe_identity(value["recipe"]) + except artifact_io.SkillDependencyError as exc: + return _skill_dependency(exc) + recipe = artifact_io.read_verified_external_json(value["recipe"]) + intent = recipe.get("encoder_intent") + recipe_gpu = intent.get("gpu") if isinstance(intent, Mapping) else None + if isinstance(recipe_gpu, bool) or not isinstance(recipe_gpu, int) or recipe_gpu < 0: + raise EncodeError("recipe encoder_intent.gpu must be a non-negative integer") + required_surfaces = ( + (value["surface"],) if value["surface"] in SURFACES else SURFACES + ) + local_errors: dict[str, str] = {} + if value.get("environment") is not None: + environment_identity: Mapping[str, Any] | None = value["environment"] + environment = artifact_io.validate_environment_envelope( + artifact_io.read_verified_external_json(value["environment"]), + label="encode", + error_type=EncodeError, + required_surfaces=required_surfaces, + ) + else: + if value.get("capability_report") is not None: + raise EncodeError( + "capability_report requires the exact supplied environment to which it is bound" + ) + environment_identity = None + environment, local_errors = sample_provenance.build_local_runtime_binding( + required_surfaces, + pynvc_interpreter=value.get("pynvc_interpreter"), + gpu=recipe_gpu, + timeout_seconds=value["timeout_seconds"], + ) + if recipe_gpu != environment.get("selected_gpu"): + raise EncodeError( + "recipe encoder_intent.gpu must equal environment selected_gpu" + ) + root = _workspace(Path(workspace)) + frame_count = _validate_raw(recipe, value["input"]) + raw_path = artifact_io.verify_external_artifact(value["input"]) + plan, capability_details = _surface_plan( + value["surface"], + recipe, + environment, + value["buffer_mode"], + recipe_identity=value["recipe"], + environment_identity=environment_identity, + capability_identity=value.get("capability_report"), + local_errors=local_errors, + ) + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": RESULT_KIND, + "status": "blocked", + "classification": plan["classification"], + "selected_surfaces": plan["selected_surfaces"], + "surface_plan": plan, + "capability_validation": capability_details, + "request": { + "surface": value["surface"], + "buffer_mode": value["buffer_mode"], + "timeout_seconds": value["timeout_seconds"], + "recipe": value["recipe"], + "capability_report": value.get("capability_report"), + "environment": value.get("environment"), + "runtime_authority": ( + "supplied_setup_environment" + if value.get("environment") is not None + else "consumer_local_binding" + ), + "pynvc_interpreter": value.get("pynvc_interpreter"), + "input": value["input"], + }, + "operations": {}, + "output_artifacts": {}, + "reasons": list(plan["reasons"]), + } + selected = plan["selected_surfaces"] + if value["surface"] in SURFACES and not selected: + selected_surface = value["surface"] + readiness = ( + _native_readiness(environment) + if selected_surface == "native" + else _pynvc_readiness(environment) + ) + setup_reasons = [ + *readiness, + *( + [local_errors[selected_surface]] + if selected_surface in local_errors + else [] + ), + ] + if setup_reasons: + return _setup_dependency( + selected_surface, "; ".join(dict.fromkeys(setup_reasons)) + ) + if value["surface"] == "both": + for surface in SURFACES: + if surface not in selected: + result["operations"][surface] = { + "surface": surface, + "status": "blocked", + "reasons": list(capability_details[surface]["reasons"]), + } + if not selected: + result["status"] = ( + "selection_required" if plan["classification"] == "selection_required" else "blocked" + ) + return result + service = runner or command_runner.CommandRunner(root) + native_auth, pynvc_auth, launch = _providers() + prepared: dict[str, tuple[Path, dict[str, Any], Path | None, dict[str, Any]]] = {} + authentications: dict[str, Any] = {} + for surface in selected: + branch = artifact_io.create_private_workspace(root / surface) + environment_identity = _stage_environment(root, branch, environment) + config_identity, config_path = _configuration(branch, surface, recipe) + prepared[surface] = (branch, config_identity, config_path, environment_identity) + # Attempt every selected authentication before the first codec launch. + for surface in selected: + try: + branch, _config_identity, _config_path, environment_identity = prepared[surface] + authenticate = native_auth if surface == "native" else pynvc_auth + authentications[surface] = authenticate( + environment_workspace=branch, + environment_identity=environment_identity, + workspace=branch, + samples=SAMPLES[surface], + runner=service, + timeout_seconds=value["timeout_seconds"], + report_path=branch / "sample-provenance.json", + ) + except Exception as exc: # pylint: disable=broad-exception-caught + reason = f"preflight authentication failed: {type(exc).__name__}: {exc}" + result["reasons"].append(f"{surface}: {reason}") + result["operations"][surface] = { + "surface": surface, + "status": "blocked", + "reasons": [reason], + } + verified = 0 + for surface in (item for item in selected if item in authentications): + branch, config_identity, config_path, _environment_identity = prepared[surface] + try: + operation = _execute_branch( + surface=surface, + root=root, + branch=branch, + authentication=authentications[surface], + recipe=recipe, + request=value, + runner=service, + config_identity=config_identity, + config_path=config_path, + raw_path=raw_path, + frame_count=frame_count, + launch=launch, + ) + verified += 1 + result["operations"][surface] = operation + result["output_artifacts"][surface] = operation["encode"]["output"] + except Exception as exc: # pylint: disable=broad-exception-caught + reason = f"{type(exc).__name__}: {exc}" + result["operations"][surface] = { + "surface": surface, + "status": "operation_failed", + "reasons": [reason], + } + result["reasons"].append(f"{surface}: {reason}") + if verified == len(selected) and plan["classification"] == "ready": + result["status"] = "operation_verified" + elif verified: + result["status"] = "partial" + elif any(item.get("status") == "blocked" for item in result["operations"].values()): + result["status"] = "blocked" + else: + result["status"] = "operation_failed" + return result + + +def main(argv: list[str] | None = None) -> int: + """Run the thin JSON-only encode CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + output = artifact_io.preflight_fresh_output(args.workspace, args.output) + request = artifact_io.strict_json_loads(args.request.read_bytes()) + result = run_encode_request(request, workspace=args.workspace) + root = ( + _workspace(args.workspace) + if result.get("status") in {"input_required", "dependency_required"} + else artifact_io.resolve_private_workspace(args.workspace) + ) + artifact_io.write_fresh_json(root, output, result) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 if result["status"] == "operation_verified" else 2 + except (OSError, ValueError) as exc: + print(json.dumps({"status": "error", "error": str(exc)}, sort_keys=True)) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-pipeline/scripts/pipeline_controller.py b/skills/jetson-video-pipeline/scripts/pipeline_controller.py new file mode 100644 index 00000000..7f19acef --- /dev/null +++ b/skills/jetson-video-pipeline/scripts/pipeline_controller.py @@ -0,0 +1,2685 @@ +#!/usr/bin/env python3 +"""Run compact, authenticated Video SDK pipeline routes and verify handoffs.""" + +# The route handlers deliberately keep their complete producer/consumer facts +# local so boundary evidence stays reviewable. +# pylint: disable=too-many-arguments,too-many-locals,too-many-lines + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import importlib +import json +import math +import re +import stat +import sys +from collections import Counter +from pathlib import Path +from typing import Any, Callable, Mapping, Sequence +from urllib.parse import urlparse + +if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise SystemExit("invoke this producer with isolated Python: python3 -I") + +# Isolated mode excludes the script directory. Add only this skill's fixed +# private module directory; sibling skills remain subprocess boundaries. +_SCRIPT_DIR = Path(__file__).absolute().parent +if str(_SCRIPT_DIR) not in sys.path: + sys.path.insert(0, str(_SCRIPT_DIR)) + +# pylint: disable=wrong-import-position,import-error +import _pipeline_provenance as sample_provenance # noqa: E402 +import _pipeline_runtime as artifact_io # noqa: E402 +# pylint: enable=wrong-import-position,import-error + +command_runner = artifact_io +pynvc_sample_provenance = sample_provenance + +SCHEMA_VERSION = "1.0" +REQUEST_KIND = "nvcodec-pipeline-request" +RESULT_KIND = "nvcodec-pipeline-result" +IDENTITY_KEYS = {"schema_version", "kind", "path", "size_bytes", "sha256"} +BOUNDARY_KEYS = { + "producer", "consumer", "artifact_path", "artifact_size_bytes", + "artifact_sha256", "status", "reason", +} +ROUTE_TABLE = { + "encode_decode": "_run_encode_decode", + "native_transcode": "_run_native_transcode", + "pynvc_segments": "_run_pynvc_segments", + "container_triage": "_run_container_triage", + "av1_verify": "_run_av1_verify", + "acceptance": "_run_acceptance", +} +COMMAND_KEYS = { + "schema_version", "kind", "command_id", "stage", "phase", "argv", "cwd", + "environment_keys", "started_at", "ended_at", "duration_seconds", + "approval_wait_seconds", "timeout_seconds", "exit_code", "timed_out", + "launch_error", "stdout", "stderr", "stdout_tail", "stderr_tail", +} +NATIVE_DECODE = re.compile(r"^Total frame decoded: ([0-9]+)$", re.MULTILINE) +NATIVE_TRANSCODE_LEGACY = re.compile(r"\(#totFrames=([0-9]+)\)") +NATIVE_TRANSCODE_CURRENT = re.compile( + r"^Total frame transcoded: ([0-9]+)$", re.MULTILINE +) +NATIVE_SAVED = re.compile(r"^Saved in file (.+) of (8|10) bit depth$", re.MULTILINE) +NATIVE_ENCODE = re.compile(r"^Total frames encoded: ([0-9]+)$", re.MULTILINE) +PYNVC_ALL_DECODE = re.compile( + r"^Successfully decoded all ([0-9]+) frames to (.+)$", re.MULTILINE +) +PYNVC_LIMITED_DECODE = re.compile( + r"^Successfully decoded requested ([0-9]+) frames to (.+)$", re.MULTILINE +) +SEGMENT_CREATED = re.compile(r"^\u2713 Created: (.+)$", re.MULTILINE) +SEGMENT_SUMMARY = re.compile( + r"^Summary: ([0-9]+) segments created successfully$", re.MULTILINE +) +EXPLICIT_FAILURE = sample_provenance.OFFICIAL_SAMPLE_FAILURE_PATTERN +FAILURE_DETAIL_LIMIT = 200 +ACCEPTANCE_STATIC_FILES = ( + "report.md", + "capability-report.md", + "review-findings.md", + "timing.json", + "task-results.jsonl", + "commands.jsonl", + "evidence/summary.json", + "evidence/representative-content.json", +) +ACCEPTANCE_TASKS = ( + "readiness", + "capabilities", + "content", + "p4_recipe", + "p5_recipe", + "p4_encode_decode", + "p5_encode_decode", + "p4_benchmark", + "p5_benchmark", + "handoffs", +) +ACCEPTANCE_LEDGER_REFERENCES = ("commands", "task_results", "timing") +REPRESENTATIVE_CONTENT_FIELDS = ( + "source_url", "license", "attribution", "path", "size_bytes", "sha256", +) +MEDIA_INPUT_ROUTES = { + "encode_decode", + "native_transcode", + "pynvc_segments", + "container_triage", + "av1_verify", + "acceptance", +} +DIRECT_INPUT_ROUTES = {"native_transcode", "pynvc_segments", "av1_verify"} + + +def _acceptance_files(request: Mapping[str, Any]) -> tuple[str, ...]: + filename = request.get("validation_result_filename") + if ( + not isinstance(filename, str) + or Path(filename).name != filename + or not filename.endswith(".json") + or filename in {"summary.json", "representative-content.json"} + ): + raise PipelineError( + "validation_result_filename must be one fresh non-reserved JSON filename" + ) + return (*ACCEPTANCE_STATIC_FILES, f"evidence/{filename}") + + +class PipelineError(ValueError): + """A pipeline request or operation violates the customer contract.""" + + +class SurfaceInputRequired(PipelineError): + """One exact selector is needed to authenticate a manually installed SDK.""" + + def __init__(self, field: str, surface: str, reason: str): + super().__init__(reason) + self.field = field + self.surface = surface + self.reason = reason + + +class SurfaceSetupRequired(PipelineError): + """The selected SDK surface failed independent local authentication.""" + + def __init__(self, surface: str, reason: str): + super().__init__(reason) + self.surface = surface + self.reason = reason + + +class BoundaryFailure(PipelineError): + """One named producer-to-consumer boundary failed.""" + + def __init__(self, producer: str, consumer: str, path: str, reason: str): + super().__init__(reason) + self.producer = producer + self.consumer = consumer + self.path = path + self.reason = reason + + +def _installed_sibling(name: str) -> bool: + """Return whether the lexical installed catalog contains one sibling.""" + candidate = Path(__file__).absolute().parents[2] / name / "SKILL.md" + try: + details = candidate.lstat() + resolved_root = candidate.parent.resolve(strict=True) + resolved = candidate.resolve(strict=True) + except OSError: + return False + return ( + stat.S_ISREG(details.st_mode) + and not stat.S_ISLNK(details.st_mode) + and resolved_root.is_dir() + and resolved == resolved_root / "SKILL.md" + ) + + +_providers_cache: tuple[Callable[..., Any], ...] | None = None # pylint: disable=invalid-name + + +def _encode_controller() -> Any: + """Load the encode owner only for a route that consumes encode contracts.""" + try: + return importlib.import_module("encode_controller") + except ModuleNotFoundError as exc: + if exc.name == "encode_controller": + raise PipelineError( + "pipeline encode controller is missing; reinstall jetson-video-pipeline" + ) from exc + raise + + +def _run_encode_request(*args: Any, **kwargs: Any) -> Any: + """Delegate without importing the encode owner on unrelated routes.""" + return _encode_controller().run_encode_request(*args, **kwargs) + + +def _providers() -> tuple[Callable[..., Any], ...]: + global _providers_cache # pylint: disable=global-statement + if _providers_cache is None: + _providers_cache = ( + _run_encode_request, + sample_provenance.authenticate_native_samples, + pynvc_sample_provenance.authenticate_pynvc_samples, + sample_provenance.run_authenticated, + ) + return _providers_cache + + +def _workspace(path: Path) -> Path: + requested = Path(path).expanduser() + if requested.exists(): + root = artifact_io.resolve_private_workspace(requested) + if any(root.iterdir()): + raise PipelineError("pipeline workspace must be fresh and empty") + return root + return artifact_io.create_private_workspace(requested) + + +def _timeout(value: Any, maximum: float) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PipelineError("timeout_seconds must be a positive finite number") + seconds = float(value) + if not math.isfinite(seconds) or seconds <= 0 or seconds > maximum: + raise PipelineError(f"timeout_seconds must be in (0, {maximum}]") + return seconds + + +def _request(value: Any, timeout_ceiling: float) -> dict[str, Any]: + if not isinstance(value, Mapping): + raise PipelineError("request must be a JSON object") + request = dict(value) + if request.get("schema_version") != SCHEMA_VERSION or request.get("kind") != REQUEST_KIND: + raise PipelineError( + f"request identity must be schema {SCHEMA_VERSION} kind {REQUEST_KIND}" + ) + if request.get("route") not in ROUTE_TABLE: + raise PipelineError(f"route must be one of {sorted(ROUTE_TABLE)}") + request["mode"] = request.get("mode", "execute") + if request["mode"] not in {"dry_run", "execute"}: + raise PipelineError("mode must be dry_run or execute") + request["timeout_seconds"] = _timeout( + request.get("timeout_seconds", timeout_ceiling), timeout_ceiling + ) + return request + + +def _exact_http_url(value: Any, *, label: str) -> str: + """Return one unchanged, credential-free HTTP(S) URL.""" + if ( + not isinstance(value, str) + or not value + or value != value.strip() + or "\\" in value + or any( + character.isspace() + or ord(character) < 0x20 + or ord(character) == 0x7F + for character in value + ) + ): + raise PipelineError(f"{label} must be one exact HTTP(S) URL") + try: + parsed = urlparse(value) + hostname = parsed.hostname + port = parsed.port + username = parsed.username + password = parsed.password + except (UnicodeError, ValueError) as exc: + raise PipelineError(f"{label} must be one exact HTTP(S) URL") from exc + if not ( + parsed.scheme in {"http", "https"} + and parsed.netloc + and hostname + and username is None + and password is None + and (port is None or 1 <= port <= 65535) + ): + raise PipelineError(f"{label} must be one exact HTTP(S) URL") + return value + + +def _input_required(reason: str) -> dict[str, Any]: + """Return the common no-work result for absent meaningful media.""" + return { + "status": "input_required", + "gate": "media_input", + "reason": reason, + "accepted_inputs": ["local_path", "http_url", "https_url"], + "next_action": "provide_media_path_or_url", + "synthetic_input_allowed": False, + "retrieval_attempted": False, + "operations": {}, + "planned_operations": [], + "no_mutation_performed": True, + } + + +def _media_preflight( # pylint: disable=too-many-return-statements,too-many-branches + request: Mapping[str, Any], +) -> dict[str, Any] | None: + """Resolve missing user media before any route creates a workspace or starts work.""" + route = request["route"] + if route not in MEDIA_INPUT_ROUTES: + return None + if route == "container_triage": + supplied_input = request.get("input") + supplied_url = request.get("source_url") + if supplied_input is not None and supplied_url is not None: + raise PipelineError( + "container triage requires exactly one user source: input or source_url" + ) + if supplied_input is None and supplied_url is None: + return _input_required("container triage requires user-provided media") + gate = request.get("target_eligibility") + if isinstance(gate, Mapping) and gate.get("eligible") is False: + _eligible, reasons = _target_gate(request) + return { + "status": "blocked", + "gate": "target_eligibility", + "reasons": reasons, + "retrieval_attempted": False, + "operations": {}, + "planned_operations": [], + "no_mutation_performed": True, + } + if supplied_input is None: + _exact_http_url( + supplied_url, label="container source_url" + ) + return None + if route in DIRECT_INPUT_ROUTES: + if request.get("input") is None: + return _input_required(f"{route} requires user-provided media") + return None + if route == "encode_decode": + supplied = request.get("encode_request_ref") + if supplied is None: + return _input_required( + "encode/decode requires a user-media-bound encode request" + ) + reference = _external( + supplied, label="encode_request_ref", + schema="1.0", kinds=("nvcodec-encode-request",), + ) + encode_request = artifact_io.read_verified_external_json(reference) + if not isinstance(encode_request, Mapping): + raise PipelineError("encode_request_ref must contain a JSON object") + if ( + encode_request.get("schema_version") != "1.0" + or encode_request.get("kind") != "nvcodec-encode-request" + ): + raise PipelineError("encode request has an invalid schema or kind") + if encode_request.get("input") is None: + result = _input_required("encode/decode requires user-provided media") + result["encode_request"] = reference + return result + return None + raw_references = request.get("references") + if ( + not isinstance(raw_references, Mapping) + or not raw_references + or raw_references.get("content") is None + ): + return _input_required( + "acceptance references require user-provided content evidence" + ) + return None + + +def _planned_workspace(path: Path) -> Path: + """Resolve a proposed workspace locator without creating it.""" + requested = Path(path).expanduser() + if requested.exists(): + root = artifact_io.resolve_private_workspace(requested) + if any(root.iterdir()): + raise PipelineError("pipeline workspace must be fresh and empty") + return root + try: + parent = requested.parent.resolve(strict=True) + except (OSError, RuntimeError) as exc: + raise PipelineError("pipeline workspace parent is unavailable") from exc + return parent / requested.name + + +def _external( + value: Any, *, label: str, schema: str | None = None, + kinds: Sequence[str] | None = None, +) -> dict[str, Any]: + if not isinstance(value, Mapping) or set(value) != IDENTITY_KEYS: + raise PipelineError(f"{label} must be an exact portable artifact identity") + identity = dict(value) + if schema is not None and identity.get("schema_version") != schema: + raise PipelineError(f"{label} must use schema {schema}") + if kinds is not None and identity.get("kind") not in kinds: + raise PipelineError(f"{label} kind must be one of {list(kinds)}") + try: + artifact_io.verify_external_artifact(identity) + except (OSError, ValueError) as exc: + raise PipelineError(f"{label} is not current: {exc}") from exc + return identity + + +def _bind_environment_gpu( + environment: Mapping[str, Any], gpu: Any, *, label: str +) -> int: + if isinstance(gpu, bool) or not isinstance(gpu, int) or gpu < 0: + raise PipelineError(f"{label} gpu must be a non-negative integer") + if environment.get("selected_gpu") != gpu: + raise PipelineError(f"{label} gpu must equal environment selected_gpu") + return gpu + + +def _environment( + request: Mapping[str, Any], + *, + required_surfaces: Sequence[str] = artifact_io.SURFACE_ORDER, + selected_gpu: int | None = None, + local_errors: dict[str, str] | None = None, +) -> tuple[dict[str, Any] | None, dict[str, Any]]: + supplied = request.get("environment") + if supplied is not None: + identity = _external( + supplied, label="environment", + schema=artifact_io.ENVIRONMENT_SCHEMA_VERSION, + kinds=(artifact_io.ENVIRONMENT_KIND,), + ) + value = artifact_io.validate_environment_envelope( + artifact_io.read_verified_external_json(identity), + label="pipeline", + error_type=PipelineError, + required_surfaces=required_surfaces, + ) + else: + gpu = request.get("gpu", 0) if selected_gpu is None else selected_gpu + if isinstance(gpu, bool) or not isinstance(gpu, int) or gpu < 0: + raise PipelineError("selected gpu must be a non-negative integer") + if "pynvc" in required_surfaces: + interpreter = request.get("pynvc_interpreter") + if ( + not isinstance(interpreter, str) + or not interpreter + or interpreter != interpreter.strip() + or not Path(interpreter).is_absolute() + or str(Path(interpreter)) != interpreter + ): + raise SurfaceInputRequired( + "pynvc_interpreter", + "pynvc", + "provide one canonical absolute Python interpreter for the existing " + "PyNvVideoCodec environment; no environment scan is performed", + ) + value, errors = sample_provenance.build_local_runtime_binding( + required_surfaces, + pynvc_interpreter=request.get("pynvc_interpreter"), + gpu=gpu, + timeout_seconds=float(request.get("timeout_seconds", 600)), + ) + if local_errors is not None: + local_errors.update(errors) + if len(required_surfaces) == 1 and required_surfaces[0] in errors: + raise SurfaceSetupRequired(required_surfaces[0], errors[required_surfaces[0]]) + identity = None + # Direct pipeline routes select exactly one SDK. Convert structurally + # valid but unready supplied evidence into the same setup handoff as local + # authentication. Multi-surface encode planning deliberately handles its + # two surfaces independently in encode_controller._surface_plan. + if len(required_surfaces) == 1: + surface = required_surfaces[0] + defects = ( + artifact_io.native_surface_defects(value) + if surface == "native" + else artifact_io.pynvc_surface_defects(value) + ) + if defects: + raise SurfaceSetupRequired(surface, "; ".join(defects)) + if selected_gpu is not None: + _bind_environment_gpu(value, selected_gpu, label="request") + elif "gpu" in request: + _bind_environment_gpu(value, request["gpu"], label="request") + return identity, value + + +def _pynvc_full_samples_preflight( + request: Mapping[str, Any], +) -> dict[str, Any] | None: + """Block Torch-dependent Py routes before workspace creation or launch.""" + route = request["route"] + if route == "container_triage": + if request.get("surface") != "pynvc": + return None + elif route != "pynvc_segments": + return None + identity, environment = _environment(request, required_surfaces=("pynvc",)) + reasons = artifact_io.pynvc_full_sample_defects(environment) + if not reasons: + return None + installed = _installed_sibling("jetson-video-setup") + action = ( + "use jetson-video-setup to provision a new full-samples " + "PyNvVideoCodec environment, then retry" + if installed + else "install jetson-video-setup, provision a new full-samples " + "PyNvVideoCodec environment, then retry" + ) + return { + "status": "dependency_required", + "gate": "pynvc_full_samples", + "surface": "pynvc", + "required_profile": "full-samples", + "reasons": reasons, + "dependency": { + "skill": "jetson-video-setup", + "installed": installed, + "needed_for": "PyNvVideoCodec full-samples environment provisioning", + "next_action": action, + }, + "retrieval_attempted": False, + "operations": {}, + "planned_operations": [], + "planned_output_paths": [], + "validated_identities": ( + {"environment": identity} if identity is not None else {} + ), + "runtime_authority": ( + "supplied_setup_environment" + if identity is not None + else "consumer_local_binding" + ), + "no_mutation_performed": True, + } + + +def _recipe(request: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: + identity = _external( + request.get("recipe"), label="recipe", schema="2.0", + kinds=("nvcodec-recipe",), + ) + value = artifact_io.read_verified_external_json(identity) + try: + artifact_io.validate_recipe_identity(identity) + except artifact_io.SkillDependencyError: + # Preserve the public composition handoff. SkillDependencyError is a + # ValueError subclass, so the generic wrapper below would otherwise + # erase its structured dependency_required result. + raise + except ValueError as exc: + raise PipelineError(f"recipe validation failed: {exc}") from exc + return identity, value + + +def _stage_environment(branch: Path, environment: Mapping[str, Any]) -> dict[str, Any]: + return artifact_io.write_fresh_json( + branch, branch / "environment.json", dict(environment) + ) + + +def _promote(root: Path, child: Path, identity: Mapping[str, Any]) -> dict[str, Any]: + path = artifact_io.verify_artifact(child, identity) + return artifact_io.snapshot_artifact( + root, path, schema_version=identity["schema_version"], kind=identity["kind"] + ) + + +def _command_text(runner: Any, result: Mapping[str, Any]) -> tuple[str, str]: + runner_workspace = artifact_io.resolve_private_workspace(Path(runner.workspace)) + return ( + artifact_io.read_verified_text( + runner_workspace, result["stdout"], max_bytes=16 * 1024 * 1024 + ), + artifact_io.read_verified_text( + runner_workspace, result["stderr"], max_bytes=16 * 1024 * 1024 + ), + ) + + +def _first_failure_line(stdout: str, stderr: str) -> str | None: + """Return one bounded, printable line that matched the failure vocabulary.""" + for output in (stdout, stderr): + for line in output.splitlines(): + if EXPLICIT_FAILURE.search(line): + printable = "".join( + character if character.isprintable() else " " + for character in line + ) + return re.sub(r"\s+", " ", printable).strip()[:FAILURE_DETAIL_LIMIT] + return None + + +def _command( + result: Any, *, runner: Any, argv: Sequence[str], cwd: Path, + timeout: float, stage: str, phase: str, +) -> tuple[dict[str, Any], str, str]: + expected = { + "schema_version": "1", "kind": "command-result", "argv": list(argv), + "cwd": str(cwd), "timeout_seconds": float(timeout), "stage": stage, + "phase": phase, + } + if ( + not isinstance(result, Mapping) + or set(result) != COMMAND_KEYS + or any(result.get(key) != value for key, value in expected.items()) + ): + raise PipelineError("command result does not bind the exact launch") + if ( + result.get("exit_code") != 0 + or result.get("timed_out") is not False + or result.get("launch_error") is not None + ): + raise PipelineError( + f"command failed: exit={result.get('exit_code')} " + f"timeout={result.get('timed_out')} launch={result.get('launch_error')}" + ) + for key in ("duration_seconds", "approval_wait_seconds"): + number = result.get(key) + if ( + isinstance(number, bool) + or not isinstance(number, (int, float)) + or not math.isfinite(number) + or number < 0 + ): + raise PipelineError(f"command {key} is invalid") + if result["approval_wait_seconds"] != 0: + raise PipelineError("non-interactive command cannot report approval wait") + if not isinstance(result["environment_keys"], list) or not all( + isinstance(item, str) for item in result["environment_keys"] + ): + raise PipelineError("command environment-key evidence is invalid") + stdout, stderr = _command_text(runner, result) + failure_line = _first_failure_line(stdout, stderr) + if failure_line is not None: + raise PipelineError( + "command output contains an explicit failure marker: " + f"{failure_line}" + ) + return dict(result), stdout, stderr + + +def _launch( + token: Any, suffix: Sequence[str], *, branch: Path, runner: Any, + timeout: float, stage: str, phase: str, +) -> tuple[dict[str, Any], str, str]: + launch = _providers()[3] + result = launch( + token, list(suffix), workspace=branch, runner=runner, cwd=branch, + timeout_seconds=timeout, stage=stage, phase=phase, + ) + return _command( + result, runner=runner, argv=[*token.launcher, *suffix], cwd=branch, + timeout=timeout, stage=stage, phase=phase, + ) + + +def _authenticate( + surface: str, samples: Sequence[str], *, root: Path, branch: Path, + environment: Mapping[str, Any], runner: Any, timeout: float, +) -> tuple[Any, dict[str, Any]]: + environment_identity = _stage_environment(branch, environment) + provider = _providers()[1] if surface == "native" else _providers()[2] + authentication = provider( + environment_workspace=branch, environment_identity=environment_identity, + workspace=branch, samples=tuple(samples), runner=runner, + timeout_seconds=timeout, report_path=branch / "sample-provenance.json", + ) + report = _promote(root, branch, authentication.report_identity) + return authentication, report + + +def _native_decode_count(text: str, expected_path: Path | None = None) -> int: + matches = NATIVE_DECODE.findall(text) + if len(matches) != 1 or int(matches[0]) <= 0: + raise PipelineError("AppDec must report one positive 'Total frame decoded' marker") + if expected_path is not None: + saved = re.findall(r"^Saved in file (.+) in .+ format$", text, re.MULTILINE) + if saved != [str(expected_path)]: + raise PipelineError("AppDec saved-output marker does not name the requested output") + return int(matches[0]) + + +def _pynvc_decode_count(text: str, expected_path: Path) -> int: + matches = PYNVC_ALL_DECODE.findall(text) + PYNVC_LIMITED_DECODE.findall(text) + if len(matches) != 1: + raise PipelineError("Py decode must report exactly one successful frame marker") + count, path = matches[0] + if int(count) <= 0 or path != str(expected_path): + raise PipelineError("Py decode marker has a nonpositive count or wrong output path") + return int(count) + + +def _decode( + surface: str, authentication: Any, encoded: Path, output: Path, *, + root: Path, branch: Path, runner: Any, timeout: float, gpu: int, + expected_frames: int | None = None, +) -> dict[str, Any]: + if output.exists(): + raise PipelineError(f"decode output is not fresh: {output}") + if surface == "native": + sample = "AppDec" + suffix = ["-i", str(encoded), "-o", str(output), "-gpu", str(gpu)] + else: + sample = "samples/advanced/decode.py" + suffix = ["-i", str(encoded), "-o", str(output), "-g", str(gpu), "-d", "0"] + if expected_frames is not None: + suffix.extend(["-f", str(expected_frames)]) + command, stdout, _stderr = _launch( + authentication.token(sample), suffix, branch=branch, runner=runner, + timeout=timeout, stage=f"{surface}-decode", phase="verify", + ) + frames = ( + _native_decode_count(stdout, output) + if surface == "native" + else _pynvc_decode_count(stdout, output) + ) + if expected_frames is not None and frames != expected_frames: + raise PipelineError( + f"decode produced {frames} frames but producer reported {expected_frames}" + ) + identity = artifact_io.snapshot_artifact( + root, output, schema_version="1", kind="nvcodec-decoded-frames" + ) + if identity["size_bytes"] <= 0: + raise PipelineError("decode produced no frame bytes") + return {"command": command, "frames": frames, "output": identity, "independent": True} + + +def _boundary( + producer: str, consumer: str, identity: Mapping[str, Any], *, + status: str, reason: str = "", +) -> dict[str, Any]: + return { + "producer": producer, "consumer": consumer, + "artifact_path": identity.get("path", ""), + "artifact_size_bytes": identity.get("size_bytes"), + "artifact_sha256": identity.get("sha256"), + "status": status, "reason": reason, + } + + +def _run_encode_decode( # pylint: disable=too-many-branches,too-many-statements + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + encode_request_ref = _external( + request.get("encode_request_ref"), label="encode_request_ref", + schema="1.0", kinds=("nvcodec-encode-request",), + ) + encode_request = artifact_io.read_verified_external_json(encode_request_ref) + if not isinstance(encode_request, Mapping): + raise PipelineError("encode_request_ref must contain a JSON object") + child = artifact_io.create_private_workspace(root / "encode-decode") + encoded = _providers()[0]( + encode_request, workspace=child, runner=runner, timeout_seconds=timeout + ) + if not isinstance(encoded, Mapping) or encoded.get("kind") != "nvcodec-encode-result": + raise PipelineError("encode controller returned an invalid result") + artifact_io.verify_external_artifact(encode_request_ref) + encoded_result_ref = artifact_io.write_fresh_json( + root, child / "encode-controller-result.json", dict(encoded) + ) + operations: dict[str, Any] = {} + boundaries: list[dict[str, Any]] = [] + verified = 0 + for surface in encoded.get("selected_surfaces", []): + operation = encoded.get("operations", {}).get(surface) + if not isinstance(operation, Mapping) or operation.get("status") != "operation_verified": + reason = ( + "; ".join(operation.get("reasons", [])) + if isinstance(operation, Mapping) else "missing operation" + ) + boundaries.append({ + "producer": f"{surface}-encode", "consumer": f"{surface}-decode", + "artifact_path": "", "artifact_size_bytes": None, + "artifact_sha256": None, "status": "failed", "reason": reason, + }) + operations[surface] = ( + dict(operation) if isinstance(operation, Mapping) + else {"status": "operation_failed"} + ) + continue + encode_data = operation.get("encode", {}) + decode_data = operation.get("decode", {}) + encoded_identity = encode_data.get("output") + decoded_identity = decode_data.get("output") + try: + if ( + not isinstance(encoded_identity, Mapping) + or not isinstance(decoded_identity, Mapping) + ): + raise PipelineError("operation omits encode or decode output identity") + promoted_encoded = _promote(root, child, encoded_identity) + promoted_decoded = _promote(root, child, decoded_identity) + if ( + decode_data.get("independent") is not True + or decode_data.get("frames", 0) <= 0 + or decode_data.get("consumed_bitstream_sha256") != promoted_encoded["sha256"] + ): + raise PipelineError("independent decode did not consume the exact encoded SHA-256") + artifact_io.verify_artifact(root, promoted_encoded) + boundaries.append( + _boundary( + f"{surface}-encode", f"{surface}-decode", promoted_encoded, + status="verified", + ) + ) + operations[surface] = { + "status": "operation_verified", "encoded": promoted_encoded, + "decoded": promoted_decoded, "encode_frames": encode_data.get("frames"), + "decode_frames": decode_data.get("frames"), + } + verified += 1 + except (OSError, ValueError) as exc: + boundaries.append( + _boundary( + f"{surface}-encode", f"{surface}-decode", + encoded_identity if isinstance(encoded_identity, Mapping) else {}, + status="failed", reason=str(exc), + ) + ) + operations[surface] = {"status": "operation_failed", "reasons": [str(exc)]} + encoded_operations = encoded.get("operations", {}) + if isinstance(encoded_operations, Mapping): + for surface in ("native", "pynvc"): + operation = encoded_operations.get(surface) + if surface not in operations and isinstance(operation, Mapping): + operations[surface] = dict(operation) + selected = encoded.get("selected_surfaces", []) + if encoded.get("status") in { + "selection_required", + "input_required", + "dependency_required", + }: + status = "selection_required" + if encoded.get("status") != "selection_required": + status = str(encoded["status"]) + elif not selected: + status = "blocked" + elif verified == len(selected) and encoded.get("classification") == "ready": + status = "complete" + elif verified: + status = "partial" + else: + status = "failed" + result = { + "status": status, "surface_status": encoded.get("status"), + "selected_surfaces": list(selected), "operations": operations, + "boundaries": boundaries, "encode_request": encode_request_ref, + "encode_result": encoded_result_ref, "encode_workspace": str(child), + } + if isinstance(encoded.get("surface_plan"), Mapping): + result["surface_plan"] = dict(encoded["surface_plan"]) + if isinstance(encoded.get("capability_validation"), Mapping): + result["surface_evaluations"] = dict(encoded["capability_validation"]) + for key in ("gate", "surface", "required_field", "reason", "next_action", "dependency"): + if key in encoded: + result[key] = encoded[key] + return result + + +def _run_native_transcode( + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + _recipe_identity, recipe = _recipe(request) + content = _external(request.get("input"), label="input video") + input_path = artifact_io.verify_external_artifact(content) + intent = recipe["encoder_intent"] + _environment_identity, environment = _environment( + request, + required_surfaces=("native",), + selected_gpu=intent.get("gpu"), + ) + _bind_environment_gpu(environment, intent.get("gpu"), label="recipe") + if intent.get("codec") != "hevc" or recipe["projections"]["native"]["status"] != "exact": + raise PipelineError("native_transcode requires an exact HEVC native recipe") + branch = artifact_io.create_private_workspace(root / "native-transcode") + authentication, provenance = _authenticate( + "native", ("AppTrans", "AppDec"), root=root, branch=branch, + environment=environment, runner=runner, timeout=timeout, + ) + output = branch / "transcoded.hevc" + decoded = branch / "transcoded-decoded.yuv" + options = _transcode_options(recipe) + suffix = [ + "-i", str(input_path), "-o", str(output), "-gpu", str(intent["gpu"]), + *options, + ] + return _execute_native_transcode( + authentication=authentication, suffix=suffix, output=output, + decoded=decoded, root=root, branch=branch, runner=runner, + timeout=timeout, intent=intent, content=content, provenance=provenance, + ) + + +def _transcode_options(recipe: Mapping[str, Any]) -> list[str]: + """Project schema-2 encoder controls onto AppTrans without raw-input flags.""" + projection = recipe["projections"]["native"]["cli_options"] + options: list[str] = [] + index = 0 + while index < len(projection): + option = projection[index] + if option == "-temporalaq": + options.append(option) + index += 1 + continue + if index + 1 >= len(projection): + raise PipelineError("native recipe projection has an incomplete option") + value = projection[index + 1] + if option not in {"-s", "-if", "-gpu"}: + options.extend([option, value]) + index += 2 + return options + + +def _execute_native_transcode( + *, authentication: Any, suffix: Sequence[str], output: Path, decoded: Path, + root: Path, branch: Path, runner: Any, timeout: float, + intent: Mapping[str, Any], content: Mapping[str, Any], provenance: Mapping[str, Any], +) -> dict[str, Any]: + """Launch and verify the already-prepared AppTrans-to-AppDec boundary.""" + try: + command, stdout, _stderr = _launch( + authentication.token("AppTrans"), suffix, branch=branch, runner=runner, + timeout=timeout, stage="native-transcode", phase="execute", + ) + except (OSError, ValueError) as exc: + raise BoundaryFailure( + "AppTrans", "AppDec", str(output), str(exc) + ) from exc + frame_matches = ( + NATIVE_TRANSCODE_LEGACY.findall(stdout) + + NATIVE_TRANSCODE_CURRENT.findall(stdout) + ) + saved_matches = NATIVE_SAVED.findall(stdout) + if len(frame_matches) != 1 or int(frame_matches[0]) <= 0: + raise BoundaryFailure( + "AppTrans", "AppDec", str(output), + "AppTrans must report exactly one positive frame marker", + ) + bit_depth = "10" if intent["format"] in {"P010", "P210", "YUV444_16BIT"} else "8" + if saved_matches != [(str(output), bit_depth)]: + raise BoundaryFailure( + "AppTrans", "AppDec", str(output), + "AppTrans saved-output marker is missing or mismatched", + ) + frames = int(frame_matches[0]) + encoded = artifact_io.snapshot_artifact( + root, output, schema_version="1", kind="nvcodec-hevc-bitstream" + ) + if encoded["size_bytes"] <= 0: + raise BoundaryFailure("AppTrans", "AppDec", str(output), "AppTrans output is empty") + artifact_io.verify_external_artifact(content) + artifact_io.verify_artifact(root, encoded) + try: + decode = _decode( + "native", authentication, output, decoded, root=root, branch=branch, + runner=runner, timeout=timeout, gpu=int(intent["gpu"]), + expected_frames=frames, + ) + except (OSError, ValueError) as exc: + raise BoundaryFailure( + "AppTrans", "AppDec", str(output), str(exc) + ) from exc + artifact_io.verify_artifact(root, encoded) + return { + "status": "complete", "surface": "native", "provenance": provenance, + "producer": {"sample": "AppTrans", "command": command, "frames": frames, "output": encoded}, + "consumer": {"sample": "AppDec", **decode, "consumed_sha256": encoded["sha256"]}, + "boundaries": [_boundary("AppTrans", "AppDec", encoded, status="verified")], + } + + +def _protected_data(token: Any, filename: str) -> Path: + matches: list[Path] = [] + for seal in token.protected_files: + identity = seal.identity + path = artifact_io.verify_external_artifact(identity) + if path.name == filename: + matches.append(path) + if len(matches) != 1: + raise PipelineError(f"authenticated sample must bind exactly one {filename}") + return matches[0] + + +def _segment_rows(path: Path) -> list[tuple[float, float]]: + rows: list[tuple[float, float]] = [] + try: + lines = path.read_text(encoding="utf-8", errors="strict").splitlines() + except (OSError, UnicodeError) as exc: + raise PipelineError(f"cannot read authenticated segments.txt: {exc}") from exc + for number, line in enumerate(lines, 1): + if not line.strip(): + continue + fields = line.split() + if len(fields) != 2: + raise PipelineError(f"segments.txt line {number} must have two values") + try: + start, end = (float(item) for item in fields) + except ValueError as exc: + raise PipelineError(f"segments.txt line {number} is not numeric") from exc + if not all(math.isfinite(item) for item in (start, end)) or start < 0 or start >= end: + raise PipelineError(f"segments.txt line {number} has an invalid range") + rows.append((start, end)) + if not rows: + raise PipelineError("authenticated segments.txt is empty") + return rows + + +def _record_unmaterialized_segment_failure( + outputs: list[dict[str, Any]], + boundaries: list[dict[str, Any]], + *, + expected_path: str, + reason: str, +) -> None: + outputs.append( + { + "expected_segment_path": expected_path, + "status": "operation_failed", + "reason": reason, + } + ) + boundaries.append( + _boundary( + "create_video_segments.py", + "advanced/decode.py", + {"path": expected_path}, + status="failed", + reason=reason, + ) + ) + + +def _run_pynvc_segments( + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + content = _external(request.get("input"), label="input video") + input_path = artifact_io.verify_external_artifact(content) + gpu = request.get("gpu", 0) + if type(gpu) is not int or gpu < 0: # pylint: disable=unidiomatic-typecheck + raise PipelineError("gpu must be a non-negative integer") + _environment_identity, environment = _environment( + request, required_surfaces=("pynvc",), selected_gpu=gpu + ) + _bind_environment_gpu(environment, gpu, label="request") + branch = artifact_io.create_private_workspace(root / "pynvc-segments") + samples = ("samples/basic/create_video_segments.py", "samples/advanced/decode.py") + authentication, provenance = _authenticate( + "pynvc", samples, root=root, branch=branch, environment=environment, + runner=runner, timeout=timeout, + ) + segment_token = authentication.token(samples[0]) + schedule = _protected_data(segment_token, "segments.txt") + configuration = _protected_data(segment_token, "transcode_config.json") + rows = _segment_rows(schedule) + template = branch / "segment.mp4" + expected = [ + branch / f"segment_{start:g}_{end:g}.mp4" for start, end in rows + ] + if any(path.exists() for path in expected): + raise PipelineError("segment outputs must be fresh") + suffix = [ + "-i", str(input_path), "-s", str(schedule), "-c", str(configuration), + "-o", str(template), "-g", str(gpu), + ] + command, stdout, _stderr = _launch( + segment_token, suffix, branch=branch, runner=runner, timeout=timeout, + stage="pynvc-segment", phase="execute", + ) + summary = SEGMENT_SUMMARY.findall(stdout) + created = SEGMENT_CREATED.findall(stdout) + if summary != [str(len(rows))] or created != [str(path) for path in expected]: + raise BoundaryFailure( + "create_video_segments.py", "advanced/decode.py", str(template), + "segment markers do not exactly match authenticated segments.txt rows", + ) + outputs: list[dict[str, Any]] = [] + boundaries: list[dict[str, Any]] = [] + verified = 0 + for index, segment in enumerate(expected, 1): + try: + identity = artifact_io.snapshot_artifact( + root, segment, schema_version="1", kind="nvcodec-video-segment" + ) + except (OSError, ValueError) as exc: + expected_path = segment.relative_to(root).as_posix() + _record_unmaterialized_segment_failure( + outputs, + boundaries, + expected_path=expected_path, + reason=str(exc), + ) + continue + if identity["size_bytes"] <= 0: + boundaries.append( + _boundary( + "create_video_segments.py", "advanced/decode.py", identity, + status="failed", reason="segment is empty", + ) + ) + outputs.append({"segment": identity, "status": "operation_failed"}) + continue + try: + artifact_io.verify_artifact(root, identity) + decode = _decode( + "pynvc", authentication, segment, branch / f"segment-{index}.yuv", + root=root, branch=branch, runner=runner, timeout=timeout, gpu=gpu, + ) + artifact_io.verify_artifact(root, identity) + decode["consumed_sha256"] = identity["sha256"] + outputs.append({"segment": identity, "status": "operation_verified", "decode": decode}) + boundaries.append( + _boundary( + "create_video_segments.py", "advanced/decode.py", identity, + status="verified", + ) + ) + verified += 1 + except (OSError, ValueError) as exc: + outputs.append({"segment": identity, "status": "operation_failed", "reason": str(exc)}) + boundaries.append( + _boundary( + "create_video_segments.py", "advanced/decode.py", identity, + status="failed", reason=str(exc), + ) + ) + artifact_io.verify_external_artifact(content) + return { + "status": ( + "complete" if verified == len(expected) + else ("partial" if verified else "failed") + ), + "surface": "pynvc", "provenance": provenance, + "schedule": artifact_io.snapshot_external_artifact( + schedule, schema_version="1", kind="pynvc-segment-schedule" + ), + "producer": {"sample": samples[0], "command": command, "segments": len(expected)}, + "segments": outputs, "boundaries": boundaries, + } + + +def _target_gate(request: Mapping[str, Any]) -> tuple[bool, list[str]]: + value = request.get("target_eligibility") + if not isinstance(value, Mapping) or set(value) != {"eligible", "reasons"}: + raise PipelineError("target_eligibility must contain exactly eligible and reasons") + eligible = value.get("eligible") + reasons = value.get("reasons") + if not isinstance(eligible, bool) or not isinstance(reasons, list) or not all( + isinstance(reason, str) and reason for reason in reasons + ): + raise PipelineError("target_eligibility fields are invalid") + if eligible and reasons: + raise PipelineError("eligible target must not contain failure reasons") + if not eligible and not reasons: + reasons = ["target is not an eligible released Jetson route"] + return eligible, list(reasons) + + +def _retrieve( + request: Mapping[str, Any], *, root: Path, branch: Path, runner: Any, + timeout: float, +) -> tuple[dict[str, Any], Path, dict[str, Any] | None]: + supplied = request.get("input") + if supplied is not None: + identity = _external(supplied, label="container input") + return identity, artifact_io.verify_external_artifact(identity), None + url = _exact_http_url( + request.get("source_url"), label="container source_url" + ) + name = request.get("content_name", "content.mp4") + if not isinstance(name, str) or not name or Path(name).name != name: + raise PipelineError("content_name must be one plain filename") + destination = branch / name + curl = Path("/usr/bin/curl") + argv = [ + str(curl), "--fail", "--silent", "--show-error", + "--output", str(destination), "--write-out", "%{http_code}", url, + ] + result = runner.run( + argv, cwd=branch, env={ + "LANG": "C.UTF-8", "LC_ALL": "C.UTF-8", + "PATH": "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + }, timeout_seconds=timeout, stage="content-retrieval", phase="prepare", + ) + command, stdout, _stderr = _command( + result, runner=runner, argv=argv, cwd=branch, timeout=timeout, + stage="content-retrieval", phase="prepare", + ) + if re.fullmatch(r"2[0-9]{2}", stdout.strip()) is None: + raise PipelineError( + "content retrieval must complete without an HTTP redirect" + ) + identity = artifact_io.snapshot_artifact( + root, destination, schema_version="1", kind="nvcodec-video-content" + ) + if identity["size_bytes"] <= 0: + raise PipelineError("retrieved content is empty") + return identity, destination, command + + +def _run_container_triage( + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + eligible, reasons = _target_gate(request) + if not eligible: + return { + "status": "blocked", "gate": "target_eligibility", "reasons": reasons, + "retrieval_attempted": False, "operations": {}, "boundaries": [], + } + surface = request.get("surface") + if not isinstance(surface, str) or surface not in {"native", "pynvc"}: + raise PipelineError("container_triage surface must be native or pynvc") + gpu = request.get("gpu", 0) + if type(gpu) is not int or gpu < 0: # pylint: disable=unidiomatic-typecheck + raise PipelineError("gpu must be a non-negative integer") + _environment_identity, environment = _environment( + request, required_surfaces=(surface,), selected_gpu=gpu + ) + _bind_environment_gpu(environment, gpu, label="request") + branch = artifact_io.create_private_workspace(root / "container-triage") + sample = "AppDec" if surface == "native" else "samples/advanced/decode.py" + authentication, provenance = _authenticate( + surface, (sample,), root=root, branch=branch, environment=environment, + runner=runner, timeout=timeout, + ) + content, path, retrieval = _retrieve( + request, root=root, branch=branch, runner=runner, timeout=timeout + ) + output = branch / "decoded.yuv" + try: + decode = _decode( + surface, authentication, path, output, root=root, branch=branch, + runner=runner, timeout=timeout, gpu=gpu, + ) + if Path(content["path"]).is_absolute(): + artifact_io.verify_external_artifact(content) + else: + artifact_io.verify_artifact(root, content) + decode["consumed_sha256"] = content["sha256"] + raw_video = artifact_io.snapshot_external_artifact( + output, schema_version="1", kind="nvcodec-raw-video" + ) + artifact_io.verify_external_artifact(raw_video) + decode["raw_video"] = raw_video + boundary = _boundary(sample, sample, content, status="verified") + boundary["producer"] = f"{sample} intrinsic libavformat demux" + boundary["consumer"] = f"{sample} NVDEC decode" + return { + "status": "complete", "gate": "decode", + "retrieval_attempted": retrieval is not None, + "retrieval": retrieval, "content": content, "provenance": provenance, + "capability_query": request.get("decoder_capability"), + "demux": { + "sample": sample, "implementation": "intrinsic-libavformat", + "status": "operation_verified", + }, + "decode": decode, "boundaries": [boundary], + } + except (OSError, ValueError) as exc: + boundary = _boundary(sample, sample, content, status="failed", reason=str(exc)) + boundary["producer"] = f"{sample} intrinsic libavformat demux" + boundary["consumer"] = f"{sample} NVDEC decode" + return { + "status": "failed", "gate": "container_or_decode", + "retrieval_attempted": retrieval is not None, + "retrieval": retrieval, "content": content, "provenance": provenance, + "capability_query": request.get("decoder_capability"), + "reason": str(exc), "boundaries": [boundary], + } + + +def _ivf( + path: Path, *, width: int, height: int, expected_frames: int +) -> dict[str, Any]: + try: + structure = _encode_controller()._structure( # pylint: disable=protected-access + path, "av1", width, height, expected_frames + ) + except (OSError, ValueError) as exc: + raise PipelineError(str(exc)) from exc + return { + **structure, + "walked_frames": structure["walked_frame_count"], + "exact_eof": True, + } + + +def _run_av1_verify( + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + _recipe_identity, recipe = _recipe(request) + raw = _external( + request.get("input"), label="raw input", schema="1", kinds=("nvcodec-raw-video",) + ) + raw_path = artifact_io.verify_external_artifact(raw) + intent = recipe["encoder_intent"] + _environment_identity, environment = _environment( + request, + required_surfaces=("native",), + selected_gpu=intent.get("gpu"), + ) + _bind_environment_gpu(environment, intent.get("gpu"), label="recipe") + frame_count = intent.get("frame_count") + if ( + intent.get("codec") != "av1" + or isinstance(frame_count, bool) + or not isinstance(frame_count, int) + or frame_count <= 0 + ): + raise PipelineError("av1_verify requires an AV1 recipe with positive frame_count") + if recipe["projections"]["native"]["status"] != "exact": + raise PipelineError("av1_verify requires an exact native projection") + branch = artifact_io.create_private_workspace(root / "av1-verify") + authentication, provenance = _authenticate( + "native", ("AppEncCuda", "AppDec"), root=root, branch=branch, + environment=environment, runner=runner, timeout=timeout, + ) + operations: dict[str, Any] = {} + boundaries: list[dict[str, Any]] = [] + verified = 0 + for mode, flag in (("host", "0"), ("vidmem", "1")): + output = branch / f"av1-{mode}.av1" + decoded = branch / f"av1-{mode}.yuv" + suffix = [ + "-i", str(raw_path), "-o", str(output), + *recipe["projections"]["native"]["cli_options"], + "-outputInVidMem", flag, + ] + try: + command, stdout, _stderr = _launch( + authentication.token("AppEncCuda"), suffix, branch=branch, + runner=runner, timeout=timeout, stage=f"av1-{mode}-encode", + phase="execute", + ) + markers = NATIVE_ENCODE.findall(stdout) + if markers != [str(frame_count)]: + raise PipelineError("AppEncCuda did not report the exact recipe frame count") + structure = _ivf( + output, + width=int(intent["width"]), + height=int(intent["height"]), + expected_frames=frame_count, + ) + if structure["walked_frames"] != frame_count: + raise PipelineError("walked IVF frame count differs from AppEncCuda") + encoded = artifact_io.snapshot_artifact( + root, output, schema_version="1", kind="nvcodec-av1-bitstream" + ) + artifact_io.verify_artifact(root, encoded) + decode = _decode( + "native", authentication, output, decoded, root=root, branch=branch, + runner=runner, timeout=timeout, gpu=int(intent["gpu"]), + expected_frames=frame_count, + ) + artifact_io.verify_artifact(root, encoded) + decode["consumed_sha256"] = encoded["sha256"] + operations[mode] = { + "status": "operation_verified", "encode": { + "command": command, "frames": frame_count, "output": encoded, + "structure": structure, + }, "decode": decode, + } + boundaries.append(_boundary("AppEncCuda", "AppDec", encoded, status="verified")) + verified += 1 + except (OSError, ValueError) as exc: + operations[mode] = {"status": "operation_failed", "reason": str(exc)} + boundaries.append({ + "producer": "AppEncCuda", "consumer": "AppDec", + "artifact_path": str(output), "artifact_size_bytes": None, + "artifact_sha256": None, "status": "failed", "reason": str(exc), + }) + artifact_io.verify_external_artifact(raw) + return { + "status": "complete" if verified == 2 else ("partial" if verified else "failed"), + "surface": "native", "provenance": provenance, + "operations": operations, "boundaries": boundaries, + } + + +def _jsonl_values( + identity: Mapping[str, Any], label: str, +) -> tuple[list[dict[str, Any]], bytes]: + path = artifact_io.verify_external_artifact(identity) + try: + lines = path.read_bytes().splitlines() + except OSError as exc: + raise PipelineError(f"cannot read {label}: {exc}") from exc + values = [] + for number, line in enumerate(lines, 1): + if not line.strip(): + continue + value = artifact_io.strict_json_loads(line) + if not isinstance(value, dict): + raise PipelineError(f"{label} line {number} is not a JSON object") + values.append(artifact_io.canonical_json_bytes(value).rstrip(b"\n")) + if not values: + raise PipelineError(f"{label} contains no JSON objects") + payload = b"\n".join(values) + b"\n" + return [artifact_io.strict_json_loads(item) for item in values], payload + + +def _jsonl(identity: Mapping[str, Any], label: str) -> bytes: + return _jsonl_values(identity, label)[1] + + +def _stage_object(refs: Mapping[str, Mapping[str, Any]], name: str) -> dict[str, Any]: + value = artifact_io.read_verified_external_json(refs[name]) + if not isinstance(value, dict): + raise PipelineError(f"acceptance stage {name} must contain one JSON object") + return value + + +def _complete_stage(value: Mapping[str, Any], name: str) -> None: + if value.get("status") not in {"complete", "completed", "operation_verified", "ready"}: + raise PipelineError(f"acceptance stage {name} is not complete") + + +def _content_stage( + value: Mapping[str, Any], +) -> tuple[dict[str, Any], dict[str, Any]]: + record = value.get("representative_content", value) + if not isinstance(record, Mapping): + raise PipelineError("acceptance content evidence is malformed") + if record is value: + if value.get("kind") != "representative-content" or value.get("verified") is not True: + raise PipelineError("acceptance content evidence is not verified") + else: + _complete_stage(value, "content") + required = set(REPRESENTATIVE_CONTENT_FIELDS) + if not required.issubset(record): + raise PipelineError( + "acceptance content evidence omits source URL field, license, attribution, " + "path, size, or SHA-256" + ) + source_url = record["source_url"] + if source_url is not None: + _exact_http_url( + source_url, label="acceptance content source_url" + ) + if any( + not isinstance(record[field], str) + or not record[field] + or record[field] != record[field].strip() + for field in ("license", "attribution") + ): + raise PipelineError("acceptance content license and attribution must be nonempty") + if ( + isinstance(record["size_bytes"], bool) + or not isinstance(record["size_bytes"], int) + or record["size_bytes"] <= 0 + ): + raise PipelineError("acceptance content size_bytes must be positive") + if ( + not isinstance(record["path"], str) + or not isinstance(record["sha256"], str) + or re.fullmatch(r"[0-9a-f]{64}", record["sha256"]) is None + ): + raise PipelineError("acceptance content path and SHA-256 are invalid") + observed = artifact_io.snapshot_external_artifact( + Path(record["path"]), schema_version="1", kind="nvcodec-video-content" + ) + if (observed["path"], observed["size_bytes"], observed["sha256"]) != ( + record["path"], record["size_bytes"], record["sha256"], + ): + raise PipelineError("acceptance content path, size, or SHA-256 is stale") + expected_size = record.get("expected_size_bytes", record["size_bytes"]) + expected_sha = record.get("expected_sha256", record["sha256"]) + valid_expected_size = ( + not isinstance(expected_size, bool) + and isinstance(expected_size, int) + and expected_size == record["size_bytes"] + ) + valid_expected_sha = ( + isinstance(expected_sha, str) + and re.fullmatch(r"[0-9a-f]{64}", expected_sha) is not None + and expected_sha == record["sha256"] + ) + if not valid_expected_size or not valid_expected_sha: + raise PipelineError("acceptance content expected identity differs from observed identity") + metadata = {field: record[field] for field in REPRESENTATIVE_CONTENT_FIELDS} + source = { + "kind": "representative-content", + **metadata, + "expected_size_bytes": expected_size, + "expected_sha256": expected_sha, + "verified": True, + } + return metadata, source + + +def _recipe_stage( + value: Mapping[str, Any], + preset: str, + identity: Mapping[str, Any], +) -> None: + try: + artifact_io.validate_recipe_identity(identity) + except artifact_io.SkillDependencyError: + raise + except ValueError as exc: + raise PipelineError(f"acceptance {preset} recipe is invalid: {exc}") from exc + observed = value.get("encoder_intent", {}).get("preset") + if not isinstance(observed, str) or observed.lower() != preset: + raise PipelineError(f"acceptance {preset} recipe does not select {preset.upper()}") + + +def _workspace_identity( + value: Any, + workspace: Path, + *, + label: str, + kinds: Sequence[str], + schema: str = "1", + owner: Path | None = None, +) -> dict[str, Any]: + if ( + not isinstance(value, Mapping) + or set(value) != IDENTITY_KEYS + or value.get("schema_version") != schema + or value.get("kind") not in kinds + ): + raise PipelineError(f"{label} must be one exact artifact identity") + try: + artifact_path = artifact_io.verify_artifact(workspace, value) + if owner is not None: + artifact_path.relative_to(artifact_io.resolve_private_workspace(owner)) + except (OSError, ValueError) as exc: + raise PipelineError(f"{label} is not current: {exc}") from exc + return dict(value) + + +def _stage_workspace( + value: Mapping[str, Any], + identity: Mapping[str, Any], + preset: str, +) -> tuple[Path, Path]: + child = value.get("encode_workspace") + if not isinstance(child, str): + raise PipelineError(f"acceptance {preset} encode workspace is missing") + try: + child_path = artifact_io.resolve_private_workspace(Path(child)) + root = artifact_io.resolve_private_workspace(child_path.parent) + result_path = artifact_io.verify_external_artifact(identity) + result_path.relative_to(root) + except (OSError, ValueError) as exc: + raise PipelineError( + f"acceptance {preset} encode workspace/result ownership is invalid: {exc}" + ) from exc + if child_path != root / "encode-decode": + raise PipelineError(f"acceptance {preset} encode workspace is inconsistent") + return root, child_path + + +def _exact_boundary( + value: Any, + *, + producer: str, + consumer: str, + identity: Mapping[str, Any], + label: str, +) -> dict[str, Any]: + expected = _boundary(producer, consumer, identity, status="verified") + if ( + not isinstance(value, Mapping) + or set(value) != BOUNDARY_KEYS + or dict(value) != expected + ): + raise PipelineError(f"{label} does not match its exact encoded artifact") + return expected + + +def _encode_decode_stage( + value: Mapping[str, Any], + preset: str, + *, + stage_identity: Mapping[str, Any], + recipe_identity: Mapping[str, Any], + recipe: Mapping[str, Any], +) -> dict[str, Any]: + if ( + value.get("schema_version") != "1.0" + or value.get("kind") != "nvcodec-pipeline-result" + or value.get("route") != "encode_decode" + ): + raise PipelineError(f"acceptance {preset} encode/decode envelope is invalid") + _complete_stage(value, f"{preset}_encode_decode") + root, encode_workspace = _stage_workspace(value, stage_identity, preset) + encode_request = _external( + value.get("encode_request"), + label=f"acceptance {preset} encode request", + schema="1.0", + kinds=("nvcodec-encode-request",), + ) + request_value = artifact_io.read_verified_external_json(encode_request) + if not isinstance(request_value, Mapping): + raise PipelineError(f"acceptance {preset} encode request is invalid") + if request_value.get("recipe") != dict(recipe_identity): + raise PipelineError( + f"acceptance {preset} encode request did not consume its accepted recipe" + ) + input_identity = _external( + request_value.get("input"), + label=f"acceptance {preset} encode input", + schema="1", + kinds=("nvcodec-raw-video",), + ) + try: + _encode_controller()._validate_raw( # pylint: disable=protected-access + recipe, input_identity + ) + except ValueError as exc: + raise PipelineError( + f"acceptance {preset} encode input differs from its recipe: {exc}" + ) from exc + _workspace_identity( + value.get("encode_result"), + root, + label=f"acceptance {preset} nested encode result", + kinds=("nvcodec-encode-result",), + schema="1.0", + owner=encode_workspace, + ) + operations = value.get("operations") + if not isinstance(operations, Mapping) or not operations: + raise PipelineError(f"acceptance {preset} encode/decode operations are missing") + selected = value.get("selected_surfaces") + if not isinstance(selected, list) or set(selected) != set(operations): + raise PipelineError( + f"acceptance {preset} selected surfaces and operations differ" + ) + boundaries = value.get("boundaries") + if not isinstance(boundaries, list) or len(boundaries) != len(selected): + raise PipelineError(f"acceptance {preset} boundaries are incomplete") + by_producer = { + item.get("producer"): item + for item in boundaries + if isinstance(item, Mapping) + } + if len(by_producer) != len(boundaries): + raise PipelineError(f"acceptance {preset} boundaries have duplicate producers") + verified_boundaries = [] + expected_frames = recipe["encoder_intent"].get("frame_count") + for surface in selected: + operation = operations[surface] + if ( + surface not in {"native", "pynvc"} + or not isinstance(operation, Mapping) + or set(operation) != { + "status", "encoded", "decoded", "encode_frames", "decode_frames", + } + or operation.get("status") != "operation_verified" + ): + raise PipelineError( + f"acceptance {preset} {surface} encode/decode is not operation_verified" + ) + encoded = _workspace_identity( + operation.get("encoded"), + root, + label=f"acceptance {preset} {surface} encoded artifact", + kinds=(f"nvcodec-{recipe['encoder_intent']['codec']}-bitstream",), + owner=encode_workspace, + ) + decoded = _workspace_identity( + operation.get("decoded"), + root, + label=f"acceptance {preset} {surface} decoded artifact", + kinds=("nvcodec-decoded-frames",), + owner=encode_workspace, + ) + if ( + encoded["size_bytes"] <= 0 + or decoded["size_bytes"] <= 0 + or operation.get("encode_frames") != expected_frames + or operation.get("decode_frames") != expected_frames + ): + raise PipelineError( + f"acceptance {preset} encode/decode outputs or frame counts are invalid" + ) + verified_boundaries.append(_exact_boundary( + by_producer.get(f"{surface}-encode"), + producer=f"{surface}-encode", + consumer=f"{surface}-decode", + identity=encoded, + label=f"acceptance {preset} {surface} boundary", + )) + return {"input": input_identity, "boundaries": verified_boundaries} + + +def _pynvc_benchmark_config( + branch: Mapping[str, Any], + arguments: Sequence[str], + recipe: Mapping[str, Any], + preset: str, +) -> None: + """Bind the exact -json operand to the accepted recipe projection.""" + option_index = arguments.index("-json") + if option_index + 1 >= len(arguments): + raise PipelineError( + f"acceptance {preset} pynvc benchmark -json option lacks its value" + ) + config_operand = Path(arguments[option_index + 1]) + if not config_operand.is_absolute(): + raise PipelineError( + f"acceptance {preset} pynvc benchmark config path is not absolute" + ) + try: + config_workspace = artifact_io.resolve_private_workspace(config_operand.parent) + config_identity = _workspace_identity( + branch.get("generated_config"), + config_workspace, + label=f"acceptance {preset} pynvc generated config", + kinds=("pynvc-encoder-config",), + ) + config_path = artifact_io.verify_artifact(config_workspace, config_identity) + except (OSError, ValueError) as exc: + raise PipelineError( + f"acceptance {preset} pynvc benchmark config is invalid: {exc}" + ) from exc + if ( + config_identity["path"] != config_operand.name + or config_path != config_operand + ): + raise PipelineError( + f"acceptance {preset} pynvc benchmark config differs from -json operand" + ) + projection = recipe.get("projections", {}).get("pynvc", {}) + if ( + not isinstance(projection, Mapping) + or projection.get("status") != "exact" + ): + raise PipelineError( + f"acceptance {preset} pynvc benchmark has no exact accepted projection" + ) + expected_config = projection.get("config") + try: + config_matches = ( + isinstance(expected_config, Mapping) + and artifact_io.read_verified_bytes(config_workspace, config_identity) + == artifact_io.canonical_json_bytes(dict(expected_config)) + ) + except (OSError, ValueError) as exc: + raise PipelineError( + f"acceptance {preset} pynvc benchmark config is stale: {exc}" + ) from exc + if not config_matches: + raise PipelineError( + f"acceptance {preset} pynvc benchmark config differs from accepted recipe" + ) + + +def _benchmark_recipe_option( + surface: str, + branch: Mapping[str, Any], + arguments: Sequence[str], + recipe: Mapping[str, Any], + preset: str, +) -> None: + option = "-preset" if surface == "native" else "-json" + if arguments.count(option) != 1: + raise PipelineError( + f"acceptance {preset} {surface} benchmark must contain " + f"one {option} recipe option" + ) + if surface == "pynvc": + _pynvc_benchmark_config(branch, arguments, recipe, preset) + return + option_index = arguments.index(option) + if ( + option_index + 1 >= len(arguments) + or arguments[option_index + 1].lower() != preset + ): + raise PipelineError( + f"acceptance {preset} native benchmark preset differs from recipe" + ) + + +def _benchmark_stage( # pylint: disable=too-many-branches + value: Mapping[str, Any], + preset: str, + *, + recipe_identity: Mapping[str, Any], + recipe: Mapping[str, Any], + input_identity: Mapping[str, Any], +) -> None: + if ( + value.get("kind") != "nvcodec-benchmark-result" + or value.get("status") != "completed" + ): + raise PipelineError(f"acceptance {preset} benchmark is not completed") + variants = value.get("variants") + matching = [ + item for item in variants if isinstance(item, Mapping) + and str(item.get("name", "")).lower() == preset + ] if isinstance(variants, list) else [] + if len(matching) != 1: + raise PipelineError( + f"acceptance benchmark must contain exactly one {preset.upper()} variant" + ) + if matching[0].get("recipe") != dict(recipe_identity): + raise PipelineError( + f"acceptance {preset} benchmark did not consume its accepted recipe" + ) + _external( + matching[0]["recipe"], + label=f"acceptance {preset} benchmark recipe", + schema="2.0", + kinds=("nvcodec-recipe",), + ) + surfaces = matching[0].get("surfaces") + if not isinstance(surfaces, Mapping) or not surfaces: + raise PipelineError(f"acceptance {preset} benchmark surfaces are missing") + if any(surface not in {"native", "pynvc"} for surface in surfaces): + raise PipelineError(f"acceptance {preset} benchmark contains an unknown surface") + for surface, branch in surfaces.items(): + if not isinstance(branch, Mapping) or branch.get("status") != "completed": + raise PipelineError(f"acceptance {preset} {surface} benchmark is incomplete") + input_record = branch.get("input") + if ( + not isinstance(input_record, Mapping) + or input_record.get("identity") != dict(input_identity) + ): + raise PipelineError( + f"acceptance {preset} {surface} benchmark did not consume " + "the encode/decode input" + ) + _external( + input_record["identity"], + label=f"acceptance {preset} {surface} benchmark input", + schema="1", + kinds=("nvcodec-raw-video",), + ) + intent = recipe["encoder_intent"] + for key in ("width", "height", "frame_count", "fps", "codec", "format"): + observed_key = "frames" if key == "frame_count" else key + if input_record.get(observed_key) != intent.get(key): + raise PipelineError( + f"acceptance {preset} {surface} benchmark input differs " + "from the recipe" + ) + authority = branch.get("sample_authority") + launcher = authority.get("launcher") if isinstance(authority, Mapping) else None + arguments = branch.get("benchmark_arguments") + if ( + not isinstance(launcher, list) + or not launcher + or not isinstance(arguments, list) + or any(not isinstance(item, str) or "\0" in item for item in [*launcher, *arguments]) + ): + raise PipelineError( + f"acceptance {preset} {surface} benchmark exact argv is missing" + ) + _benchmark_recipe_option(surface, branch, arguments, recipe, preset) + expected_argv = [*launcher, *arguments] + measured = branch.get("measured_runs") + if not isinstance(measured, list) or len(measured) < 3: + raise PipelineError( + f"acceptance {preset} {surface} benchmark lacks warmup, measurements, or summary" + ) + _benchmark_command( + branch.get("warmup"), "warmup", expected_argv, + f"acceptance {preset} {surface} warmup", + ) + fps_values: list[float] = [] + megapixel_values: list[float] = [] + for index, run in enumerate(measured, 1): + if not isinstance(run, Mapping) or run.get("repetition") != index: + raise PipelineError( + f"acceptance {preset} {surface} benchmark repetitions are not sequential" + ) + fps_values.append(_positive_finite(run.get("fps"), "FPS", preset, surface)) + megapixel_values.append( + _positive_finite( + run.get("megapixels_per_second"), "megapixels/second", preset, surface + ) + ) + _benchmark_command( + run.get("command"), "measure", expected_argv, + f"acceptance {preset} {surface} measurement {index}", + ) + _benchmark_summary( + branch.get("summary"), fps_values, megapixel_values, preset, surface + ) + + +def _positive_finite(value: Any, label: str, preset: str, surface: str) -> float: + if isinstance(value, bool) or not isinstance(value, (int, float)): + raise PipelineError( + f"acceptance {preset} {surface} benchmark {label} must be positive and finite" + ) + observed = float(value) + if not math.isfinite(observed) or observed <= 0: + raise PipelineError( + f"acceptance {preset} {surface} benchmark {label} must be positive and finite" + ) + return observed + + +def _benchmark_command( + value: Any, phase: str, expected_argv: Sequence[str], label: str, +) -> None: + if not isinstance(value, Mapping) or set(value) != COMMAND_KEYS: + raise PipelineError(f"{label} must retain one exact command result") + argv = value.get("argv") + exit_code = value.get("exit_code") + successful_exit = ( + isinstance(exit_code, int) + and not isinstance(exit_code, bool) + and exit_code == 0 + ) + successful = ( + value.get("schema_version") == "1" + and value.get("kind") == "command-result" + and value.get("phase") == phase + and isinstance(argv, list) + and argv == list(expected_argv) + and bool(argv) + and all(isinstance(item, str) and "\0" not in item for item in argv) + and successful_exit + and value.get("timed_out") is False + and value.get("launch_error") is None + ) + if not successful: + raise PipelineError( + f"{label} command phase, argv, exit code, or timeout evidence is invalid" + ) + + +def _benchmark_summary( + value: Any, + fps_values: Sequence[float], + megapixel_values: Sequence[float], + preset: str, + surface: str, +) -> None: + if not isinstance(value, Mapping) or value.get("repetitions") != len(fps_values): + raise PipelineError(f"acceptance {preset} {surface} benchmark summary is inconsistent") + for key, observed_values in ( + ("fps", fps_values), ("megapixels_per_second", megapixel_values), + ): + metrics = value.get(key) + if not isinstance(metrics, Mapping) or set(metrics) != {"mean", "minimum", "maximum"}: + raise PipelineError( + f"acceptance {preset} {surface} benchmark {key} summary is incomplete" + ) + expected = { + "mean": sum(observed_values) / len(observed_values), + "minimum": min(observed_values), + "maximum": max(observed_values), + } + for name, expected_value in expected.items(): + actual = _positive_finite(metrics.get(name), f"summary {key}.{name}", preset, surface) + if not math.isclose(actual, expected_value, rel_tol=1e-12, abs_tol=1e-12): + raise PipelineError( + f"acceptance {preset} {surface} benchmark {key} summary is inconsistent" + ) + + +def _verified_boundaries( + value: Mapping[str, Any], + name: str, + expected: Sequence[Mapping[str, Any]] | None = None, +) -> None: + boundaries = value.get("boundaries") + if not isinstance(boundaries, list) or not boundaries: + raise PipelineError(f"acceptance {name} boundaries are missing") + if any( + not isinstance(boundary, Mapping) + or set(boundary) != BOUNDARY_KEYS + or boundary.get("status") != "verified" + for boundary in boundaries + ): + raise PipelineError(f"acceptance {name} contains an unverified boundary") + if expected is not None: + def canonical(item: Mapping[str, Any]) -> str: + return json.dumps( + dict(item), sort_keys=True, separators=(",", ":"), + allow_nan=False, + ) + + if Counter(map(canonical, boundaries)) != Counter(map(canonical, expected)): + raise PipelineError( + "acceptance handoffs differ from the freshly verified " + "encode/decode boundaries" + ) + + +def _task_results( + identity: Mapping[str, Any], refs: Mapping[str, Mapping[str, Any]], +) -> tuple[list[dict[str, Any]], bytes]: + rows, payload = _jsonl_values(identity, "task_results") + by_task: dict[str, dict[str, Any]] = {} + for number, row in enumerate(rows, 1): + if set(row) != {"task_id", "status", "evidence"}: + raise PipelineError( + f"task_results line {number} fields must be task_id, status, evidence" + ) + task_id = row["task_id"] + evidence = row["evidence"] + if task_id not in ACCEPTANCE_TASKS or task_id in by_task: + raise PipelineError(f"task_results line {number} has duplicate or unknown task_id") + if row["status"] != "complete": + raise PipelineError(f"task_results task {task_id} is not complete") + if ( + not isinstance(evidence, list) + or not evidence + or any(not isinstance(name, str) or name not in refs for name in evidence) + or len(set(evidence)) != len(evidence) + ): + raise PipelineError(f"task_results task {task_id} has invalid evidence names") + by_task[task_id] = row + missing = sorted(set(ACCEPTANCE_TASKS) - set(by_task)) + if missing: + raise PipelineError(f"task_results is missing required tasks {missing}") + return rows, payload + + +def _validate_acceptance_stages( + refs: Mapping[str, Mapping[str, Any]], +) -> tuple[ + dict[str, dict[str, Any]], list[dict[str, Any]], bytes, + dict[str, Any], dict[str, Any], +]: + values = {name: _stage_object(refs, name) for name in ACCEPTANCE_TASKS} + _complete_stage(values["readiness"], "readiness") + _complete_stage(values["capabilities"], "capabilities") + content_metadata, content_source = _content_stage(values["content"]) + recipes = { + preset: values[f"{preset}_recipe"] for preset in ("p4", "p5") + } + for preset in ("p4", "p5"): + _recipe_stage(recipes[preset], preset, refs[f"{preset}_recipe"]) + p4_intent = dict(recipes["p4"]["encoder_intent"]) + p5_intent = dict(recipes["p5"]["encoder_intent"]) + p4_intent.pop("preset", None) + p5_intent.pop("preset", None) + if p4_intent != p5_intent: + raise PipelineError("acceptance P4/P5 recipes must differ only by preset") + encode_results = { + preset: _encode_decode_stage( + values[f"{preset}_encode_decode"], + preset, + stage_identity=refs[f"{preset}_encode_decode"], + recipe_identity=refs[f"{preset}_recipe"], + recipe=recipes[preset], + ) + for preset in ("p4", "p5") + } + if encode_results["p4"]["input"] != encode_results["p5"]["input"]: + raise PipelineError("acceptance P4/P5 inputs differ") + if any( + content_source[key] != encode_results["p4"]["input"][key] + for key in ("path", "size_bytes", "sha256") + ): + raise PipelineError( + "acceptance content is not the exact raw artifact consumed by encode" + ) + expected_boundaries = [] + for preset in ("p4", "p5"): + expected_boundaries.extend(encode_results[preset]["boundaries"]) + _benchmark_stage( + values[f"{preset}_benchmark"], + preset, + recipe_identity=refs[f"{preset}_recipe"], + recipe=recipes[preset], + input_identity=encode_results[preset]["input"], + ) + _complete_stage(values["handoffs"], "handoffs") + _verified_boundaries( + values["handoffs"], "handoffs", expected=expected_boundaries + ) + rows, task_payload = _task_results(refs["task_results"], refs) + return values, rows, task_payload, content_metadata, content_source + + +def _reference_records(refs: Mapping[str, Mapping[str, Any]]) -> list[dict[str, Any]]: + return [ + { + "name": name, "path": identity["path"], + "size_bytes": identity["size_bytes"], "sha256": identity["sha256"], + "schema_version": identity["schema_version"], "kind": identity["kind"], + } + for name, identity in sorted(refs.items()) + ] + + +def _markdown(title: str, records: Sequence[Mapping[str, Any]]) -> bytes: + lines = [f"# {title}", "", "| Evidence | Size | SHA-256 |", "|---|---:|---|"] + lines.extend( + f"| `{item['name']}` | {item['size_bytes']} | `{item['sha256']}` |" + for item in records + ) + return ("\n".join(lines) + "\n").encode("utf-8") + + +def _run_acceptance( + request: Mapping[str, Any], *, root: Path, runner: Any, timeout: float, +) -> dict[str, Any]: + del runner, timeout + raw_refs = request.get("references") + if not isinstance(raw_refs, Mapping) or not raw_refs: + raise PipelineError("acceptance requires a non-empty references object") + refs = { + name: _external(identity, label=f"references.{name}") + for name, identity in raw_refs.items() + if isinstance(name, str) and name + } + if len(refs) != len(raw_refs): + raise PipelineError("acceptance reference names must be non-empty strings") + required = set(ACCEPTANCE_TASKS) | set(ACCEPTANCE_LEDGER_REFERENCES) + missing = sorted(required - set(refs)) + if missing: + raise PipelineError(f"acceptance references are missing {missing}") + files = _acceptance_files(request) + validation_file = files[-1] + ( + _stage_values, task_rows, task_payload, content_metadata, content_source, + ) = _validate_acceptance_stages(refs) + evidence = artifact_io.create_private_workspace(root / "evidence") + del evidence + records = _reference_records(refs) + setup = [item for item in records if item["name"] in {"readiness", "capabilities", "content"}] + handoffs = [item for item in records if "handoff" in item["name"]] + payloads: dict[str, bytes] = { + "report.md": _markdown("Jetson Video SDK acceptance", records), + "capability-report.md": _markdown("Capability evidence", setup), + "review-findings.md": _markdown("Review findings", handoffs or records), + "task-results.jsonl": task_payload, + "commands.jsonl": _jsonl(refs["commands"], "commands"), + } + timing_value = artifact_io.read_verified_external_json(refs["timing"]) + if not isinstance(timing_value, dict): + raise PipelineError("timing reference must contain a JSON object") + envelopes = { + "timing.json": { + "schema_version": "1", "kind": "nvcodec-acceptance-timing", + "source": refs["timing"], "value": timing_value, + }, + "evidence/summary.json": { + "schema_version": "1", "kind": "nvcodec-acceptance-evidence", + "representative_content": content_metadata, + }, + "evidence/representative-content.json": content_source, + } + outputs: dict[str, dict[str, Any]] = {} + for relative in files[:-1]: + path = root / relative + if relative == "evidence/representative-content.json": + outputs[relative] = artifact_io.write_fresh_bytes( + root, path, artifact_io.canonical_json_bytes(envelopes[relative]), + schema_version="1", kind="representative-content", + ) + elif relative in envelopes: + outputs[relative] = artifact_io.write_fresh_json( + root, path, envelopes[relative] + ) + else: + kind = ( + "nvcodec-acceptance-jsonl" if relative.endswith(".jsonl") + else "nvcodec-acceptance-markdown" + ) + outputs[relative] = artifact_io.write_fresh_bytes( + root, path, payloads[relative], schema_version="1", kind=kind + ) + for identity in refs.values(): + artifact_io.verify_external_artifact(identity) + if ( + (root / "evidence" / "manifest-sha256.txt").exists() + or (root / "inventory.json").exists() + ): + raise PipelineError( + "acceptance assembler must not create the sealing manifest or inventory" + ) + return { + "status": "complete", "references": records, "outputs": outputs, + "validated_tasks": [row["task_id"] for row in task_rows], + "validation_result_path": validation_file, + "assembler_owned_files": list(files), + "seal_pending": True, "manifest_created": False, "inventory_created": False, + "boundaries": [], + } + + +def _planned_operation( + sample: str, arguments: Sequence[str], output: Path | None = None, +) -> dict[str, Any]: + """Describe one exact authenticated-sample suffix without launching it.""" + value: dict[str, Any] = { + "sample": sample, "arguments": list(arguments), + "launcher": "resolved_by_sample_authentication", + } + if output is not None: + value["output_path"] = str(output) + return value + + +def _dry_run( # pylint: disable=too-many-branches,too-many-statements,protected-access + request: Mapping[str, Any], root: Path, +) -> dict[str, Any]: + """Validate one route and return a read-only launch plan.""" + route = request["route"] + planned: list[dict[str, Any]] = [] + outputs: list[str] = [] + identities: dict[str, Any] = {} + selected_surface_plan: dict[str, Any] | None = None + surface_evaluations: dict[str, Any] | None = None + if route == "encode_decode": + reference = _external( + request.get("encode_request_ref"), label="encode_request_ref", + schema="1.0", kinds=("nvcodec-encode-request",), + ) + encode_request = artifact_io.read_verified_external_json(reference) + if not isinstance(encode_request, Mapping): + raise PipelineError("encode_request_ref must contain a JSON object") + if ( + encode_request.get("schema_version") != "1.0" + or encode_request.get("kind") != "nvcodec-encode-request" + ): + raise PipelineError("encode request has an invalid schema or kind") + surface = encode_request.get("surface") + if ( + not isinstance(surface, str) + or surface not in {"native", "pynvc", "auto", "both"} + ): + raise PipelineError("encode request has an invalid surface") + buffer_mode = encode_request.get("buffer_mode", "gpu") + if ( + not isinstance(buffer_mode, str) + or buffer_mode not in {"cpu", "gpu"} + ): + raise PipelineError("encode request has an invalid buffer mode") + recipe_identity = _external( + encode_request.get("recipe"), label="encode recipe", schema="2.0", + kinds=("nvcodec-recipe",), + ) + capability_identity = ( + _external( + encode_request.get("capability_report"), + label="encode capability report", + schema=artifact_io.CAPABILITY_REPORT_SCHEMA_VERSION, + kinds=(artifact_io.CAPABILITY_REPORT_KIND,), + ) + if encode_request.get("capability_report") is not None + else None + ) + recipe = artifact_io.read_verified_external_json(recipe_identity) + artifact_io.validate_recipe_identity(recipe_identity) + intent = recipe.get("encoder_intent", {}) + recipe_gpu = intent.get("gpu") if isinstance(intent, Mapping) else None + local_auto_without_pynvc = ( + surface == "auto" + and encode_request.get("environment") is None + and encode_request.get("pynvc_interpreter") is None + ) + if surface in {"native", "pynvc"}: + required_surfaces = (surface,) + elif surface == "auto" and local_auto_without_pynvc: + # An unavailable optional Py selector must not block a valid native + # auto route. The unexamined peer is disclosed below. + required_surfaces = ("native",) + else: + required_surfaces = artifact_io.SURFACE_ORDER + local_errors: dict[str, str] = {} + environment_identity, environment_value = _environment( + { + "environment": encode_request.get("environment"), + "pynvc_interpreter": encode_request.get("pynvc_interpreter"), + "timeout_seconds": request["timeout_seconds"], + }, + required_surfaces=required_surfaces, + selected_gpu=recipe_gpu, + local_errors=local_errors, + ) + input_identity = _external( + encode_request.get("input"), label="encode input", schema="1", + kinds=("nvcodec-raw-video",), + ) + _bind_environment_gpu( + environment_value, + recipe_gpu, + label="recipe", + ) + if local_auto_without_pynvc: + local_errors["pynvc"] = ( + "PyNvVideoCodec was not evaluated because no exact absolute " + "pynvc_interpreter was supplied" + ) + try: + surface_plan, capability_details = _encode_controller()._surface_plan( + surface, + recipe, + environment_value, + buffer_mode, + recipe_identity=recipe_identity, + environment_identity=environment_identity, + capability_identity=capability_identity, + local_errors=local_errors, + ) + except ValueError as exc: + raise PipelineError(f"encode surface planning failed: {exc}") from exc + if not surface_plan["selected_surfaces"]: + return { + "status": surface_plan["classification"], + "surface_plan": surface_plan, + "surface_evaluations": capability_details, + "planned_operations": [], "planned_output_paths": [], + "validated_identities": { + "encode_request": reference, + **( + {"environment": environment_identity} + if environment_identity is not None + else {} + ), + }, + "no_mutation_performed": True, + } + selected_surface_plan = surface_plan + surface_evaluations = capability_details + codec = recipe["encoder_intent"]["codec"] + raw_path = Path(input_identity["path"]) + surfaces = surface_plan["selected_surfaces"] + for surface in surfaces: + branch = root / "encode-decode" / surface + encoded = branch / f"encoded.{codec}" + decoded = branch / "decoded.yuv" + if surface == "native": + projection = recipe["projections"]["native"]["cli_options"] + encode_args = ["-i", str(raw_path), "-o", str(encoded), *projection] + decode_args = [ + "-i", str(encoded), "-o", str(decoded), "-gpu", + str(recipe["encoder_intent"]["gpu"]), + ] + samples = ("AppEncCuda", "AppDec") + else: + intent = recipe["encoder_intent"] + configuration = branch / "encode-config.json" + encode_args = [ + "-i", str(raw_path), "-o", str(encoded), "-s", + f"{intent['width']}x{intent['height']}", "-m", + buffer_mode, "-if", intent["format"], + "-f", str(intent["frame_count"]), "-g", str(intent["gpu"]), + "-c", codec, "-json", str(configuration), + ] + decode_args = [ + "-i", str(encoded), "-o", str(decoded), "-g", str(intent["gpu"]), + "-d", "0", "-f", str(intent["frame_count"]), + ] + samples = ("samples/basic/encode.py", "samples/advanced/decode.py") + planned.extend(( + _planned_operation(samples[0], encode_args, encoded), + _planned_operation(samples[1], decode_args, decoded), + )) + outputs.extend((str(encoded), str(decoded))) + identities["encode_request"] = reference + if environment_identity is not None: + identities["environment"] = environment_identity + if capability_identity is not None: + identities["capability_report"] = capability_identity + elif route == "native_transcode": + identities["recipe"], recipe = _recipe(request) + identities["input"] = _external(request.get("input"), label="input video") + if ( + recipe["encoder_intent"].get("codec") != "hevc" + or recipe["projections"]["native"]["status"] != "exact" + ): + raise PipelineError( + "native_transcode requires an exact HEVC native recipe" + ) + output = root / "native-transcode" / "transcoded.hevc" + decoded = root / "native-transcode" / "transcoded-decoded.yuv" + intent = recipe["encoder_intent"] + environment_identity, _environment_value = _environment( + request, + required_surfaces=("native",), + selected_gpu=intent.get("gpu"), + ) + if environment_identity is not None: + identities["environment"] = environment_identity + _bind_environment_gpu(_environment_value, intent.get("gpu"), label="recipe") + transcode_args = [ + "-i", identities["input"]["path"], "-o", str(output), "-gpu", + str(intent["gpu"]), *_transcode_options(recipe), + ] + planned.extend(( + _planned_operation("AppTrans", transcode_args, output), + _planned_operation( + "AppDec", [ + "-i", str(output), "-o", str(decoded), "-gpu", + str(intent["gpu"]), + ], decoded, + ), + )) + outputs.extend((str(output), str(decoded))) + elif route == "pynvc_segments": + identities["input"] = _external(request.get("input"), label="input video") + template = root / "pynvc-segments" / "segment.mp4" + gpu = request.get("gpu", 0) + if type(gpu) is not int or gpu < 0: # pylint: disable=unidiomatic-typecheck + raise PipelineError("gpu must be a non-negative integer") + environment_identity, _environment_value = _environment( + request, required_surfaces=("pynvc",), selected_gpu=gpu + ) + if environment_identity is not None: + identities["environment"] = environment_identity + _bind_environment_gpu(_environment_value, gpu, label="request") + schedule = "${AUTHENTICATED_SAMPLE_DIR}/segments.txt" + configuration = "${AUTHENTICATED_SAMPLE_DIR}/transcode_config.json" + planned.extend(( + _planned_operation( + "samples/basic/create_video_segments.py", + [ + "-i", identities["input"]["path"], "-s", schedule, + "-c", configuration, "-o", str(template), "-g", str(gpu), + ], template, + ), + _planned_operation( + "samples/advanced/decode.py", + ["for each output derived from authenticated segments.txt"], + ), + )) + outputs.append(str(template)) + elif route == "container_triage": + eligible, reasons = _target_gate(request) + if not eligible: + return { + "status": "blocked", "gate": "target_eligibility", + "reasons": reasons, "planned_operations": [], + "planned_output_paths": [], "validated_identities": {}, + "no_mutation_performed": True, + } + surface = request.get("surface") + if not isinstance(surface, str) or surface not in {"native", "pynvc"}: + raise PipelineError("container_triage surface must be native or pynvc") + gpu_value = request.get("gpu", 0) + if ( + type(gpu_value) is not int # pylint: disable=unidiomatic-typecheck + or gpu_value < 0 + ): + raise PipelineError("gpu must be a non-negative integer") + environment_identity, _environment_value = _environment( + request, required_surfaces=(surface,), selected_gpu=gpu_value + ) + if environment_identity is not None: + identities["environment"] = environment_identity + _bind_environment_gpu(_environment_value, gpu_value, label="request") + if request.get("input") is not None: + identities["input"] = _external( + request.get("input"), label="container input" + ) + source = identities["input"]["path"] + else: + _exact_http_url( + request.get("source_url"), label="container source_url" + ) + source = str(root / "container-triage" / request.get("content_name", "content.mp4")) + output = root / "container-triage" / "decoded.yuv" + sample = "AppDec" if surface == "native" else "samples/advanced/decode.py" + gpu = str(gpu_value) + arguments = ["-i", source, "-o", str(output)] + arguments.extend(["-gpu", gpu] if surface == "native" else ["-g", gpu, "-d", "0"]) + planned.append(_planned_operation(sample, arguments, output)) + outputs.append(str(output)) + elif route == "av1_verify": + identities["recipe"], recipe = _recipe(request) + identities["input"] = _external( + request.get("input"), label="raw input", schema="1", + kinds=("nvcodec-raw-video",), + ) + intent = recipe["encoder_intent"] + environment_identity, _environment_value = _environment( + request, + required_surfaces=("native",), + selected_gpu=intent.get("gpu"), + ) + if environment_identity is not None: + identities["environment"] = environment_identity + _bind_environment_gpu(_environment_value, intent.get("gpu"), label="recipe") + frame_count = intent.get("frame_count") + if ( + intent.get("codec") != "av1" + or isinstance(frame_count, bool) + or not isinstance(frame_count, int) + or frame_count <= 0 + ): + raise PipelineError( + "av1_verify requires an AV1 recipe with positive frame_count" + ) + if recipe["projections"]["native"]["status"] != "exact": + raise PipelineError("av1_verify requires an exact native projection") + for mode, flag in (("host", "0"), ("vidmem", "1")): + output = root / "av1-verify" / f"av1-{mode}.av1" + decoded = root / "av1-verify" / f"av1-{mode}.yuv" + planned.extend(( + _planned_operation( + "AppEncCuda", [ + "-i", identities["input"]["path"], "-o", str(output), + *recipe["projections"]["native"]["cli_options"], + "-outputInVidMem", flag, + ], output, + ), + _planned_operation( + "AppDec", [ + "-i", str(output), "-o", str(decoded), "-gpu", + str(recipe["encoder_intent"]["gpu"]), + ], decoded, + ), + )) + outputs.extend((str(output), str(decoded))) + else: + raw_refs = request.get("references") + if not isinstance(raw_refs, Mapping) or not raw_refs: + raise PipelineError("acceptance requires a non-empty references object") + identities = { + name: _external(identity, label=f"references.{name}") + for name, identity in raw_refs.items() + if isinstance(name, str) and name + } + required = set(ACCEPTANCE_TASKS) | set(ACCEPTANCE_LEDGER_REFERENCES) + if len(identities) != len(raw_refs) or required - set(identities): + raise PipelineError("acceptance references are incomplete") + _validate_acceptance_stages(identities) + _jsonl(identities["commands"], "commands") + timing_value = artifact_io.read_verified_external_json(identities["timing"]) + if not isinstance(timing_value, dict): + raise PipelineError("timing reference must contain a JSON object") + outputs.extend( + str(root / relative) for relative in _acceptance_files(request) + ) + result = { + "status": "planned", "planned_operations": planned, + "planned_output_paths": outputs, "validated_identities": identities, + "no_mutation_performed": True, + } + if selected_surface_plan is not None: + result["surface_plan"] = selected_surface_plan + result["surface_evaluations"] = surface_evaluations + if selected_surface_plan["classification"] != "ready": + result["status"] = "partial" + return result + + +def run_pipeline_request( + request: Mapping[str, Any], *, workspace: Path, runner: Any = None, + timeout_seconds: float = 600, +) -> dict[str, Any]: + """Execute one route without fallback and return honest boundary status.""" + ceiling = _timeout(timeout_seconds, float(timeout_seconds)) + value = _request(request, ceiling) + base = { + "schema_version": SCHEMA_VERSION, "kind": RESULT_KIND, + "route": value["route"], "mode": value["mode"], + "status": "failed", "boundaries": [], + } + workspace_created = False + try: + preflight = _media_preflight(value) + if preflight is not None: + base.update(preflight) + return base + preflight = _pynvc_full_samples_preflight(value) + if preflight is not None: + base.update(preflight) + return base + if value["mode"] == "dry_run": + root = _planned_workspace(Path(workspace)) + base.update(_dry_run(value, root)) + return base + root = _workspace(Path(workspace)) + workspace_created = True + service = runner or command_runner.CommandRunner(root) + handler = globals()[ROUTE_TABLE[value["route"]]] + try: + result = handler( + value, root=root, runner=service, timeout=value["timeout_seconds"] + ) + base.update(result) + except BoundaryFailure as exc: + base.update({ + "status": "failed", "reason": exc.reason, + "boundaries": [{ + "producer": exc.producer, "consumer": exc.consumer, + "artifact_path": exc.path, "artifact_size_bytes": None, + "artifact_sha256": None, "status": "failed", "reason": exc.reason, + }], + }) + except artifact_io.SkillDependencyError as exc: + base.update({ + "status": "dependency_required", + "gate": "skill_dependency", + "dependency": { + "skill": exc.skill, + "needed_for": exc.needed_for, + "reason": str(exc), + "next_action": f"install_{exc.skill}_and_retry_stage", + }, + "no_mutation_performed": not workspace_created, + }) + except SurfaceInputRequired as exc: + base.update({ + "status": "input_required", + "gate": "surface_selector", + "surface": exc.surface, + "required_field": exc.field, + "reason": exc.reason, + "next_action": f"provide_{exc.field}", + "no_mutation_performed": not workspace_created, + }) + except SurfaceSetupRequired as exc: + installed = _installed_sibling("jetson-video-setup") + action = ( + f"use jetson-video-setup for the {exc.surface} surface and retry" + if installed + else "install jetson-video-setup, configure the selected surface, and retry" + ) + base.update({ + "status": "dependency_required", + "gate": "sdk_setup", + "surface": exc.surface, + "reason": exc.reason, + "dependency": { + "skill": "jetson-video-setup", + "installed": installed, + "needed_for": f"{exc.surface} SDK installation or repair", + "next_action": action, + }, + "no_mutation_performed": not workspace_created, + }) + return base + + +def main(argv: list[str] | None = None) -> int: + """Run one JSON request and write one fresh JSON result.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--request", type=Path, required=True) + parser.add_argument("--workspace", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + try: + output = artifact_io.preflight_fresh_output(args.workspace, args.output) + request = artifact_io.strict_json_loads(args.request.read_bytes()) + if not isinstance(request, Mapping): + raise PipelineError("request must be a JSON object") + acceptance_output: Path | None = None + if request.get("route") == "acceptance": + planned_root = args.workspace.expanduser().resolve(strict=False) + validation_file = _acceptance_files(request)[-1] + expected = planned_root / validation_file + requested = output + if requested != expected: + raise PipelineError( + "acceptance --output must be exactly " + f"evidence/{Path(validation_file).name} within --workspace" + ) + acceptance_output = requested + result = run_pipeline_request(request, workspace=args.workspace) + if result.get("no_mutation_performed") is True: + root = _workspace(args.workspace) + if acceptance_output is not None: + artifact_io.create_private_workspace(root / "evidence") + else: + root = artifact_io.resolve_private_workspace(args.workspace) + artifact_io.write_fresh_json(root, output, result) + if request.get("route") == "acceptance" and result["status"] == "complete": + observed = { + path.relative_to(root).as_posix() + for path in root.rglob("*") + if path.is_file() + } + expected = set(_acceptance_files(request)) + if observed != expected: + raise PipelineError( + "acceptance CLI did not produce exactly the nine pre-seal files" + ) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + return 0 if result["status"] in {"complete", "planned"} else 2 + except (OSError, ValueError) as exc: + print(json.dumps({"status": "error", "error": str(exc)}, sort_keys=True)) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-pipeline/scripts/validate_representative_content_summary.py b/skills/jetson-video-pipeline/scripts/validate_representative_content_summary.py new file mode 100644 index 00000000..dd735e2d --- /dev/null +++ b/skills/jetson-video-pipeline/scripts/validate_representative_content_summary.py @@ -0,0 +1,597 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Validate compact-review representative-content metadata against fresh evidence.""" + +# pylint: disable=missing-function-docstring,too-many-branches,too-many-locals,too-many-statements +# pylint: disable=unidiomatic-typecheck + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import stat +import sys +from pathlib import Path +from typing import Any +from urllib.parse import urlsplit + +if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise SystemExit("invoke this producer with isolated Python: python3 -I") + +SCHEMA_VERSION = "1.0" +KIND = "nvcodec-representative-content-summary-validation" +MAX_JSON_BYTES = 1024 * 1024 +READ_CHUNK_BYTES = 1024 * 1024 +SHA256_PATTERN = re.compile(r"[0-9a-f]{64}") +REPRESENTATIVE_CONTENT_FIELDS = ( + "source_url", + "license", + "attribution", + "path", + "size_bytes", + "sha256", +) +SOURCE_ARTIFACT_FIELDS = ( + "kind", + *REPRESENTATIVE_CONTENT_FIELDS, + "expected_size_bytes", + "expected_sha256", + "verified", +) + + +class ReviewValidationError(ValueError): + """An input cannot be safely read as current review evidence.""" + + +def _reject_json_constant(value: str) -> None: + raise ValueError(f"non-RFC JSON numeric constant is forbidden: {value}") + + +def strict_json_dumps(value: Any, **kwargs: Any) -> str: + """Render strict finite JSON with caller-selected deterministic options.""" + _require_finite_json(value) + return json.dumps(value, allow_nan=False, **kwargs) + + +def exclusive_write_bytes( + path: Path, content: bytes, *, mode: int = 0o600 +) -> Path: + """Create one fresh regular file without following path symlinks.""" + requested = path.expanduser().absolute() + if not requested.parent.is_dir(): + raise FileNotFoundError( + f"evidence parent directory does not exist: {requested.parent}" + ) + current = Path(requested.anchor) + for part in requested.parent.parts[1:]: + current /= part + if current.is_symlink(): + raise ValueError( + f"evidence parent contains a symlink component: {current}" + ) + resolved = requested.parent.resolve(strict=True) / requested.name + flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL + if hasattr(os, "O_NOFOLLOW"): + flags |= os.O_NOFOLLOW + descriptor = os.open(resolved, flags, mode) + try: + with os.fdopen(descriptor, "wb") as handle: + descriptor = -1 + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + except Exception: + resolved.unlink(missing_ok=True) + raise + finally: + if descriptor >= 0: + os.close(descriptor) + return resolved + + +def _reject_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError(f"duplicate JSON object key is forbidden: {key}") + result[key] = value + return result + + +def _strict_json_object(raw: bytes, label: str) -> dict[str, Any]: + try: + value = json.loads( + raw.decode("utf-8"), + parse_constant=_reject_json_constant, + object_pairs_hook=_reject_duplicate_keys, + ) + _require_finite_json(value) + except (UnicodeError, json.JSONDecodeError, ValueError, RecursionError) as exc: + raise ReviewValidationError(f"{label} cannot be parsed strictly: {exc}") from exc + if not isinstance(value, dict): + raise ReviewValidationError(f"{label} must contain one JSON object") + return value + + +def _require_finite_json(value: Any, location: str = "$") -> None: + if isinstance(value, float) and not math.isfinite(value): + raise ValueError(f"non-finite JSON number is forbidden at {location}") + if isinstance(value, dict): + for key, item in value.items(): + _require_finite_json(item, f"{location}.{key}") + elif isinstance(value, list): + for index, item in enumerate(value): + _require_finite_json(item, f"{location}[{index}]") + + +def _valid_canonical_absolute_path_text(value: Any) -> bool: + if not isinstance(value, str) or not value or "\x00" in value: + return False + try: + value.encode("utf-8") + candidate = Path(value) + except (UnicodeError, ValueError): + return False + return candidate.is_absolute() and str(candidate) == value + + +def _canonical_existing_path(path: Path, label: str) -> Path: + path_text = str(path) + if not _valid_canonical_absolute_path_text(path_text): + raise ReviewValidationError( + f"{label} path must already be one canonical absolute path" + ) + requested = Path(path_text) + current = Path(requested.anchor) + try: + for part in requested.parts[1:]: + current /= part + if current.is_symlink(): + raise ReviewValidationError( + f"{label} path contains a symlink component: {current}" + ) + resolved = requested.resolve(strict=True) + except ReviewValidationError: + raise + except (OSError, RuntimeError, UnicodeError, ValueError) as exc: + raise ReviewValidationError(f"{label} does not resolve: {exc}") from exc + if requested != resolved: + raise ReviewValidationError( + f"{label} path must already be one canonical absolute path: {requested}" + ) + return resolved + + +def _open_anchored_regular(path: Path, label: str) -> tuple[int, int]: + """Open a file through a held, no-symlink parent-directory chain.""" + directory_flags = ( + os.O_RDONLY + | getattr(os, "O_DIRECTORY", 0) + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + ) + file_flags = ( + os.O_RDONLY + | getattr(os, "O_NOFOLLOW", 0) + | getattr(os, "O_CLOEXEC", 0) + ) + parent_descriptor = -1 + descriptor = -1 + try: + parent_descriptor = os.open(path.anchor, directory_flags) + for component in path.parts[1:-1]: + child = os.open(component, directory_flags, dir_fd=parent_descriptor) + os.close(parent_descriptor) + parent_descriptor = child + if not path.name: + raise ReviewValidationError(f"{label} must name one regular file") + descriptor = os.open(path.name, file_flags, dir_fd=parent_descriptor) + opened = os.fstat(descriptor) + named = os.stat(path.name, dir_fd=parent_descriptor, follow_symlinks=False) + if ( + not stat.S_ISREG(opened.st_mode) + or not stat.S_ISREG(named.st_mode) + or (opened.st_dev, opened.st_ino) != (named.st_dev, named.st_ino) + ): + raise ReviewValidationError( + f"{label} must be one stable, non-symlink regular file" + ) + return descriptor, parent_descriptor + except ReviewValidationError: + if descriptor >= 0: + os.close(descriptor) + if parent_descriptor >= 0: + os.close(parent_descriptor) + raise + except (OSError, UnicodeError, ValueError) as exc: + if descriptor >= 0: + os.close(descriptor) + if parent_descriptor >= 0: + os.close(parent_descriptor) + raise ReviewValidationError(f"{label} cannot be opened safely: {exc}") from exc + + +def _read_authenticated_file( + path: Path, + label: str, + *, + capture: bool, + expected_size: int | None = None, +) -> tuple[dict[str, Any], bytes | None]: + resolved = _canonical_existing_path(path, label) + descriptor, parent_descriptor = _open_anchored_regular(resolved, label) + + captured = bytearray() if capture else None + try: + initial = os.fstat(descriptor) + if not stat.S_ISREG(initial.st_mode): + raise ReviewValidationError(f"{label} must be a regular file") + if expected_size is not None and initial.st_size != expected_size: + raise ReviewValidationError( + f"{label} current size differs before hashing: " + f"expected {expected_size}, observed {initial.st_size}" + ) + if capture and initial.st_size > MAX_JSON_BYTES: + raise ReviewValidationError( + f"{label} exceeds the {MAX_JSON_BYTES}-byte strict JSON limit" + ) + digest = hashlib.sha256() + observed_size = 0 + while True: + chunk = os.read(descriptor, READ_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + observed_size += len(chunk) + if captured is not None: + if observed_size > MAX_JSON_BYTES: + raise ReviewValidationError( + f"{label} exceeds the {MAX_JSON_BYTES}-byte strict JSON limit" + ) + captured.extend(chunk) + terminal = os.fstat(descriptor) + identity_fields = ("st_dev", "st_ino", "st_size", "st_mtime_ns", "st_ctime_ns") + if any( + getattr(initial, field) != getattr(terminal, field) + for field in identity_fields + ): + raise ReviewValidationError(f"{label} changed while it was being read") + try: + named = os.stat( + resolved.name, + dir_fd=parent_descriptor, + follow_symlinks=False, + ) + _canonical_existing_path(resolved, label) + visible = resolved.lstat() + except ReviewValidationError: + raise + except (OSError, UnicodeError, ValueError) as exc: + raise ReviewValidationError( + f"{label} disappeared after it was read: {exc}" + ) from exc + for observed in (named, visible): + if ( + not stat.S_ISREG(observed.st_mode) + or (observed.st_dev, observed.st_ino) + != (terminal.st_dev, terminal.st_ino) + ): + raise ReviewValidationError( + f"{label} path changed while it was being read" + ) + if observed_size != terminal.st_size: + raise ReviewValidationError(f"{label} size changed while it was being read") + finally: + os.close(descriptor) + os.close(parent_descriptor) + + identity = { + "path": str(resolved), + "size_bytes": observed_size, + "sha256": digest.hexdigest(), + "device": terminal.st_dev, + "inode": terminal.st_ino, + "mode": terminal.st_mode, + "mtime_ns": terminal.st_mtime_ns, + "ctime_ns": terminal.st_ctime_ns, + } + return identity, bytes(captured) if captured is not None else None + + +def _nonempty_text(value: Any) -> bool: + return isinstance(value, str) and bool(value) and value == value.strip() + + +def _valid_source_url(value: Any) -> bool: + if value is None: + return True + if not _nonempty_text(value): + return False + try: + value.encode("utf-8") + except UnicodeError: + return False + if "\\" in value or any( + character.isspace() + or ord(character) < 0x20 + or ord(character) == 0x7F + for character in value + ): + return False + try: + parsed = urlsplit(value) + hostname = parsed.hostname + port = parsed.port + username = parsed.username + password = parsed.password + except (UnicodeError, ValueError): + return False + return ( + parsed.scheme in {"http", "https"} + and bool(parsed.netloc) + and bool(hostname) + and username is None + and password is None + and (port is None or 1 <= port <= 65535) + ) + + +def _valid_positive_integer(value: Any) -> bool: + return type(value) is int and value > 0 + + +def _valid_sha256(value: Any) -> bool: + return isinstance(value, str) and SHA256_PATTERN.fullmatch(value) is not None + + +def _metadata_errors(record: Any, label: str) -> list[str]: + errors: list[str] = [] + expected = set(REPRESENTATIVE_CONTENT_FIELDS) + if not isinstance(record, dict): + return [f"{label} must be an exact representative-content object"] + if set(record) != expected: + missing = sorted(expected - set(record)) + extra = sorted(set(record) - expected) + errors.append(f"{label} fields differ: missing={missing}, extra={extra}") + if not _valid_source_url(record.get("source_url")): + errors.append( + f"{label}.source_url must be null for user-local input or one safe HTTP(S) URL" + ) + for field in ("license", "attribution"): + if not _nonempty_text(record.get(field)): + errors.append(f"{label}.{field} must be one nonempty trimmed string") + path_value = record.get("path") + if not _valid_canonical_absolute_path_text(path_value): + errors.append(f"{label}.path must be one nonempty canonical absolute path") + if not _valid_positive_integer(record.get("size_bytes")): + errors.append(f"{label}.size_bytes must be one exact positive integer") + if not _valid_sha256(record.get("sha256")): + errors.append(f"{label}.sha256 must be one lowercase 64-hex digest") + return errors + + +def _source_artifact_errors(source: Any) -> list[str]: + errors: list[str] = [] + expected = set(SOURCE_ARTIFACT_FIELDS) + if not isinstance(source, dict): + return ["representative-content source artifact must be one JSON object"] + if set(source) != expected: + missing = sorted(expected - set(source)) + extra = sorted(set(source) - expected) + errors.append( + "representative-content source artifact fields differ: " + f"missing={missing}, extra={extra}" + ) + if source.get("kind") != "representative-content": + errors.append("representative-content source artifact kind is invalid") + if source.get("verified") is not True: + errors.append("representative-content source artifact verified must be true") + metadata = { + field: source.get(field) for field in REPRESENTATIVE_CONTENT_FIELDS + } + errors.extend(_metadata_errors(metadata, "representative-content source metadata")) + expected_size = source.get("expected_size_bytes") + if not _valid_positive_integer(expected_size): + errors.append("source expected_size_bytes must be one exact positive integer") + elif type(source.get("size_bytes")) is int and expected_size != source.get("size_bytes"): + errors.append("source expected_size_bytes differs from size_bytes") + expected_sha = source.get("expected_sha256") + if not _valid_sha256(expected_sha): + errors.append("source expected_sha256 must be one lowercase 64-hex digest") + elif isinstance(source.get("sha256"), str) and expected_sha != source.get("sha256"): + errors.append("source expected_sha256 differs from sha256") + return errors + + +def _exact_json_equal(left: Any, right: Any) -> bool: + if type(left) is not type(right): + return False + if isinstance(left, dict): + return set(left) == set(right) and all( + _exact_json_equal(left[key], right[key]) for key in left + ) + if isinstance(left, list): + return len(left) == len(right) and all( + _exact_json_equal(left_item, right_item) + for left_item, right_item in zip(left, right) + ) + return left == right + + +def validate_representative_content_summary( + summary_path: Path, + source_artifact_path: Path, +) -> tuple[dict[str, Any], int]: + """Freshly authenticate both JSON inputs and the representative content file.""" + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "kind": KIND, + "status": "invalid", + "valid": False, + "validation_mode": "strict_fresh_read_only", + "inputs_mutated": False, + } + errors: list[str] = [] + summary: dict[str, Any] | None = None + source: dict[str, Any] | None = None + + try: + summary_identity, summary_raw = _read_authenticated_file( + summary_path, "summary artifact", capture=True + ) + result["summary_artifact"] = summary_identity + assert summary_raw is not None + summary = _strict_json_object(summary_raw, "summary artifact") + except (OSError, ReviewValidationError) as exc: + errors.append(str(exc)) + + try: + source_identity, source_raw = _read_authenticated_file( + source_artifact_path, "representative-content source artifact", capture=True + ) + result["source_artifact"] = source_identity + assert source_raw is not None + source = _strict_json_object(source_raw, "representative-content source artifact") + except (OSError, ReviewValidationError) as exc: + errors.append(str(exc)) + + summary_metadata: Any = None + if summary is not None: + summary_metadata = summary.get("representative_content") + if "representative_content" not in summary: + errors.append("summary artifact is missing representative_content") + errors.extend(_metadata_errors(summary_metadata, "summary representative_content")) + + source_metadata: dict[str, Any] | None = None + if source is not None: + errors.extend(_source_artifact_errors(source)) + source_metadata = { + field: source.get(field) for field in REPRESENTATIVE_CONTENT_FIELDS + } + + if isinstance(summary_metadata, dict) and source_metadata is not None: + for field in REPRESENTATIVE_CONTENT_FIELDS: + if not _exact_json_equal(summary_metadata.get(field), source_metadata.get(field)): + errors.append( + f"summary representative_content.{field} differs from source artifact" + ) + + content_identity: dict[str, Any] | None = None + if source_metadata is not None: + content_path = source_metadata.get("path") + content_size = source_metadata.get("size_bytes") + content_sha = source_metadata.get("sha256") + if ( + _valid_canonical_absolute_path_text(content_path) + and _valid_positive_integer(content_size) + and _valid_sha256(content_sha) + ): + try: + content_identity, _unused = _read_authenticated_file( + Path(content_path), + "representative content", + capture=False, + expected_size=content_size, + ) + result["content_artifact"] = content_identity + except (OSError, ReviewValidationError) as exc: + errors.append(str(exc)) + else: + if content_identity["size_bytes"] != content_size: + errors.append( + "representative content current size differs from source artifact" + ) + if content_identity["sha256"] != content_sha: + errors.append( + "representative content current SHA-256 differs from source artifact" + ) + + if not errors and isinstance(summary_metadata, dict): + result.update( + { + "status": "verified", + "valid": True, + "representative_content": summary_metadata, + "errors": [], + "exit_code": 0, + } + ) + return result, 0 + + result.update({"errors": errors, "exit_code": 2}) + return result, 2 + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description=__doc__, + epilog=( + "Exit 0: verified; exit 2: invalid or unauthenticated input; " + "exit 3: validator or fresh-output failure. Inputs are direct JSON files; " + "stdin and command-stream substitution are not accepted." + ), + ) + parser.add_argument( + "--summary", + type=Path, + required=True, + help="Canonical absolute path to the compact review evidence/summary.json", + ) + parser.add_argument( + "--source-artifact", + type=Path, + required=True, + help="Canonical absolute path to the original representative-content.json", + ) + parser.add_argument( + "--output", + type=Path, + help="Optional fresh JSON result path; an existing path is never overwritten", + ) + return parser + + +def _error_result(reason: str) -> dict[str, Any]: + return { + "schema_version": SCHEMA_VERSION, + "kind": KIND, + "status": "error", + "valid": False, + "validation_mode": "strict_fresh_read_only", + "inputs_mutated": False, + "errors": [reason], + "exit_code": 3, + } + + +def main() -> int: + args = build_parser().parse_args() + try: + result, code = validate_representative_content_summary( + args.summary, args.source_artifact + ) + except Exception as exc: # pylint: disable=broad-exception-caught + result = _error_result(f"unexpected validator failure: {exc}") + code = 3 + + rendered = strict_json_dumps(result, sort_keys=True, separators=(",", ":")) + "\n" + if args.output is not None: + try: + exclusive_write_bytes(args.output, rendered.encode("utf-8")) + except (OSError, ValueError) as exc: + failed = _error_result(f"fresh validation output could not be created: {exc}") + print(strict_json_dumps(failed, sort_keys=True, separators=(",", ":"))) + return 3 + print(rendered, end="") + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-pipeline/skill-card.md b/skills/jetson-video-pipeline/skill-card.md new file mode 100644 index 00000000..2c953bc3 --- /dev/null +++ b/skills/jetson-video-pipeline/skill-card.md @@ -0,0 +1,83 @@ +## Description:
+Execute and verify Jetson Video Codec SDK or PyNvVideoCodec encode/decode, transcode, segmentation, container decode, AV1, or acceptance workflows with exact artifact handoffs.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0
+## Use Case:
+Developers and engineers use this skill to execute official-sample codec stages on NVIDIA Jetson devices and prove that every consumer used the exact artifact produced by the preceding stage, including encode-then-decode verification, native transcode, PyNvVideoCodec segments, container decode triage, AV1 operation verification, and customer acceptance packages.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Pipeline workflow](references/pipeline-workflow.md)
+- [Official sample contract](references/official-sample-contract.md)
+- [Agent Skills](https://agentskills.io/)
+ + +## Skill Output:
+**Output Type(s):** [Shell commands, JSON results, Artifact verification]
+**Output Format:** [JSON with structured pipeline result and workspace artifacts]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [Produces provenance-tracked workspace with hashed artifacts and structured result JSON]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+2 evaluation tasks (2 positive) from skill-evaluator-dataset-snapshot/1, each run in an isolated k8s-sandbox pod.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Checks for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Checks final-answer correctness against the reference answer.
+- Discoverability: Checks whether the expected skill was found and executed.
+- Effectiveness: Checks whether the user's goal was achieved and expected workflow behavior was followed.
+- Efficiency: Checks routing quality, workspace-aware skill reads, and productive tool use.
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 50% → 100% (+50 points) | 35% → 71% (+36 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 40% → 100% (+60 points) | 40% → 30% (-10 points) | +| Discoverability | 50% → 100% (+50 points) | 22% → 94% (+72 points) | +| Effectiveness | 30% → 100% (+70 points) | 0% → 30% (+30 points) | +| Efficiency | 31% → 100% (+69 points) | 11% → 100% (+89 points) | + +## Skill Version(s):
+e61c045 (source: git SHA, committed 2026-08-10)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/jetson-video-pipeline/skill.oms.sig b/skills/jetson-video-pipeline/skill.oms.sig new file mode 100644 index 00000000..8b6d4ac0 --- /dev/null +++ b/skills/jetson-video-pipeline/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiamV0c29uLXZpZGVvLXBpcGVsaW5lIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjhjMWVjMTQ5NjE2N2ZhZGYzZjY5YzU1MzhmOThhNDUxM2YzNjJmYjZlN2ZjNGIyMmMxMjJkY2JlMjYyZTIyZDciCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJyZXNvdXJjZXMiOiBbCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmMjAxZmY0OTg1NGMwNDI5ZWVjNTY1N2M0NjcyM2MzZjExNTY0YWUyMDkyOGNmZDRiZDAxMzAwNjgxZGRiNzBkIiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJlYmM2Nzc3Yjg2NjIwMDlmMjgwZWE5N2NlNjVjZGNhODlhYTgyMzkzZmJlZTgzOTQ4NjllZmFiNzAyODI3OTE1IiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImUwZjg2OGRlZjQzZDk1ZjQzOWFkNmY5MTBhOTk2OWVmYTEwZjgwMGNhZTZiNGE0ZjdiNDk0ZmIyZWNlZmFhNTQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICI2NjcyM2MxNzQ1MWYxOGFlOTk1YmU5NDNjNTIzMzVjYzAxYzhiOGYxYTE2YjI0MjkzMjI3YjY2MmM0NDc5MjM4IiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL29mZmljaWFsLXNhbXBsZS1jb250cmFjdC5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjEwNjNmYzdlODgwZTRhOTlkMDRlZDE1ZTg0NDg3OTg0NzcwMWM3MjFkYWU1NGM4ZTRhNDFhZmI5ZTk5MzQwOGQiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcGlwZWxpbmUtd29ya2Zsb3cubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkYzJkN2YzZjkzMmE4YjI2YjJkODgwOTEzZjc1OGQyZWZkZTg1NzBlOTZlZWFkNjVkOWJhMzhkMzZhZjQwYWRhIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL19waXBlbGluZV9wcm92ZW5hbmNlLnB5IgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZDFmYmM2MWQ1NjkzYTcwMzdhZDQ2ZDM5ZTEzZGUxZTk0NWE5ZGVhNDNlNjZkMTBhMjMyOTI3ZDRhYTM1NzZiMiIsCiAgICAgICAgIm5hbWUiOiAic2NyaXB0cy9fcGlwZWxpbmVfcnVudGltZS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjViNjUwYjIwYzM1ZWYwZTE4YTc0MzVhMWY2NmE2NzhjZDZjOTQwNWRjYzQ3ZjQ3OWEwYjI4YjUwMjE0NTQ5MjQiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvZW5jb2RlX2NvbnRyb2xsZXIucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJkNzRkNThjOWVkOTdmNjBkOTY4MWVmNDJiMzkwNWYxZWRkYTFlMWY3MWE1MzQ1ODdkNzgwYzVmOWFkMTkyZmEzIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3BpcGVsaW5lX2NvbnRyb2xsZXIucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICJmN2M0MWQ3NGZmMzUyZDFiZTExZWYzZTMzNzY4NTM4NGRmMzhiODdlMzZhY2FkMjE0YjJiMTJiZjQxOTI2MjZlIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3ZhbGlkYXRlX3JlcHJlc2VudGF0aXZlX2NvbnRlbnRfc3VtbWFyeS5weSIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImU1Y2YwYzk5MThmOTMwYjAyNzJkZGQ0NzhkOWY4OTY5Y2Q1YzY5YWI2NWYzOTA4ODMwMzBiMjAyMmY4MGY0MjYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXQiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRodWIiCiAgICAgIF0sCiAgICAgICJoYXNoX3R5cGUiOiAic2hhMjU2IiwKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMQDfOa1xPvPyPlrPJ/OQI1JdpPqfe+u9qiFSONb6/KbJRpnwM53r2ak9nFnDy6JTm78CMBO1pfNqzVintZsAJVzz8U4einF2JM3zJsmbsmgGStRW4a5S8ZumFMpX51QrPmsOcg==","keyid":""}]}} \ No newline at end of file diff --git a/skills/jetson-video-recipe/BENCHMARK.md b/skills/jetson-video-recipe/BENCHMARK.md new file mode 100644 index 00000000..4996bd5e --- /dev/null +++ b/skills/jetson-video-recipe/BENCHMARK.md @@ -0,0 +1,98 @@ +# Skill Benchmark: jetson-video-recipe + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `jetson-video-recipe` +- Evaluation date: 2026-08-10 +- Evaluator version: `1.1.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 3 evaluation tasks (3 positive) +- Dataset digest: `sha256:cb2ef860b7b7cf59666c858c733f9b9fab262a8b7c52227977b0d53f6ae69da4` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 64% → 95% (+31 points) | 53% → 95% (+42 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 100% → 100% (±0 points) | 87% → 100% (+13 points) | +| Discoverability | 33% → 100% (+67 points) | 33% → 79% (+46 points) | +| Effectiveness | 74% → 94% (+21 points) | 45% → 100% (+55 points) | +| Efficiency | 15% → 83% (+67 points) | 1% → 95% (+94 points) | + +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. + +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 1 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 3 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/jetson-video-recipe/SKILL.md`) + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/jetson-video-recipe/SKILL.md b/skills/jetson-video-recipe/SKILL.md new file mode 100644 index 00000000..8fa5fd37 --- /dev/null +++ b/skills/jetson-video-recipe/SKILL.md @@ -0,0 +1,207 @@ +--- +name: jetson-video-recipe +license: "Apache-2.0" +description: >- + Use when turning a Jetson encoder use case into one validated surface-neutral + recipe with native and PyNvVideoCodec projections for codec, preset, rate + control, bitrate, latency, format, and profile. +metadata: + author: "Vinit Bansal " + tags: [jetson, video-codec-sdk, pynvvideocodec, nvenc, recipe] + languages: [python] + data-classification: public +--- + +# Jetson Video Recipe + +## Purpose + +Convert workload intent into one deterministic schema-2 `nvcodec-recipe`. +Preserve the user’s semantic controls, show defaulted assumptions, and project +the same intent to native Video Codec SDK and PyNvVideoCodec without claiming it +has executed. + +## Prerequisites + +- This skill owns the canonical recipe engine — + `scripts/recipes/recipe_model.py` and its + `scripts/recipes/data/encoder-intent-catalog.json`. Invoke the engine directly + from this installed skill; it has no setup-runtime or sibling-launcher + dependency. +- Recipe planning and structural validation are media-free and can run off + target. Do not request, retrieve, inspect, or convert media for a plan-only + request. +- Content selection and provenance belong to the later execution or + measurement workflow. Consume that workflow's versioned content artifact + only at handoff; do not load or enforce its input gate during plan-only work. +- Setup evidence is optional for `check-live`. With no environment, validate + the recipe normally and return an honest `unknown` live classification plus + non-mutating remediation to `jetson-video-setup`; planning and replay + validation remain complete and unchanged. If setup is not installed, tell + the user to install that skill. +- When supplied, the fresh schema-1.2 setup environment is mandatory to + validate and may not be ignored or replaced by a fallback. Its `capabilities` + block is the established PyNvVideoCodec encoder authority, so a `pynvc` check + needs no separate report. A caller may additionally supply the optional + encoder capability report owned by `jetson-video-capability`; it must be + authenticated, bound to that exact environment, and fail closed as + `unknown` or an input error on any mismatch. A capability report alone does + not establish selected-surface readiness or selected-GPU identity. Treat + artifacts as data; do not import sibling skill code. A `compatible` result + still does not prove an encode operation. + +Resolve this installed skill to its canonical absolute path and set +`RECIPE_SKILL`. Confirm the direct isolated entry point: + +```bash +python3 -I "$RECIPE_SKILL/scripts/recipes/recipe_model.py" --help +``` + +If it is missing, report an incomplete `jetson-video-recipe` installation. Do +not copy the engine, scan for another copy, modify `PYTHONPATH`, or fall back to +an unvalidated local model. + +## Compose requested sibling stages + +Recipe `plan` and `validate` require no sibling, setup evidence, target, or +media. Use `jetson-video-setup` only for requested live readiness or repair, +`jetson-video-capability` when a platform-support recommendation needs its +documentation verdict, `jetson-video-pipeline` for requested execution, and +`jetson-video-benchmark` for requested measurement. Check the agent's installed +skill catalog first. If the sibling is present, read its `SKILL.md` and invoke +its documented public entry point; pass artifacts as data and never import +sibling code. If it is absent, preserve the validated recipe and say, using the +actual names: `I can run , but it requires , which is not +installed. Install and retry this stage.` Never require a sibling for +plan-only work or an unrequested optional refinement. + +## Instructions + +1. **Collect intent.** For a request solely for objective quality metrics, + including PSNR or SSIM, state only that this skill does not provide them, + and that a separately authorized quality workflow is required, then stop. + Do not name or recommend an external tool, and do not offer to configure or + run the comparison; do not request media, probe, install anything, or + launch an operation. Resolve mutually exclusive rate-control intent before + collecting any other omitted field. In particular, when CQ and an average + bitrate are both supplied, explain the conflict, ask only whether to keep CQ + or the average bitrate, and stop. Do not reinterpret the bitrate as a cap or + ask for use case, resolution, frame rate, format, GPU, profile, preset, or + another field until the user resolves that choice. Otherwise resolve the + use case (`conferencing`, `live_streaming`, `vod`, `archival`, or + `lossless`), codec, width, height, raw input format, integer frame rate, GPU, + preset/tuning, rate-control or encoder quality priority, and any explicit + latency, profile, or buffering constraints. Resolve frame count only when + later execution or measurement needs it. Ask before assigning an + unqualified “low latency” request to a use case. Treat profile as a + bitstream/downstream-compatibility control separate from preset: preserve an + explicit profile, but when it is omitted leave it SDK-selected and never + invent a named profile. +2. **Write one intent JSON.** Keep caller values separate from defaults. Put + only caller-specified control values in the intent and leave every omitted + control to the authenticated use-case catalog. Do not turn qualitative + wording into guessed overrides: for example, “low-latency live streaming” + selects `live_streaming`; it does not by itself request `bf=0` or disabled + multipass. Never construct drifting native and Python intents. +3. **Plan with the recipe engine:** + + ```bash + python3 -I "$RECIPE_SKILL/scripts/recipes/recipe_model.py" \ + plan --intent "$INTENT_JSON" --output "$RECIPE_JSON" + ``` + +4. **Replay validation before use:** + + ```bash + python3 -I "$RECIPE_SKILL/scripts/recipes/recipe_model.py" \ + validate --recipe "$RECIPE_JSON" + ``` + + Do not hand-edit a generated recipe. Regenerate it from an updated intent. +5. **Optionally classify a live projection.** Run `check-live` for the selected + surface. The environment option is optional: + + ```bash + python3 -I "$RECIPE_SKILL/scripts/recipes/recipe_model.py" \ + check-live --recipe "$RECIPE_JSON" \ + --surface native --output "$LIVE_CHECK_JSON" + ``` + + For an exact projection, omission intentionally returns `unknown` with setup + remediation; it never invents readiness. To resolve the live result, repeat + with `--environment "$ENVIRONMENT_JSON"`. Repeat independently for `pynvc` + when requested. The Py check reads the environment's schema-1.2 + `capabilities` block. `--capability-report "$CAPABILITY_REPORT_JSON"` is an + optional Py refinement only when that environment is also supplied; it must + be bound to the same artifact. It replaces only the selected Py encoder API + evidence, not the environment's readiness facts, so omit it for native. + Missing optional evidence never fails, but supplied evidence must validate + and never silently falls back. A `compatible` result means the projection + and live Py evidence agree; it is not operation proof. Inspect the emitted + classification, not only the process exit code: an exact native projection + whose authenticated AppEncCuda run is deferred returns `unknown` with exit + code `0` and must never be reported as compatible or ready. + A Py CPU-buffer compatibility check requires the default smoke dependency + subset; GPU-buffer mode additionally requires the exact Torch facts provided + only by a validated `full-samples` environment. +6. **Return the recipe and assumptions.** Report schema/kind, exact portable + artifact identity, canonical encoder intent, native and PyNv projections, + projection losses, defaulted values, rationale, and any facts still needed + before execution. +7. **Stop before media work.** This skill never invokes `AppEncCuda`, `AppDec`, + PyNvVideoCodec sample applications, benchmark helpers, or pipeline + controllers. Route execution to `jetson-video-pipeline` and performance + measurement to `jetson-video-benchmark`. + +Use [recipes-workflow.md](references/recipes-workflow.md) for the request and +output contract and +[recipes-knobs-and-constraints.md](references/recipes-knobs-and-constraints.md) +for exact accepted values and surface limitations. + +## Recommendation rules + +Apply the tuning, preset, and matched-measurement rules in +[Tuning and preset](references/recipes-knobs-and-constraints.md#tuning-and-preset), +and the profile, format, and projection rules in +[Profile selection](references/recipes-knobs-and-constraints.md#profile-selection). + +- For a named platform, treating a codec as a recipe candidate is a support + claim. Consume the capability workflow's authenticated documentation verdict + first, exclude documentation-unsupported codecs, and preserve an unknown + verdict as unknown rather than offering it as supported. API fields or a + conditional “capability-gated” candidate do not override an unsupported + documentation verdict. +- If a recommendation publishes a documentation-based support verdict, consume + the capability result and reproduce every authenticated candidate row and + its count; never reinterpret a partial subset. + +## Available Scripts + +| Script | Purpose | Arguments | +|---|---|---| +| `scripts/recipes/recipe_model.py` | Plan, replay-validate, or live-check one canonical recipe and its native/PyNv projections. | Invoke directly with `python3 -I`; use the `plan`, `validate`, or `check-live` subcommand and inspect `--help`. | + +## Troubleshooting + +- Reject malformed, legacy, hybrid, duplicate-key, non-finite, or determinism- + mismatched recipe documents. +- Preserve an exact unrepresentable control as a per-surface projection loss. + For explicit `both`, do not hide the blocked peer or silently drop the + control. +- Do not choose an execution surface for an `auto` request. Preserve both + projections and hand runtime selection to `jetson-video-benchmark` or + `jetson-video-pipeline`, where live eligibility can be evaluated. +- Treat missing live fields as `unknown` and explicit negative fields as + `unsupported`. Missing selected-surface prerequisites include remediation to + `jetson-video-setup`; tell the user to install that skill if it is absent. + Neither state changes the portable recipe itself. + +## Limitations + +- Planning and validation do not establish installation readiness, + documentation support, live availability, output quality, or performance. +- This skill produces elementary encoder configuration only; container, + transcode, segmentation, decode verification, and artifact handoffs belong + to `jetson-video-pipeline`. +- Objective quality measurement, including PSNR and SSIM, is outside this + skill. diff --git a/skills/jetson-video-recipe/evals/evals.json b/skills/jetson-video-recipe/evals/evals.json new file mode 100644 index 00000000..eed577b4 --- /dev/null +++ b/skills/jetson-video-recipe/evals/evals.json @@ -0,0 +1,41 @@ +{ + "skill_name": "jetson-video-recipe", + "evals": [ + { + "id": "recipe-live-streaming-both", + "prompt": "Give me one H.264 1080p60, 6 Mbps CBR low-latency live-streaming recipe for native Video Codec SDK and PyNvVideoCodec. Do not run it.", + "expected_output": "One validated live-streaming recipe projected to both SDKs with explicit defaults.", + "assertions": [ + "Preserves H.264, 1920x1080, 60 fps, CBR, and 6000000 bps in one canonical intent", + "Resolves to P4, low-latency tuning, GOP 60, one B-frame, zero lookahead, full-resolution multipass, and a 3000000-bit VBV buffer", + "Projects the same intent to native and PyNvVideoCodec without claiming execution, measured quality, or performance" + ], + "expected_skill": "jetson-video-recipe", + "expected_workflow": "recipes" + }, + { + "id": "recipe-high-profile-projection-loss", + "prompt": "Prepare an H.264 1080p archival recipe for native Video Codec SDK and PyNvVideoCodec. High profile is required. Do not run it.", + "expected_output": "One archival recipe that preserves High profile natively and exposes the PyNvVideoCodec projection loss.", + "assertions": [ + "Preserves H.264 High profile as an explicit canonical control and expresses it in the native projection", + "Reports the released PyNvVideoCodec sample projection as unrepresentable because it cannot express a named profile", + "Does not drop or downgrade High profile, claim both projections are exact, or claim execution or performance" + ], + "expected_skill": "jetson-video-recipe", + "expected_workflow": "recipes" + }, + { + "id": "recipe-cq-bitrate-conflict", + "prompt": "Plan an H.264 encoder recipe with CQ 22 and a 6 Mbps bitrate. Do not run it.", + "expected_output": "A clear conflict result with no fabricated or silently modified recipe.", + "assertions": [ + "Explains that CQ is a target-quality VBR control and conflicts with an average bitrate in the shared recipe contract", + "Does not silently discard CQ, reinterpret the bitrate as a cap, or produce a validated native or PyNvVideoCodec projection", + "Launches no media, codec, or benchmark operation and asks only for information needed after the conflict is resolved" + ], + "expected_skill": "jetson-video-recipe", + "expected_workflow": "recipes" + } + ] +} diff --git a/skills/jetson-video-recipe/references/recipes-knobs-and-constraints.md b/skills/jetson-video-recipe/references/recipes-knobs-and-constraints.md new file mode 100644 index 00000000..c4c39f77 --- /dev/null +++ b/skills/jetson-video-recipe/references/recipes-knobs-and-constraints.md @@ -0,0 +1,160 @@ +# NVENC/PyVC knob checklist + +## Contents + +- [Tuning and preset](#tuning-and-preset) +- [Profile selection](#profile-selection) +- [Rate control and buffering](#rate-control-and-buffering) +- [Latency and quality controls](#latency-and-quality-controls) +- [Surface gates](#surface-gates) +- [Runtime reconfiguration](#runtime-reconfiguration) + +## Tuning and preset + +The supplied parser defaults `preset` to P4 and, for SDK-10+ presets, defaults an omitted +`tuning_info` to `NV_ENC_TUNING_INFO_HIGH_QUALITY` (`NvEncoderClInterface.cpp:696-724`). It does +not make the workload low latency merely because P1/P2 is selected. Multi-pass encoding is +separately defaulted to disabled unless `multipass` is provided (`:1063-1066`). Use the typed +value `ultra_high_quality` for UHQ. The wrapper preserves that typed value as intent evidence but +translates the executed native CLI and PyNvVideoCodec sample JSON to the parser token `uhq`; +`ultra_high_quality` must never reach the released sample parser unchanged. Public-2.1 Python UHQ +is restricted to HEVC/AV1, Turing-or-newer hardware, and a loaded extension recording linked NVENC +API 12.2 or newer; the `_121` surface does not contain the UHQ token and must remain blocked. Thor +satisfies the generation prerequisite. Other GPU-generation claims remain operation-verified rather +than inferred when authoritative generation evidence is unavailable. + +Treat P1–P7 as candidates along a speed/quality trade-off within a +codec/tuning/hardware context. Do not call one faster or higher quality without +matched measurements. For later measurement, let the owning benchmark or +pipeline workflow validate its versioned content-provenance artifact and +measure adjacent presets on the same user-selected content and frame range. + +Without matched measurements on identical representative content, do not present qualitative or +numeric cross-codec storage, compression, quality, or output-size rankings as measured or proven. +A documentation/support-based option may be labeled only as a starting candidate for matched +measurement; words such as "better", "best", and "largest" remain unmeasured claims unless the +labeled evidence proves that comparison. +When H.264 and HEVC are both documentation-supported for an 8-bit storage-priority comparison, +HEVC Main is the first matched-measurement candidate and H.264 is the compatibility baseline. +This orders measurement; it does not call either codec a measured storage, compression, quality, +or output-size winner. +For an explicit native HEVC profile, select Main10 when 10-bit output is required and downstream +decoder/container/playback compatibility is confirmed. NVENC can accept authenticated 8-bit 4:2:0 +input for Main10 and convert internally; P010 is the exact no-bit-depth-conversion input path. Use +Main for ordinary 8-bit 4:2:0 output or ask for the missing facts. The released PyNvVideoCodec 2.1 +sample cannot express `profile`; report that projection limitation instead of claiming that Main or +Main10 was applied. + +Use only `p1` through `p7`, exact lowercase tuning values `high_quality`, `low_latency`, +`ultra_low_latency`, `lossless`, or `ultra_high_quality`, and multipass values `disabled`, `qres`, +or `fullres`. +Invalid parser strings can fall back to defaults, so reject them rather than reporting the +requested string as the applied setting. +Schema-1 canonical config and schema-2 `encoder_intent` presets remain lowercase `p1` through +`p7`. The schema-2 Python projection, like public-2.1 executed JSON, uses +uppercase `P1` through `P7`, while native AppEnc CLI argv remains lowercase. Evidence must preserve +both the canonical typed config and the executed config; these are explicit token translations, not +intent changes. Any parser fallback diagnostic is an operation failure. + +## Profile selection + +Profile is a bitstream-tool and downstream-compatibility choice; it is independent of preset. +Do not call a profile faster, higher quality, or more storage-efficient without matched +measurements. Preserve an explicit caller profile exactly. When the caller and downstream contract +do not require one, omit it and report the result as SDK-selected/default rather than guessing or +claiming a named profile. + +| Codec/profile | Selection guidance | Required constraint or gate | +|---|---|---| +| H.264 Baseline | Choose only for a decoder or transport that explicitly requires this restricted legacy profile. | 8-bit 4:2:0; resolved `bf=0`; live profile support and operation verification. | +| H.264 Main | Choose when the downstream contract explicitly requires Main. | 8-bit 4:2:0; live profile support and operation verification. | +| H.264 High | Compatibility-oriented candidate for modern 8-bit 4:2:0 H.264 consumers; this is not a quality claim. | 8-bit 4:2:0; live profile support and operation verification. | +| H.264 High444 | Choose for an 8-bit 4:4:4 bitstream requirement or as one prerequisite of native H.264 lossless. It may also carry 8-bit 4:2:0; the name does not require 4:4:4 input. | Reject 10-bit input in this released Jetson recipe path. When 4:4:4 is requested, require matching input/capability. Lossless also requires QP zero, CONSTQP, transform bypass from the lossless path, and positive lossless capability. | +| HEVC Main | Ordinary 8-bit 4:2:0 output candidate. | Matching codec/profile support and operation verification. | +| HEVC Main10 | Choose when 10-bit 4:2:0 output is required and the downstream consumer supports it. NVENC can convert verified 8-bit input internally, but P010 is the exact no-bit-depth-conversion input path. | Positive 10-bit capability, authenticated input depth, downstream compatibility, and operation verification. | +| HEVC FRExt | Choose for native 4:2:2 or 4:4:4 output, at 8 or 10 bits. | Matching NV16/P210/YUV444/YUV444_16BIT semantics and exact chroma/bit-depth capabilities. | +| AV1 Main | The AppEnc AV1 profile covers 4:2:0 at 8 or 10 bits. | 4:2:0 only; 10-bit needs the exact capability. Release-matched product documentation still governs support. | + +The released PyNvVideoCodec 2.1 sample cannot express a named `profile`. Every explicit profile is +therefore a structured Python projection loss, never a silently dropped control. An emitted +bitstream may later reveal the runtime-selected profile, but recipe planning alone does not prove +that profile. Parser acceptance also does not prove a profile/format pair operates; keep structural +validation, live capability, operation, and emitted-bitstream proof separate. + +## Rate control and buffering + +- `rc=cbr`: bounded-rate starting point for live workloads; set bitrate, max bitrate, and VBV deliberately. +- `rc=vbr`: quality/size starting point when bitrate may vary. +- `rc=constqp`: constant-QP path; the QP value uses the separate `constqp` option. Do not combine with bitrate targets. +- Gate the selected mode against `supported_ratecontrol_modes` from the exact codec query: + positive mask presence proves const-QP, bit `0x1` proves VBR, and bit `0x2` proves CBR. Missing, + malformed, or non-positive evidence leaves support unknown. +- RC strings are lowercase-only. An unrecognized value, including `CQP`, falls back to CBR (`NvEncoderClInterface.cpp:912-926`). +- Supplying `bitrate` sets CBR unless an explicit valid `rc` later overrides it (`:1050-1108`). +- Supplying `cq` zeros average and maximum bitrate on the public Python parser; shared recipes + therefore reject CQ with either bitrate field (`:1096-1103`). CQ is a VBR target-quality value: + shared recipes use exact integer `0..51` for H.264/HEVC and `0..63` for AV1. +- A direct typed native AppEncPerf/AppEncCuda request may express finite fractional CQ in that + codec range and may pair it with `maxbitrate` as a cap, but not with average `bitrate`. This + native-only expressiveness must not be copied into shared public-2.1 Python recipe JSON. +- `constqp` uses one integer or `P,B,I` integers without spaces. Use `0..51` for H.264/HEVC and + `0..255` for AV1. Do not retain bitrate, maximum-bitrate, VBV, or CQ keys in const-QP mode. +- `qmin`: do not generate in this release because the parser fetches `qmin` and then parses `initqp` at `NvEncoderClInterface.cpp:1119-1122`. + +## Latency and quality controls + +- `bf`: gate against exact `num_max_bframes`. The resolver enforces `bf=0` for + `ultra_low_latency`; use `low_latency` when B-frames are required. +- `lookahead`: gate on `support_lookahead` and enforce the API range + `0..(31 - bf)`. The resolver enforces zero for `ultra_low_latency`. +- `multipass`: `disabled`, quarter-resolution, or full-resolution choices must be throughput-tested. Do not infer support from `support_multiple_ref_frames`; they are different concepts. +- `aq`: spatial AQ strength is 1–15 when enabled. `temporalaq` is a distinct option and is gated on `support_temporal_aq`. +- `temporalaq` is enabled by key presence in the supplied parser. Emit `1`/`true` to enable it; + omit the key to disable it—do not emit zero or false. +- `gop`: choose from recovery/random-access needs and frame rate; require `gop >= bf + 1`. A longer + GOP is not automatically better. +- `fps`: the installed public PyNvVideoCodec 2.1 sample/config contract and the native projection + require an exact positive JSON integer. Reject decimal numbers, decimal strings, and fraction + strings, and cap native `std::stoi` fields (`fps`, `gop`, `bf`) at `INT_MAX=2147483647`. + Bitrate/VBV integer fields are capped at `UINT32_MAX=4294967295`. Do not generalize fractional + handling observed in a newer local/development source tree to the authenticated public 2.1 wheel. +- Intra refresh is unsupported through rel_2_1 PyVC kwargs. Native NVENC + structs/caps expose it, but `NvEncoderClInterface` only prints the native + fields and never parses a public kwarg + (`:1375-1377,1438-1440,1745-1747`). Report it unrepresentable on the + selected released projection and stop without proposing another interface. + +## Surface gates + +- `YUV420`, `ARGB`, and `ABGR` are recipe/standalone encoder-input surfaces, not current + independent-decode pipeline target formats. Native `YUV420` pipeline output would require an + explicit `AppDec -outplanar` route that is not implemented, public-2.1 PyNvVideoCodec + `advanced/decode.py` cannot emit planar `YUV420`, and `AppDec` cannot emit matching RGB for + `ARGB`/`ABGR`. Resolving an encoder recipe for one of these formats does not claim a complete + content-to-independent-decode pipeline. +- `profile` is native-AppEnc-only because public PyNvVideoCodec 2.1 does not parse it. Native + profiles are codec-specific: H.264 `baseline|main|high|high444`; HEVC + `main|main10|frext`; AV1 `main`. H.264 Baseline requires resolved `bf=0`; reject a positive + caller or catalog/default B-frame value instead of silently rewriting it. Lookahead remains + independently gated. Use [Profile selection](#profile-selection) for format, depth, lossless, + omitted-profile, and downstream guidance. +- CPU-buffer sample readiness requires NumPy plus exact `pycuda==2026.1`; GPU-buffer mode also + requires the validated CUDA-enabled Torch record. + +- P010/P016 and other >8-bit surfaces require `support_10bit_encode`. +- AV1 encode is structurally limited to 4:2:0 input. Reject NV16, P210, YUV444, and + YUV444_16BIT before capability interpretation even when feature fields are positive. P010 + remains eligible when `support_10bit_encode` is positive; H.264/HEVC 4:2:2 and 4:4:4 remain + live-capability gated. +- H.264/HEVC 4:4:4 surfaces require `support_yuv444_encode`. +- The public 2.1 GPU utility implements H.264/HEVC YUV444 as three full-resolution planes and + assigns a `width * height * 3` frame size; do not block this path when the live 4:4:4 capability + is positive. +- H.264/HEVC NV16/P210 4:2:2 encoder surfaces require `support_yuv422_encode`; absence from an + API-12.1-linked module is unknown, not unsupported. `P216` is decoder-output naming and is not + accepted as an encode input here. +- 4:2:0 widths and heights must be even. + +## Runtime reconfiguration + +Do not generalize from the full native API. The supplied Python binding's `structEncodeReconfigureParams` exposes rate-control mode, multipass, average/max bitrate, VBV size/initial delay, frame-rate numerator/denominator, and target quality (`PyNvEncoder.cpp:1172-1198`). Any requested field outside that Python surface requires source/API verification and may require recreating the session. diff --git a/skills/jetson-video-recipe/references/recipes-workflow.md b/skills/jetson-video-recipe/references/recipes-workflow.md new file mode 100644 index 00000000..b8b3dbc6 --- /dev/null +++ b/skills/jetson-video-recipe/references/recipes-workflow.md @@ -0,0 +1,210 @@ +# Encoder recipe workflow + +Use this workflow to turn one workload intent into a deterministic, +surface-neutral encoder recipe. Planning and structural validation do not use +media and never launch a codec operation. + +## Contents + +- [Owner and CLI](#owner-and-cli) +- [Intent contract](#intent-contract) +- [Recipe contract](#recipe-contract) +- [Workflow](#workflow) +- [Live compatibility](#live-compatibility) +- [Handoffs](#handoffs) + +## Owner and CLI + +The canonical recipe engine is +`jetson-video-recipe/scripts/recipes/recipe_model.py`, owned by this recipe +skill. Invoke its public CLI directly from the canonical installed skill path: + +```text +python3 -I /scripts/recipes/recipe_model.py --help + +python3 -I /scripts/recipes/recipe_model.py \ + plan --intent intent.json --output nvcodec-recipe.json + +python3 -I /scripts/recipes/recipe_model.py \ + validate --recipe nvcodec-recipe.json + +python3 -I /scripts/recipes/recipe_model.py \ + check-live --recipe nvcodec-recipe.json \ + --surface native --output recipe-live-native.json + +python3 -I /scripts/recipes/recipe_model.py \ + check-live --recipe nvcodec-recipe.json \ + --environment nvcodec-environment.json \ + --capability-report nvcodec-encoder-capability.json \ + --surface pynvc --output recipe-live-pynvc.json +``` + +`--surface` accepts exactly `native` or `pynvc`. `--environment` is optional for +both live checks. With no environment, an exact projection remains valid but +its live classification is honestly `unknown` and includes non-mutating +remediation to `jetson-video-setup`. This does not change `plan` or `validate`. +If setup is not installed, tell the user to install that skill. + +When an environment is supplied, the native check validates that selected +surface and never requires a capability report; its authenticated AppEncCuda +operation remains deferred. The Py check does not require a separate report: +the `capabilities` block of the schema-1.2 environment is the established +encoder-capability authority and is fully functional on its own. +`--capability-report` is an optional Py-only refinement — omit it and the +environment's own block is used; supply it only with a fresh environment to +which it is bound. A report supplied without an environment cannot establish +selected-surface readiness or selected-GPU identity. Omit it for native. +Use `--buffer-mode cpu` or `--buffer-mode gpu` for the selected PyNv execution +path; omission preserves the `gpu` default and its Torch requirement. +`check-live` exits `0` for a compatible projection or a native `unknown` whose +authenticated operation remains deferred, `2` for an unsupported projection or +other unresolved live evidence, and `3` for malformed input. Never describe the +native exit-`0` deferred case as compatible or ready. + +This skill owns `scripts/recipes/recipe_model.py` and its catalog. Do not invoke +the engine from a copied path, insert another scripts directory into +`PYTHONPATH`, import it from a sibling skill, or recreate it in a consumer. A +missing entry point is an incomplete recipe-skill installation. + +## Intent contract + +The intent is a strict JSON object. Require only `use_case`, positive integer +`width`, and positive integer `height`. `use_case` is `conferencing`, +`live_streaming`, `vod`, `archival`, or `lossless`. + +Leave omitted fields to the resolver: `codec` defaults to `h264`, `format` to +`NV12`, `fps` to `30`, GPU to `0`, and `output_requirement` to +`elementary_stream`. `frame_count` (or its `frames` alias) is optional for +planning but must be an exact positive integer before execution. Include only +caller-specified profile, preset, tuning, rate-control, bitrate, maximum +bitrate, VBV, GOP, B-frame, lookahead, AQ, temporal-AQ, multipass, CQ, or +constant-QP controls. + +An unqualified “low latency” request does not select a use case. Ask whether it +means conferencing, live streaming, or another latency contract before +defaulting controls. + +Recipe planning is media-free. Do not ask for a path or URL, probe media, +retrieve content, or create raw frames to produce this object. + +## Recipe contract + +The output is one `schema_version: "2.0"`, `kind: "nvcodec-recipe"` object. It +contains: + +- the canonical `encoder_intent`; +- exact native and PyNvVideoCodec projections; +- per-surface projection status and any projection losses; +- caller-supplied and defaulted values with provenance; +- assumptions and rationale; +- a deterministic catalog identity. + +The native and Python projections are representations of the same intent, not +independent recipes. Do not delete a requested control merely to make the +projections look identical. + +Validation reconstructs the recipe from its recorded intent and catalog and +requires exact deterministic equality. Regenerate instead of editing the +artifact in place. + +## Workflow + +1. Normalize the user request into one intent JSON. +2. Run the public CLI's `plan` subcommand to create a fresh output. +3. Run its `validate` subcommand on the exact resulting file. +4. Return the recipe identity, assumptions, defaults, native projection, Python + projection, and any unrepresentable controls. +5. Stop. Do not select media, build samples, encode, decode, benchmark, or + claim the configuration ran. + +Follow [recipes-knobs-and-constraints.md](recipes-knobs-and-constraints.md) for +accepted values, parser spellings, integer bounds, and surface limitations. + +## Live compatibility + +`check-live` can run with no setup evidence. It still replay-validates the exact +recipe, then returns `classification: unknown` plus a `jetson-video-setup` +remediation for an otherwise exact projection. It performs no probe, install, +repair, surface discovery, or recipe mutation. A capability report supplied by +itself does not resolve that result. A structurally unrepresentable projection +remains `unsupported` independently of live evidence. + +- When supplied, `--environment` is the setup-probe environment, exactly + `kind: "nvcodec-environment"`, `schema_version: "1.2"`, and `mode: "live"`. + Read from it: `selected_gpu` and the matching `nvidia_smi.gpus[]` record for + GPU identity, the selected native or Py surface, and — when Py is selected — + Python readiness from + `installation.python.packages..{status, version, requirement_satisfied}`, + where `torch` additionally carries `cuda_build`, `cuda_available`, and + `sample_readiness`. +- `--capability-report` is the optional capability-owned Py refinement, + `kind: "nvcodec-encoder-capability-report"`, `schema_version: "1.0"` report. + When one is supplied, encoder evidence is read from it: `authority`, `gpu`, + `evidence_classification.encode`, `linked_nvenc_api`, `encode.`, and + `documentation_crosscheck`. When it is omitted, that evidence comes from the + environment's own established `capabilities` block, which is fully functional + on its own. + Its absence never fails a check. When present, it requires the environment + and affects only `pynvc`; omit it for `native`. + +Every supplied artifact is mandatory to validate and is never silently ignored +or replaced with an evidence-free fallback. Strict JSON or recipe input errors +fail as input errors; unavailable, wrong-schema, stale, mismatched, or incomplete +live authority remains closed as `unknown` or `unsupported` according to the +explicit evidence. In particular, a supplied invalid capability report does +not fall back to the environment's `capabilities` block. + +A report is valid only for the environment it was produced from. It records +that artifact's path, size, raw-byte SHA-256, canonical-JSON SHA-256, schema, +mode, selected GPU, lexical interpreter, and authenticated Py identity. Each +field is compared in its own domain against the supplied environment; any +mismatch fails closed as `unknown`. This rejects an environment/report swap +and any change to the environment between report production and consumption. + +Codec/API support and installation readiness are different questions with +different owners; both are in scope for a live compatibility check. An explicit +negative dependency readiness is `unsupported`; missing or malformed dependency +evidence is `unknown`. Both include non-mutating remediation to +`jetson-video-setup` when selected-surface prerequisites need setup work. If the +setup skill is absent, tell the user to install it. + +`GetEncoderCaps` evidence is `status: "capability_reported"` with +`supported: null` and `operation_status: "not_tested"`; `capability_reported` is +never support. Final-answer precedence is API query, then official operation, +then NVIDIA documentation: an applicable `documentation_crosscheck.support` of +`No` is the final `unsupported` verdict for any codec and outranks a positive +query, and the discrepancy is recorded alongside it. + +Pass artifacts as data; never import the producing skill's implementation. The +command classifies one projection from exact fields: + +- `compatible`: requested controls are expressible and required positive fields + are present; +- `unsupported`: an exact required field is explicitly negative or the surface + cannot represent a requested control; +- `unknown`: authority, field, or current evidence is missing. + +Compatibility is not readiness or operation proof. The later pipeline must +authenticate and run the selected official sample and independently decode the +exact output before claiming an encode succeeded. + +Evaluate native and PyNv projections independently. For explicit `both`, +preserve both outcomes and every blocked reason. + +## Handoffs + +- `jetson-video-setup` optionally supplies the schema-1.2 environment artifact + that resolves otherwise unknown selected-surface readiness. + `jetson-video-capability` owns the optional encoder capability report that + `check-live` may additionally consume, plus authenticated documentation + evidence. A + recommendation consumes those results when it needs a support statement. +- `jetson-video-benchmark` consumes the exact validated recipe artifact and + holds all non-compared controls constant. +- `jetson-video-pipeline` consumes the exact validated recipe artifact for + official sample execution and artifact handoff verification. + +Every consumer receives the original canonical recipe path, byte size, and +SHA-256 and rehashes it before use. Consumers exchange versioned artifacts or +public-CLI results; they do not import this skill. A copied summary or rewritten +projection is not a valid handoff. diff --git a/skills/jetson-video-recipe/scripts/recipes/data/encoder-intent-catalog.json b/skills/jetson-video-recipe/scripts/recipes/data/encoder-intent-catalog.json new file mode 100644 index 00000000..a1464199 --- /dev/null +++ b/skills/jetson-video-recipe/scripts/recipes/data/encoder-intent-catalog.json @@ -0,0 +1,52 @@ +{ + "schema_version": "1.0", + "kind": "nvcodec-encoder-intent-catalog", + "version": "1.0-surface-neutral", + "recipes": { + "conferencing": { + "encoder_intent_defaults": {"preset": "p3", "tuning_info": "ultra_low_latency", "rc": "cbr", "gop": 30, "bf": 0, "multipass": "disabled", "lookahead": 0, "bitrate": 3000000, "maxbitrate": 3000000, "vbvbufsize": 300000}, + "projections": { + "native": {"sample_id": "native-appenc-cuda", "sample": "AppEncCuda", "release_family": "13.0"}, + "pynvc": {"sample_id": "pynvc-basic-encode", "sample": "samples/basic/encode.py", "version": "2.1.0", "official_sample_baseline": "advanced/encode_config_lowlatency.json"} + }, + "assumptions": ["latency is prioritized over compression efficiency", "the application can tolerate strict CBR behavior"], + "rationale": {"tuning_info": "starts from the NVENC ultra-low-latency tuning path", "bf": "removes B-frame reordering delay", "lookahead": "removes lookahead queueing", "vbvbufsize": "small starting buffer; measure under network jitter"} + }, + "live_streaming": { + "encoder_intent_defaults": {"preset": "p4", "tuning_info": "low_latency", "rc": "cbr", "gop": 60, "bf": 1, "multipass": "fullres", "lookahead": 0, "bitrate": 6000000, "maxbitrate": 6000000, "vbvbufsize": 3000000}, + "projections": { + "native": {"sample_id": "native-appenc-cuda", "sample": "AppEncCuda", "release_family": "13.0"}, + "pynvc": {"sample_id": "pynvc-basic-encode", "sample": "samples/basic/encode.py", "version": "2.1.0", "official_sample_baseline": "advanced/encode_config_lowlatency.json"} + }, + "assumptions": ["bounded bitrate and moderate latency matter", "one B frame is acceptable only after latency validation"], + "rationale": {"tuning_info": "low-latency starting point", "rc": "bounded-rate starting point", "multipass": "quality-oriented candidate that must be throughput-tested", "gop": "roughly two seconds at the default 30 fps"} + }, + "vod": { + "encoder_intent_defaults": {"preset": "p6", "tuning_info": "high_quality", "rc": "vbr", "gop": 120, "bf": 3, "multipass": "fullres", "lookahead": 20, "aq": 8, "bitrate": 8000000, "maxbitrate": 12000000, "vbvbufsize": 12000000}, + "projections": { + "native": {"sample_id": "native-appenc-cuda", "sample": "AppEncCuda", "release_family": "13.0"}, + "pynvc": {"sample_id": "pynvc-basic-encode", "sample": "samples/basic/encode.py", "version": "2.1.0", "official_sample_baseline": "advanced/encode_config.json"} + }, + "assumptions": ["offline throughput is secondary to compression efficiency", "input surfaces remain valid while lookahead consumes them"], + "rationale": {"preset": "quality-leaning candidate", "rc": "allows content-dependent bitrate", "lookahead": "candidate for better frame decisions at added latency and memory", "aq": "moderate spatial AQ starting point"} + }, + "archival": { + "encoder_intent_defaults": {"preset": "p7", "tuning_info": "high_quality", "rc": "vbr", "gop": 250, "bf": 3, "multipass": "fullres", "lookahead": 28, "aq": 8, "bitrate": 20000000, "maxbitrate": 30000000, "vbvbufsize": 30000000}, + "projections": { + "native": {"sample_id": "native-appenc-cuda", "sample": "AppEncCuda", "release_family": "13.0"}, + "pynvc": {"sample_id": "pynvc-basic-encode", "sample": "samples/basic/encode.py", "version": "2.1.0", "official_sample_baseline": "advanced/encode_config.json"} + }, + "assumptions": ["quality and size dominate real-time throughput", "the archive workflow accepts long GOPs"], + "rationale": {"preset": "slowest quality-leaning starting point", "gop": "long-GOP candidate; random access requirements may require reduction", "multipass": "quality candidate to validate against throughput budget"} + }, + "lossless": { + "encoder_intent_defaults": {"preset": "p3", "tuning_info": "lossless", "rc": "constqp", "constqp": "0,0,0", "gop": 30, "bf": 0, "multipass": "disabled", "lookahead": 0}, + "projections": { + "native": {"sample_id": "native-appenc-cuda", "sample": "AppEncCuda", "release_family": "13.0"}, + "pynvc": {"sample_id": "pynvc-basic-encode", "sample": "samples/basic/encode.py", "version": "2.1.0", "official_sample_baseline": "basic/encode_config.json"} + }, + "assumptions": ["the selected codec/profile/surface supports the requested lossless path", "large output size is acceptable"], + "rationale": {"rc": "constant zero QP candidate", "tuning_info": "lossless tuning path", "bf": "simple reorder-free baseline"} + } + } +} diff --git a/skills/jetson-video-recipe/scripts/recipes/recipe_model.py b/skills/jetson-video-recipe/scripts/recipes/recipe_model.py new file mode 100644 index 00000000..14fa0468 --- /dev/null +++ b/skills/jetson-video-recipe/scripts/recipes/recipe_model.py @@ -0,0 +1,1620 @@ +#!/usr/bin/env python3 +"""Resolve and validate the sole surface-neutral NVCodec recipe contract.""" + +# This single domain owner keeps recipe parsing, resolution, projection, and +# validation on one schema contract. +# pylint: disable=too-many-lines + +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import stat +import sys +from pathlib import Path +from typing import Any + +if not sys.flags.ignore_environment or not sys.flags.no_user_site: + raise SystemExit("invoke this producer with isolated Python: python3 -I") + +SCHEMA_VERSION = "2.0" +KIND = "nvcodec-recipe" +ENVIRONMENT_KIND = "nvcodec-environment" +ENVIRONMENT_SCHEMA_VERSION = "1.2" +# Python-surface readiness required by ``check-live``, read from the established +# schema-1.2 ``installation.python.packages`` map. Torch applies to the GPU +# buffer mode only. +PYNVC_DEPENDENCIES = ( + ("numpy", "NumPy", {}), + ("pycuda", "PyCUDA", {"version": "2026.1"}), +) +TORCH_GPU_DEPENDENCY = ( + "torch", + "Torch GPU mode", + { + "version": "2.9.1+cu130", + "cuda_build": "13.0", + "cuda_available": True, + "sample_readiness": "ready", + }, +) +# Artifact identity of the optional capability-owned PyNvVideoCodec +# GetEncoderCaps report. Pinned in this one place; see ``_capability_report``. +CAPABILITY_REPORT_IDENTITY = { + "kind": "nvcodec-encoder-capability-report", + "schema_version": "1.0", +} +# A supplied report is bound to the exact environment artifact it was produced +# from. +CAPABILITY_BINDING_FIELDS = ( + "path", + "size_bytes", + "sha256", + "canonical_sha256", + "kind", + "schema_version", + "mode", + "selected_gpu", + "interpreter", + "interpreter_identity", + "pynvc_identity", +) +# Encoder capability evidence has two accepted producers, and each keeps its own +# "no encode was attempted" spelling. The accepted value is therefore selected +# by source rather than merged, so neither producer can borrow the other's +# vocabulary and neither may ever assert that an operation ran. +REPORT_OPERATION_STATUS = "not_tested" +ENVIRONMENT_OPERATION_STATUS = "not_verified_by_probe" +CATALOG_PATH = Path(__file__).resolve().parent / "data" / "encoder-intent-catalog.json" +CATALOG_REFERENCE = "scripts/recipes/data/encoder-intent-catalog.json" +INT_MAX = (1 << 31) - 1 +UINT32_MAX = (1 << 32) - 1 + +CORE_KEYS = { + "use_case", + "codec", + "width", + "height", + "format", + "fps", + "gpu", + "output_requirement", + "frame_count", + "frames", +} +CONTROL_KEYS = { + "preset", + "tuning_info", + "rc", + "gop", + "bf", + "multipass", + "lookahead", + "aq", + "temporalaq", + "cq", + "constqp", + "vbvbufsize", + "profile", + "bitrate", + "maxbitrate", +} +ALLOWED_KEYS = CORE_KEYS | CONTROL_KEYS +FORMATS = { + "NV12", + "YUV420", + "NV16", + "YUV444", + "P010", + "P210", + "YUV444_16BIT", + "ARGB", + "ABGR", +} +TEN_BIT_FORMATS = {"P010", "P210", "YUV444_16BIT"} +YUV422_FORMATS = {"NV16", "P210"} +YUV444_FORMATS = {"YUV444", "YUV444_16BIT"} +KNOWN_YUV_FORMATS = { + "NV12", + "YUV420", + "NV16", + "YUV444", + "P010", + "P210", + "YUV444_16BIT", +} +CODECS = {"h264", "hevc", "av1"} +CODEC_PROFILES = { + "h264": {"baseline", "main", "high", "high444"}, + "hevc": {"main", "main10", "frext"}, + "av1": {"main"}, +} +PRESETS = {f"p{index}" for index in range(1, 8)} +TUNINGS = { + "high_quality", + "low_latency", + "ultra_low_latency", + "lossless", + "ultra_high_quality", +} +RATE_CONTROLS = {"cbr", "vbr", "constqp"} +MULTIPASS = {"disabled", "qres", "fullres"} +PROFILES = set().union(*CODEC_PROFILES.values()) + +# These are exact AppEncCuda ``-if`` spellings from the released sample. In +# particular, 4:2:2 and 4:4:4 are formats, not invented ``-422``/``-444`` +# switches. RGB spellings are included only where AppEncCuda advertises them. +NATIVE_FORMATS = { + "NV12": "nv12", + "YUV420": "iyuv", + "NV16": "nv16", + "YUV444": "yuv444", + "P010": "p010", + "P210": "p210", + "YUV444_16BIT": "yuv444p16", + "ARGB": "bgra", + "ABGR": "abgr", +} +NATIVE_TUNINGS = { + "high_quality": "hq", + "low_latency": "lowlatency", + "ultra_low_latency": "ultralowlatency", + "lossless": "lossless", + "ultra_high_quality": "uhq", +} +PY_CONFIG_KEYS = ( + "codec", + "fps", + "preset", + "tuning_info", + "rc", + "gop", + "bf", + "multipass", + "lookahead", + "aq", + "temporalaq", + "cq", + "constqp", + "vbvbufsize", + "bitrate", + "maxbitrate", +) +NATIVE_OPTIONS = ( + ("preset", "-preset"), + ("tuning_info", "-tuninginfo"), + ("rc", "-rc"), + ("profile", "-profile"), + ("fps", "-fps"), + ("gop", "-gop"), + ("bf", "-bf"), + ("multipass", "-multipass"), + ("bitrate", "-bitrate"), + ("maxbitrate", "-maxbitrate"), + ("vbvbufsize", "-vbvbufsize"), + ("lookahead", "-lookahead"), + ("aq", "-aq"), + ("constqp", "-constqp"), + ("cq", "-cq"), +) + + +class RecipeError(ValueError): + """An input cannot be represented by the exact recipe contract.""" + + +def _validate_explicit_profile_format(codec: str, profile: str, raw_format: str) -> None: + """Reject named profiles that cannot preserve the known YUV format.""" + if raw_format not in KNOWN_YUV_FORMATS: + return + allowed = { + ("h264", "baseline"): {"NV12", "YUV420"}, + ("h264", "main"): {"NV12", "YUV420"}, + ("h264", "high"): {"NV12", "YUV420"}, + ("h264", "high444"): {"NV12", "YUV420", "YUV444"}, + ("hevc", "main"): {"NV12", "YUV420"}, + ("hevc", "main10"): {"NV12", "YUV420", "P010"}, + ("hevc", "frext"): {"NV16", "P210", "YUV444", "YUV444_16BIT"}, + ("av1", "main"): {"NV12", "YUV420", "P010"}, + }.get((codec, profile)) + if allowed is not None and raw_format not in allowed: + raise RecipeError( + f"profile {profile} cannot preserve format {raw_format} for {codec}; " + f"allowed known YUV formats are {sorted(allowed)}" + ) + + +def _reject_constant(token: str) -> None: + raise RecipeError(f"non-finite JSON constant is prohibited: {token}") + + +def _strict_object_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise RecipeError(f"duplicate JSON object key: {key}") + result[key] = value + return result + + +def _strict_json_text(payload: str) -> Any: + return json.loads( + payload, + object_pairs_hook=_strict_object_pairs, + parse_constant=_reject_constant, + ) + + +def strict_json_load(path: Path) -> Any: + """Read strict JSON, rejecting duplicate keys and non-finite numbers.""" + try: + return _strict_json_text(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError) as exc: + raise RecipeError(f"cannot read strict JSON from {path}: {exc}") from exc + + +def canonical_sha256(value: Any) -> str: + """Hash one strict JSON value using its canonical representation.""" + try: + payload = json.dumps( + value, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + except (TypeError, ValueError) as exc: + raise RecipeError(f"value is not strict JSON: {exc}") from exc + return hashlib.sha256(payload).hexdigest() + + +def _integer(value: Any, name: str, minimum: int, maximum: int) -> int: + if isinstance(value, bool) or not isinstance(value, int) or not minimum <= value <= maximum: + raise RecipeError(f"{name} must be an integer in {minimum}..{maximum}") + return value + + +def _enum(value: Any, name: str, choices: set[str]) -> str: + if not isinstance(value, str) or value not in choices: + raise RecipeError(f"{name} must be one of {sorted(choices)}") + return value + + +def _enabled(value: Any) -> bool: + return value is True or (isinstance(value, int) and not isinstance(value, bool) and value == 1) + + +def _validate_constqp(value: Any, codec: str) -> str: + if isinstance(value, int) and not isinstance(value, bool): + values = [value] + elif isinstance(value, str): + parts = value.split(",") + if len(parts) not in {1, 3} or any(not part.isdigit() for part in parts): + raise RecipeError("constqp must be one integer or three comma-separated integers") + values = [int(part) for part in parts] + else: + raise RecipeError("constqp must be one integer or three comma-separated integers") + maximum = 255 if codec == "av1" else 51 + if any(not 0 <= item <= maximum for item in values): + raise RecipeError(f"constqp values must be in 0..{maximum} for {codec}") + return ",".join(str(item) for item in values) + + +def _load_catalog() -> tuple[dict[str, Any], str]: # pylint: disable=too-many-branches + catalog = strict_json_load(CATALOG_PATH) + if not isinstance(catalog, dict) or set(catalog) != { + "schema_version", + "kind", + "version", + "recipes", + }: + raise RecipeError("encoder intent catalog has an invalid exact shape") + if catalog["schema_version"] != "1.0" or catalog["kind"] != "nvcodec-encoder-intent-catalog": + raise RecipeError("encoder intent catalog identity is invalid") + if not isinstance(catalog["version"], str) or not catalog["version"]: + raise RecipeError("encoder intent catalog version is invalid") + recipes = catalog["recipes"] + if not isinstance(recipes, dict) or not recipes: + raise RecipeError("encoder intent catalog recipes are invalid") + for use_case, item in recipes.items(): + if not isinstance(use_case, str) or not isinstance(item, dict): + raise RecipeError("encoder intent catalog recipe is invalid") + required = {"encoder_intent_defaults", "projections", "assumptions", "rationale"} + if set(item) != required: + raise RecipeError(f"catalog recipe {use_case!r} has an invalid exact shape") + defaults = item["encoder_intent_defaults"] + if not isinstance(defaults, dict) or not defaults: + raise RecipeError(f"catalog recipe {use_case!r} defaults are invalid") + if set(defaults) - CONTROL_KEYS: + raise RecipeError(f"catalog recipe {use_case!r} defaults contain core/unknown fields") + projections = item["projections"] + if not isinstance(projections, dict) or set(projections) != {"native", "pynvc"}: + raise RecipeError(f"catalog recipe {use_case!r} projections are invalid") + native = projections["native"] + pynvc = projections["pynvc"] + if native != { + "sample_id": "native-appenc-cuda", + "sample": "AppEncCuda", + "release_family": "13.0", + }: + raise RecipeError(f"catalog recipe {use_case!r} native projection is unapproved") + if ( + not isinstance(pynvc, dict) + or pynvc.get("sample_id") != "pynvc-basic-encode" + or pynvc.get("sample") != "samples/basic/encode.py" + or pynvc.get("version") != "2.1.0" + ): + raise RecipeError(f"catalog recipe {use_case!r} Py projection is unapproved") + if not isinstance(item["assumptions"], list) or not all( + isinstance(value, str) and value for value in item["assumptions"] + ): + raise RecipeError(f"catalog recipe {use_case!r} assumptions are invalid") + if not isinstance(item["rationale"], dict) or not all( + isinstance(key, str) and isinstance(value, str) and value + for key, value in item["rationale"].items() + ): + raise RecipeError(f"catalog recipe {use_case!r} rationale is invalid") + return catalog, hashlib.sha256(CATALOG_PATH.read_bytes()).hexdigest() + + +def _normalize(raw: Any, catalog: dict[str, Any]) -> tuple[dict[str, Any], list[dict[str, Any]]]: + if not isinstance(raw, dict): + raise RecipeError("intent must be a JSON object") + canonical_sha256(raw) + unknown = sorted(set(raw) - ALLOWED_KEYS) + if unknown: + raise RecipeError(f"intent contains unsupported fields: {unknown}") + missing = sorted({"use_case", "width", "height"} - set(raw)) + if missing: + raise RecipeError(f"intent is missing required fields: {missing}") + use_case = raw["use_case"] + if not isinstance(use_case, str) or use_case not in catalog["recipes"]: + raise RecipeError(f"use_case must be one of {sorted(catalog['recipes'])}") + normalized = dict(raw) + defaults: list[dict[str, Any]] = [] + for key, value, rule in ( + ("codec", "h264", "codec defaults to h264"), + ("format", "NV12", "input format defaults to NV12"), + ("fps", 30, "fps defaults to 30"), + ("gpu", 0, "GPU ordinal defaults to 0"), + ("output_requirement", "elementary_stream", "output defaults to an elementary stream"), + ): + if key not in normalized: + normalized[key] = value + defaults.append( + { + "path": f"/encoder_intent/{key}", + "value": value, + "source": "resolver", + "rule": rule, + } + ) + normalized["codec"] = _enum(str(normalized["codec"]).lower(), "codec", CODECS) + normalized["format"] = _enum(str(normalized["format"]).upper(), "format", FORMATS) + normalized["width"] = _integer(normalized["width"], "width", 1, INT_MAX) + normalized["height"] = _integer(normalized["height"], "height", 1, INT_MAX) + normalized["fps"] = _integer(normalized["fps"], "fps", 1, INT_MAX) + normalized["gpu"] = _integer(normalized["gpu"], "gpu", 0, INT_MAX) + if normalized["format"] in {"NV12", "YUV420", "P010"} and ( + normalized["width"] % 2 or normalized["height"] % 2 + ): + raise RecipeError(f"{normalized['format']} requires even width and height") + if normalized["format"] in YUV422_FORMATS and normalized["width"] % 2: + raise RecipeError(f"{normalized['format']} requires even width") + if normalized["codec"] == "av1" and normalized["format"] in ( + YUV422_FORMATS | YUV444_FORMATS + ): + raise RecipeError(f"AV1 does not support {normalized['format']} encoder input") + if normalized["output_requirement"] != "elementary_stream": + raise RecipeError("output_requirement must be elementary_stream") + frame_values = [ + _integer(normalized[key], key, 1, INT_MAX) + for key in ("frame_count", "frames") + if key in normalized + ] + if frame_values: + frame_count = frame_values[0] + if any(value != frame_count for value in frame_values[1:]): + raise RecipeError("frame_count and frames must match") + normalized["frame_count"] = frame_count + normalized.pop("frames", None) + return normalized, defaults + + +def _controls( # pylint: disable=too-many-branches,too-many-statements + raw: dict[str, Any], + normalized: dict[str, Any], + catalog_item: dict[str, Any], + defaults: list[dict[str, Any]], +) -> dict[str, Any]: + controls = dict(catalog_item["encoder_intent_defaults"]) + sources = { + key: { + "source": "catalog", + "catalog_pointer": (f"/recipes/{normalized['use_case']}/encoder_intent_defaults/{key}"), + } + for key in controls + } + for key in CONTROL_KEYS: + if key in normalized: + controls[key] = normalized[key] + sources.pop(key, None) + controls["rc"] = _enum(controls.get("rc"), "rc", RATE_CONTROLS) + rc = controls["rc"] + if "cq" in raw: + if "rc" in raw and rc != "vbr": + raise RecipeError("cq requires vbr rate control") + conflicting = sorted(set(raw) & {"bitrate", "maxbitrate"}) + if conflicting: + raise RecipeError(f"cq conflicts with {conflicting}") + if rc == "constqp": + for key in ("bitrate", "maxbitrate", "vbvbufsize", "cq"): + if key not in raw: + controls.pop(key, None) + sources.pop(key, None) + else: + if "constqp" not in raw: + controls.pop("constqp", None) + sources.pop("constqp", None) + if "cq" in raw: + if "rc" not in raw: + controls["rc"] = "vbr" + rc = "vbr" + sources["rc"] = { + "source": "derived", + "derived_from": ["/intent/cq"], + "rule": "cq selects VBR when rc is omitted", + } + for key in ("bitrate", "maxbitrate"): + if key not in raw: + controls.pop(key, None) + sources.pop(key, None) + if "bitrate" in raw and rc != "constqp" and "cq" not in controls: + # Validate the caller value before using it to derive VBV defaults. A + # quoted JSON number is an ordinary authoring error and must follow the + # recipe error contract instead of escaping as a Python ``TypeError``. + bitrate = _integer(controls["bitrate"], "bitrate", 1, UINT32_MAX) + controls["bitrate"] = bitrate + if "maxbitrate" not in raw: + controls["maxbitrate"] = bitrate + sources["maxbitrate"] = { + "source": "derived", + "derived_from": ["/intent/bitrate"], + "rule": "maxbitrate follows caller bitrate when omitted", + } + if "vbvbufsize" not in raw: + seconds = 0.1 if normalized["use_case"] == "conferencing" else 0.5 + if normalized["use_case"] not in {"conferencing", "live_streaming"}: + seconds = 1.0 + controls["vbvbufsize"] = max(1, int(bitrate * seconds)) + sources["vbvbufsize"] = { + "source": "derived", + "derived_from": ["/intent/bitrate", "/intent/use_case"], + "rule": f"VBV defaults to {seconds:g} seconds of caller bitrate", + } + controls["preset"] = _enum(controls.get("preset"), "preset", PRESETS) + controls["tuning_info"] = _enum(controls.get("tuning_info"), "tuning_info", TUNINGS) + controls["multipass"] = _enum(controls.get("multipass"), "multipass", MULTIPASS) + for key, minimum, maximum in ( + ("gop", 1, INT_MAX), + ("bf", 0, INT_MAX), + ("lookahead", 0, 31), + ("aq", 1, 15), + ("cq", 0, 63 if normalized["codec"] == "av1" else 51), + ("bitrate", 1, UINT32_MAX), + ("maxbitrate", 1, UINT32_MAX), + ("vbvbufsize", 1, UINT32_MAX), + ): + if key in controls: + controls[key] = _integer(controls[key], key, minimum, maximum) + if "temporalaq" in controls and not _enabled(controls["temporalaq"]): + raise RecipeError("temporalaq must be true or integer 1") + if "constqp" in controls: + controls["constqp"] = _validate_constqp(controls["constqp"], normalized["codec"]) + if "profile" in controls: + controls["profile"] = _enum(controls["profile"], "profile", PROFILES) + if controls["profile"] not in CODEC_PROFILES[normalized["codec"]]: + raise RecipeError( + f"profile {controls['profile']} is invalid for codec {normalized['codec']}" + ) + _validate_explicit_profile_format( + normalized["codec"], controls["profile"], normalized["format"] + ) + if ( + normalized["codec"] == "h264" + and controls.get("profile") == "baseline" + and controls["bf"] > 0 + ): + raise RecipeError( + f"H.264 Baseline profile requires bf=0; resolved bf={controls['bf']}" + ) + if controls["gop"] < controls["bf"] + 1: + raise RecipeError("gop must be at least bf + 1") + if controls["lookahead"] > 31 - controls["bf"]: + raise RecipeError("lookahead must be at most 31 - bf") + if controls["tuning_info"] == "ultra_low_latency" and (controls["bf"] or controls["lookahead"]): + raise RecipeError("ultra_low_latency requires bf=0 and lookahead=0") + rc = controls["rc"] + if rc == "constqp": + if "constqp" not in controls: + raise RecipeError("constqp rate control requires constqp") + conflicting = sorted(set(controls) & {"bitrate", "maxbitrate", "vbvbufsize", "cq"}) + if conflicting: + raise RecipeError(f"constqp conflicts with {conflicting}") + elif "constqp" in controls: + raise RecipeError("constqp values require constqp rate control") + if rc in {"cbr", "vbr"} and "cq" not in controls and "bitrate" not in controls: + raise RecipeError(f"{rc} requires bitrate") + if rc == "cbr" and controls.get("maxbitrate", controls.get("bitrate")) != controls.get( + "bitrate" + ): + raise RecipeError("CBR maxbitrate must equal bitrate") + if ( + rc == "vbr" + and "bitrate" in controls + and controls.get("maxbitrate", controls["bitrate"]) < controls["bitrate"] + ): + raise RecipeError("VBR maxbitrate must be at least bitrate") + if normalized["codec"] == "h264" and controls["tuning_info"] == "lossless": + if rc != "constqp": + raise RecipeError("H.264 lossless requires constqp rate control") + if controls.get("constqp") not in {"0", "0,0,0"}: + raise RecipeError("H.264 lossless requires every constqp component to be zero") + if controls.get("profile") not in {None, "high444"}: + raise RecipeError("H.264 lossless requires an omitted or high444 profile") + for key, source in sources.items(): + if key in controls: + defaults.append({"path": f"/encoder_intent/{key}", "value": controls[key], **source}) + return controls + + +def _loss(path: str, value: Any, sample: str, reason: str) -> dict[str, Any]: + return {"path": path, "value": value, "sample": sample, "reason": reason} + + +def _projections( + intent: dict[str, Any], sample_catalog: dict[str, Any] +) -> tuple[dict[str, Any], dict[str, list[dict[str, Any]]]]: + native_losses: list[dict[str, Any]] = [] + pynvc_losses: list[dict[str, Any]] = [] + native_options = [ + "-s", + f"{intent['width']}x{intent['height']}", + "-if", + NATIVE_FORMATS[intent["format"]], + "-gpu", + str(intent["gpu"]), + "-codec", + intent["codec"], + ] + for key, option in NATIVE_OPTIONS: + if key not in intent: + continue + value = intent[key] + if key == "tuning_info": + value = NATIVE_TUNINGS[value] + native_options.extend([option, str(value).lower()]) + if _enabled(intent.get("temporalaq")): + native_options.append("-temporalaq") + pynvc_config = {key: intent[key] for key in PY_CONFIG_KEYS if key in intent} + pynvc_config["gpu_id"] = intent["gpu"] + if "preset" in pynvc_config: + pynvc_config["preset"] = pynvc_config["preset"].upper() + if pynvc_config.get("tuning_info") == "ultra_high_quality": + if intent["codec"] in {"hevc", "av1"}: + pynvc_config["tuning_info"] = "uhq" + else: + pynvc_config.pop("tuning_info") + pynvc_losses.append( + _loss( + "/encoder_intent/tuning_info", + intent["tuning_info"], + "pynvc-basic-encode", + "public PyNvVideoCodec 2.1 UHQ is restricted to HEVC and AV1", + ) + ) + if "profile" in intent: + pynvc_losses.append( + _loss( + "/encoder_intent/profile", + intent["profile"], + "pynvc-basic-encode", + "the released PyNvVideoCodec 2.1 encode parser has no profile option", + ) + ) + projections = { + "native": { + "status": "exact" if not native_losses else "unrepresentable", + "sample": dict(sample_catalog["native"]), + "cli_options": native_options, + "frame_limit": { + "mode": "exact_input" if "frame_count" in intent else "all_input", + "frames": intent.get("frame_count"), + }, + }, + "pynvc": { + "status": "exact" if not pynvc_losses else "unrepresentable", + "sample": { + key: sample_catalog["pynvc"][key] for key in ("sample_id", "sample", "version") + }, + "official_sample_baseline": sample_catalog["pynvc"]["official_sample_baseline"], + "config": pynvc_config, + "arguments": { + "size": f"{intent['width']}x{intent['height']}", + "format": intent["format"], + "gpu": intent["gpu"], + "codec": intent["codec"], + "frame_count": intent.get("frame_count"), + }, + }, + } + return projections, {"native": native_losses, "pynvc": pynvc_losses} + + +def _rationale(catalog_rationale: dict[str, str], intent: dict[str, Any]) -> dict[str, str]: + """Render request-bound rationale instead of retaining stale catalog prose.""" + result = { + key: value + for key, value in catalog_rationale.items() + if key in intent + } + duration = intent["gop"] / intent["fps"] + result["gop"] = ( + f"{intent['gop']} frames at {intent['fps']} fps spans approximately " + f"{duration:.3f} seconds; validate random-access requirements" + ) + result["bf"] = ( + "disables B-frame reordering" + if intent["bf"] == 0 + else f"allows {intent['bf']} B frames; validate reorder latency" + ) + result["lookahead"] = ( + "disables lookahead queueing" + if intent["lookahead"] == 0 + else f"uses a {intent['lookahead']}-frame lookahead; validate latency and memory" + ) + result["preset"] = ( + f"uses the resolved {intent['preset']} candidate; validate operation and" + " measure throughput" + ) + result["tuning_info"] = ( + f"uses the resolved {intent['tuning_info']} tuning path; validate it on the " + "selected surface" + ) + result["rc"] = ( + f"uses resolved {intent['rc']} rate control; validate its exact bitrate or " + "constant-quality behavior" + ) + result["multipass"] = ( + f"uses resolved multipass={intent['multipass']}; measure its throughput impact" + ) + if "aq" in intent: + result["aq"] = f"uses spatial AQ strength {intent['aq']}" + if "vbvbufsize" in intent: + result["vbvbufsize"] = ( + f"uses resolved VBV size {intent['vbvbufsize']} bits; validate buffering " + "against the latency budget" + ) + return result + + +def _assumptions(catalog_assumptions: list[str], intent: dict[str, Any]) -> list[str]: + """Keep catalog context only when it still matches resolved controls.""" + result = [] + for value in catalog_assumptions: + lowered = value.lower() + if "b frame" in lowered or "b-frame" in lowered: + continue + if "lookahead consumes" in lowered and intent["lookahead"] == 0: + continue + if "strict cbr" in lowered and intent["rc"] != "cbr": + continue + result.append(value) + if intent["bf"] > 0: + result.append(f"{intent['bf']} B frames are acceptable only after latency validation") + return result + + +def resolve_recipe(raw: Any) -> dict[str, Any]: + """Build the one supported deterministic schema-2 recipe.""" + catalog, catalog_sha = _load_catalog() + normalized, defaults = _normalize(raw, catalog) + item = catalog["recipes"][normalized["use_case"]] + controls = _controls(raw, normalized, item, defaults) + intent = { + key: normalized[key] + for key in ( + "codec", + "gpu", + "width", + "height", + "format", + "fps", + "output_requirement", + "frame_count", + ) + if key in normalized + } + intent.update(controls) + projections, losses = _projections(intent, item["projections"]) + provided = sorted(raw) + body = { + "intent": normalized, + "encoder_intent": intent, + "projections": projections, + "projection_losses": losses, + "catalog": { + "path": CATALOG_REFERENCE, + "sha256": catalog_sha, + "version": catalog["version"], + "use_case": normalized["use_case"], + }, + "defaults": { + "provided_intent_keys": provided, + "entries": sorted(defaults, key=lambda item: item["path"]), + }, + "assumptions": _assumptions(item["assumptions"], intent), + "rationale": _rationale(item["rationale"], intent), + } + digests = { + "intent_sha256": canonical_sha256(intent), + "native_projection_sha256": canonical_sha256( + {"projection": projections["native"], "losses": losses["native"]} + ), + "pynvc_projection_sha256": canonical_sha256( + {"projection": projections["pynvc"], "losses": losses["pynvc"]} + ), + "contract_sha256": canonical_sha256(body), + } + return { + "schema_version": SCHEMA_VERSION, + "kind": KIND, + "status": ( + "candidate" + if any(item["status"] == "exact" for item in projections.values()) + else "blocked" + ), + **body, + "digests": digests, + } + + +def validate_recipe(recipe: Any) -> None: + """Replay deterministic derivation, raising on legacy, hybrid, or drift.""" + if not isinstance(recipe, dict): + raise RecipeError("recipe must be a JSON object") + if recipe.get("schema_version") != SCHEMA_VERSION or recipe.get("kind") != KIND: + raise RecipeError(f"recipe identity must be schema {SCHEMA_VERSION} kind {KIND}") + if set(recipe) & {"surface", "pynvc_config", "official_sample_baseline"}: + raise RecipeError( + "legacy recipe fields are prohibited; migration adapters are not supported" + ) + try: + defaults = recipe.get("defaults", {}) + if not isinstance(defaults, dict): + raise RecipeError("recipe defaults must be a JSON object") + provided = defaults.get("provided_intent_keys") + resolved = recipe.get("intent") + if ( + not isinstance(provided, list) + or not all(isinstance(key, str) for key in provided) + or provided != sorted(provided) + or len(provided) != len(set(provided)) + or not isinstance(resolved, dict) + ): + raise RecipeError("recipe defaults/provided keys are invalid") + raw = {} + for key in provided: + source = "frame_count" if key == "frames" else key + if source not in resolved: + raise RecipeError(f"provided key {key!r} has no resolved value") + raw[key] = resolved[source] + expected = resolve_recipe(raw) + except (OSError, RecipeError) as exc: + raise RecipeError(str(exc)) from exc + if recipe != expected: + raise RecipeError("recipe differs from deterministic intent/catalog replay") + + +def _unknown( + reasons: list[str], *, authenticated_dry_run_deferred: bool = True +) -> dict[str, Any]: + return { + "classification": "unknown", + "authenticated_dry_run_deferred": authenticated_dry_run_deferred, + "reasons": reasons, + } + + +def _setup_skill_path() -> Path: + """Return the setup marker in the lexical installed-skill catalog.""" + return Path(__file__).absolute().parents[3] / "jetson-video-setup" / "SKILL.md" + + +def _setup_dependency(*, surface: str) -> dict[str, Any]: + """Describe whether the canonical setup sibling can resolve live readiness.""" + if surface not in {"native", "pynvc"}: + raise RecipeError("setup dependency surface must be native or pynvc") + candidate = _setup_skill_path() + try: + details = candidate.lstat() + resolved_root = candidate.parent.resolve(strict=True) + resolved = candidate.resolve(strict=True) + except OSError: + installed = False + else: + installed = bool( + stat.S_ISREG(details.st_mode) + and not stat.S_ISLNK(details.st_mode) + and resolved_root.is_dir() + and resolved == resolved_root / "SKILL.md" + ) + action = ( + f"use jetson-video-setup to configure or repair the {surface} surface, " + "then retry this live compatibility check" + if installed + else f"install jetson-video-setup as a sibling skill, use it to configure " + f"or repair the {surface} surface, then retry this live compatibility check" + ) + return { + "skill": "jetson-video-setup", + "installed": installed, + "needed_for": f"{surface} live-compatibility prerequisites", + "next_action": action, + } + + +def _setup_remediation( + result: dict[str, Any], *, surface: str, reason: str +) -> dict[str, Any]: + result["remediation"] = { + "route": "jetson-video-setup", + "surface": surface, + "mutation_performed": False, + "reason": reason, + "dependency": _setup_dependency(surface=surface), + } + return result + + +def _dependency_record_reasons( + record: Any, label: str, exact: dict[str, Any] +) -> tuple[list[str], list[str]]: + """Classify one schema-1.2 package entry into negatives and missing evidence.""" + if not isinstance(record, dict): + return [], [f"{label} installation evidence is absent or malformed"] + unsupported: list[str] = [] + unknown: list[str] = [] + status = record.get("status") + if status is None: + unknown.append(f"{label} installation status is unavailable") + elif isinstance(status, str) and status in {"missing", "not_installed", "absent"}: + unsupported.append(f"{label} is explicitly not installed") + elif status != "installed": + unknown.append(f"{label} installation status is {status!r}") + satisfied = record.get("requirement_satisfied") + if satisfied is False: + unsupported.append(f"{label} requirement is explicitly unsatisfied") + elif satisfied is not True: + unknown.append(f"{label} requirement evidence is unavailable") + for key, expected in exact.items(): + value = record.get(key) + if value is None: + unknown.append(f"{label} {key} evidence is unavailable") + elif value != expected: + unsupported.append(f"{label} {key}={value!r} does not match required {expected!r}") + return unsupported, unknown + + +def _dependency_reasons( + environment: dict[str, Any], buffer_mode: str +) -> tuple[list[str], list[str]]: + """Separate explicit dependency negatives from missing or malformed evidence. + + Schema 1.2 carries Python readiness as ``installation.python.packages`` — + ``{status, version, requirement_satisfied}`` per entry, plus ``cuda_build``, + ``cuda_available``, and ``sample_readiness`` for torch. + """ + node: Any = environment + for key, absent in ( + ("installation", "Python installation evidence is absent or malformed"), + ("python", "Python environment evidence is absent or malformed"), + ("packages", "Python package evidence is absent or malformed"), + ): + node = node.get(key) if isinstance(node, dict) else None + if not isinstance(node, dict): + return [], [absent] + required = list(PYNVC_DEPENDENCIES) + if buffer_mode == "gpu": + required.append(TORCH_GPU_DEPENDENCY) + unsupported: list[str] = [] + unknown: list[str] = [] + for name, label, exact in required: + negatives, missing = _dependency_record_reasons(node.get(name), label, exact) + unsupported.extend(negatives) + unknown.extend(missing) + return unsupported, unknown + + +def _capability_report(report: Any) -> dict[str, Any] | None: + """Accept the capability-owned encoder report, gated by its artifact identity. + + Artifact identity is pinned in ``CAPABILITY_REPORT_IDENTITY`` and read here + and only here, so adopting a future identity stays a one-line change. + """ + if not isinstance(report, dict): + return None + if any(report.get(key) != value for key, value in CAPABILITY_REPORT_IDENTITY.items()): + return None + return report + + +def _encoder_authority( + environment: dict[str, Any], capability_report: Any +) -> tuple[dict[str, Any] | None, str]: + """Select the encoder capability authority and its producer's operation spelling. + + ``--capability-report`` is optional and additive. When one is supplied it is + the authority for every encoder fact and must additionally prove it was + produced from this exact environment artifact. When none is supplied the + established schema-1.2 ``capabilities`` block of the environment is the + authority; that is the baseline path, so an absent report is never a failure. + Either way the evidence is still an API query and is classified as such. + """ + if capability_report is None: + caps = environment.get("capabilities") + return (caps if isinstance(caps, dict) else None), ENVIRONMENT_OPERATION_STATUS + return _capability_report(capability_report), REPORT_OPERATION_STATUS + + +def _text(value: Any) -> bool: + return isinstance(value, str) and bool(value) and value == value.strip() + + +def _absolute(value: Any) -> bool: + return _text(value) and Path(str(value)).is_absolute() + + +def _environment_verdict( # pylint: disable=too-many-locals,too-many-return-statements + environment: Any, *, gpu: int, surface: str +) -> tuple[str, str] | None: + """Validate the envelope and classify only explicit selected-surface negatives.""" + if ( + not isinstance(environment, dict) + or environment.get("schema_version") != ENVIRONMENT_SCHEMA_VERSION + or environment.get("kind") != ENVIRONMENT_KIND + or environment.get("mode") != "live" + ): + return "unknown", ( + "environment is not an exact live schema-" + f"{ENVIRONMENT_SCHEMA_VERSION} {ENVIRONMENT_KIND}" + ) + selected = environment.get("selected_gpu") + if isinstance(selected, bool) or selected != gpu: + return "unknown", "recipe GPU does not match environment selected_gpu" + requested_runtime = environment.get("requested_runtime") + if requested_runtime not in {surface, "both"}: + return "unknown", ( + f"environment requested_runtime does not include the selected {surface} surface" + ) + if surface == "native": + installation = environment.get("installation") + native = installation.get("native_sdk") if isinstance(installation, dict) else None + package = native.get("package") if isinstance(native, dict) else None + installed = isinstance(native, dict) and ( + native.get("installed") is True or native.get("status") == "installed" + ) + if ( + not installed + or not isinstance(package, dict) + or package.get("name") != "nvidia-video-codec-sdk" + or package.get("status") != "installed" + ): + reason = "environment does not report an installed native Video Codec SDK surface" + explicit = ( + ( + isinstance(native, dict) + and ( + native.get("installed") is False + or native.get("status") in {"missing", "partial", "not_ready"} + ) + ) + or ( + isinstance(package, dict) + and package.get("status") == "missing" + ) + ) + else: + readiness = environment.get("readiness") + layers = readiness.get("layers") if isinstance(readiness, dict) else None + native_readiness = layers.get("native_sdk") if isinstance(layers, dict) else None + if ( + not isinstance(native_readiness, dict) + or native_readiness.get("installation") != "ready" + ): + reason = ( + "environment does not report native Video Codec SDK " + "installation readiness" + ) + explicit = isinstance(native_readiness, dict) and ( + native_readiness.get("installation") in {"missing", "not_ready"} + ) + else: + return None + return ("unsupported" if explicit else "unknown"), reason + pynvc = environment.get("pynvc") + if not isinstance(pynvc, dict) or pynvc.get("imported") is not True: + classification = ( + "unsupported" + if isinstance(pynvc, dict) and pynvc.get("imported") is False + else "unknown" + ) + return classification, "environment does not report an imported PyNvVideoCodec surface" + identity = pynvc.get("identity") + if not isinstance(identity, dict) or identity.get("status") != "verified": + classification = ( + "unsupported" + if isinstance(identity, dict) + and identity.get("status") in {"missing", "not_ready", "unverified"} + else "unknown" + ) + return classification, "environment does not report a verified PyNvVideoCodec identity" + return None + + +def _declared_pynvc_identity(environment: dict[str, Any]) -> dict[str, Any]: + """Read the schema-1.2 ``pynvc.identity`` block, or an empty required subset.""" + surface = environment.get("pynvc") + identity = surface.get("identity") if isinstance(surface, dict) else None + return identity if isinstance(identity, dict) else {} + + +def _declared_distribution_version(declared: dict[str, Any]) -> Any: + """Read the manifest's PyNvVideoCodec distribution version by either spelling. + + The established 1.2 producer carries it as ``identity.distribution``; the + flattened ``identity.version`` is an additive refinement. Both mean the + installed distribution version, so either satisfies the comparison. + """ + if "version" in declared: + return declared.get("version") + distribution = declared.get("distribution") + if isinstance(distribution, dict): + return distribution.get("version") + return distribution + + +def _identity_subset_matches(recorded: Any, observed: Any) -> bool: + """Compare one authenticated identity by required fields, permitting additions.""" + if not isinstance(recorded, dict) or not isinstance(observed, dict): + return False + return all(recorded.get(key) == observed.get(key) for key in ("path", "size_bytes", "sha256")) + + +def _pynvc_identity_matches(binding: dict[str, Any], declared: Any) -> bool: + """Bind the report's authenticated live import to the environment Py identity. + + ``declared.version`` and ``declared.interpreter_identity`` are additive + refinements, not requirements: the established producer spells the + distribution version ``distribution`` and publishes no resolved interpreter + identity. Either version spelling satisfies the comparison, and a resolved + interpreter identity is compared only when the manifest carries one — but + when carried it is held exactly, so a refinement can only tighten. + """ + identity = binding.get("pynvc_identity") + if not isinstance(identity, dict) or not isinstance(declared, dict): + return False + expected = ( + (identity.get("status"), "verified"), + (identity.get("authentication"), "local_wheel_record_ownership"), + (identity.get("interpreter"), declared.get("interpreter")), + (identity.get("sys_prefix"), declared.get("sys_prefix")), + (identity.get("version"), _declared_distribution_version(declared)), + ) + observed_interpreter = declared.get("interpreter_identity") + if any(recorded != observed for recorded, observed in expected) or ( + observed_interpreter is not None + and ( + not _identity_subset_matches( + identity.get("interpreter_identity"), observed_interpreter + ) + or not _identity_subset_matches( + binding.get("interpreter_identity"), observed_interpreter + ) + ) + ): + return False + module, observed_module = identity.get("module"), declared.get("module") + extension, observed_extension = identity.get("extension"), declared.get("extension") + if not isinstance(module, dict) or not isinstance(observed_module, dict): + return False + if ( + module.get("path") != observed_module.get("path") + or module.get("version") != observed_module.get("version") + ): + return False + if not isinstance(extension, dict) or not isinstance(observed_extension, dict): + return False + return ( + extension.get("path") == observed_extension.get("path") + and extension.get("sha256") == observed_extension.get("sha256") + and _absolute(extension.get("loaded_path")) + and _absolute(observed_extension.get("loaded_path")) + and os.path.realpath(extension["loaded_path"]) + == os.path.realpath(observed_extension["loaded_path"]) + ) + + +def _binding_reason( + report: dict[str, Any], environment: dict[str, Any], environment_path: Path | None +) -> str | None: + """Reject a report that was not produced from this exact environment artifact. + + The raw and canonical digests have different domains: ``sha256`` covers the + exact file bytes, while ``canonical_sha256`` covers the parsed JSON value. + The remaining fields bind schema/GPU and the authenticated live Py import. + """ + if environment_path is None: + raise RecipeError( + "capability report binding requires the exact environment artifact path" + ) + binding = report.get("environment") + if not isinstance(binding, dict) or any( + key not in binding for key in CAPABILITY_BINDING_FIELDS + ): + return "capability report does not record the environment it was produced from" + try: + raw_environment = environment_path.read_bytes() + except OSError as exc: + raise RecipeError(f"cannot read the environment artifact bytes: {exc}") from exc + try: + parsed_environment = _strict_json_text(raw_environment.decode("utf-8")) + except (UnicodeError, json.JSONDecodeError, RecipeError): + return "capability report is bound to a malformed environment artifact" + if parsed_environment != environment: + return "capability report is bound to a different environment artifact" + declared = _declared_pynvc_identity(environment) + expected: list[tuple[Any, Any]] = [ + (binding["sha256"], hashlib.sha256(raw_environment).hexdigest()), + (binding["canonical_sha256"], canonical_sha256(parsed_environment)), + (binding["size_bytes"], len(raw_environment)), + (binding["path"], os.path.realpath(environment_path)), + (binding["kind"], environment.get("kind")), + (binding["schema_version"], environment.get("schema_version")), + (binding["mode"], environment.get("mode")), + (binding["selected_gpu"], environment.get("selected_gpu")), + (binding["interpreter"], declared.get("interpreter")), + ] + if any(recorded != observed for recorded, observed in expected) or not ( + _pynvc_identity_matches(binding, declared) + ): + return "capability report is bound to a different environment artifact" + return None + + +def _documentation_verdict(record: dict[str, Any]) -> str | None: + """Read the published documentation cross-check verdict for one codec.""" + crosscheck = record.get("documentation_crosscheck") + if not isinstance(crosscheck, dict): + return None + support = crosscheck.get("support") + return support.lower() if isinstance(support, str) else None + + +def _rc_supported(mask: Any, rc: str) -> bool | None: + if isinstance(mask, bool) or not isinstance(mask, int) or mask <= 0: + return None + if rc == "constqp": + return True + return bool(mask & (0x1 if rc == "vbr" else 0x2)) + + +def _api_version(value: Any) -> tuple[int, int] | None: + if not isinstance(value, str): + return None + fields = value.split(".") + if len(fields) != 2 or any(not field.isdigit() for field in fields): + return None + return int(fields[0]), int(fields[1]) + + +def _selected_gpu_name(environment: dict[str, Any], gpu: int) -> str | None: + """Name the environment's selected GPU only when it is the exact requested one.""" + selected = environment.get("selected_gpu") + if isinstance(selected, bool) or selected != gpu: + return None + inventory = environment.get("nvidia_smi") + records = inventory.get("gpus") if isinstance(inventory, dict) else None + if not isinstance(records, list): + return None + for record in records: + if isinstance(record, dict) and record.get("index") == gpu: + name = record.get("name") + return name if isinstance(name, str) and name else None + return None + + +def _uhq_reason( + caps: dict[str, Any], environment: dict[str, Any], gpu: int +) -> dict[str, Any] | None: + """Gate ultra-high-quality tuning on linked-NVENC API and generation evidence.""" + linked_api = caps.get("linked_nvenc_api", {}) + linked_version = _api_version( + linked_api.get("value") if isinstance(linked_api, dict) else None + ) + if ( + not isinstance(linked_api, dict) + or linked_api.get("status") != "observed" + or linked_version is None + ): + return _unknown(["PyNvVideoCodec UHQ linked-NVENC API evidence is unavailable"]) + if linked_version < (12, 2): + return { + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": ["public PyNvVideoCodec 2.1 UHQ requires linked NVENC API 12.2 or newer"], + } + gpu_name = _selected_gpu_name(environment, gpu) + if gpu_name is None or "thor" not in gpu_name.lower(): + return _unknown( + [ + "PyNvVideoCodec UHQ requires authenticated Turing-or-newer generation " + "evidence; only Thor is established by this skill" + ] + ) + return None + + +def _bound_reasons( + values: dict[str, Any], intent: dict[str, Any] +) -> tuple[list[str], list[str]]: + """Compare requested geometry, rate control, and B frames against queried bounds.""" + unsupported: list[str] = [] + unknown: list[str] = [] + for key, requested, relation in ( + ("width_min", intent["width"], "min"), + ("width_max", intent["width"], "max"), + ("height_min", intent["height"], "min"), + ("height_max", intent["height"], "max"), + ): + bound = values.get(key) + if isinstance(bound, bool) or not isinstance(bound, int) or bound < 0: + unknown.append(f"{key} is unavailable") + elif relation == "min" and requested < bound: + unsupported.append(f"{key}={bound} excludes requested {requested}") + elif relation == "max" and requested > bound: + unsupported.append(f"{key}={bound} excludes requested {requested}") + rc = _rc_supported(values.get("supported_ratecontrol_modes"), intent["rc"]) + if rc is False: + unsupported.append(f"rate control {intent['rc']} is not reported") + elif rc is None: + unknown.append("supported_ratecontrol_modes is unavailable") + if intent["bf"] > 0: + maximum = values.get("num_max_bframes") + if isinstance(maximum, bool) or not isinstance(maximum, int) or maximum < 0: + unknown.append("num_max_bframes is unavailable") + elif intent["bf"] > maximum: + unsupported.append(f"bf={intent['bf']} exceeds num_max_bframes={maximum}") + return unsupported, unknown + + +def _gate_reasons( + values: dict[str, Any], unavailable: Any, intent: dict[str, Any] +) -> tuple[list[str], list[str]]: + """Require an exact positive capability flag for each requested optional control.""" + unsupported: list[str] = [] + unknown: list[str] = [] + gates = ( + (intent["lookahead"] > 0, "support_lookahead", "lookahead"), + (_enabled(intent.get("temporalaq")), "support_temporal_aq", "temporal AQ"), + ("vbvbufsize" in intent, "support_custom_vbv_buf_size", "custom VBV"), + (intent["format"] in TEN_BIT_FORMATS, "support_10bit_encode", "10-bit"), + (intent["format"] in YUV422_FORMATS, "support_yuv422_encode", "4:2:2"), + (intent["format"] in YUV444_FORMATS, "support_yuv444_encode", "4:4:4"), + (intent["tuning_info"] == "lossless", "support_lossless_encode", "lossless"), + ) + for required, key, label in gates: + if not required: + continue + value = values.get(key) + if value == 0: + unsupported.append(f"{label} requires {key}=1 but query reports 0") + elif value != 1: + reason = ( + unavailable.get(key, {}).get("reason") if isinstance(unavailable, dict) else None + ) + unknown.append(f"{label} capability is unavailable" + (f": {reason}" if reason else "")) + return unsupported, unknown + + +def _field_reasons( + record: dict[str, Any], intent: dict[str, Any] +) -> tuple[list[str], list[str]]: + """Compare requested controls against the exact queried capability fields.""" + values = record["values"] + bounded, bounded_unknown = _bound_reasons(values, intent) + gated, gated_unknown = _gate_reasons(values, record.get("unavailable_fields", {}), intent) + return bounded + gated, bounded_unknown + gated_unknown + + +def _codec_record_verdict(record: Any, operation_status: str) -> dict[str, Any] | None: + """Apply the evidence precedence to one codec record before reading its fields. + + Precedence is API query, then official operation, then documentation. An + applicable documentation "No" is the final verdict for any codec and outranks + a positive query, but the disagreement is never dropped. A positive query is + ``capability_reported`` with ``supported: null`` and an ``operation_status`` + that explicitly denies an operation; it is never product support. Returning + ``None`` means the record survived every gate and its fields may be read. + """ + if not isinstance(record, dict): + return _unknown(["exact codec capability record is absent"]) + if record.get("status") == "unsupported" or record.get("supported") is False: + return { + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": ["exact codec query reports unsupported"], + } + if _documentation_verdict(record) == "no": + reasons = [ + "NVIDIA documentation reports this codec unsupported; documentation is the " + "final product-support authority" + ] + if record.get("status") == "capability_reported": + reasons.append( + "recorded discrepancy: GetEncoderCaps reports capability_reported while " + "documentation reports No" + ) + return { + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": reasons, + } + if not ( + record.get("status") == "capability_reported" + and record.get("supported") is None + and record.get("session_status") == "opened_by_GetEncoderCaps" + and record.get("operation_status") == operation_status + and isinstance(record.get("values"), dict) + ): + return _unknown(["exact codec GetEncoderCaps record is incomplete or malformed"]) + return None + + +def validate_live_capabilities( # pylint: disable=too-many-branches,too-many-locals + # pylint: disable=too-many-return-statements,too-many-arguments + recipe: dict[str, Any], + environment: Any | None, + capability_report: Any, + surface: str, + *, + environment_path: Path | None, + buffer_mode: str = "gpu", +) -> dict[str, Any]: + """Classify one projection from exact live environment and capability facts. + + ``environment`` is the schema-1.2 setup artifact, read for ``kind``, + ``schema_version``, ``mode``, ``selected_gpu``, ``nvidia_smi``, ``pynvc`` and + ``installation.python.packages``. Encoder facts come from the environment's + own ``capabilities`` block, or from ``capability_report`` when one is + supplied. Codec/API support and installation readiness are different + questions and keep different sources. + + ``capability_report`` is optional and additive on every surface; pass ``None`` + when it was not supplied. A supplied report is accepted only when it is bound + to this exact environment artifact, which is why ``environment_path`` + accompanies the parsed environment. + """ + validate_recipe(recipe) + if surface not in {"native", "pynvc"}: + raise RecipeError("surface must be native or pynvc") + buffer_mode = _enum(buffer_mode, "buffer_mode", {"cpu", "gpu"}) + if surface == "native" and capability_report is not None: + raise RecipeError( + "--capability-report is a PyNvVideoCodec-only refinement and is invalid " + "for the native surface" + ) + if recipe["projections"][surface]["status"] != "exact": + return { + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": ["recipe has blocking projection losses"], + } + intent = recipe["encoder_intent"] + if environment is None: + reason = ( + f"live {surface} environment evidence is absent; recipe planning and " + "validation remain complete but live compatibility is unresolved" + ) + if capability_report is not None: + _capability_report(capability_report) + reason += ( + "; a capability report alone does not authenticate selected-surface " + "readiness or selected-GPU identity" + ) + return _setup_remediation( + _unknown([reason], authenticated_dry_run_deferred=False), + surface=surface, + reason="produce fresh selected-surface readiness evidence", + ) + environment_verdict = _environment_verdict( + environment, gpu=intent["gpu"], surface=surface + ) + if environment_verdict is not None: + classification, environment_reason = environment_verdict + result = _unknown( + [environment_reason], authenticated_dry_run_deferred=False + ) + if classification == "unsupported": + result["classification"] = classification + result["reasons"][0] += ( + "; this is explicit live selected-surface SDK readiness, not a " + "codec/product-support verdict" + ) + return _setup_remediation( + result, + surface=surface, + reason="refresh or repair selected-surface readiness evidence", + ) + if surface == "native": + return _unknown( + [ + "native AppEncCuda has no equivalent authenticated GetEncoderCaps " + "authority; require its authenticated dry run" + ] + ) + dependency_failures, dependency_unknown = _dependency_reasons(environment, buffer_mode) + if dependency_failures: + return _setup_remediation({ + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": dependency_failures + dependency_unknown, + }, surface=surface, reason="repair selected-surface prerequisites") + if dependency_unknown: + return _setup_remediation( + _unknown(dependency_unknown), + surface=surface, + reason="probe or repair selected-surface prerequisites", + ) + caps, operation_status = _encoder_authority(environment, capability_report) + expected_classification = { + "evidence_source_type": "api_query_helper", + "official_sample": False, + "api_symbol": "PyNvVideoCodec.GetEncoderCaps", + } + evidence = caps.get("evidence_classification") if caps is not None else None + if ( + caps is None + or caps.get("authority") != "pynvc" + or caps.get("gpu") != intent["gpu"] + or not isinstance(evidence, dict) + or evidence.get("encode") != expected_classification + ): + return _unknown( + ["capability authority, exact GPU, or GetEncoderCaps classification is invalid"] + ) + if capability_report is not None: + binding_reason = _binding_reason(caps, environment, environment_path) + if binding_reason is not None: + return _unknown([binding_reason]) + if intent["tuning_info"] == "ultra_high_quality": + uhq_reason = _uhq_reason(caps, environment, intent["gpu"]) + if uhq_reason is not None: + return uhq_reason + encode = caps.get("encode") + record = encode.get(intent["codec"]) if isinstance(encode, dict) else None + verdict = _codec_record_verdict(record, operation_status) + if verdict is not None: + return verdict + unsupported, unknown = _field_reasons(record, intent) + if unsupported: + return { + "classification": "unsupported", + "authenticated_dry_run_deferred": False, + "reasons": unsupported + unknown, + } + if unknown: + return _unknown(unknown) + return { + "classification": "compatible", + "authenticated_dry_run_deferred": True, + "reasons": [ + "GetEncoderCaps reports all requested fields; an authenticated official " + "encode is still required" + ], + } + + +def _write_exclusive(path: Path, payload: dict[str, Any]) -> None: + destination = Path(os.path.abspath(os.path.expanduser(str(path)))) + if destination.parent.resolve(strict=True) != destination.parent or os.path.lexists( + destination + ): + raise RecipeError("output parent must be resolved and output must be fresh") + descriptor = os.open(destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + with os.fdopen(descriptor, "w", encoding="utf-8") as stream: + json.dump(payload, stream, indent=2, sort_keys=True, allow_nan=False) + stream.write("\n") + + +def main(argv: list[str] | None = None) -> int: + """Run the thin recipe CLI.""" + parser = argparse.ArgumentParser(description=__doc__) + subparsers = parser.add_subparsers(dest="command", required=True) + plan = subparsers.add_parser("plan", help="resolve intent into the sole schema-2 recipe") + plan.add_argument("--intent", type=Path, required=True) + plan.add_argument("--output", type=Path, required=True) + validate = subparsers.add_parser("validate", help="replay and validate one recipe") + validate.add_argument("--recipe", type=Path, required=True) + live = subparsers.add_parser( + "check-live", help="classify one recipe projection from live facts" + ) + live.add_argument("--recipe", type=Path, required=True) + live.add_argument( + "--environment", + type=Path, + help=( + "optional live schema-1.2 setup environment; omission preserves the " + "offline recipe and returns honest unresolved live compatibility" + ), + ) + live.add_argument( + "--capability-report", + type=Path, + help=( + "optional capability-owned PyNvVideoCodec GetEncoderCaps report bound to" + " --environment; when omitted the environment's own capabilities block is" + " the encoder authority" + ), + ) + live.add_argument("--surface", choices=("native", "pynvc"), required=True) + live.add_argument("--buffer-mode", choices=("cpu", "gpu"), default="gpu") + live.add_argument("--output", type=Path) + args = parser.parse_args(argv) + try: + if args.command == "plan": + result = resolve_recipe(strict_json_load(args.intent)) + _write_exclusive(args.output, result) + elif args.command == "validate": + validate_recipe(strict_json_load(args.recipe)) + result = {"valid": True, "errors": []} + else: + result = validate_live_capabilities( + strict_json_load(args.recipe), + ( + strict_json_load(args.environment) + if args.environment is not None + else None + ), + ( + strict_json_load(args.capability_report) + if args.capability_report is not None + else None + ), + args.surface, + environment_path=args.environment, + buffer_mode=args.buffer_mode, + ) + if args.output: + _write_exclusive(args.output, result) + print(json.dumps(result, indent=2, sort_keys=True, allow_nan=False)) + classification = result.get("classification") + if not result.get("valid", True) or classification == "unsupported": + return 2 + if classification == "unknown" and ( + getattr(args, "surface", None) != "native" + or result.get("authenticated_dry_run_deferred") is not True + ): + return 2 + return 0 + except (OSError, RecipeError) as exc: + print(json.dumps({"status": "error", "error": str(exc)}, sort_keys=True)) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-recipe/skill-card.md b/skills/jetson-video-recipe/skill-card.md new file mode 100644 index 00000000..a1a84d96 --- /dev/null +++ b/skills/jetson-video-recipe/skill-card.md @@ -0,0 +1,82 @@ +## Description:
+Use when turning a Jetson encoder use case into one validated surface-neutral recipe with native and PyNvVideoCodec projections for codec, preset, rate control, bitrate, latency, format, and profile.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0
+## Use Case:
+Developers and engineers converting encoder workload intent into validated NVENC and PyNvVideoCodec recipes for NVIDIA Jetson devices.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [No]
+**Credential Type(s):** [None]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [Recipes Workflow](references/recipes-workflow.md)
+- [Recipes Knobs and Constraints](references/recipes-knobs-and-constraints.md)
+ + +## Skill Output:
+**Output Type(s):** [JSON, Shell commands]
+**Output Format:** [JSON recipe artifacts with shell command examples]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+3 evaluation tasks (3 positive) run in isolated sandbox pods.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Whether the skill is safe to use, checking for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the final answer is correct against the reference answer.
+- Discoverability: Whether the right skill was found and activated when needed.
+- Effectiveness: Whether the skill helps the agent complete the user's goal and expected workflow.
+- Efficiency: Whether the skill avoids wasted tool or skill usage through routing quality and productive tool use.
+ +Underlying evaluation signals used in this run:
+- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 64% → 95% (+31 points) | 53% → 95% (+42 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 100% → 100% (±0 points) | 87% → 100% (+13 points) | +| Discoverability | 33% → 100% (+67 points) | 33% → 79% (+46 points) | +| Effectiveness | 74% → 94% (+21 points) | 45% → 100% (+55 points) | +| Efficiency | 15% → 83% (+67 points) | 1% → 95% (+94 points) | + +## Skill Version(s):
+e61c045 (source: git SHA, committed 2026-08-10)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/jetson-video-recipe/skill.oms.sig b/skills/jetson-video-recipe/skill.oms.sig new file mode 100644 index 00000000..dfdf4156 --- /dev/null +++ b/skills/jetson-video-recipe/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiamV0c29uLXZpZGVvLXJlY2lwZSIsCiAgICAgICJkaWdlc3QiOiB7CiAgICAgICAgInNoYTI1NiI6ICI4YmY0ZTgwYjBiZDZjNmZhZTgxYjhmMWQ1MDI0OTQ0YjAwMGU0MWEwODNhNTY5YTcxNmNlNzFhMzg5NWM0YzhmIgogICAgICB9CiAgICB9CiAgXSwKICAicHJlZGljYXRlVHlwZSI6ICJodHRwczovL21vZGVsX3NpZ25pbmcvc2lnbmF0dXJlL3YxLjAiLAogICJwcmVkaWNhdGUiOiB7CiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiMmJmMDFkMTA2N2JkNjBlM2UyN2IwNWQ2MGZkMmNkMmI3ZjEyOGU5ZGM4ZTVmNDRkYWNmZGYwZDlkNmVkMzVmYyIsCiAgICAgICAgIm5hbWUiOiAiQkVOQ0hNQVJLLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiZTEyNGIxZmViZThhZjgyNWI4NzgwNjZkZWZjNDk1NTJmZGRiZGNmMjI4OGMyYTI3NzAyN2Y2ZjUwYjc2YzU2OSIsCiAgICAgICAgIm5hbWUiOiAiU0tJTEwubWQiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIwNmQ0ZjA5ZTBmNTNlNGRjZTliOWY3NGM2ZDQ1MjM1YmFjYzBmNWQ0MjA2ZmQ4NWYwNmFhMDgyOWI3YzM0MzA2IiwKICAgICAgICAibmFtZSI6ICJldmFscy9ldmFscy5qc29uIgogICAgICB9LAogICAgICB7CiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJkaWdlc3QiOiAiNTYzNmMzN2RjYzljNzFjM2NkYWUwYjViZmU4NzUxMDc0ZDhkNTNkYTQzMjNjN2NhYzYyNTg1ZjI1ZWI3MTBlMSIsCiAgICAgICAgIm5hbWUiOiAicmVmZXJlbmNlcy9yZWNpcGVzLWtub2JzLWFuZC1jb25zdHJhaW50cy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImU3ZGUyMTc4YWYxMjQwOTk2NzgzMjU5Mzg1MzhiZGJjZmRhMTcwZDM3YTA5Zjk3ZjFlNjFiZTdhZGVjNDRjYWQiLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvcmVjaXBlcy13b3JrZmxvdy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogImRiMzQ2NzNmZDYyYWRmMGNiZTQ5YjllNTc0NWU0ZTdjYzEyOGY0ZDg4ZDEyNDFhZjJhNDE3NzIzNzg0ZjM2NTUiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVjaXBlcy9kYXRhL2VuY29kZXItaW50ZW50LWNhdGFsb2cuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IiwKICAgICAgICAiZGlnZXN0IjogIjJkZGZmNGQ0ZTQ4NTZmYjFmN2FiODcxMzMwMmU5MzVjYTU1OWIxMmJlZjFkNWE3ZmFiMmI5YjRiZjM4ODkzMjciLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvcmVjaXBlcy9yZWNpcGVfbW9kZWwucHkiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgImRpZ2VzdCI6ICIxYTlmZGFiODcwZWEyNzFmMmE4ODY5NWUxMWE3ODA1NDA1ZWJlYTBiOTc5YWM2YTc3MjJiMjkzM2RjMDA5OGJhIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIgogICAgICB9CiAgICBdLAogICAgInNlcmlhbGl6YXRpb24iOiB7CiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaWdub3JlX3BhdGhzIjogWwogICAgICAgICIuZ2l0aHViIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGlnbm9yZSIKICAgICAgXSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZQogICAgfQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGQCMGUqaeDFJRBvlSnEEJOtj69OsiU7H5N+dqj9pjv59ZaL+uase2kMvYmy7pKUw8bmHAIwcLCB6kYtKfnPze8juyCCZOUpkCrqIQvtKqm4+hCGtmzaoGXrxTutQMA5567znbMg","keyid":""}]}} \ No newline at end of file diff --git a/skills/jetson-video-setup/BENCHMARK.md b/skills/jetson-video-setup/BENCHMARK.md new file mode 100644 index 00000000..167ad935 --- /dev/null +++ b/skills/jetson-video-setup/BENCHMARK.md @@ -0,0 +1,99 @@ +# Skill Benchmark: jetson-video-setup + +> ✅ **Overall verdict: PASS — Recommended for publication** + +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. + +## Evaluation Metadata + +- Skill: `jetson-video-setup` +- Evaluation date: 2026-08-10 +- Evaluator version: `1.1.2` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 4 evaluation tasks (4 positive) +- Dataset digest: `sha256:da01efb8a143276d497b34e33588f31b5cb1d0f128bfa23256ee67a894ccd664` (skill-evaluator-dataset-snapshot/1) +- Attempts per task: 1 +- Environment: `k8s-sandbox` +- Tier 3 evidence: required for publication + +Each task attempt ran in its own isolated sandbox pod. + +## What This Report Answers + +The three-tier evaluation checks whether the skill: + +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. + +## Results at a Glance + +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 36% → 81% (+45 points) | 24% → 83% (+58 points) | +| Security | 50% → 100% (+50 points) | 0% → 100% (+100 points) | +| Correctness | 35% → 75% (+40 points) | 50% → 95% (+45 points) | +| Discoverability | 38% → 98% (+61 points) | 33% → 84% (+52 points) | +| Effectiveness | 24% → 46% (+22 points) | 13% → 46% (+32 points) | +| Efficiency | 32% → 85% (+53 points) | 26% → 90% (+63 points) | + +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. + +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. + +## Tier Status + +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 2 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 4 task(s) | + +## Findings and Observations + +
+Show detailed findings and successful checks + +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/jetson-video-setup/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/jetson-video-setup/SKILL.md`) + +
+ +## Scoring Methodology + +
+Show dimension definitions, source signals, and thresholds + +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | + +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. + +Signals present in this run: + +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. + +
+ +## Freshness + +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/jetson-video-setup/SKILL.md b/skills/jetson-video-setup/SKILL.md new file mode 100644 index 00000000..04b09662 --- /dev/null +++ b/skills/jetson-video-setup/SKILL.md @@ -0,0 +1,181 @@ +--- +name: jetson-video-setup +license: "Apache-2.0" +description: >- + Use when installing, repairing, probing, or verifying native NVIDIA Video + Codec SDK or PyNvVideoCodec on Jetson with official encode-to-decode samples, + including registered-environment recovery. +metadata: + author: "Vinit Bansal " + tags: [jetson, video-codec-sdk, pynvvideocodec, setup, nvenc, nvdec] + languages: [python] + data-classification: public +--- + +# Jetson Video Setup + +## Purpose + +Probe, install, and independently verify the native NVIDIA Video Codec SDK and +PyNvVideoCodec surfaces on a live Jetson. Setup owns installation readiness, +not codec-support verdicts, recipes, benchmarks, or application pipelines. + +## Read before acting + +- Read [setup-workflow.md](references/setup-workflow.md) for surface selection + and the probe → plan → apply → verify order. +- Read [setup-install.md](references/setup-install.md) before any APT, venv, or + pip mutation. +- Read [setup-output-contract.md](references/setup-output-contract.md) before + consuming or reporting an artifact. + +## Select the surface + +Before step 1 or any probe, resolve the requested surface. "Video Codec SDK", +"VC SDK", "native SDK", or `nvidia-video-codec-sdk` selects native; +"PyNvVideoCodec", "PyNv", "PySDK", or Python selects PyNvVideoCodec. Match a +named product before considering the bare phrase: "Video Codec SDK" is the +native product name even though it contains the words "video SDK". A genuinely +bare "video SDK" setup, install, operation, readiness, or report-only request +is ambiguous: ask only whether the user wants native Video Codec SDK, +PyNvVideoCodec, or both, then stop before probing, acting, or describing future +probes, checks, installation steps, or report contents. Report-only intent +alone does not select a surface or authorize broadening to both. + +Select both only when explicitly requested, and reuse the selection for the +rest of the request. One narrow exception applies to a consumer skill's +`auto` selection gate: that consumer may invoke setup's read-only probe with +`--runtime both` solely to evaluate both candidates. This does not select both +for installation, verification, execution, or the final report. + +Keep the selected surfaces independent. A native failure must not suppress an +actionable Python surface, and a Python failure must not suppress native. +Report aggregate `both` readiness only after both verification chains pass. + +## Compose requested sibling stages + +Setup's probe, plan, install, and verification workflow requires no sibling +skill. When a complex request also asks for product capability, recipe, +performance, or pipeline work, add only the corresponding +`jetson-video-capability`, `jetson-video-recipe`, `jetson-video-benchmark`, or +`jetson-video-pipeline` stage. Check the agent's installed skill catalog first. +If the sibling is present, read its `SKILL.md` and invoke its documented public +entry point; pass artifacts as data and never import sibling code. If it is +absent, preserve completed setup results and say, using the actual names: `I +can run , but it requires , which is not installed. Install + and retry this stage.` Never acquire a sibling for an unrequested +stage. + +## Workflow + +1. Confirm execution is on the Jetson. On a non-Jetson host, produce guidance + only and make no live readiness claim. +2. Probe the selected surface with `probe_nvcodec.py --runtime + native|pynvc|both --output ...`. Use `both` only when the request explicitly + selects both surfaces or for the narrow read-only consumer `auto` candidate + check above. The probe is read-only. Reauthenticate a saved artifact with + the same CLI's `--reauthenticate` action. +3. For PyNvVideoCodec, use the fixed validated-venv registry or an exact + user-supplied interpreter. Never scan for or guess a venv. If the user says + PyNvVideoCodec is already installed but supplies no exact path and the + registry is not ready, ask for the path before provisioning anything. A + missing registered interpreter makes that registry not ready; a registered + interpreter that cannot be launched blocks the selected Py surface. Never + scan or fall back to another environment. +4. Generate an install plan with `plan_install.py`, then run + `plan_install.py validate PLAN`. A report-only request stops after the + probe; `plan-only` never authorizes mutation. Use `setup-install` intent only + for an explicit install/setup request; that request authorizes only the + complete unchanged batches in the reviewed plan. +5. Execute only literal commands from the reviewed `setup-install` plan. + Invoke every published `argv` verbatim as the current user, including steps marked `privilege: "root"`; never prefix `sudo`, because `plan_install.py` owns the authorized internal `sudo -n` escalation for APT operations. + `plan_install.py` owns APT refresh, preview, and apply actions; + `lock_pip_reports.py` owns clean-venv creation and the locked pip apply. + APT execution regenerates the canonical plan and rechecks live candidate, + origin, source, and simulation evidence before mutation. +6. Re-probe the completed surface. Run `verify_native.py` for native or + `verify_pynvc_sample.py` for Python. Each setup proof uses the installed + release's official samples to encode one 640×360 NV12 frame to H.264, then + independently decode that fresh bitstream. Native, and Python under + `--profile full-samples`, decode to exactly 345,600 bytes. The default + Python profile `pynvc-smoke` decodes one bounded frame with + `advanced/decode_perf.py`, which writes no raw output, so it proves frame + production only. A consumer that genuinely needs Torch — Python + encode-benchmark, pipeline, or the full raw-decode proof — is blocked under + `pynvc-smoke`; say so and name the remedy: provision a `full-samples` venv + explicitly with `plan_install.py --profile full-samples`. Exit zero alone is + never proof: require the profile's exact positive markers and counts. Only a + passing verifier may promote the selected surface from probe `partial` to a + final ready verdict. +7. After a ready Python verification, publish the fixed registry only with + `verify_pynvc_sample.py --register-current --output READY_REPORT`. +8. Report the detected Jetson Linux release, product versions, independent + surface verdicts, blockers, and artifact identities. + +Use `--fresh-setup` only when the user explicitly requests a new setup or +reinstall. It never authorizes removing working base packages. A fresh Python +setup also requires a unique, previously absent `--venv`. That `--venv` must be +an absolute path under a durable location, for example +`/home/ubuntu/.venvs/nvcodec-fresh`; never place it in the current working +directory or any transient run, session, or evidence tree, because the registry +you publish outlives that directory. Relative `--output` names resolve against +the working directory, so write setup reports somewhere equally durable. + +## Direct setup scripts + +Run every public CLI under `python3 -I` and inspect its `--help` before building +arguments. + +| File | Public responsibility | +|---|---| +| `scripts/setup/probe_nvcodec.py` | Emit or reauthenticate the read-only live `nvcodec-environment` schema 1.2 artifact. | +| `scripts/setup/plan_install.py` | Plan and validate selected components; execute only its own reviewed APT refresh/preview/apply actions. | +| `scripts/setup/lock_pip_reports.py` | Create a new venv and materialize/apply the authenticated pip lock. | +| `scripts/setup/verify_native.py` | Build package-owned `AppEncCuda`/`AppDec` and verify the fixed native encode→decode smoke. | +| `scripts/setup/verify_pynvc_sample.py` | Authenticate and run wheel-owned Python encode/decode samples, emit the readiness artifact, and authenticate the validated-venv registry chain. | +| `scripts/setup/setup_contract.py` | Private common mechanics for these setup CLIs: strict JSON, bounded commands, and public-APT binding; never invoke it as a CLI. | + +There is no setup dispatcher. Invoke these five public CLIs directly. Setup +must not import Python code from another skill, and another skill must not +import setup's private implementation. + +## Readiness and scope + +- Inventory or import presence is not operational proof. +- `operation_verified` requires both official operations, their positive + markers, and a fresh nonempty bitstream. Native and Python `full-samples` + additionally require the exact decoded frame count and raw-output size; + Python `pynvc-smoke` instead requires its two exact one-frame production + markers and claims no raw decoded artifact. +- Exit zero or output-file creation alone is insufficient. +- Setup emits only bounded baseline Py API-query observations and raw native + sample summaries as supporting readiness evidence. It does not emit the + complete decoder tuple matrix or a product-support verdict; use + `jetson-video-capability` for those questions. +- Use `jetson-video-recipe`, `jetson-video-benchmark`, and + `jetson-video-pipeline` for configuration, measurement, and handoff work. +- A local probe proves only the detected stack and minimum release gate; it + does not prove release currency or the newest release compatible with this + target. Call a release `latest` or `newest compatible` only when successfully + retrieved current official NVIDIA documentation, recorded with URL and + retrieval date, establishes both release currency and compatibility with the + authenticated target identity. Otherwise report newest-compatible as + `unknown` and point to the official compatibility documentation; local APT + state, a failed source, or either fact alone is insufficient. +- For a quality-only request such as PSNR or SSIM, state that setup does not + provide it and that a separately authorized quality workflow is required, + then stop; do not install, invoke, name, recommend, or offer to set + up an external quality tool. + +## Safety + +- Accept native SDK/CUDA packages only from the configured, + signature-authenticated stock public NVIDIA Jetson source + (`repo.download.nvidia.com/jetson/common` or `/som`, exact `rNN.N/main`). + Base prerequisites may use another already configured, + signature-authenticated APT origin. Bind every candidate to its exact source + record and never add or change a source or key. +- Keep credentials out of argv, logs, artifacts, stdout, and stderr. +- Preserve exact plan, package, interpreter, artifact, and source identities. +- Use fresh output/work/build paths. Never overwrite evidence or reuse it after + a reflash, driver/package change, or venv replacement. diff --git a/skills/jetson-video-setup/evals/evals.json b/skills/jetson-video-setup/evals/evals.json new file mode 100644 index 00000000..f4310de0 --- /dev/null +++ b/skills/jetson-video-setup/evals/evals.json @@ -0,0 +1,53 @@ +{ + "skill_name": "jetson-video-setup", + "evals": [ + { + "id": "setup-native-only-smoke", + "prompt": "Install only the native Video Codec SDK and run an H.264 encode and decode smoke test.", + "expected_output": "Native-only setup verified with the official native encoder and decoder.", + "assertions": [ + "Selects only the native SDK and does not install PyNvVideoCodec", + "Probes and validates the target before changes, then uses AppEncCuda followed by AppDec", + "Reports ready only after the decoder produces a frame from the generated bitstream" + ], + "expected_skill": "jetson-video-setup", + "expected_workflow": "setup" + }, + { + "id": "setup-pynvc-only-smoke", + "prompt": "Install only PyNvVideoCodec in a new isolated Python environment and run an H.264 encode and decode smoke test.", + "expected_output": "Python-only setup verified with wheel-owned encoder and decoder samples.", + "assertions": [ + "Selects only PyNvVideoCodec and does not install the native SDK developer package", + "Creates a new isolated Python environment, reports its exact interpreter, and uses that interpreter", + "Reports ready only after the wheel-owned encoder output is independently decoded" + ], + "expected_skill": "jetson-video-setup", + "expected_workflow": "setup" + }, + { + "id": "setup-reuse-pynvc-smoke", + "prompt": "PyNvVideoCodec is already installed. Reuse it and run an H.264 encode and decode smoke test.", + "expected_output": "The existing validated Python environment is reused without reinstallation and passes the smoke test.", + "assertions": [ + "Uses the registered environment or asks for the exact interpreter path instead of scanning for environments", + "Does not reinstall packages or create another environment when the existing one is valid", + "Uses the wheel-owned encoder followed by an independent decoder before reporting ready" + ], + "expected_skill": "jetson-video-setup", + "expected_workflow": "setup" + }, + { + "id": "setup-ambiguous-surface-clarification", + "prompt": "Is my environment ready to run video SDK work right now? Probe the target and report readiness. (Report-only.)", + "expected_output": "A clarification asking whether to inspect native Video Codec SDK, PyNvVideoCodec, or both, with no target probe or action before the user chooses.", + "assertions": [ + "Treats the bare phrase video SDK as ambiguous even though the request is report-only", + "Asks whether the user wants native Video Codec SDK, PyNvVideoCodec, or both", + "Stops before target inspection, probing, planning, installation, building, or codec operations" + ], + "expected_skill": "jetson-video-setup", + "expected_workflow": "setup" + } + ] +} diff --git a/skills/jetson-video-setup/references/setup-install.md b/skills/jetson-video-setup/references/setup-install.md new file mode 100644 index 00000000..3d6b9411 --- /dev/null +++ b/skills/jetson-video-setup/references/setup-install.md @@ -0,0 +1,244 @@ +# Public installation workflow + +## Contents + +- [Product boundaries](#product-boundaries) +- [Authorization and planning](#authorization-and-planning) +- [Stock public APT policy](#stock-public-apt-policy) +- [Native Video Codec SDK](#native-video-codec-sdk) +- [PyNvVideoCodec](#pynvvideocodec) +- [Fresh setup](#fresh-setup) +- [Official operation proof](#official-operation-proof) + +## Product boundaries + +| Surface | Acquisition | Setup proof | +|---|---|---| +| Native Video Codec SDK | `nvidia-video-codec-sdk` 13.0.x from the configured Jetson APT release | Package-owned `AppEncCuda` → package-owned `AppDec` | +| PyNvVideoCodec | `PyNvVideoCodec==2.1.0` and its dependencies in an isolated venv | Wheel-owned `samples/basic/encode.py` → `samples/advanced/decode_perf.py` by default, or `samples/advanced/decode.py` with `--profile full-samples` | + +These are independent products. A PyNvVideoCodec request must not install the +native developer package, and a native request must not create or modify a +Python environment. CUDA, driver, compiler, SDK, and Python package versions +are separate facts; do not require their version strings to match. + +Record the detected Jetson Linux release and apply the documented minimum +release gate, but do not impose an exact JetPack/L4T revision. Setup readiness +comes from the applicable installed-artifact operation proof, not an API +capability query. + +## Authorization and planning + +Always probe before planning. Plan only the selected `native-sdk`, `pynvc`, or +explicitly both components. + +- `--request-intent plan-only` creates a review artifact and authorizes no + protected command. +- `--request-intent setup-install` preauthorizes only unchanged commands in the + reviewed plan for the selected components. +- `--fresh-setup` requires the complete fresh action for each nonblocked + selected surface. + +The plan groups protected commands into these deterministic batches: + +| Batch | Purpose | +|---|---| +| `apt-metadata-refresh` | One authorized metadata refresh, followed by mandatory re-probe/re-plan | +| `native-sdk-install` | Candidate-pinned native package preview and apply | +| `pynvc-bootstrap` | Required candidate-pinned CUDA/Python bootstrap and new-venv creation | +| `pynvc-resolution` | Isolated, non-applying pip resolver reports | +| `pynvc-install` | Authenticated lock materialization and exact locked apply | + +Review the plan with `plan_install.py validate`. Execute only the exact +`refresh`, `preview`, or `apply` argv emitted by that plan. Before APT +mutation, `plan_install.py` requires the literal command to occur in the +reviewed sibling `nvcodec-install-plan.json`, reauthenticates the bound inputs, +regenerates the canonical plan, and requires its digest to match. It then +rechecks candidate/source policy and repeats the safe simulation immediately +before apply. + +Any change in component, setup mode, command, package, candidate, origin, +source file, dependency set, version, or archive hash invalidates the reviewed +authorization. Re-probe, re-plan, show the difference, and obtain authorization +for the changed scope. + +## Stock public APT policy + +Accept the native SDK or selected CUDA build candidate only when all of these are +true: + +- its configured source is exactly + `https://repo.download.nvidia.com/jetson/common` or + `https://repo.download.nvidia.com/jetson/som`; +- its suite/component is exactly `rNN.N/main`; +- APT signature enforcement is active; +- no `trusted`, insecure, weak, downgrade-to-insecure, or unauthenticated + bypass is enabled; +- the candidate origin is bound to the exact configured-source record and + source-file SHA-256 captured by the probe; +- the candidate remains unchanged at apply time. + +Do not accept an internal mirror, HTTP NVIDIA source, wrong suite/component, +unsigned candidate, trust bypass, or unbound origin. Base Ubuntu prerequisites +such as `pkg-config`, `python3-venv`, and `python3-dev` may use another already +configured APT origin only when its package-signature chain, exact source +record, and source-file SHA-256 are bound by the probe. Never add, replace, or +edit a repository or signing key. + +When local metadata has no candidate but configured sources exist, the plan +may emit one `plan_install.py refresh` action. After it succeeds, discard the +old environment and plan, probe again, and build a new plan bound to the +refresh receipt. Do not refresh repeatedly or carry a pre-refresh candidate +forward. + +Every APT apply uses an exact `package=version`, `--no-remove`, noninteractive +form after a matching safe simulation. A removal, downgrade, candidate drift, +origin drift, or unsafe simulation blocks the apply. + +## Native Video Codec SDK + +Use only a stable public 13.0.x +`nvidia-video-codec-sdk=`. A normal installed 13.0.x +surface can take the verify-only route. An explicitly fresh native setup emits +the candidate-pinned simulation/apply pair with `--reinstall`; it does not +uninstall the working package. + +The final native probe must show: + +- the package name, installed status, and version; +- canonical SDK root; +- complete AppDec `pkg-config` prerequisites for `libavcodec`, + `libavformat`, `libavutil`, and `libswresample`; +- installed CUDA root/version; +- authenticated `cmake`, C++ compiler, `nvcc`, `pkg-config`, and CMake + generator records. + +If a prerequisite is absent, install it only when the selected plan contains +its exact authenticated APT candidate. Never infer missing development +packages from an incomplete or timed-out query. + +`verify_native.py` reauthenticates package version/ownership and +`dpkg --verify`, builds only `AppEncCuda` and `AppDec` in a fresh user-owned +directory, and rejects unresolved or stub CUDA/NVENC/NVDEC linkage. + +## PyNvVideoCodec + +### Reuse + +Prefer the exact validated registry environment. A ready +`existing-isolated-environment` route performs no venv, resolver, pip, CUDA, +or native-SDK install; it runs fresh official-sample verification only. + +If the registry is absent/stale and the user identifies an existing +installation, require the exact lexical interpreter or venv. Probe and verify +that candidate directly. Never search the filesystem, guess among venvs, or +reinstall a valid candidate. + +### New environment + +Create only a previously absent target with +`lock_pip_reports.py create-venv`. An existing directory, partial target, or +symlink is blocked and is never deleted, repaired, or reused in place. + +The Python route does not depend on `nvidia-video-codec-sdk`. It may install +only its own authenticated prerequisites: + +- the version-matched public `cuda-minimal-build-13-N` and + `libcurand-dev-13-N` packages when the top-level CUDA build probe is absent. + Derive `13-N` from the authenticated `cuda-toolkit` release candidate and + authenticate both selected candidates independently. PyCUDA 2026.1 enables + CURAND in its default source build; do not install the full `cuda-toolkit` + meta-package merely to compile it; +- Python venv support or development headers when their top-level probe fields + are missing; +- exact `PyNvVideoCodec==2.1.0`; +- exact `pycuda==2026.1`; +- `numpy>=1.24` and the narrow support set recorded in the plan; +- CUDA-enabled `torch==2.9.1+cu130` under `--profile full-samples` only. The + default `pynvc-smoke` profile installs no Torch and never reaches the cu130 + extra index, because neither official sample it runs imports Torch. + +A minimal CUDA build plus CURAND bootstrap is nonterminal: after applying it, discard the plan, +probe again, and re-plan before creating the venv. Scope `PATH`, `CPATH`, and +`LIBRARY_PATH` to the PyCUDA build only; never expose a stub directory through +runtime loader variables. + +Run the plan's isolated pip dry-runs and retain their JSON reports. +`lock_pip_reports.py materialize-apply` then: + +1. verifies each required root and merges duplicate records only when package, + version, filename, type, and SHA-256 agree; +2. accepts only credential-free HTTPS artifacts from its fixed host allowlist; +3. pins each URL and SHA-256, with exact policy for the one permitted PyCUDA + source archive; +4. writes a fresh lock and manifest; +5. reauthenticates the lock, resolver environment, artifact origins, redirect + destinations, and source hash before mutation; +6. installs hash-checked wheels before the PyCUDA source stage, with no + dependency resolution; +7. verifies installed versions and requires `pip check` to pass; +8. emits `pynvc_wheel` evidence for the locked PyNvVideoCodec wheel. + +The compact apply contract emits persisted wheel/source stage reports, +installed versions, `pip_check`, and `pynvc_wheel` evidence. + +After the final Python probe, run the official operation proof with +`verify_pynvc_sample.py --register-current`. It publishes the registry only +after safely writing a ready verification result. A failed proof or +publication attempt preserves the previous registry. + +## Fresh setup + +Use fresh mode only when the user explicitly requests another complete setup +attempt. + +- Native: reinstall the exact live 13.0.x candidate without removing working + base packages, then re-probe and verify. +- Python: require an explicit unique absent `--venv` at an absolute durable + path such as `/home/ubuntu/.venvs/nvcodec-fresh` — never inside the working + directory or a transient run/evidence tree, since the published registry + outlives it — perform the complete resolver/locked apply, then re-probe, + verify, and register. Write the setup reports to an equally durable directory. +- Both: keep each plan and proof independent. Use deterministic native-then- + Python execution when both are actionable. A blocked peer does not suppress + the other. +- Shared APT refresh: refresh once, then discard and regenerate all selected + package plans. + +Never reuse prior plans, resolver reports, locks, build directories, work +directories, or operation artifacts for a fresh attempt. + +## Official operation proof + +Both proofs use one generated 640×360 8-bit NV12 frame (345,600 bytes), encode +H.264 with the installed release's official sample, then independently decode +that fresh bitstream with the corresponding official decoder. The Python decoder +is selected by profile: the default `pynvc-smoke` runs +`samples/advanced/decode_perf.py`, and `full-samples` runs +`samples/advanced/decode.py`. + +Require: + +- authenticated installed sample ownership; +- zero child exit codes and no recognized failure marker; +- the exact positive marker for one encoded frame before decoder launch; +- a fresh, regular, nonempty bitstream with SHA-256; +- `software_fallback=false`. + +Native, and Python under `full-samples`, additionally require the exact positive +marker for one decoded frame and a fresh 345,600-byte decoded NV12 output with +SHA-256. `pynvc-smoke` instead requires both anchored `decode_perf.py` markers, +each exactly once, and rejects any worker error, warning, or traceback; it +writes no raw output, so it claims no decoded artifact and proves frame +production only. + +Output-file creation alone is not proof, and for `pynvc-smoke` a zero exit code +alone is not proof either, because the sample swallows worker exceptions. Do not +require the decoded pixels to match the input byte-for-byte because H.264 is +lossy. Setup does not use an external software codec or quality-measurement +tool. + +## Primary sources + +- https://docs.nvidia.com/video-technologies/video-codec-sdk/13.0/read-me/index.html +- https://docs.nvidia.com/video-technologies/pynvvideocodec/read-me/index.html diff --git a/skills/jetson-video-setup/references/setup-output-contract.md b/skills/jetson-video-setup/references/setup-output-contract.md new file mode 100644 index 00000000..e6923039 --- /dev/null +++ b/skills/jetson-video-setup/references/setup-output-contract.md @@ -0,0 +1,334 @@ +# Setup output contracts + +## Contents + +- [Common rules](#common-rules) +- [`nvcodec-environment` 1.2](#nvcodec-environment-12) +- [Install-plan and APT artifacts](#install-plan-and-apt-artifacts) +- [Python environment and lock artifacts](#python-environment-and-lock-artifacts) +- [Native verification](#native-verification) +- [PyNvVideoCodec verification and registry](#pynvvideocodec-verification-and-registry) + +## Common rules + +Public setup CLIs emit one strict JSON document to stdout. Persisted outputs are +fresh regular files created without replacing an existing path. Preserve each +artifact's path, size, and SHA-256 at consumer boundaries. + +Normalized exits are: + +- `0`: the requested positive operation completed; a completed read-only + inventory also exits 0 even when a surface is absent; +- `2`: a completed blocked, negative, unknown, unsupported, or operation-failed + result; +- `3`: malformed/unsafe input, a refused write, or an internal contract error. + +An `argparse` usage error exits `2` and writes no result artifact. A child +process's original return code remains in its stage evidence. + +## `nvcodec-environment` 1.2 + +`probe_nvcodec.py` owns the only setup environment artifact: + +```json +{ + "schema_version": "1.2", + "kind": "nvcodec-environment", + "generated_at": "...", + "mode": "live", + "requested_runtime": "native", + "platform": {}, + "selected_gpu": 0, + "nvidia_smi": {}, + "libraries": {}, + "driver_nvenc_api": {}, + "pynvc": {}, + "installation": {}, + "capabilities": {}, + "warnings": [], + "readiness": {}, + "command_evidence": [] +} +``` + +These sixteen keys are the closed top-level set; an unknown top-level key is a +validation error. Nine are required: `schema_version`, `kind`, `platform`, +`selected_gpu`, `requested_runtime`, `pynvc`, `installation`, `capabilities`, +and `readiness`. `requested_runtime` is a **string**, exactly `native`, +`pynvc`, or `both` — never a list. + +The artifact does not serialize `target`, `requested_surfaces`, `surfaces`, +`cuda`, `python`, or `apt` as top-level keys. `surfaces` may exist only as an +internal normalized representation inside the producer; no consumer reads it +from the artifact. Native surface facts are published under +`installation.native_sdk`, the Python surface under `pynvc`, Python readiness +under `installation.python.packages`, and APT and CUDA observations under +`installation.apt` and `installation.cuda_toolkit`. + +`native` and `pynvc` stay independent: either may be absent when not selected, +or present and not installed, without changing its peer. + +Consumers validate their required subset and ignore additive optional keys. +They reject an unknown schema major and any non-`live` artifact. A ready Python +probe preserves its baseline API-query fields for downstream compatibility; +those fields are not operation proof or a product-support verdict. Complete +capability matrices and operational classification belong to the capability skill. + +### Required top-level identity + +| Key | Required content | +|---|---| +| `platform` | `jetson` boolean, `machine` matching `^(aarch64\|arm64)$`, and `jetson_linux.{version, release_line, compatibility.status}` evidence against the minimum release | +| `selected_gpu` | Non-negative integer; GPU identity is the matching `nvidia_smi.gpus[]` record with that `index` | +| `requested_runtime` | Exactly one of the strings `native`, `pynvc`, `both` | + +### Required `readiness` block + +`readiness.state` is `ready`, `partial`, or `not_ready`. `readiness.layers` carries one layer +per requested surface — `native_sdk` and `pynvc` — each with its own `status`, +`installation`, and `operation` values. Report the layers separately; the probe +proves no codec operation, so a satisfied surface is `partial`, never `ready`, +and there is no aggregate-both readiness claim at this level. + +### Required `installation.apt` block + +`installation.apt` records local APT observations only: + +- `apt_cache`: canonical executable path when available; +- `signature_enforced`: boolean; +- `source_files`: configured source paths; +- `sources`: each source path and SHA-256; +- `candidates`: package-name → candidate record. + +A candidate record contains `package`, `query_status`, exact `query_argv`, +`query_exit_code`, `installed`, `candidate`, `candidate_origins`, and +`public_origin`. When native is requested, the +`nvidia-video-codec-sdk` candidate record is required even if no candidate is +visible. + +An actionable `public_origin` binds the candidate to: + +- `https://repo.download.nvidia.com/jetson/common` or `/som`; +- exact `rNN.N/main`; +- `authentication=apt-signature-chain`; +- exact `configured_source_sha256` and `configured_sources` records; +- no trust bypass. + +An absent or untrusted candidate remains inventory and cannot create an APT +apply command. + +### Required `installation.cuda_toolkit` block + +`installation.cuda_toolkit` is target identity used by Python setup; it is not +borrowed from the native surface. + +- Always require `status`, either `installed` or `absent`. +- When installed, require `version`, canonical `root`, and non-empty absolute + `environment_prefixes.PATH`, `.CPATH`, and `.LIBRARY_PATH` lists. +- When absent, `version` and `root` are null and the prefix lists are empty. + +### Required `installation.python` block + +`installation.python.executable` is the interpreter that produced the artifact, +after any validated-venv delegation; it must be an absolute, executable file +and it drives `@python-from`. For a new Python environment, also require: + +- `system_executable`: the observed canonical system Python when available; +- `venv_module`: `ok` or `missing`; +- `development_headers`: `ok` or `missing`. + +These fields select authenticated bootstrap packages at plan time. They do not +authorize scanning for another interpreter. + +Python readiness is `installation.python.packages..{status, version, +requirement_satisfied}`; when Torch is present it additionally carries +`cuda_build`, `cuda_available`, and `sample_readiness`. Torch is required only +by the `full-samples` profile; under the default `pynvc-smoke` profile its +absence is expected and is not a readiness failure. + +### Required `installation.native_sdk` block + +Always require `installed` as a boolean. When true, require: + +- `package.{name,status,version}`, where name is + `nvidia-video-codec-sdk` and status is `installed`; +- canonical `sdk_root`; +- `build_prerequisites.{status,unresolved_modules}` for the AppDec modules; +- `cuda.{status,version,root}`; +- `tools.cmake`, `tools.cxx`, `tools.nvcc`, `tools.pkg_config`, and + `tools.generator`, each with `path`, `version`, and `sha256`; +- `tools.generator.name`, either `Ninja` or `Unix Makefiles`. + +### Required top-level `pynvc` block + +The Python surface is published at top level as `pynvc`. Its established fields +include `imported`, `module`, `module_file`, `module_version`, +`distribution_version`, `linked_nvenc_api`, `identity`, and `errors`. +Consumers validate their required subset and permit additive provenance fields. + +Always require `imported` as a boolean. When it is true in a `live` artifact, +require a verified `identity`: + +- `identity.status` is `verified`, and otherwise carries only `status` and + `reason`; +- `identity.interpreter` is the exact lexical interpreter and must equal + `installation.python.executable`; +- `identity.sys_prefix`, `identity.dist_info_path`; +- `identity.distribution.version` and `identity.module.version`, both exactly + `2.1.0`; +- `identity.extension` for the loaded extension, whose recorded loaded path + equals the hashed extension path. + +`module_version` must match `distribution_version`. `linked_nvenc_api` records +the linked NVENC API level. The verifier, not the probe, authenticates wheel +`RECORD` ownership of the files it executes. + +Python dependency readiness is not part of this block; it is +`installation.python.packages` as described above. + +### Reauthentication result + +`probe_nvcodec.py --reauthenticate ENVIRONMENT` emits: + +```json +{ + "kind": "nvcodec-environment-validation", + "schema_version": "1.0", + "valid": true, + "errors": [] +} +``` + +It validates required structure and rehashes the recorded native tools and +Python interpreter/extension identities. It performs no mutation. + +## Install-plan and APT artifacts + +`plan_install.py` emits `kind=nvcodec-install-plan`, +`schema_version=1.5`. Its required contract includes: + +- `generated_from`: bound environment path, size, SHA-256, kind/mode/schema, + and canonical JSON digest; +- `plan_inputs`: environment, exact selected components, request intent, + optional venv, fresh flag, and optional refresh receipt; +- independent `components.native-sdk` and/or `components.pynvc`; +- per-component `plan_status`, selected route, blockers, artifacts, exact + commands, and final validation commands; +- `preflight_commands`, sequential `execution_batches`, and intent-scoped + authorization; +- `overall_status`, `mutated=false`, and `plan_digest`. + +Component status is `installed`, `ready_to_review`, or `blocked`. A blocked +component has no command. A mixed two-surface plan stays actionable for its +nonblocked peer. `both` readiness is not a plan status. + +`plan_install.py validate PLAN` emits +`nvcodec-install-plan-validation` schema 1.0 and checks the plan document, +including its digest and authorization consistency, without mutation. + +Transaction actions accept only literal commands present in the reviewed +sibling plan. They regenerate the canonical plan from its bound inputs and +require the same digest before execution. Refresh/environment binding is +performed by the planner through `--apt-refresh-receipt`; the resulting +transaction records `authorizing_plan_digest`. Direct planless transactions +are not supported and carry no separate refresh/environment assertion flags. +APT artifacts use schema 1.0: + +- `nvcodec-apt-refresh-receipt`; +- `nvcodec-apt-simulation-receipt`, including live policy evidence, + transaction summary, and `safe_to_apply`; +- `nvcodec-apt-apply-report`, including the authorizing plan digest, reviewed + receipt identity, immediate repeated simulation, exact command evidence, + and final `safe` result. + +## Python environment and lock artifacts + +`lock_pip_reports.py create-venv` emits +`nvcodec-clean-venv-creation` schema 1.0. `status=created` proves a previously +absent target was reserved and populated without system site packages. +`target_exists` is a complete negative result; the path is not deleted or +modified. A partial failure is preserved and never described as rollback. + +`materialize-apply` produces: + +- `nvcodec-pip-lock-manifest` schema 1.0 with interpreter/environment + contract, resolver-report identities, requirements, allowed hosts, exact + artifact name/version/URL/filename/type/SHA-256 records, lock identity, and + install policy; +- fresh wheel/source pip stage reports with bounded command evidence; +- `nvcodec-pip-lock-apply` schema 1.0 with source preflight, installed + versions, `pip_check`, and `pynvc_wheel` evidence for the locked + PyNvVideoCodec wheel. + +Successful apply status is `applied_verified`. A post-mutation failure is +reported as mutation-possible or verification-failed, never clean rollback. + +## Native verification + +`verify_native.py` emits `kind=nvcodec-native-verification`, +`schema_version=1.5`. + +The artifact binds: + +- the environment identity and native surface; +- live package version, ownership, and `dpkg --verify` evidence; +- official sample build commands and binary identities; +- real CUDA/NVENC/NVDEC linkage with stub rejection; +- the generated one-frame 640×360 NV12 input; +- exact encode/decode argv, markers, exit codes, log tails, bitstream identity, + decoded-output identity, and failure reasons. + +`ready=true` and `status=operation_verified` require both official operations, +exactly one reported frame at each stage, a fresh nonempty bitstream, and an +exact 345,600-byte decoded NV12 output. A missing prerequisite is `unknown`; a +launched sample failure is `operation_failed`. + +## PyNvVideoCodec verification and registry + +`verify_pynvc_sample.py` emits +`kind=nvcodec-pynvc-sample-verification`, `schema_version=1.3`. + +The artifact binds: + +- environment path/hash/canonical digest and selected GPU; +- exact lexical interpreter, prefix, distribution/module version, and loaded + extension identity; +- wheel `RECORD` ownership for the imported module, loaded extension, official + encoder/decoder, helpers, and configuration; +- the `verification_profile` and exactly its dependency subset: NumPy and + PyCUDA readiness always, plus CUDA-enabled Torch readiness under + `full-samples`; +- the generated one-frame 640×360 NV12 input; +- exact child argv, output logs, positive markers, bitstream identity, decoded + output identity, and reasons. + +Only the known PyNvVideoCodec 2.1.0 stale `RECORD` row for its native extension +may use the narrow recorded mismatch exception; all other executed files must +match their wheel rows. + +`ready=true` and `status=operation_verified` require both official operations, +one encoded frame, and a fresh nonempty H.264 bitstream. The decode proof is the +profile's own: `full-samples` runs `advanced/decode.py` and still requires the +exact 345,600-byte decoded NV12 output, while the default `pynvc-smoke` runs +`advanced/decode_perf.py`, which writes no raw output and therefore carries no +decoded-output identity. Its proof is frame production: both anchored markers, +each exactly once, with no worker error, warning, or traceback. A zero exit code +alone is never sufficient, because the sample swallows worker exceptions. + +The fixed registry is: + +```text +$HOME/.local/state/jetson-videosdk/current-pynvc.json +``` + +Its schema is `jetson-videosdk/current-pynvc/1`. It records the exact ready +verification-artifact identity and the frozen verified Python surface. +`verify_pynvc_sample.py --register-current` is the only publisher. It writes +the `nvcodec-pynvc-sample-verification` schema 1.3 result before publication, +refuses a non-ready verification, and preserves any prior registry on failure. + +A normal PyNv probe reauthenticates the registry, bound verification artifact, +surface, and exact interpreter, then delegates to that lexical interpreter. +It never scans venv directories or falls back to another interpreter. Keep the +registry-bound verification and environment artifacts in persistent storage; +missing or changed evidence makes the registry not ready. diff --git a/skills/jetson-video-setup/references/setup-workflow.md b/skills/jetson-video-setup/references/setup-workflow.md new file mode 100644 index 00000000..c212aa36 --- /dev/null +++ b/skills/jetson-video-setup/references/setup-workflow.md @@ -0,0 +1,244 @@ +# Setup and readiness workflow + +## Contents + +- [Select scope](#select-scope) +- [Classify intent](#classify-intent) +- [Probe](#1-probe) +- [Plan and validate](#2-plan-and-validate) +- [Execute the reviewed plan](#3-execute-the-reviewed-plan) +- [Verify native](#4-verify-native) +- [Verify PyNvVideoCodec](#5-verify-pynvvideocodec) +- [Report](#6-report) + +Use this workflow on the Jetson target. The skill does not open SSH sessions, +copy itself, or treat a GPU-name inference as live evidence. + +## Select scope + +Before target inspection or the first probe, resolve the requested surface. +Named native and Python products select only their own surface. A bare “video +SDK” setup, install, operation, readiness, or report-only request is ambiguous: +ask whether the user wants native Video Codec SDK, PyNvVideoCodec, or both, +then stop before probing or acting. “Video Codec SDK” is the native product +name, not the bare ambiguous phrase. Report-only intent alone does not select a +surface or authorize broadening to both. + +Map the choice to commands as follows: + +| User scope | Probe runtime | Plan component | +|---|---|---| +| Native Video Codec SDK | `native` | `native-sdk` | +| PyNvVideoCodec/PySDK/Python | `pynvc` | `pynvc` | +| Explicitly both | `both` | Repeat both component flags | + +Use `--runtime both` only when the request explicitly selects both surfaces, +or when a consumer's `auto` gate invokes this read-only probe solely to +authenticate both candidates. That internal candidate probe does not authorize +installing, verifying, executing, or reporting both surfaces. + +Evaluate the two surfaces independently. Native Video Codec SDK 13.0.x and +PyNvVideoCodec 2.1.0 are separate products; PyNvVideoCodec does not require the +native developer package. + +## Classify intent + +- **Report-only/readiness audit:** after the surface is selected, run only its + read-only probe and reauthentication. Do not build, install, create a venv, + or launch codec operations. A bare “video SDK” request remains ambiguous: + ask which surface to inspect and stop before probing. +- **Verify usability:** run the applicable non-installing official-sample + verifier only when the user authorizes its build/work/output writes. +- **Plan-only:** emit and validate a plan, then stop. The plan marks protected + commands as requiring confirmation and cannot authorize a transaction. +- **Setup/install:** use `--request-intent setup-install`. The explicit request + authorizes only the unchanged batches in the reviewed plan. +- **Fresh setup:** additionally use `--fresh-setup`. For Python, require an + explicit unique, absent `--venv` at an absolute durable path such as + `/home/ubuntu/.venvs/nvcodec-fresh`, never in the working directory or a + transient run/evidence tree; the registry outlives that directory. Write the + `--output` reports somewhere equally durable. Preserve working base packages. + +## 1. Probe + +Run the selected live probe into a fresh, persistent attempt directory: + +```bash +python3 -I scripts/setup/probe_nvcodec.py \ + --runtime native \ + --gpu 0 \ + --output nvcodec-environment-before.json +``` + +Use `pynvc` or `both` for the corresponding explicitly selected scope. The +probe reads local target, APT, CUDA, Python, package, tool, and import state. +It does not refresh APT, contact a package index, install, build, or launch a +codec. It returns an inventory even when a requested surface is not installed. + +Reauthenticate a saved environment with the same public CLI: + +```bash +python3 -I scripts/setup/probe_nvcodec.py \ + --reauthenticate nvcodec-environment-before.json +``` + +This emits `nvcodec-environment-validation`. Do not use a separate validator. +A valid result proves the recorded structure and file identities still match; +it does not prove an encode or decode operation. + +### Exact PyNvVideoCodec interpreter + +The normal `pynvc` probe resolves the fixed registry at +`$HOME/.local/state/jetson-videosdk/current-pynvc.json` and delegates only to +its authenticated lexical interpreter. It never scans common venv locations +or falls back to system Python. + +If the registry is absent or stale and the user says PyNvVideoCodec already +exists, require the exact interpreter or venv path. Probe that lexical +interpreter explicitly as the setup candidate: + +```bash + -I scripts/setup/probe_nvcodec.py \ + --runtime pynvc \ + --gpu 0 \ + --setup-candidate \ + --output nvcodec-environment-candidate.json +``` + +Then run `--reauthenticate` through that same interpreter. Reuse it if valid. +If no path is supplied, ask for one; do not scan, guess, or immediately create +a replacement. Use `--setup-candidate` only for this exact supplied +interpreter or the interpreter created by an authorized setup plan. + +## 2. Plan and validate + +Generate a plan for only the selected component: + +```bash +python3 -I scripts/setup/plan_install.py \ + nvcodec-environment-before.json \ + --component native-sdk \ + --request-intent setup-install \ + --output nvcodec-install-plan.json + +python3 -I scripts/setup/plan_install.py \ + validate nvcodec-install-plan.json +``` + +For PyNvVideoCodec, use `--component pynvc`. For explicit `both`, repeat +`--component` in native-then-Python order. Use `--request-intent plan-only` +when no mutation is authorized. + +The `validate` action checks the plan kind/schema, digest, component states, +and authorization fields without mutation. Before an APT transaction, +`plan_install.py` additionally reloads the reviewed sibling +`nvcodec-install-plan.json`, reauthenticates its bound inputs, regenerates the +canonical plan, and requires the same digest. Never execute serialized or +hand-edited caller argv. + +If an authorized metadata refresh is required, execute only the plan's emitted +`plan_install.py refresh` command. Discard the old plan, create a fresh probe, +and regenerate it with the exact successful refresh receipt. A refresh can +change all APT candidates, so re-plan every selected package surface. + +## 3. Execute the reviewed plan + +Read [setup-install.md](setup-install.md), inspect every exact argv, and execute +the plan's batches sequentially. The public owners are: + +- `plan_install.py refresh|preview|apply` for APT transactions; +- `lock_pip_reports.py create-venv|materialize-apply` for the isolated Python + environment and locked pip application; +- `verify_pynvc_sample.py --register-current` for the operation proof and + registry publication. + +Do not reconstruct commands from prose. Preserve the reviewed plan's working +directory and artifact names. Stop on command, candidate, origin, dependency, +hash, setup-mode, or scope drift. + +APT packages are eligible only from an already configured, normally +signature-verified source at +`https://repo.download.nvidia.com/jetson/common` or `/som`, exact +`rNN.N/main`, with no trust bypass. Never add, replace, or repair a repository +or signing key. + +A blocked component carries no executable command. Continue a separately +actionable selected peer unless a shared APT refresh invalidates both plans. + +## 4. Verify native + +After native installation or validated reuse, create a fresh native-only +environment and reauthenticate it, then run: + +```bash +python3 -I scripts/setup/verify_native.py \ + --environment nvcodec-environment-after-native.json \ + --sdk-root /opt/nvidia/video-codec-sdk \ + --build-dir ./nvcodec-native-build \ + --work-dir ./nvcodec-native-smoke \ + --build --run-encode --run-decode \ + --gpu 0 \ + --output nvcodec-native-verification.json +``` + +The verifier reauthenticates package ownership, `dpkg --verify`, required +tools, AppDec prerequisites, and real CUDA/NVENC/NVDEC linkage. It builds only +package-owned `AppEncCuda` and `AppDec`. It launches `AppDec` only after the +current `AppEncCuda` run proves exactly one encoded frame, its exact output +marker, and a fresh nonempty H.264 bitstream. + +Native readiness requires one decoded frame and an exact 345,600-byte +640×360 NV12 output. H.264 is lossy, so source and decoded pixel hashes need +not match. + +## 5. Verify PyNvVideoCodec + +For a new environment, execute the plan's exact clean-venv, resolver, and +locked-apply commands. For a valid existing environment, skip all installation +steps and verify only. + +Run the final probe and verifier through the exact selected lexical +interpreter: + +```bash + -I scripts/setup/probe_nvcodec.py \ + --runtime pynvc --gpu 0 --setup-candidate \ + --output nvcodec-environment-after-python.json + + -I scripts/setup/verify_pynvc_sample.py \ + --environment nvcodec-environment-after-python.json \ + --work-dir ./nvcodec-pynvc-smoke \ + --gpu 0 \ + --register-current \ + --output nvcodec-pynvc-verification.json +``` + +The verifier authenticates the imported module, loaded extension, and every +executed official sample/helper/config file against the installed wheel. It +runs wheel-owned encode before independent decode. Readiness always requires the +exact one-frame encode marker and a fresh nonempty H.264 bitstream. Under +`--profile full-samples` it also requires `advanced/decode.py`'s exact +345,600-byte decoded NV12 output; under the default `pynvc-smoke` it requires +`advanced/decode_perf.py`'s two anchored markers, each exactly once, with no +worker error, warning, or traceback, and produces no decoded output file. + +`verify_pynvc_sample.py --register-current` is the only publisher. It first +writes the complete verification result, then publishes the registry only +when that result has `ready=true` and `status=operation_verified`. A failed +proof or publication preserves the previous registry. + +## 6. Report + +Report: + +- target and Jetson Linux release; +- selected surface(s); +- installed/candidate native package and PyNvVideoCodec versions; +- exact Python interpreter for the Python surface; +- independent native and Python inventory and operation verdicts; +- every blocker without suppressing an actionable peer; +- paths, sizes, and SHA-256 values for retained artifacts. + +Use `operation_verified` only for a completed official encode→decode proof. +Do not derive codec support from setup inventory. Route capability questions +to `jetson-video-capability`. diff --git a/skills/jetson-video-setup/references/video-content.md b/skills/jetson-video-setup/references/video-content.md new file mode 100644 index 00000000..3ba22e03 --- /dev/null +++ b/skills/jetson-video-setup/references/video-content.md @@ -0,0 +1,83 @@ +# Video content policy + +This setup-owned reference defines the shared media-selection and evidence +policy for every Jetson video skill. Setup owns the policy and deterministic +smoke fixture; benchmark and pipeline workflows own normal media execution. + +## Route matrix + +| Workflow | Content rule | +|---|---| +| Setup/readiness verification | A deterministic synthetic raw fixture is allowed; user media is not required. | +| Capability API query | Media-free. A minimal capability-operation smoke test may use setup's deterministic fixture. | +| Recipe planning, validation, or live compatibility check | Media-free. Stop before media work. | +| Documentation-only benchmark estimate | Media-free. Do not launch a controller or claim a measurement. | +| Live benchmark `encode`, `decode`, `compare`, or `camera_capacity` | Require exact user-selected media under the input gate below. | +| Pipeline `encode_decode`, `native_transcode`, `pynvc_segments`, `container_triage`, `av1_verify`, or `acceptance` | Require exact user-selected media under the input gate below. | +| Direct recipe-bound encode/decode execution | Require exact user-selected media, except for the capability-smoke fixture above. | +| Pipeline `content-summary` validation | Consume the already bound external media and evidence; never select or acquire replacement media. | + +Never use synthetic, uniform, black, or generated content for recipe +execution, performance/benchmarking, a content-sensitive comparison, or a +pipeline. + +## Input gate + +Recipe planning and validation, capability queries, and documentation-only +benchmark estimates are media-free. Before any recipe execution, live +benchmark dry run or execution, content-sensitive comparison, or pipeline dry +run or execution, require the user to supply exactly one of: + +- an absolute target-local media path; or +- one exact HTTP(S) media URL. + +If neither is present, return `input_required` with +`next_action: provide_media_path_or_url`, ask the user for one, and pause before probing, +retrieving, converting, constructing a controller request, or launching an operation. Never browse +for, select, recommend, or retrieve media from a bundled catalog or an alternate source. +The controller's `synthetic_input_allowed: false` field is an agent-routing rule, not a byte-content +classifier: an artifact identity alone does not prove who selected the file or how its frames were +created. Bind request construction to the user's supplied path or URL and never turn the setup +fixture into an execution input. + +Treat user selection as operation input, not as proof of copyright or license rights. Preserve +license and attribution exactly when evidenced or supplied. Use the literal string `unknown` when +either is unknown; never infer or invent it. + +## Resolve the selected source + +- For a local path, require a canonical absolute regular-file path on the target, keep the source + read-only, and record its current size and SHA-256. Set `source_url` to JSON `null`. +- For a URL, first prove validated target eligibility and an authenticated released route for the + intended operation. Retrieve only the exact user-supplied HTTP(S) URL into the fresh workspace. + Record that exact requested URL as `source_url` and preserve the retrieval command evidence. + Never substitute a mirror, nearby catalog entry, or different agent-selected URL without asking + the user. +- If the source is missing, unreadable, empty, changes identity, or cannot be retrieved, report the + evidenced input failure and ask for a replacement. Do not relabel it `unsupported` or + `operation_failed`. + +## Use in workflows + +- Classify the selected media as raw, elementary-stream, or container input before choosing a + released sample. Invoke only the authenticated, release-owned NVIDIA sample selected by the + workflow for demux, decode, or raw-frame production. +- For an encoder run, create raw frames only through an approved NVIDIA sample that can write the + required format. Record the source and derived SHA-256, geometry, format, frame rate, frame count, + and decode evidence. Do not pass a container file to a raw-input runner. +- If the selected NVIDIA sample does not accept the user-selected source, ask for another local + path or URL, or report the route blocked. Do not choose fallback media or use FFmpeg to replace + codec work. +- Use the same user-selected content and frame range across comparisons. The controllers validate + byte identity, frame alignment, metadata, and official-sample evidence; they do not infer scene + complexity, representativeness, ownership, or nonuniformity. Never promote the setup fixture into + a performance input. Report results only for the exact user-selected workload and do not + generalize it into a complete workload suite. + +## Evidence + +For every selected input, preserve `source_url` (the exact user URL, or JSON +`null` for local media), honest `license` and `attribution` strings (the literal +`unknown` is allowed), canonical path, byte size, observed SHA-256, +codec/container, dimensions, frame rate, pixel format, bit depth, and frame +count. Preserve a published source checksum only when it is actually available. diff --git a/skills/jetson-video-setup/scripts/setup/lock_pip_reports.py b/skills/jetson-video-setup/scripts/setup/lock_pip_reports.py new file mode 100644 index 00000000..3af95d11 --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/lock_pip_reports.py @@ -0,0 +1,1000 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Create the isolated venv, then materialize and apply a hash-bound pip lock. + +Run directly as ``python3 -I lock_pip_reports.py ...``. ``create-venv`` builds one new +isolated environment, never deleting or reusing a target; ``materialize-apply`` writes the +reviewed lock plus its manifest and applies that exact digest. Every artifact is created +exclusively and is never rewritten. This helper owns the lock, the manifest, the apply report, +and the PyNvVideoCodec wheel evidence -- "the wheel whose SHA-256 matched the lock". +Authenticating the extension the interpreter actually loads against that wheel's ``RECORD`` +belongs to ``verify_pynvc_sample.py``, which also owns validated-venv registry publication +through its own ``--register-current``. +""" + +# pylint: disable=missing-function-docstring,too-many-arguments,too-many-positional-arguments + +from __future__ import annotations + +import argparse +import contextlib +import importlib +import importlib.metadata +import json +import os +import re +import shutil +import sys +import tempfile +import time +import venv +from pathlib import Path, PurePosixPath +from typing import Any +from urllib.parse import unquote, urlsplit, urlunsplit +from urllib.request import HTTPRedirectHandler, Request, build_opener + +_SETUP_DIR = Path(__file__).resolve().parent +if str(_SETUP_DIR) not in sys.path: + sys.path.insert(0, str(_SETUP_DIR)) + +from setup_contract import ( # noqa: E402 # pylint: disable=wrong-import-position + file_identity, read_json, require_isolated, + run_command, sha256_bytes, system_executable, utc_now, write_new_bytes, write_new_json) + +SCHEMA_VERSION = "1.0" +DEFAULT_ALLOWED_HOSTS = ("files.pythonhosted.org", "download.pytorch.org", + "download-r2.pytorch.org", "pypi.nvidia.com") +# The one permitted source artifact: SHA-256-pinned bytes, so inspecting its declared build and +# runtime dependencies cannot add assurance. pip enforces them via --check-build-dependencies. +PYCUDA_SOURCE_POLICY = { + "name": "pycuda", "version": "2026.1", + "url": ("https://files.pythonhosted.org/packages/75/93/" + "36ffd54430924dcb03a88c97519d56589e573d0dcc999c77a4c4d8a90306/pycuda-2026.1.tar.gz"), + "sha256": "759516160628ba06f32ce7e563e3f5b9214691dc9528a03ea99ea1073f4e14ba"} +INSTALL_CONTRACT = { + "no_dependency_resolution": True, "hash_checking": True, "venv_reuse": False, + "direct_https_artifacts_only": True, "source_preflight_before_mutation": True, + "wheel_before_source": True, "build_isolation": False, "build_dependencies_checked": True, + "source_archive_hash_verified": True, "proxy_forwarded_to_children": True, + "build_environment": "explicit PATH/CPATH/LIBRARY_PATH scoped to the build subprocess"} +# Internal markers: creation proves the applying venv; applied is written once, barring a redo. +CLEAN_VENV_MARKER_NAME, CLEAN_VENV_MARKER_KIND = ".nvcodec-clean-venv.json", ( + "nvcodec-clean-venv-marker") +APPLIED_MARKER_NAME, APPLIED_MARKER_KIND = ".nvcodec-clean-venv-applied.json", ( + "nvcodec-clean-venv-applied-marker") +# run_command's allowlist excludes PIP_*, so this is the complete pip config for every child. +_PIP_ENV = {"PIP_CONFIG_FILE": os.devnull, "PIP_DISABLE_PIP_VERSION_CHECK": "1", + "PIP_NO_INPUT": "1"} +_SYSTEM_BUILD_TOOLS = frozenset({"gcc", "g++", "make"}) +_SOURCE_ARCHIVE_LIMIT, _CHILD_TIMEOUT = 64 * 1024 * 1024, 3600.0 +_STREAM_TAIL_BYTES = 8000 +_SHA256 = re.compile(r"[0-9a-f]{64}") +_NAME = re.compile(r"[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?") +# packaging helper -> submodule, resolved from packaging or pip's vendored copy. +_PACKAGING_HELPERS = {"Requirement": "requirements", "Version": "version", + "default_environment": "markers"} +_EXISTS = "The venv target already exists; it was not deleted, reused, or modified." +_APPEARED = "The venv target appeared during reservation; it was not reused or modified." +_RECOVERY = "The partial target was preserved. Choose a different new target to retry." + + +class ApplyBlockedError(ValueError): + """A complete, safely stopped apply outcome caused by external policy drift.""" + + +class _StageFailed(Exception): + """A pip stage exited non-zero, so the environment may already be mutated.""" + + def __init__(self, stage: str) -> None: + super().__init__(stage) + self.stage = stage + + +class _VerificationFailed(Exception): + """Post-mutation evidence did not match the reviewed lock.""" + + def __init__(self, stage: str, cause: BaseException) -> None: + super().__init__(str(cause)) + self.stage = stage + self.cause = cause + + +class _PipCheckFailed(Exception): + """`pip check` reported an inconsistent environment after application.""" + + +class _EnvironmentMismatch(Exception): + """Post-install check found a difference: a completed negative, so exit 2, not 3.""" + + +class _Packaging: + """Packaging helpers resolved on first use, from packaging or pip's vendored copy.""" + + def __getattr__(self, name: str) -> Any: + module = _PACKAGING_HELPERS.get(name) + if module is None: + raise AttributeError(name) + try: + source = importlib.import_module(f"packaging.{module}") + except ImportError: + source = importlib.import_module(f"pip._vendor.packaging.{module}") + value = getattr(source, name) + setattr(self, name, value) + return value + + +_pkg = _Packaging() + + +def _absolute(path: Any) -> Path: + """Normalize dot segments without following a target symlink.""" + return Path(os.path.abspath(os.fspath(Path(path).expanduser()))) + + +def _digest(path: Any, label: str) -> str: + return file_identity(path, label=label)["sha256"] + + +def _emit(path: Path) -> None: + # Replay the persisted artifact verbatim, so stdout and disk can never disagree. + sys.stdout.write(path.read_text(encoding="utf-8")) + + +def _render(value: Any) -> str: + """Render one JSON document for stdout only; every on-disk artifact uses write_new_json.""" + return json.dumps(value, allow_nan=False, sort_keys=True, indent=2) + + +def _prepared_parent(path: Path) -> Path: + """Create every missing parent privately, rejecting a symlinked path component.""" + requested = _absolute(path) + current = Path(requested.anchor) + for part in requested.parent.parts[1:]: + current /= part + if current.is_symlink() or (current.exists() and not current.is_dir()): + raise ValueError(f"path parent component is a symlink or not a directory: {current}") + current.mkdir(mode=0o700, exist_ok=True) + return requested.parent.resolve(strict=True) / requested.name + + +def _fresh_output_path(path: Any, *, label: str) -> Path: + """Refuse an output path that already exists, before anything is created.""" + requested = _absolute(path) + if not requested.parent.is_dir(): + raise FileNotFoundError(f"{label} parent does not exist: {requested.parent}") + resolved = requested.parent.resolve(strict=True) / requested.name + if os.path.lexists(resolved): + raise FileExistsError(f"{label} must be a fresh path: {resolved}") + return resolved + + +def _assert_distinct_paths(paths: list[Path]) -> None: + resolved = [_absolute(path) for path in paths] + if len(set(resolved)) != len(resolved): + raise ValueError("resolver, lock, manifest, and report paths must be distinct") + + +def _canonical_name(name: str) -> str: + """PEP 503 normalization, independent of whether packaging itself is importable.""" + return re.sub(r"[-_.]+", "-", name).lower() + + +def _validated_name(name: str) -> str: + if not _NAME.fullmatch(name): + raise ValueError(f"invalid package name in pip report: {name!r}") + return _canonical_name(name) + + +def _load_report(path: Path) -> dict[str, Any]: + data = read_json(Path(path)) + if not isinstance(data, dict) or data.get("version") != "1": + raise ValueError(f"unsupported pip report version in {path}") + if (not isinstance(data.get("pip_version"), str) or not data["pip_version"] + or not isinstance(data.get("environment"), dict) or not data["environment"]): + raise ValueError(f"pip report lacks a pip_version and target environment: {path}") + return data + + +def _resolution_contract(reports: list[Path]) -> dict[str, Any]: + contracts = [{"pip_version": data["pip_version"], "environment": data["environment"]} + for data in (_load_report(path) for path in reports)] + if any(contract != contracts[0] for contract in contracts[1:]): + raise ValueError("resolver reports do not describe the same pip/environment contract") + return contracts[0] + + +def _archive_sha256(download_info: Any) -> str: + archive = download_info.get("archive_info") if isinstance(download_info, dict) else None + if not isinstance(archive, dict): + raise ValueError("pip report entry has no archive_info") + hashes = archive.get("hashes") + digest = hashes.get("sha256") if isinstance(hashes, dict) else None + if not digest: + value = archive.get("hash") + digest = value.split("=", 1)[1] if isinstance(value, str) and "=" in value else None + if not isinstance(digest, str) or not _SHA256.fullmatch(digest.lower()): + raise ValueError("pip report entry has no exact SHA-256 archive hash") + return digest.lower() + + +def _locked_url(url: str, digest: str, allowed_hosts: set[str]) -> str: + """Bind an artifact to default-port HTTPS on an allowlisted, credential-free origin.""" + parsed = urlsplit(url) + if parsed.scheme != "https" or (parsed.hostname or "").lower() not in allowed_hosts: + raise ValueError(f"unapproved artifact origin: {url!r}") + if parsed.username or parsed.password: + raise ValueError("artifact URL must not contain credentials") + if parsed.port not in {None, 443}: + raise ValueError("artifact URL must use the default HTTPS port") + if parsed.query: + raise ValueError("artifact URL must not contain an unstable query string") + return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", f"sha256={digest}")) + + +class _LockedRedirectHandler(HTTPRedirectHandler): + """Validate every resolved redirect before urllib opens the next URL.""" + + def __init__(self, digest: str, allowed_hosts: set[str]) -> None: + super().__init__() + self._digest = digest + self._allowed_hosts = set(allowed_hosts) + + def redirect_request(self, req, fp, code, msg, headers, newurl): + try: + _locked_url(newurl, self._digest, self._allowed_hosts) + except ValueError: + fp.close() + raise + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _artifact_type(url: str) -> str: + path = unquote(urlsplit(url).path).lower() + if path.endswith(".whl"): + return "wheel" + if path.endswith((".tar.gz", ".tar.bz2", ".tar.xz", ".zip")): + return "source" + raise ValueError(f"unsupported Python artifact type: {url!r}") + + +def _artifact_filename(url: str) -> str: + filename = PurePosixPath(unquote(urlsplit(url).path)).name + if not filename: + raise ValueError(f"artifact URL has no filename: {url!r}") + return filename + + +def _locked_requirement(name: str, url: str, digest: str, allowed_hosts: set[str]) -> str: + return f"{name} @ {_locked_url(url, digest, allowed_hosts)} --hash=sha256:{digest}" + + +def lock_text(artifacts: list[dict[str, Any]]) -> str: + return "".join(f"{item['locked_requirement']}\n" for item in artifacts) + + +def _artifact_identity(artifacts: list[dict[str, Any]]) -> set[tuple[str, str, str]]: + return {(item["normalized_name"], item["version"], item["sha256"]) for item in artifacts} + + +def _report_entry(entry: Any, report: Path, allowed_hosts: set[str]) -> dict[str, Any]: + metadata = entry.get("metadata") if isinstance(entry, dict) else None + download = entry.get("download_info") if isinstance(entry, dict) else None + if not isinstance(metadata, dict) or not isinstance(download, dict): + raise ValueError(f"pip report entry lacks metadata/download_info: {report}") + name, version, url = metadata.get("name"), metadata.get("version"), download.get("url") + if not all(isinstance(value, str) and value for value in (name, version, url)): + raise ValueError(f"pip report entry lacks name/version/url: {report}") + _pkg.Version(version) + if not isinstance(entry.get("requested"), bool): + raise ValueError(f"pip report entry has no requested flag: {report}") + if entry.get("is_yanked") is True: + raise ValueError(f"pip report selected a yanked artifact: {name}=={version}") + digest = _archive_sha256(download) + return {"name": name, "normalized_name": _validated_name(name), "version": version, + "url": url, "filename": _artifact_filename(url), "sha256": digest, + "artifact_type": _artifact_type(url), "requested": entry["requested"], + "locked_requirement": _locked_requirement(name, url, digest, allowed_hosts)} + + +def _report_records(reports: list[Path], allowed_hosts: set[str]) -> dict[str, dict[str, Any]]: + records: dict[str, dict[str, Any]] = {} + for report in reports: + installs = _load_report(report).get("install") + if not isinstance(installs, list) or not installs: + raise ValueError(f"pip report has no resolved install set: {report}") + for entry in installs: + record = _report_entry(entry, report, allowed_hosts) + previous = records.setdefault(record["normalized_name"], record) + # Merge only the same wheel bytes/metadata. When approved mirrors expose the + # same artifact, retain the lexically first URL for order-independent output. + if previous is not record and any(previous[key] != record[key] for key in ( + "version", "sha256", "artifact_type", "filename")): + raise ValueError(f"conflicting resolved artifacts for {record['name']}") + previous["requested"] = previous["requested"] or record["requested"] + canonical_url = min(previous["url"], record["url"]) + previous["url"] = canonical_url + previous["locked_requirement"] = _locked_requirement( + previous["name"], canonical_url, previous["sha256"], allowed_hosts + ) + return records + + +def collect_artifacts(reports: list[Path], requirements: list[str], + allowed_hosts: set[str]) -> list[dict[str, Any]]: + """Return one exact artifact per normalized package and enforce the reviewed requirements.""" + if not requirements: + raise ValueError("at least one policy requirement is required") + records = _report_records(reports, allowed_hosts) + required: set[str] = set() + for raw in requirements: + requirement = _pkg.Requirement(raw) + required.add(_canonical_name(requirement.name)) + record = records.get(_canonical_name(requirement.name)) + if not record: + raise ValueError(f"required package is absent from reports: {requirement.name}") + if requirement.url: + raise ValueError("policy requirements must use version specifiers, not direct URLs") + if not record["requested"]: + raise ValueError(f"policy root was not requested in a report: {requirement.name}") + if requirement.specifier and not requirement.specifier.contains(record["version"], + prereleases=True): + raise ValueError(f"resolved {record['name']}=={record['version']} violates {raw}") + if (_pkg.Version(record["version"]).is_prerelease + and str(requirement.specifier) != f"=={record['version']}"): + raise ValueError(f"prerelease {record['name']} requires an exact policy pin") + unexpected = sorted(name for name, item in records.items() + if item["requested"] and name not in required) + if unexpected: + raise ValueError(f"reports contain unexpected requested roots: {unexpected}") + loose = sorted(name for name, item in records.items() + if not item["requested"] and _pkg.Version(item["version"]).is_prerelease) + if loose: + raise ValueError(f"transitive prerelease artifacts are forbidden: {loose}") + return [records[name] for name in sorted(records)] + + +def _source_names(source_requirements: list[str]) -> set[str]: + names: set[str] = set() + for raw in source_requirements: + requirement = _pkg.Requirement(raw) + if requirement.url: + raise ValueError("source policy requirements must not use direct URLs") + names.add(_canonical_name(requirement.name)) + return names + + +def _validate_source_policy(artifacts: list[dict[str, Any]], source_requirements: list[str]): + expected = _source_names(source_requirements) + observed = {item["normalized_name"] for item in artifacts if item["artifact_type"] == "source"} + if observed != expected: + raise ValueError("source artifact set does not match the explicit source policy: " + f"expected {sorted(expected)}, observed {sorted(observed)}") + + +def _policy_source_artifacts(source_requirements: list[str], + allowed_hosts: set[str]) -> list[dict[str, Any]]: + if not source_requirements: + return [] + if source_requirements != ["pycuda==2026.1"]: + raise ValueError("the only permitted source policy is exact pycuda==2026.1") + name, url, digest = (PYCUDA_SOURCE_POLICY[key] for key in ("name", "url", "sha256")) + return [{"name": name, "normalized_name": name, "version": PYCUDA_SOURCE_POLICY["version"], + "url": url, "filename": _artifact_filename(url), "sha256": digest, + "artifact_type": "source", "requested": True, + "locked_requirement": _locked_requirement(name, url, digest, allowed_hosts)}] + + +def _venv_outcome(result: dict[str, Any], status: str, exit_code: int, + **fields: Any) -> tuple[dict[str, Any], int]: + result.update({"status": status, **fields}) + return result, exit_code + + +def create_clean_venv(target: Path) -> tuple[dict[str, Any], int]: + """Create a reserved, absent target; an existing one is reported, never deleted or reused.""" + target = _absolute(target) + base = str(Path(getattr(sys, "_base_executable", None) or sys.executable).resolve()) + result: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, "kind": "nvcodec-clean-venv-creation", + "target": str(target), "creator_executable": sys.executable, "base_executable": base, + "reuse_allowed": False, "delete_existing": False, "system_site_packages": False} + exists = {"created": False, "mutated": False, "target_preexisted": True} + if os.path.lexists(target): + return _venv_outcome(result, "target_exists", 2, **exists, reason=_EXISTS) + try: + target = _prepared_parent(target) + target.mkdir(mode=0o700) + except FileExistsError: + return _venv_outcome(result, "target_exists", 2, **exists, reason=_APPEARED) + except Exception as exc: # pylint: disable=broad-exception-caught + return _venv_outcome(result, "creation_failed", 3, created=False, mutated=False, + target_preexisted=False, reason=str(exc)) + try: + reserved = target.stat() + # venv and Debian's patched ensurepip write to stdout; this CLI's stdout is JSON only. + with contextlib.redirect_stdout(sys.stderr): + venv.EnvBuilder(system_site_packages=False, clear=False, symlinks=os.name != "nt", + upgrade=False, with_pip=True, prompt="nvcodec", + upgrade_deps=False).create(target) + current = target.stat() + if target.is_symlink() or (current.st_dev, current.st_ino) != (reserved.st_dev, + reserved.st_ino): + raise RuntimeError("reserved venv target identity changed during creation") + interpreter = target / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + configuration = target / "pyvenv.cfg" + if not interpreter.is_file() or not configuration.is_file(): + raise RuntimeError("venv creation did not produce its interpreter and pyvenv.cfg") + if configuration.is_symlink() or not configuration.resolve( + strict=True).is_relative_to(target): + raise RuntimeError("pyvenv.cfg must be a regular file inside the reserved target") + # Mirrors this record with the resolved target path; only the apply stage reads it. + marker = write_new_json(target / CLEAN_VENV_MARKER_NAME, { + **result, "kind": CLEAN_VENV_MARKER_KIND, "created_at": utc_now(), + "target": str(target), "target_python": str(interpreter), + "target_preexisted": False}) + return _venv_outcome( + result, "created", 0, created=True, mutated=True, target_preexisted=False, + target_python=str(interpreter), configuration=str(configuration), marker=str(marker), + marker_sha256=_digest(marker, "clean venv marker"), completed_at=utc_now()) + # SystemExit is deliberate: Debian/Ubuntu patch venv to sys.exit() with no python3-venv. + except (Exception, SystemExit) as exc: # pylint: disable=broad-exception-caught + return _venv_outcome(result, "creation_failed_partial", 3, created=False, mutated=True, + target_preexisted=False, target_exists=target.exists(), + reason=f"{type(exc).__name__}: {exc}", recovery=_RECOVERY) + + +def _assert_clean_venv() -> Path: + """Prove the applying interpreter is the fresh, never-applied venv holding only pip tooling.""" + if sys.prefix == getattr(sys, "base_prefix", sys.prefix): + raise ValueError("locked application requires the newly created virtual environment") + marker_path = Path(sys.prefix) / CLEAN_VENV_MARKER_NAME + if marker_path.is_symlink() or not marker_path.is_file(): + raise ValueError("new virtual environment has no clean-creation marker") + if os.path.lexists(Path(sys.prefix) / APPLIED_MARKER_NAME): + raise ValueError("this virtual environment already consumed a locked apply") + marker = read_json(marker_path) + if (not isinstance(marker, dict) or marker.get("schema_version") != SCHEMA_VERSION + or marker.get("kind") != CLEAN_VENV_MARKER_KIND + or marker.get("target_preexisted") is not False): + raise ValueError("clean virtual-environment marker is invalid") + unexpected = sorted({_canonical_name(item.metadata.get("Name", "")) + for item in importlib.metadata.distributions()} - {"pip", "setuptools"}) + if unexpected: + raise ValueError(f"new virtual environment is not clean: {unexpected}") + return marker_path + + +def _seal_applied_venv(manifest_digest: str, wheel_evidence: dict[str, Any] | None) -> str: + """Create the one-shot applied marker, so a second apply into this venv is refused.""" + return _digest(write_new_json(Path(sys.prefix) / APPLIED_MARKER_NAME, { + "schema_version": SCHEMA_VERSION, "kind": APPLIED_MARKER_KIND, + "apply_status": "applied_verified", "apply_completed_at": utc_now(), + "manifest_sha256": manifest_digest, "pynvc_wheel": wheel_evidence}), "applied marker") + + +def materialize(reports: list[Path], requirements: list[str], source_requirements: list[str], + lock_path: Path, manifest_path: Path) -> dict[str, Any]: + allowed_hosts = set(DEFAULT_ALLOWED_HOSTS) + lock_path = _fresh_output_path(lock_path, label="pip lock") + manifest_path = _fresh_output_path(manifest_path, label="pip lock manifest") + identities = [file_identity(path, label="pip resolver report") for path in reports] + resolved = [Path(item["path"]) for item in identities] + if len(set(resolved)) != len(resolved): + raise ValueError("resolver report paths must be unique") + _assert_distinct_paths([*resolved, lock_path, manifest_path]) + source_names = _source_names(source_requirements) + artifacts = collect_artifacts( + resolved, [item for item in requirements + if _canonical_name(_pkg.Requirement(item).name) not in source_names], + allowed_hosts) + artifacts.extend(_policy_source_artifacts(source_requirements, allowed_hosts)) + artifacts.sort(key=lambda item: item["normalized_name"]) + _validate_source_policy(artifacts, source_requirements) + contract = _resolution_contract(resolved) + if contract["pip_version"] != importlib.metadata.version("pip"): + raise ValueError("resolver reports were created by a different pip version") + if contract["environment"] != _pkg.default_environment(): + raise ValueError("resolver reports describe a different Python/platform environment") + lock_path = write_new_bytes(lock_path, lock_text(artifacts).encode("utf-8")) + manifest = { + "schema_version": SCHEMA_VERSION, "kind": "nvcodec-pip-lock-manifest", + "generated_at": utc_now(), "interpreter": sys.executable, + "requirements": requirements, "source_requirements": source_requirements, + "allowed_hosts": sorted(allowed_hosts), "resolution_contract": contract, + "resolution_reports": [{"path": item["path"], "sha256": item["sha256"]} + for item in identities], + "lock": {"path": str(lock_path), "sha256": _digest(lock_path, "pip lock")}, + "artifacts": artifacts, "install_contract": INSTALL_CONTRACT} + write_new_json(manifest_path, manifest) + return manifest + + +def _authenticated_manifest_artifacts(manifest: dict[str, Any]) -> list[dict[str, Any]]: + """Re-enforce the origin/hash policy on every artifact this apply will install.""" + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list) or not artifacts: + raise ValueError("lock manifest has no artifacts") + allowed_hosts = set(manifest["allowed_hosts"]) + for item in artifacts: + if not isinstance(item, dict): + raise ValueError("lock manifest artifact record is invalid") + name, url = str(item.get("name", "")), str(item.get("url", "")) + digest = str(item.get("sha256", "")) + if _validated_name(name) != item.get("normalized_name"): + raise ValueError(f"lock manifest artifact name is invalid: {name!r}") + if not _SHA256.fullmatch(digest): + raise ValueError(f"lock manifest artifact has no exact SHA-256: {name}") + if (item.get("locked_requirement") != _locked_requirement(name, url, digest, allowed_hosts) + or item.get("filename") != _artifact_filename(url) + or item.get("artifact_type") != _artifact_type(url)): + raise ValueError(f"lock manifest artifact is not bound to its reviewed URL: {name}") + _validate_source_policy(artifacts, manifest["source_requirements"]) + return artifacts + + +def _bound_file(record: Any, label: str) -> Path: + """Return a recorded path only while the file on disk still hashes to its recorded digest.""" + path = record.get("path") if isinstance(record, dict) else None + if not isinstance(path, str) or not path: + raise ValueError(f"lock manifest has no {label} path") + if _digest(Path(path), label) != record.get("sha256"): + raise ValueError(f"{label} drift: {path}") + return Path(path) + + +def verify_manifest(manifest_path: Path) -> dict[str, Any]: + """Authenticate the manifest, its lock, and the environment that is applying it.""" + manifest = read_json(Path(manifest_path)) + if not isinstance(manifest, dict) or manifest.get("kind") != "nvcodec-pip-lock-manifest": + raise ValueError("not an nvcodec-pip-lock-manifest") + if manifest.get("schema_version") != SCHEMA_VERSION: + raise ValueError("unsupported pip lock manifest schema") + if manifest.get("interpreter") != sys.executable: + raise ValueError("lock manifest was created by a different Python interpreter") + if sorted(manifest.get("allowed_hosts") or ()) != sorted(DEFAULT_ALLOWED_HOSTS): + raise ValueError("lock manifest allowed_hosts differs from the reviewed setup policy") + for key in ("requirements", "source_requirements"): + values = manifest.get(key) + if not isinstance(values, list) or not all(isinstance(value, str) for value in values): + raise ValueError(f"lock manifest {key} are invalid") + contract = manifest.get("resolution_contract") + if not isinstance(contract, dict): + raise ValueError("lock manifest has no resolver environment contract") + if contract.get("pip_version") != importlib.metadata.version("pip"): + raise ValueError("the applying pip version differs from the resolver pip version") + if contract.get("environment") != _pkg.default_environment(): + raise ValueError("the applying Python/platform differs from the resolver environment") + records = manifest.get("resolution_reports") + if not isinstance(records, list) or not records: + raise ValueError("lock manifest has no resolution reports") + for record in records: + _bound_file(record, "resolver report") + lock_path = _bound_file(manifest.get("lock"), "pip lock") + if lock_path.read_text(encoding="utf-8") != lock_text( + _authenticated_manifest_artifacts(manifest)): + raise ValueError("pip lock content does not match the reviewed artifact set") + return manifest + + +def _download_authenticated_source(artifact: dict[str, Any], destination: Path, + allowed_hosts: set[str]) -> dict[str, Any]: + """Fetch one locked sdist, validating every redirect target and then the exact hash.""" + parsed = urlsplit(_locked_url(artifact["url"], artifact["sha256"], allowed_hosts)) + filename = PurePosixPath(unquote(parsed.path)).name + if not filename or _artifact_type(parsed.path) != "source": + raise ValueError("source artifact URL has no supported archive filename") + destination.mkdir(mode=0o700, exist_ok=True) + request = Request(urlunsplit((parsed.scheme, parsed.netloc, parsed.path, "", "")), + headers={"User-Agent": "jetson-videosdk-skill/1.0"}) + chunks: list[bytes] = [] + observed = 0 + opener = build_opener(_LockedRedirectHandler(artifact["sha256"], allowed_hosts)) + with opener.open(request, timeout=120) as response: # nosec B310 + # Defense in depth for any response path that bypasses redirect handling. + _locked_url(response.geturl(), artifact["sha256"], allowed_hosts) + for block in iter(lambda: response.read(1024 * 1024), b""): + observed += len(block) + if observed > _SOURCE_ARCHIVE_LIMIT: + raise ValueError("source archive exceeds the reviewed size limit") + chunks.append(block) + payload = b"".join(chunks) + if sha256_bytes(payload) != artifact["sha256"]: + raise ValueError("downloaded source does not match its reviewed SHA-256") + return {"path": str(write_new_bytes(destination / filename, payload)), + "sha256": artifact["sha256"], "size_bytes": observed, "source_url": artifact["url"]} + + +def _parse_build_environment(values: list[str]) -> dict[str, str]: + allowed = {"PATH", "CPATH", "LIBRARY_PATH"} + result: dict[str, str] = {} + for value in values: + name, separator, content = value.partition("=") + if not separator or name not in allowed or not content or name in result: + raise ValueError("build environment must contain one non-empty PATH, CPATH, and" + " LIBRARY_PATH assignment") + result[name] = content + if set(result) != allowed: + raise ValueError("PyCUDA source builds require PATH, CPATH, and LIBRARY_PATH") + for name, content in result.items(): + if any(not value or not Path(value).is_absolute() for value in content.split(":")): + raise ValueError(f"PyCUDA {name} entries must be non-empty absolute paths") + return result + + +def _build_tool_identities(environment: dict[str, str]) -> dict[str, Any]: + """Identify the tools this build will use; system compilers must be the canonical ones.""" + identities: dict[str, Any] = {} + for name in ("nvcc", "gcc", "g++", "make"): + located = shutil.which(name, path=environment.get("PATH")) + if not located: + raise ValueError(f"authenticated PyCUDA build tool is unavailable: {name}") + invocation, identity_path = Path(located), Path(located) + if name in _SYSTEM_BUILD_TOOLS: + identity_path = Path(system_executable(name)) + if invocation.parent != Path("/usr/bin") or invocation.resolve(strict=True) != ( + identity_path): + raise ValueError( + f"PyCUDA build tool {name} is not the canonical system executable") + identities[name] = {"invocation": str(invocation), **file_identity( + identity_path, label=f"PyCUDA build tool {name}")} + return identities + + +def _pip_argv(*arguments: str) -> list[str]: + """The only interpreter/pip prefix any child command in this module may use.""" + return [sys.executable, "-I", "-m", "pip", "--isolated", *arguments] + + +def _pip_install_argv(report_path: Path, requirements_path: Path, *, no_index: bool, + build: bool = False) -> list[str]: + """The only permitted install shape: hash-checked, dependency-free, forced reinstall.""" + return _pip_argv("install", "--disable-pip-version-check", "--no-input", "--no-cache-dir", + *(["--no-index"] if no_index else []), + *(["--no-build-isolation", "--check-build-dependencies"] if build else []), + "--force-reinstall", "--no-deps", "--require-hashes", + "--report", str(report_path), "-r", str(requirements_path)) + + +def _run_child(argv: list[str], *, environment: dict[str, str], cwd: Path) -> dict[str, Any]: + started, monotonic = utc_now(), time.monotonic() + done = run_command(argv, cwd=cwd, timeout=_CHILD_TIMEOUT, extra_env=environment) + record = {"argv": argv, "cwd": str(cwd), "started_at": started, "ended_at": utc_now(), + "duration_seconds": time.monotonic() - monotonic, "returncode": done.returncode} + # Each stream is bounded to its last _STREAM_TAIL_BYTES; nothing is dropped silently. + for name in ("stdout", "stderr"): + data = getattr(done, name) or b"" + record[f"{name}_tail"] = data[-_STREAM_TAIL_BYTES:].decode("utf-8", "replace") + record[f"{name}_bytes"] = len(data) + record[f"{name}_truncated"] = len(data) > _STREAM_TAIL_BYTES + return record + + +def _validate_local_source_report(report_path: Path, artifact: dict[str, Any], + source: dict[str, Any], contract: dict[str, Any]) -> None: + """Confirm the source stage installed exactly the archive this process authenticated.""" + data = _load_report(report_path) + if {"pip_version": data["pip_version"], "environment": data["environment"]} != contract: + raise ValueError("source apply report environment differs from the resolver") + installs = data.get("install") + if not isinstance(installs, list) or len(installs) != 1: + raise ValueError("source apply report must contain exactly one artifact") + metadata = installs[0].get("metadata") or {} + download = installs[0].get("download_info") or {} + if _canonical_name(str(metadata.get("name", ""))) != artifact["normalized_name"]: + raise ValueError("source apply report has the wrong project") + if _pkg.Version(str(metadata.get("version", ""))) != _pkg.Version(artifact["version"]): + raise ValueError("source apply report has the wrong version") + if _archive_sha256(download) != source["sha256"]: + raise ValueError("source apply report has the wrong archive hash") + url = urlsplit(str(download.get("url", ""))) + if url.scheme != "file" or Path(unquote(url.path)).resolve() != Path(source["path"]): + raise ValueError("source apply report did not install the authenticated archive") + + +def _pynvc_wheel_evidence(wheels: list[dict[str, Any]], + wheel_report: Path | None) -> dict[str, Any] | None: + """Name the wheel whose SHA-256 matched the lock, installed under --require-hashes. + + Proven upstream, unrepeated: _wheel_stage equates this apply report's artifact identity with + the lock. verify_pynvc_sample.py owns RECORD authentication of the loaded extension. + """ + locked = [item for item in wheels if item["normalized_name"] == "pynvvideocodec"] + if not locked: + return None + if len(locked) != 1: + raise ValueError("reviewed lock must select exactly one PyNvVideoCodec wheel") + if wheel_report is None or not Path(wheel_report).is_file(): + raise ValueError("PyNvVideoCodec evidence requires the persisted wheel stage report") + artifact = locked[0] + return {"schema_version": SCHEMA_VERSION, "kind": "nvcodec-pynvc-wheel-evidence", + "name": artifact["name"], "normalized_name": "pynvvideocodec", + "version": artifact["version"], "filename": artifact["filename"], + "sha256": artifact["sha256"], "url": artifact["url"], "hash_checked_install": True, + "apply_report": str(wheel_report), + "installed_version": importlib.metadata.version("PyNvVideoCodec"), + "record_authority": "verify_pynvc_sample.py"} + + +def _preflight_apply_outputs(install_report: Path) -> dict[str, Path]: + """Reserve every evidence path this apply may write before anything is created.""" + report = _fresh_output_path(install_report, label="pip install report") + stem, suffix = (report.stem, report.suffix) if report.suffix else (report.name, ".json") + return {"install_report": report, + **{stage: _fresh_output_path(report.with_name(f"{stem}.{stage}{suffix}"), + label=f"{stage} stage report") + for stage in ("wheels", "sources")}} + + +class _Apply: # pylint: disable=too-many-instance-attributes + """One locked application: authenticate every artifact, then mutate.""" + + def __init__(self, manifest_path: Path, install_report: Path, expected_manifest_sha256: str, + build_environment_values: list[str], state: dict[str, bool]) -> None: + self.manifest_path = _absolute(manifest_path) + self.state = state + self.digest = _digest(self.manifest_path, "lock manifest") + if expected_manifest_sha256.lower() != self.digest: + raise ValueError("lock manifest does not match the expected SHA-256") + self.manifest = verify_manifest(self.manifest_path) + artifacts = self.manifest["artifacts"] + self.sources = [item for item in artifacts if item["artifact_type"] == "source"] + self.wheels = [item for item in artifacts if item["artifact_type"] == "wheel"] + if len(self.sources) > 1: + raise ValueError("the reviewed policy permits at most one source artifact") + if self.sources and not self.wheels: + raise ValueError("source builds require a reviewed wheel bootstrap stage") + if not self.sources and build_environment_values: + raise ValueError("build environment was supplied without a source artifact") + self.build_environment = (_parse_build_environment(build_environment_values) + if self.sources else {}) + self.paths = _preflight_apply_outputs(install_report) + self.report = self.paths["install_report"] + _assert_distinct_paths([self.manifest_path, Path(self.manifest["lock"]["path"]), + *(Path(item["path"]) + for item in self.manifest["resolution_reports"]), + self.report, self.paths["wheels"], self.paths["sources"]]) + self.marker_path = _assert_clean_venv() + self.summary: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, "kind": "nvcodec-pip-lock-apply", + "status": "running", "manifest": str(self.manifest_path), + "manifest_sha256": self.digest, "clean_venv_marker": str(self.marker_path), + "clean_venv_marker_status": "unconsumed", + "clean_venv_marker_sha256": _digest(self.marker_path, "clean venv marker"), + "started_at": utc_now(), "stages": []} + + def _stop(self, status: str, exit_code: int, **fields: Any) -> int: + self.summary.update({"status": status, "completed_at": utc_now(), **fields}) + write_new_json(self.report, self.summary) + return exit_code + + def _stage(self, stage: str, command: list[str], root: Path, environment: dict[str, str], + persistent: Path, report_path: Path, **extra: Any) -> dict[str, Any]: + record = {"stage": stage, + **_run_child(command, environment=environment, cwd=root), **extra} + self.summary["stages"].append(record) + if report_path.is_file(): + saved = write_new_bytes(persistent, report_path.read_bytes()) + record["pip_report"] = str(saved) + record["pip_report_sha256"] = _digest(saved, "persisted pip report") + if record["returncode"] != 0: + raise _StageFailed(stage) + return record + + def _wheel_stage(self, root: Path) -> list[dict[str, Any]]: + """Install every locked wheel with hashes and no dependency resolution.""" + requirements = write_new_bytes(root / "wheels.requirements.txt", + lock_text(self.wheels).encode("utf-8")) + report_path = root / "wheels.pip-report.json" + self._stage("wheels", _pip_install_argv(report_path, requirements, no_index=False), root, + _PIP_ENV, self.paths["wheels"], report_path) + try: + observed = collect_artifacts( + [report_path], [f"{item['name']}=={item['version']}" for item in self.wheels], + set(self.manifest["allowed_hosts"])) + if _resolution_contract([report_path]) != self.manifest["resolution_contract"]: + raise ValueError("apply report environment differs from resolver environment") + if _artifact_identity(observed) != _artifact_identity(self.wheels): + raise ValueError("wheels apply report differs from the reviewed lock") + except Exception as exc: # pylint: disable=broad-exception-caught + raise _VerificationFailed("wheels", exc) from exc + return observed + + def _source_stage(self, artifact: dict[str, Any], source: dict[str, Any], root: Path) -> None: + """Build and install the hash-pinned sdist with the scoped CUDA build environment.""" + report_path = root / "source.pip-report.json" + requirements = write_new_bytes(root / "source.requirements.txt", ( + f"{artifact['name']} @ {Path(source['path']).as_uri()}#sha256={source['sha256']} " + f"--hash=sha256:{source['sha256']}\n").encode("utf-8")) + environment = {**_PIP_ENV, **self.build_environment} + self._stage("sources", _pip_install_argv(report_path, requirements, no_index=True, + build=True), root, environment, + self.paths["sources"], report_path, source=source, + build_environment=self.build_environment, + build_tool_identities=_build_tool_identities(environment)) + try: + _validate_local_source_report(report_path, artifact, source, + self.manifest["resolution_contract"]) + except Exception as exc: # pylint: disable=broad-exception-caught + raise _VerificationFailed("sources", exc) from exc + + def _run_stages(self, root: Path) -> list[dict[str, Any]]: + """Authenticate every artifact before mutating, then install wheels before sources.""" + allowed_hosts = set(self.manifest["allowed_hosts"]) + try: + sources = [_download_authenticated_source(item, root / "sources", allowed_hosts) + for item in self.sources] + except (OSError, ValueError) as exc: + raise ApplyBlockedError(f"authenticated artifact preflight blocked: {exc}") from exc + self.summary["source_preflight"] = { + "status": "authenticated_before_mutation" if sources else "not_applicable", + "sources": sources} + # The one manifest re-check, across the network preflight window, before any mutation. + if _digest(self.manifest_path, "lock manifest") != self.digest: + raise ValueError("lock manifest changed during authenticated source preflight") + self.state["mutated"] = True + applied = self._wheel_stage(root) if self.wheels else [] + for artifact, source in zip(self.sources, sources): + self._source_stage(artifact, source, root) + applied.append(artifact) + return applied + + def _verify_environment(self, applied: list[dict[str, Any]]) -> dict[str, str]: + """Confirm the installed versions match the lock and that `pip check` is clean.""" + if _artifact_identity(applied) != _artifact_identity(self.manifest["artifacts"]): + raise _EnvironmentMismatch("combined apply evidence differs from the reviewed lock") + installed: dict[str, str] = {} + for item in self.manifest["artifacts"]: + observed = importlib.metadata.version(item["name"]) + if observed != item["version"]: + raise _EnvironmentMismatch(f"installed {item['name']} version {observed} differs" + f" from the locked {item['version']}") + installed[item["normalized_name"]] = observed + checked = _run_child(_pip_argv("check"), environment=_PIP_ENV, cwd=self.report.parent) + self.summary["pip_check"] = checked + if checked["returncode"] != 0: + raise _PipCheckFailed() + return installed + + def run(self) -> int: + try: + with tempfile.TemporaryDirectory(prefix="nvcodec-pip-apply-", + dir=self.report.parent) as temporary: + root = Path(temporary) + root.chmod(0o700) + self.summary["private_staging_root"] = {"path": str(root), "mode": "0700"} + applied = self._run_stages(root) + installed = self._verify_environment(applied) + evidence = _pynvc_wheel_evidence(self.wheels, + self.paths["wheels"] if self.wheels else None) + except _StageFailed as exc: + return self._stop("apply_failed_mutation_possible", 2, failed_stage=exc.stage) + except _PipCheckFailed: + return self._stop("mutated_verification_failed", 2, + verification_error="pip check reported an inconsistent environment") + except _EnvironmentMismatch as exc: + return self._stop("mutated_verification_failed", 2, verification_error=str(exc)) + except _VerificationFailed as exc: + self._stop("mutated_verification_failed", 3, failed_stage=exc.stage, + verification_error=str(exc.cause)) + raise exc.cause + except Exception as exc: # pylint: disable=broad-exception-caught + if self.state["mutated"]: + self._stop("mutated_verification_failed", 3, verification_error=str(exc)) + raise + return self._stop("applied_verified", 0, installed_versions=installed, + pynvc_wheel=evidence, + applied_marker_sha256=_seal_applied_venv(self.digest, evidence)) + + +def apply_lock(manifest_path: Path, install_report: Path, expected_manifest_sha256: str, + build_environment_values: list[str]) -> int: + """Apply a lock and emit exactly one persisted JSON result.""" + install_report = _fresh_output_path(install_report, label="pip install report") + state = {"mutated": False} + try: + result = _Apply(manifest_path, install_report, expected_manifest_sha256, + build_environment_values, state).run() + except Exception as exc: # pylint: disable=broad-exception-caught + if not install_report.is_file(): + write_new_json(install_report, { + "schema_version": SCHEMA_VERSION, "kind": "nvcodec-pip-lock-apply", + "status": ("mutated_verification_failed" if state["mutated"] + else "validation_failed_no_mutation"), + "manifest": str(_absolute(manifest_path)), "error": str(exc), + "clean_venv_marker": None, "clean_venv_marker_status": None, + "clean_venv_marker_sha256": None, + "completed_at": utc_now()}) + _emit(install_report) + return 2 if isinstance(exc, ApplyBlockedError) else 3 + _emit(install_report) + return result + + +def materialize_apply(reports: list[Path], requirements: list[str], + source_requirements: list[str], lock_path: Path, manifest_path: Path, + install_report: Path, build_environment_values: list[str]) -> int: + _preflight_apply_outputs(install_report) + materialize(reports, requirements, source_requirements, lock_path, manifest_path) + return apply_lock(manifest_path, install_report, _digest(manifest_path, "lock manifest"), + build_environment_values) + + +def _action_error(kind: str, output: Path | None, exc: BaseException, + **context: Any) -> dict[str, Any]: + return {"schema_version": SCHEMA_VERSION, "kind": kind, "status": "error", **context, + "output": str(output) if output is not None else None, "error": str(exc)} + + +def _action_main(output: Path | None, produce: Any, kind: str, **context: Any) -> int: + """Run one action under isolation, then persist and echo exactly one JSON document.""" + try: + require_isolated() + if output is not None: + _fresh_output_path(output, label="action report") + result, exit_code = produce() + except Exception as exc: # pylint: disable=broad-exception-caught + result, exit_code = _action_error(kind, output, exc, **context), 3 + try: + if output is not None: + write_new_json(output, result) + except Exception as exc: # pylint: disable=broad-exception-caught + result, exit_code = _action_error(kind, output, exc, **context), 3 + print(_render(result)) + return exit_code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + actions = parser.add_subparsers(dest="action", required=True) + venv_parser = actions.add_parser( + "create-venv", help="create one new isolated venv without deleting or reusing a target") + venv_parser.add_argument("--target", type=Path, required=True) + venv_parser.add_argument("--output", type=Path) + materialize_parser = actions.add_parser("materialize") + pipeline_parser = actions.add_parser( + "materialize-apply", help="materialize the reviewed lock and apply that exact digest") + for lock_parser in (materialize_parser, pipeline_parser): + lock_parser.add_argument("--report", action="append", type=Path, required=True) + lock_parser.add_argument("--require", action="append", default=[]) + lock_parser.add_argument("--source-require", action="append", default=[]) + lock_parser.add_argument("--allowed-host", action="append") + lock_parser.add_argument("--lock", type=Path, required=True) + lock_parser.add_argument("--manifest", type=Path, required=True) + pipeline_parser.add_argument("--install-report", type=Path, required=True) + pipeline_parser.add_argument("--build-env", action="append", default=[]) + apply_parser = actions.add_parser("apply") + apply_parser.add_argument("--manifest", type=Path, required=True) + apply_parser.add_argument("--install-report", type=Path, required=True) + apply_parser.add_argument("--expected-manifest-sha256", required=True) + apply_parser.add_argument("--build-env", action="append", default=[]) + args = parser.parse_args() + output = _absolute(args.output) if getattr(args, "output", None) is not None else None + if args.action == "create-venv": + target = _absolute(args.target) + if output is not None and (output == target or target in output.parents): + parser.error("--output must be outside the requested venv target") + return _action_main(output, lambda: create_clean_venv(target), + "nvcodec-clean-venv-error", target=str(target)) + try: + require_isolated() + if args.action == "apply": + return apply_lock(args.manifest, args.install_report, + args.expected_manifest_sha256, args.build_env) + if args.allowed_host is not None and set(args.allowed_host) != set(DEFAULT_ALLOWED_HOSTS): + raise ValueError("--allowed-host must name exactly the fixed approved host set") + if args.action == "materialize": + print(_render(materialize(args.report, args.require, args.source_require, + args.lock, args.manifest))) + return 0 + return materialize_apply(args.report, args.require, args.source_require, args.lock, + args.manifest, args.install_report, args.build_env) + except Exception as exc: # pylint: disable=broad-exception-caught + print(_render({"schema_version": SCHEMA_VERSION, "kind": "nvcodec-pip-lock-error", + "status": "error", "error": str(exc)})) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-setup/scripts/setup/plan_install.py b/skills/jetson-video-setup/scripts/setup/plan_install.py new file mode 100644 index 00000000..46fe6a65 --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/plan_install.py @@ -0,0 +1,1080 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Plan from environment 1.2 and execute APT under a canonical re-plan digest.""" +# pylint: disable=missing-function-docstring, too-many-arguments, too-many-positional-arguments +# pylint: disable=too-many-lines +# pylint: disable=too-many-branches, too-many-locals, too-many-return-statements +# pylint: disable=too-many-statements,wrong-import-position + +from __future__ import annotations + +import argparse +import json +import os +import re +import sys +import time +from datetime import datetime +from pathlib import Path, PurePosixPath +from typing import Any + +_SETUP_DIR = Path(__file__).resolve().parent +if str(_SETUP_DIR) not in sys.path: + sys.path.insert(0, str(_SETUP_DIR)) + +from setup_contract import (FULL_PROFILE, SMOKE_PROFILE, # noqa: E402 + VERIFICATION_PROFILES, authenticated_candidate_binding, canonical_json_sha256, + cuda_at_least_13, cuda_build_package, durable_venv_path, environment_contract_errors, + file_identity, native_sdk_release, normalized_cuda_evidence, normalized_native_surface, + normalized_pynvc_surface, normalized_python_bootstrap, parse_apt_sources, profile_dependencies, + public_candidate_binding, pycuda_bootstrap_packages, read_json, require_isolated, + require_selected_surfaces_live, run_command, sha256_bytes, + system_executable, transaction_sibling_path, utc_now, write_new_bytes, write_new_json) +SCHEMA_VERSION, APT_SCHEMA_VERSION, VALIDATION_SCHEMA = "1.5", "1.0", "1.0" +PLAN_KIND, ENVIRONMENT_KIND, ENVIRONMENT_MAJOR = "nvcodec-install-plan", "nvcodec-environment", "1" +COMPONENTS, SURFACES = ("native-sdk", "pynvc"), {"native-sdk": "native", "pynvc": "pynvc"} +REQUEST_INTENTS, TRANSACTION_ACTIONS = ("plan-only", "setup-install"), ("refresh", "preview", + "apply") +NVIDIA_PACKAGE, NATIVE_RELEASE_FAMILY = "nvidia-video-codec-sdk", "13.0" +CUDA_PACKAGE, VENV_PACKAGE, DEV_PACKAGE = "cuda-toolkit", "python3-venv", "python3-dev" +BUILD_ENV_NAMES = ("PATH", "CPATH", "LIBRARY_PATH") +CUDA_BOOTSTRAP_ROUTE = "cuda-minimal-build-curand-bootstrap-then-replan" +PYNVC_VERSION, PYNVC_SPEC = "2.1.0", "PyNvVideoCodec==2.1.0" +RUNTIME_SPECS = {"numpy": "numpy>=1.24", "pycuda": "pycuda==2026.1", "torch": "torch==2.9.1+cu130"} +SUPPORT_SPECS = ("setuptools>=68", "wheel>=0.41", "pytools>=2011.2", "platformdirs>=2.2.0", "mako") +DEFAULT_PYNVC_VENV = Path.home() / ".venvs" / "nvcodec" +TRANSACTION_SCRIPT, PIP_SCRIPT = str(Path(__file__).resolve()), str(_SETUP_DIR / + "lock_pip_reports.py") +TRANSACTION_PREFIX = (sys.executable, "-B", "-I", TRANSACTION_SCRIPT) +PLAN_ARTIFACT_NAME, REFRESH_RECEIPT_NAME = ("nvcodec-install-plan.json", + "nvcodec-apt-refresh-receipt.json") +REFRESH_PREFLIGHT_ID, CLEAN_VENV_REPORT = "refresh-apt-metadata", "nvcodec-clean-venv-creation.json" +BINARY_REPORT, TORCH_REPORT = "nvcodec-pip-binary-resolve.json", "nvcodec-pip-torch-resolve.json" +INSECURE_APT_PATTERNS = tuple(rf"{name}\s+\"?(?:1|true|yes)\"?" for name in ( + "Acquire::AllowInsecureRepositories", "Acquire::AllowDowngradeToInsecureRepositories", + "Acquire::AllowWeakRepositories", "APT::Get::AllowUnauthenticated")) +POLICY_FACTS = ("status", "package", "expected_candidate", "observed_candidate", "expected_origin", + "bound_origin", "candidate_origins", "configured_sources", + "configuration_bypasses", "verified") +APPROVAL_BATCH_REASONS = { + "apt-metadata-refresh": "Refresh APT metadata once, then discard and regenerate the plan.", + "native-sdk-install": "Apply the reviewed candidate-pinned native SDK APT transaction.", + "pynvc-bootstrap": "Create the new isolated Python environment.", + "pynvc-resolution": "Resolve every required Python package without applying anything.", + "pynvc-install": "Apply the reviewed Python package set after each resolver report passes."} +_PUBLIC_ORIGIN_TEXT = ("the stock signature-authenticated public NVIDIA Jetson origin" + " (repo.download.nvidia.com/jetson/{{common,som}}, rNN.N/main)") +_UNBOUND = " is not bound to " + _PUBLIC_ORIGIN_TEXT + "; no install command was generated." +BLOCKED_ROUTES = {"interpreter": ("canonical-interpreter-required", + "Setup requires the one canonical system interpreter: {}"), + "native-surface": ("native-surface-evidence-required", + "The native surface is malformed, so it cannot be" + " planned: {}"), + "native-cuda": ("cuda-toolkit-minimum-required", + "The native surface requires an observed CUDA Toolkit 13.0+;" + " saw {!r}."), + "native-installed": ("native-13.0-release-required", + "Installed native SDK {!r} is outside the supported" + " 13.0.x family."), + "native-candidate": ( + "release-apt-candidate-required", + "No nvidia-video-codec-sdk candidate is in local APT metadata; verify the" + " image's release repository and never invent one."), + "native-release": ("native-13.0-release-required", + "Observed candidate {!r} is not a supported 13.0.x release."), + "pynvc-surface": ("pynvc-surface-evidence-required", + "The Python surface is malformed, so it cannot be planned: {}"), + "pynvc-fresh-venv": ( + "fresh-venv-target-required", + "A fresh or reinstall PyNvVideoCodec setup requires an explicit unique" + " --venv target; none was supplied."), + "pynvc-reuse": ( + "existing-environment-reuse-required", + "A complete PyNvVideoCodec environment is already at {}; reuse it, or pass" + " --fresh-setup with a new --venv to authorize one."), + "pynvc-new-venv": ( + "new-venv-target-required", + "The requested venv is the observed environment at {}; choose a new path" + " because environments are never repaired or reused in place."), + "pynvc-venv-exists": ( + "fresh-venv-target-exists", + "The --venv target already exists at {}; setup never deletes, repairs, or" + " reuses an environment. Choose a new unique path."), + "pynvc-venv-location": ( + "durable-absolute-venv-required", + "The --venv target {!r} must be an absolute durable path outside the current" + " working directory and transient trees."), + "apt-architecture": ( + "target-architecture-candidate-required", + "The {} candidate resolves to APT architecture {!r}, not the Jetson target;" + " no install command was generated."), + "cuda-evidence": ( + "cuda-build-environment-required", + "PyCUDA needs the target CUDA build environment, but the" + " installation.cuda_toolkit block is malformed: {}"), + "build-tools": ( + "pycuda-system-build-tools-required", + "PyCUDA needs canonical /usr/bin gcc, g++, and make executables: {}"), + "cuda-minimum": ( + "cuda-toolkit-minimum-required", + "Observed CUDA candidate {!r} is not the matching stable CUDA 13.0+" + " release."), + "python-evidence": ( + "python-bootstrap-evidence-required", + "A new isolated environment needs the installation.python bootstrap" + " evidence, which is malformed: {}"), + "apt-candidate": ( + "authenticated-apt-candidate-required", + "No {} candidate exists in current authenticated APT metadata; verify the" + " image's repository, never add or change one."), + "base-apt-origin": ( + "authenticated-apt-origin-required", + "The {} candidate {!r} is not bound to a configured signature-authenticated" + " APT origin; no install command was generated."), + "apt-origin": ( + "authenticated-public-origin-required", "The {} candidate {!r}" + _UNBOUND), + "jetson-linux": ( + "jetson-linux-incompatible", + "No installation command is generated until a live Jetson Linux 38.5+" + " environment is observed: {}")} +SETUP_HELPERS = ("plan_install.py", "probe_nvcodec.py", "lock_pip_reports.py", "setup_contract.py", + "verify_native.py", "verify_pynvc_sample.py") +UNKNOWNS = ("External network reachability was not tested.", + "Operational readiness requires the post-install official sample artifacts.", + "Jetson Linux 38.5+ is required; exact L4T and JetPack revisions are diagnostic only.") +NATIVE_PROBE, PYNVC_PROBE = "nvcodec-environment-after-native.json", "nvcodec-env-after-python.json" +PYNVC_VERIFICATION = "nvcodec-pynvc-verify.json" +VALIDATION_STEPS = { + "native-sdk": ((str(_SETUP_DIR / "probe_nvcodec.py"), "--runtime", "native", "--output", + NATIVE_PROBE), + (str(_SETUP_DIR / "verify_native.py"), "--environment", NATIVE_PROBE, + "--sdk-root", "/opt/nvidia/video-codec-sdk", "--build-dir", + "nvcodec-native-build", "--work-dir", "nvcodec-native-smoke", "--build", + "--run-encode", "--run-decode", "--output", "nvcodec-native-verify.json")), + "pynvc": ((str(_SETUP_DIR / "probe_nvcodec.py"), "--runtime", "pynvc", "--output", PYNVC_PROBE, + "--setup-candidate"), + (str(_SETUP_DIR / "verify_pynvc_sample.py"), "--environment", PYNVC_PROBE, + "--work-dir", "nvcodec-pynvc-smoke", "--output", PYNVC_VERIFICATION, + "--register-current", "--profile", "@profile"))} +POST_INSTALL_STEPS = {"native-sdk": ("produce-final-native-probe", "verify-native-once"), + "pynvc": ("produce-final-pynvc-probe", "verify-pynvc-once-and-register")} + + +class PlanContractError(ValueError): + """No executable plan can be produced.""" + + +def _dumps(value: Any) -> str: + return json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + + +def _field(value: Any, path: tuple[str, ...]) -> Any: + for name in path: + if not isinstance(value, dict): + return None + value = value.get(name) + return value + + +def _emit(path: Path, document: dict[str, Any], code: int) -> int: + write_new_json(path, document) + print(_dumps(document)) + return code + + +def _run(argv: list[str]) -> dict[str, Any]: + started, monotonic = utc_now(), time.monotonic() + completed = run_command(argv) + return {"argv": argv, "cwd": str(Path.cwd().resolve()), "started_at": started, + "ended_at": utc_now(), "duration_seconds": time.monotonic() - monotonic, + "exit_code": completed.returncode, + "stdout": completed.stdout.decode("utf-8", errors="replace"), + "stderr": completed.stderr.decode("utf-8", errors="replace"), + "stdout_bytes": completed.stdout, "stderr_bytes": completed.stderr} + + +def _observed(evidence: dict[str, Any], prefix: Path, stem: str = "") -> dict[str, Any]: + infix = f".{stem}" if stem else "" + streams = {} + for name in ("stdout", "stderr"): + raw = evidence[f"{name}_bytes"] + written = write_new_bytes(prefix.with_name(f"{prefix.name}{infix}.{name}.log"), raw) + streams[name] = {"path": str(written), "size_bytes": len(raw), "sha256": sha256_bytes(raw)} + return {key: evidence[key] for key in ("argv", "cwd", "started_at", "ended_at", + "duration_seconds", "exit_code")} | streams + + +def _missing(node: Any, paths: Any, label: str) -> list[str]: + return [f"{label}.{'.'.join(path)} is required" + for path in paths if not str(_field(node, path) or "").strip()] + + +def _cuda_errors(cuda: Any) -> list[str]: + if not isinstance(cuda, dict): + return ["cuda must be an object"] + if cuda.get("status") not in {"installed", "absent"}: + return [f"cuda.status must be installed or absent, saw {cuda.get('status')!r}"] + if cuda["status"] == "absent": + return [] + prefixes = cuda.get("environment_prefixes") + bound = isinstance(prefixes, dict) and all( + isinstance(prefixes.get(name), list) and prefixes[name] for name in BUILD_ENV_NAMES) + return _missing(cuda, (("version",), ("root",)), "cuda") + ([] if bound else [ + f"cuda.environment_prefixes must list {', '.join(BUILD_ENV_NAMES)} when installed"]) + + +def _command(phase: str, argv: list[str], *, mutates: bool, network: bool = False, + privilege: str = "user", group: str | None = None) -> dict[str, Any]: + protected = mutates or network + if protected != (group in APPROVAL_BATCH_REASONS): + raise ValueError(f"exactly the protected commands carry an approval group: {phase}") + return {"phase": phase, "argv": argv, "authorization_status": "unassigned", + "new_approval_required": None, "execution_group": group, "mutates": mutates, + "requires_network": network, "privilege": privilege} + + +def _validation_commands(component: str, interpreter: str, + profile: str = SMOKE_PROFILE) -> list[list[str]]: + """Bind the selected profile into the post-install verification argv the plan publishes.""" + return [[interpreter, "-I", *(profile if item == "@profile" else item for item in step)] + for step in VALIDATION_STEPS[component]] + + +def _plan_commands(preflight: Any, plans: Any) -> list[tuple[str, dict[str, Any]]]: + located = [(f"preflight_commands[{index}]", item) + for index, item in enumerate(preflight if isinstance(preflight, list) else []) + if isinstance(item, dict)] + located.extend((f"{name}.commands[{index}]", item) + for name, component in (plans.items() if isinstance(plans, dict) else []) + if isinstance(component, dict) + for index, item in enumerate(component.get("commands") or []) + if isinstance(item, dict)) + return located + + +def _execution_batches(preflight: list[dict[str, Any]], plans: dict[str, dict[str, Any]], + request_intent: str) -> list[dict[str, Any]]: + if request_intent not in REQUEST_INTENTS: + raise ValueError(f"unsupported request intent: {request_intent}") + preauthorized = request_intent == "setup-install" + batches: dict[str, dict[str, Any]] = {} + for location, item in _plan_commands(preflight, plans): + if not (item["mutates"] or item["requires_network"]): + item.update({"authorization_status": "not-required", "new_approval_required": False}) + continue + item["authorization_status"] = "preauthorized" if preauthorized else "requires-confirmation" + item["new_approval_required"] = not preauthorized + group = item["execution_group"] + batch = batches.setdefault(group, { + "id": group, "authorization_status": item["authorization_status"], "phases": [], + "new_approval_required": item["new_approval_required"], "execution": "sequential", + "reason": APPROVAL_BATCH_REASONS[group], "command_locations": []}) + batch["command_locations"].append(location) + batch["phases"].append(item["phase"]) + return list(batches.values()) + + +def _component(name: str, observed: str) -> dict[str, Any]: + return {"name": name, "observed_status": observed, "plan_status": "blocked", + "selected_route": None, "artifact": {}, "blocking_reasons": [], "commands": [], + "validation_commands": []} + + +def _blocked(result: dict[str, Any], route_id: str, *detail: Any) -> dict[str, Any]: + route, reason = BLOCKED_ROUTES[route_id] + result.update({"plan_status": "blocked", "selected_route": route, "commands": [], + "blocking_reasons": [reason.format(*detail)], "validation_commands": []}) + return result + + +def _apt_install(result: dict[str, Any], apt: dict[str, Any], package: str, *, phase: str, + group: str, refresh_done: bool, reinstall: bool = False, + missing_id: str = "apt-candidate", + nvidia_release: bool = False) -> tuple[list[dict[str, Any]], str] | None: + record = _field(apt, ("candidates", package)) + record = record if isinstance(record, dict) else {} + candidate = record.get("candidate") + if not isinstance(candidate, str) or not candidate: + if refresh_done or not apt.get("source_files"): + _blocked(result, missing_id, package) + else: + result.update({"plan_status": "ready_to_review", "blocking_reasons": [], + "commands": [], "validation_commands": [], + "selected_route": "apt-metadata-refresh-then-replan", + "required_preflight_actions": [REFRESH_PREFLIGHT_ID]}) + result["artifact"]["apt_metadata"] = {"source_files": apt.get("source_files", []), + "candidate_state": "unverified_until_refresh"} + return None + argv, apt_cache = record.get("query_argv"), apt.get("apt_cache") + origin = None if (record.get("query_status") not in {"ok", "success"} + or record.get("query_exit_code") != 0 or not isinstance(apt_cache, str) + or argv != [apt_cache, "policy", package]) else ( + public_candidate_binding(record) if nvidia_release + else authenticated_candidate_binding(record)) + if origin is None: + _blocked(result, "apt-origin" if nvidia_release else "base-apt-origin", + package, candidate) + return None + if origin.get("architecture") not in ("aarch64", "arm64"): + _blocked(result, "apt-architecture", package, origin.get("architecture")) + return None + result["artifact"].setdefault("apt_installs", {})[package] = { + "package": package, "candidate": candidate, "selected_origin": origin, + "candidate_origin_evidence": {key: record.get(key) for key in ( + "query_status", "query_argv", "query_exit_code", "candidate_origins")}} + request = ["--package-spec", f"{package}={candidate}", *(["--reinstall"] if reinstall else [])] + receipt = f"{phase}-apt-simulation.json" + return [_command(f"{phase}-preview", [*TRANSACTION_PREFIX, "preview", *request, + "--receipt", receipt], + mutates=False), + _command( + phase, + [*TRANSACTION_PREFIX, "apply", *request, "--receipt", receipt, + "--output", f"{phase}-apt-apply.json"], + mutates=True, network=True, privilege="root", group=group, + )], candidate + + +def _plan_native(environment: dict[str, Any], apt: dict[str, Any], *, refresh_done: bool, + fresh_setup: bool) -> dict[str, Any]: + surface, defects = normalized_native_surface(environment) + installed = surface["installed"] + package = _field(surface, ("package",)) + package = package if isinstance(package, dict) else {} + package_installed = (package.get("name") == NVIDIA_PACKAGE + and package.get("status") == "installed") + result = _component("native-sdk", "installed" if package_installed else "missing") + if defects: + return _blocked(result, "native-surface", "; ".join(defects)) + try: + interpreter = system_executable("python3") + except (FileNotFoundError, OSError) as exc: + return _blocked(result, "interpreter", exc) + if installed: + cuda_version = str(_field(surface, ("cuda", "version")) or "") + result["artifact"] = { + "package": package.get("name"), "package_version": package.get("version"), + "sdk_root": surface.get("sdk_root"), "cuda": surface.get("cuda"), + "expected_release_family": NATIVE_RELEASE_FAMILY, + "build_prerequisites": surface.get("build_prerequisites")} + if not cuda_at_least_13(cuda_version): + return _blocked(result, "native-cuda", cuda_version) + if not native_sdk_release(package.get("version")): + return _blocked(result, "native-installed", package.get("version")) + if not fresh_setup: + result.update({"plan_status": "installed", + "selected_route": "existing-release-apt-installation", + "validation_commands": _validation_commands("native-sdk", interpreter)}) + return result + commands: list[dict[str, Any]] = [] + if not isinstance(_field(surface, ("tools", "pkg_config")), dict): + selected = _apt_install(result, apt, "pkg-config", phase="install-native-pkg-config", + group="native-sdk-install", refresh_done=refresh_done) + if selected is None: + return result + commands.extend(selected[0]) + selected = _apt_install(result, apt, NVIDIA_PACKAGE, phase="install-native-sdk", + group="native-sdk-install", refresh_done=refresh_done, + reinstall=package_installed, missing_id="native-candidate", + nvidia_release=True) + if selected is None: + return result + sdk_commands, candidate = selected + if not native_sdk_release(candidate): + return _blocked(result, "native-release", candidate) + result["artifact"].update({"package": NVIDIA_PACKAGE, "candidate": candidate, + "expected_release_family": NATIVE_RELEASE_FAMILY}) + commands.extend(sdk_commands) + result.update({"plan_status": "ready_to_review", "commands": commands, + "selected_route": ("configured-release-apt-fresh-reinstall" if fresh_setup + and package_installed else "configured-release-apt-repair" + if package_installed else "configured-release-apt"), + "validation_commands": _validation_commands("native-sdk", interpreter)}) + return result + + +def _plan_pynvc(environment: dict[str, Any], apt: dict[str, Any], venv: Path | None, *, + refresh_done: bool, fresh_setup: bool, profile: str) -> dict[str, Any]: + surface, defects = normalized_pynvc_surface(environment) + installed = surface["installed"] + result = _component("pynvc", "installed" if installed else "missing") + if defects: + return _blocked(result, "pynvc-surface", "; ".join(defects)) + required = profile_dependencies(profile) + ready = bool(installed and surface.get("version") == PYNVC_VERSION and all( + _field(surface, ("dependencies", name, "ready")) is True for name in required)) + observed = str(surface.get("sys_prefix") or "").rstrip("/") + result["artifact"] = {"requested_spec": PYNVC_SPEC, "verification_profile": profile, + "runtime_specs": {name: RUNTIME_SPECS[name] for name in required}, + "observed_version": surface.get("version"), "observed_ready": ready, + "observed_prefix": surface.get("sys_prefix")} + if venv is None and fresh_setup: + return _blocked(result, "pynvc-fresh-venv") + if venv is None: + venv = Path(observed) if ready and observed else DEFAULT_PYNVC_VENV + resolved_venv = durable_venv_path(venv) + if resolved_venv is None: + return _blocked(result, "pynvc-venv-location", str(venv)) + target = PurePosixPath(resolved_venv.as_posix()) + planned, target_python = str(target), str(target / "bin" / "python") + same_target = bool(observed and observed == planned.rstrip("/")) + if ready and not fresh_setup: + if not same_target: + return _blocked(result, "pynvc-reuse", observed) + result.update({"plan_status": "installed", "selected_route": + "existing-isolated-environment", "validation_commands": + _validation_commands("pynvc", str(surface["interpreter"]), profile)}) + return result + if installed and same_target: + return _blocked(result, "pynvc-new-venv", observed) + if os.path.lexists(planned): + return _blocked(result, "pynvc-venv-exists", planned) + cuda, python = normalized_cuda_evidence(environment), normalized_python_bootstrap(environment) + python_errors = ["python must be an object"] if not isinstance(python, dict) else [ + f"python.{name} must be ok or missing, saw {python.get(name)!r}" + for name in ("venv_module", "development_headers") + if python.get(name) not in {"ok", "missing"}] + for errors, route in ((_cuda_errors(cuda), "cuda-evidence"), + (python_errors, "python-evidence")): + if errors: + return _blocked(result, route, "; ".join(errors)) + if cuda["status"] == "absent": + toolkit_candidate = _field(apt, ("candidates", CUDA_PACKAGE, "candidate")) + packages = pycuda_bootstrap_packages(toolkit_candidate) + if not packages: + return _blocked(result, "cuda-minimum", toolkit_candidate) + commands: list[dict[str, Any]] = [] + for package in packages: + phase = ("install-pynvc-cuda-curand-dev" if package.startswith("libcurand") + else "install-pynvc-cuda-minimal-build") + selected = _apt_install(result, apt, package, phase=phase, group="pynvc-bootstrap", + refresh_done=refresh_done, nvidia_release=True) + if selected is None: + return result + planned, candidate = selected + if package.startswith("cuda-minimal-build-") and ( + cuda_build_package(candidate) != package): + return _blocked(result, "cuda-minimum", candidate) + commands.extend(planned) + result.update({"plan_status": "ready_to_review", "selected_route": CUDA_BOOTSTRAP_ROUTE, + "requires_reprobe_and_replan": True, "commands": commands, + "validation_commands": []}) + return result + try: + base = system_executable("python3") + identity = file_identity(base, label="base interpreter") + except (FileNotFoundError, OSError, ValueError) as exc: + return _blocked(result, "interpreter", exc) + system_bin = Path("/usr/bin") + try: + for name in ("gcc", "g++", "make"): + if (system_bin / name).resolve(strict=True) != Path(system_executable(name)): + raise FileNotFoundError(f"{system_bin / name} is not the canonical {name}") + except (FileNotFoundError, OSError) as exc: + return _blocked(result, "build-tools", exc) + prefixes = {name: list(cuda["environment_prefixes"][name]) for name in BUILD_ENV_NAMES} + prefixes["PATH"] = list(dict.fromkeys([*prefixes["PATH"], str(system_bin)])) + build_env = [f"{name}={':'.join(str(value) for value in prefixes[name])}" + for name in BUILD_ENV_NAMES] + result["artifact"].update({ + "cuda_build_environment": {"version": cuda["version"], "root": cuda["root"], + "environment": build_env, "scope": "PyCUDA build only"}, + "fresh_venv": { + "base_executable": base, "base_interpreter_identity": identity, "target": planned, + "target_python": target_python, "creation_report": CLEAN_VENV_REPORT, + "target_must_not_exist": True, "reuse_allowed": False, "delete_existing": False, + "fresh_setup_required": fresh_setup}}) + bootstrap: list[dict[str, Any]] = [] + for package, state, phase in ((VENV_PACKAGE, python["venv_module"], "install-python-venv"), + (DEV_PACKAGE, python["development_headers"], + "install-python-development-headers")): + if state != "missing": + continue + selected = _apt_install(result, apt, package, phase=phase, group="pynvc-bootstrap", + refresh_done=refresh_done) + if selected is None: + return result + bootstrap.extend(selected[0]) + result["artifact"].setdefault("repair_reasons", []).append( + f"{package} is absent, so the plan installs its exact authenticated candidate before" + " creating the isolated environment.") + specs = [PYNVC_SPEC, RUNTIME_SPECS["numpy"], *SUPPORT_SPECS] + resolve = [target_python, "-I", "-m", "pip", "--isolated", "install", "--no-input", + "--disable-pip-version-check", "--no-cache-dir", "--ignore-installed", "--dry-run", + "--report"] + lock = [target_python, "-I", PIP_SCRIPT, "materialize-apply", "--report", BINARY_REPORT, + *(["--report", TORCH_REPORT] if "torch" in required else [])] + for requirement in [*specs, *(RUNTIME_SPECS[name] for name in required if name != "numpy")]: + lock.extend(["--require", requirement]) + lock.extend(["--source-require", RUNTIME_SPECS["pycuda"], "--lock", + "nvcodec-pip.requirements.lock", "--manifest", "nvcodec-pip-lock-manifest.json", + "--install-report", "nvcodec-pip-locked-install.json"]) + for assignment in build_env: + lock.extend(["--build-env", assignment]) + result["commands"] = [*bootstrap, + _command("create-venv", + [base, "-I", PIP_SCRIPT, "create-venv", "--target", planned, + "--output", CLEAN_VENV_REPORT], mutates=True, + group="pynvc-bootstrap"), + _command("resolve-public-binary-wheels", + [*resolve, BINARY_REPORT, "--only-binary=:all:", *specs], + mutates=False, + network=True, group="pynvc-resolution"), + *([_command("resolve-cuda-pytorch-wheel", + [*resolve, TORCH_REPORT, "--only-binary=:all:", + "--extra-index-url", + "https://download.pytorch.org/whl/cu130", + RUNTIME_SPECS["torch"]], + mutates=False, network=True, group="pynvc-resolution")] + if "torch" in required else []), + _command("install-locked-python-artifacts", lock, mutates=True, + network=True, + group="pynvc-install")] + result.update({"plan_status": "ready_to_review", + "selected_route": ("apt-bootstrap-then-fresh-isolated-environment" if bootstrap + else "fresh-isolated-environment"), + "validation_commands": _validation_commands("pynvc", target_python, profile)}) + return result + + +def _receipt_binding(receipt_path: Path, environment: dict[str, Any], + source_path: Path) -> dict[str, Any]: + def moment(value: Any) -> datetime | None: + try: + return datetime.fromisoformat(str(value).replace("Z", "+00:00")) + except ValueError: + return None + resolved = receipt_path.expanduser().resolve() + errors, identity, receipt = [], {}, {} + if resolved.parent != source_path.parent or resolved.name != REFRESH_RECEIPT_NAME: + errors.append("refresh receipt must be the exact sibling of the refreshed probe artifact") + try: + identity = file_identity(resolved, label="refresh receipt") + parsed = read_json(resolved) + receipt = parsed if isinstance(parsed, dict) else {} + except (OSError, ValueError) as exc: + errors.append(f"refresh receipt could not be read: {exc}") + errors.extend(f"refresh receipt {name} is not {expected!r}" + for name, expected in (("kind", "nvcodec-apt-refresh-receipt"), + ("schema_version", APT_SCHEMA_VERSION), ("exit_code", 0)) + if receipt.get(name) != expected) + ended, generated = moment(receipt.get("ended_at")), moment(environment.get("generated_at")) + if ended is None: + errors.append("refresh receipt ended_at is invalid") + elif generated is not None and ended > generated: + errors.append("refresh receipt must precede the refreshed environment probe") + return {"path": str(resolved), "sha256": identity.get("sha256"), "identity": identity, + "verified": not errors, "errors": errors} + + +def _target_contract(environment: dict[str, Any]) -> dict[str, Any]: + jetson = _field(environment, ("platform", "jetson_linux")) + record = _field(jetson, ("compatibility",)) + record = record if isinstance(record, dict) else {} + return {"compatible": record.get("status") == "compatible", "status": record.get("status"), + "minimum_version": record.get("minimum_version"), + "observed_version": _field(jetson, ("version",)), + "reason": record.get("reason") or "no live Jetson Linux release was observed"} + + +def _overall(states: list[Any]) -> str: + if "blocked" in states: + return "blocked" + if states and all(item == "installed" for item in states): + return "installed" + return "ready_to_review" + + +def build_plan(source_path: Path, components: list[str] | None = None, *, venv: Path | None = None, + request_intent: str = "plan-only", fresh_setup: bool = False, + profile: str = SMOKE_PROFILE, + apt_refresh_receipt: Path | None = None) -> dict[str, Any]: + resolved = source_path.expanduser().resolve() + environment = read_json(resolved) + if profile not in VERIFICATION_PROFILES: + raise PlanContractError(f"verification profile must be one of" + f" {list(VERIFICATION_PROFILES)}; saw {profile!r}") + if not isinstance(environment, dict) or environment.get("kind") != ENVIRONMENT_KIND: + raise ValueError(f"input is not an {ENVIRONMENT_KIND} artifact") + if environment.get("mode") != "live": + raise ValueError(f"source artifact must be a live {ENVIRONMENT_KIND}") + if (errors := environment_contract_errors( + environment, include_surface_values=False)): + raise PlanContractError(f"source artifact is not a live {ENVIRONMENT_KIND} " + f"{ENVIRONMENT_MAJOR}.x: " + "; ".join(errors)) + receipt = (_receipt_binding(apt_refresh_receipt, environment, resolved) + if apt_refresh_receipt is not None else None) + if receipt is not None and not receipt["verified"]: + raise PlanContractError("APT refresh receipt validation failed: " + + "; ".join(receipt["errors"])) + selected = components or [name for name in COMPONENTS + if environment["requested_runtime"] in (SURFACES[name], "both")] + ordered = [name for name in COMPONENTS if name in set(selected)] + candidates = _field(environment, ("installation", "apt")) + apt = candidates if isinstance(candidates, dict) else {} + done, plans = receipt is not None, {} + contract, live = _target_contract(environment), {} + if not contract["compatible"]: + plans = {name: _blocked(_component(name, "unknown"), "jetson-linux", contract["reason"]) + for name in ordered} + elif "native-sdk" in ordered: + live["native-sdk"] = _plan_native(environment, apt, refresh_done=done, + fresh_setup=fresh_setup) + if contract["compatible"] and "pynvc" in ordered: + live["pynvc"] = _plan_pynvc(environment, apt, venv, refresh_done=done, + fresh_setup=fresh_setup, profile=profile) + plans.update(live) + for name, component in plans.items(): + if component["plan_status"] == "blocked" and component["commands"]: + raise ValueError(f"{name} is blocked but still contains executable commands") + parked = [name for name, value in plans.items() if value.get("required_preflight_actions")] + preflight = [{**_command("refresh-metadata", + [*TRANSACTION_PREFIX, + "refresh", "--receipt", REFRESH_RECEIPT_NAME], + mutates=True, network=True, privilege="root", + group="apt-metadata-refresh"), + "id": REFRESH_PREFLIGHT_ID, "affected_components": parked, + "next_step": "Discard this plan, re-run the live probe, and regenerate the plan" + " before installing packages."}] if parked else [] + batches = _execution_batches(preflight, plans, request_intent) + preauthorized = request_intent == "setup-install" + terminal = [name for name in COMPONENTS + if plans.get(name, {}).get("plan_status") not in {None, "blocked"} + and not plans.get(name, {}).get("required_preflight_actions") + and plans.get(name, {}).get("selected_route") != CUDA_BOOTSTRAP_ROUTE] + plan = {"schema_version": SCHEMA_VERSION, "kind": PLAN_KIND, "mutated": False, + "generated_from": {**file_identity(resolved, label="source artifact"), + "kind": environment["kind"], "mode": environment["mode"], + "schema_version": environment["schema_version"], + "canonical_sha256": canonical_json_sha256(environment)}, + "plan_inputs": { + "environment": str(resolved), "components": ordered, + "request_intent": request_intent, + "venv": str(venv) if venv is not None else None, "fresh_setup": fresh_setup, + "verification_profile": profile, + "apt_refresh_receipt": (str(apt_refresh_receipt.expanduser().resolve()) + if apt_refresh_receipt is not None else None)}, + "setup_helpers": {name: file_identity(_SETUP_DIR / name, label=name) + for name in SETUP_HELPERS}, + "target_contract": contract, "unknowns": list(UNKNOWNS), + "verification_profile": profile, + "platform": environment["platform"], "selected_gpu": environment["selected_gpu"], + "requested_components": ordered, "execution_order": ordered, "fresh_setup": fresh_setup, + "request_intent": request_intent, "apt_refresh_completed": done, + "overall_status": _overall([value["plan_status"] for value in plans.values()]), + "new_approval_required": any(item["new_approval_required"] for item in batches), + "authorization_policy": { + "mode": "intent-scoped", "normal_plan_preauthorized": preauthorized, + "renew_on_drift": ["argv", "candidate", "dependency-set", "version", "origin", + "archive-hash", "setup-mode", "verification-profile"]}, + "execution_batches": batches, "apt_refresh_receipt": receipt, + "preflight_commands": preflight, "components": plans, + "post_install_validation_sequence": [step for name in terminal + for step in POST_INSTALL_STEPS[name]]} + plan["plan_digest"] = canonical_json_sha256(plan) + return plan + + +def _require_canonical_plan(reviewed: Any) -> dict[str, Any]: + if not isinstance(reviewed, dict) or reviewed.get("kind") != PLAN_KIND: + raise ValueError("reviewed artifact is not an nvcodec install plan") + digest = reviewed.get("plan_digest") + if canonical_json_sha256({key: value for key, value in reviewed.items() + if key != "plan_digest"}) != digest: + raise ValueError("reviewed plan digest does not cover its own content") + inputs = reviewed.get("plan_inputs") + if not isinstance(inputs, dict): + raise ValueError("reviewed plan carries no recorded planner inputs") + require_selected_surfaces_live(read_json(Path(str(inputs["environment"]))), + list(inputs["components"])) + receipt = inputs.get("apt_refresh_receipt") + fresh = build_plan(Path(str(inputs["environment"])), list(inputs["components"]), + venv=Path(str(inputs["venv"])) if inputs.get("venv") else None, + request_intent=str(inputs["request_intent"]), + fresh_setup=bool(inputs["fresh_setup"]), + profile=str(inputs.get("verification_profile") or FULL_PROFILE), + apt_refresh_receipt=Path(str(receipt)) if receipt else None) + if fresh["plan_digest"] != digest: + raise ValueError("reviewed plan is stale: the canonical in-process re-plan produced a" + " different digest. Re-probe, regenerate the plan, and review it again.") + return fresh + + +def _authorizing_plan(invocation: list[str]) -> dict[str, Any]: + directory = Path.cwd().resolve() + plan_path = directory / PLAN_ARTIFACT_NAME + if plan_path.is_symlink() or not plan_path.is_file(): + raise ValueError(f"transaction actions require the reviewed sibling plan {plan_path}") + plan = _require_canonical_plan(read_json(plan_path)) + receipt_path, binding = directory / REFRESH_RECEIPT_NAME, plan.get("apt_refresh_receipt") + if receipt_path.is_file() and invocation[0] != "refresh" and ( + not isinstance(binding, dict) or binding.get("path") != str(receipt_path) + or binding.get("sha256") != file_identity(receipt_path)["sha256"]): + raise ValueError("an APT refresh completed in this working directory; re-probe to a new" + " artifact, re-plan, and review the new plan before applying") + prefix = list(TRANSACTION_PREFIX) + matches = [item for _, item in _plan_commands(plan["preflight_commands"], plan["components"]) + if (argv := list(item.get("argv", [])))[:len(prefix)] == prefix + and argv[len(prefix):] == invocation] + if len(matches) != 1: + raise ValueError("this invocation is not exactly one command of the reviewed plan") + if matches[0]["mutates"] or matches[0]["requires_network"]: + if plan.get("request_intent") != "setup-install": + raise ValueError("plan-only intent does not authorize a mutating transaction") + if matches[0]["authorization_status"] != "preauthorized": + raise ValueError("this transaction is not preauthorized by the reviewed plan") + return plan + + +def _planned_origin(plan: dict[str, Any], package: str) -> dict[str, Any]: + for component in plan["components"].values(): + artifact = component.get("artifact", {}) + installs = artifact.get("apt_installs") if isinstance(artifact, dict) else None + for record in (artifact, *(installs.values() if isinstance(installs, dict) else ())): + if (isinstance(record, dict) and record.get("package") == package + and isinstance(record.get("selected_origin"), dict)): + return record["selected_origin"] + raise ValueError(f"the reviewed plan binds no authenticated public origin for {package}") + + +def _transaction_context(args: argparse.Namespace, + invocation: list[str]) -> tuple[dict[str, Any], dict[str, Any]]: + supplied = bool(args.expected_origin_uri or args.expected_suite_component + or args.source_sha256 or args.source_entry) + if supplied and not (args.expected_origin_uri and args.expected_suite_component + and args.source_sha256): + raise ValueError("APT policy binding requires origin, suite, and source hashes") + if len(args.source_sha256) != len(args.source_entry): + raise ValueError("every --source-sha256 requires one paired --source-entry") + sources = [] + for value, entry in zip(args.source_sha256, args.source_entry, strict=True): + path, separator, digest = value.rpartition("=") + source = Path(path).expanduser() + if (not separator or not source.is_absolute() + or not re.fullmatch(r"[0-9a-f]{64}", digest)): + raise ValueError("--source-sha256 requires absolute-path=64-lowercase-hex") + if source.is_symlink() or not source.is_file(): + raise ValueError("--source-sha256 path must be a regular non-symlink source file") + sources.append({"path": str(source.resolve()), "sha256": digest, + "entry": json.loads(entry)}) + plan = _authorizing_plan(invocation) + origin = _planned_origin(plan, args.package_spec.split("=", 1)[0]) + + def key(item: dict[str, Any]) -> tuple[str, str]: + return str(item.get("path")), _dumps(item.get("entry")) + if supplied and (args.expected_origin_uri != origin.get("uri") + or args.expected_suite_component != origin.get("suite_component") + or sorted(sources, key=key) != sorted(origin.get("configured_sources", []), + key=key)): + raise ValueError("APT origin/source assertions differ from the canonical reviewed plan") + return plan, origin + + +def _apt_argv(action: str, package_spec: str = "", reinstall: bool = False) -> list[str]: + apt_get, sudo = system_executable("apt-get"), [system_executable("sudo"), "-n"] + if action == "refresh": + return [*sudo, apt_get, "-o", "APT::Update::Error-Mode=any", "update"] + extra = ["--reinstall"] if reinstall else [] + if action == "simulate": + return [apt_get, "--simulate", "--no-remove", "install", *extra, package_spec] + return [*sudo, apt_get, "--no-remove", "install", "-y", *extra, package_spec] + + +def _transaction_summary(stdout: str, stderr: str) -> dict[str, Any]: + buckets: dict[str, list[dict[str, str]]] = {"installed": [], "removed": [], "configured": []} + patterns = (("Inst", "installed"), ("Unpacking", "installed"), ("Remv", "removed"), + ("Removing", "removed"), ("Conf", "configured"), ("Setting up", "configured")) + for line in stdout.splitlines(): + for keyword, bucket in patterns: + if (match := re.match(rf"^{keyword}\s+(\S+)\s*(.*)$", line)): + buckets[bucket].append({"package": match.group(1), + "detail": match.group(2).strip()}) + break + combined = f"{stdout}\n{stderr}" + if re.search(r"The following packages will be REMOVED", combined, re.I): + buckets["removed"].append({"package": "", "detail": "removal summary"}) + return {**buckets, "downgrade_detected": bool( + re.search(r"\bDOWNGRADED\b|\bdowngrad(?:e|ed|ing)\b", combined, re.I))} + + +def _policy_origins(stdout: str, candidate: str) -> list[dict[str, Any]]: + lines = stdout.splitlines() + header = re.compile(r"^\s*(?:\*{3}\s+)?\S+\s+\d+\s*$") + row = re.compile(r"^\s*(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+Packages\s*$") + selected = rf"^\s*(?:\*{{3}}\s+)?{re.escape(candidate)}\s+\d+\s*$" + start = next((index for index, line in enumerate(lines) + if re.match(selected, line)), None) if candidate else None + origins: list[dict[str, Any]] = [] + for line in lines[start + 1:] if start is not None else []: + if header.match(line): + break + if (match := row.match(line)): + origins.append({"priority": int(match.group(1)), "uri": match.group(2).rstrip("/"), + "suite_component": match.group(3), "architecture": match.group(4)}) + return origins + + +def _policy_evidence(package: str, expected: str, origin: dict[str, Any], + prefix: Path) -> dict[str, Any]: + policy, config = _run([system_executable("apt-cache"), "policy", package]), _run( + [system_executable("apt-config"), "dump"]) + match = re.search(r"^\s*Candidate:\s*(\S+)", policy["stdout"], re.MULTILINE) + observed = match.group(1) if match and match.group(1) != "(none)" else None + sources = [] + for item in origin.get("configured_sources", []): + raw = Path(str(item.get("path"))).read_bytes() + entries = parse_apt_sources(item.get("path"), raw.decode("utf-8", errors="replace")) + sources.append({"path": item.get("path"), "expected_sha256": item.get("sha256"), + "sha256": sha256_bytes(raw), "expected_entry": item.get("entry"), + "matches": sha256_bytes(raw) == item.get("sha256"), + "entry_match_count": sum(1 for e in entries if e == item.get("entry")), + "trust_bypass": any(e.get("trust_bypass") is True for e in entries)}) + bypasses = [name for name in INSECURE_APT_PATTERNS if re.search(name, config["stdout"], re.I)] + clean = bool(sources and not bypasses and all( + item["matches"] and item["entry_match_count"] == 1 and not item["trust_bypass"] + for item in sources)) + live = _policy_origins(policy["stdout"], observed or "") + binder = (public_candidate_binding if package in {NVIDIA_PACKAGE, CUDA_PACKAGE} + or package.startswith(("cuda-minimal-build-", "libcurand-dev-")) + else authenticated_candidate_binding) + bound = binder({ + "query_status": "ok", "candidate": observed, "candidate_origins": [ + {**item, "authentication": "apt-signature-chain", + "configured_source_sha256": [entry["sha256"] for entry in sources], + "configured_sources": origin.get("configured_sources", [])} + for item in live]}) if clean and observed else None + verified = bool(policy["exit_code"] == 0 and config["exit_code"] == 0 and observed == expected + and bound is not None and bound.get("uri") == origin.get("uri") + and bound.get("suite_component") == origin.get("suite_component")) + keys = ("uri", "suite_component") + return {"status": "verified" if verified else "failed", "package": package, + "expected_candidate": expected, "observed_candidate": observed, "verified": verified, + "expected_origin": {key: origin.get(key) for key in keys}, + "bound_origin": {key: bound.get(key) for key in keys} if bound else None, + "candidate_origins": live, "configured_sources": sources, + "configuration_bypasses": bypasses, + "policy_command": _observed(policy, prefix, "policy"), + "config_command": _observed(config, prefix, "apt-config")} + + +def _simulation(package_spec: str, reinstall: bool, prefix: Path, + origin: dict[str, Any]) -> tuple[dict[str, Any], int]: + package, _, candidate = package_spec.partition("=") + policy = _policy_evidence(package, candidate, origin, prefix) + evidence = _run(_apt_argv("simulate", package_spec, reinstall)) + transaction = _transaction_summary(evidence["stdout"], evidence["stderr"]) + safe = bool(policy["verified"] and evidence["exit_code"] == 0 and not transaction["removed"] + and not transaction["downgrade_detected"]) + return {"schema_version": APT_SCHEMA_VERSION, "kind": "nvcodec-apt-simulation-receipt", + "package_spec": package_spec, "reinstall": reinstall, "policy": policy, + **_observed(evidence, prefix), "simulation_argv": evidence["argv"], + "transaction": transaction, "safe_to_apply": safe}, 0 if safe else 2 + + +def refresh(receipt_path: Path) -> int: + receipt_path = transaction_sibling_path(receipt_path, "refresh receipt", ("",)) + _authorizing_plan(sys.argv[1:]) + evidence = _run(_apt_argv("refresh")) + return _emit(receipt_path, { + "schema_version": APT_SCHEMA_VERSION, "kind": "nvcodec-apt-refresh-receipt", + **_observed(evidence, receipt_path), + "next_step": "The candidate set may have changed. Discard this plan, re-run" + " probe_nvcodec.py to a new sibling artifact, re-plan with" + " --apt-refresh-receipt, and review the new plan before installing."}, + 0 if evidence["exit_code"] == 0 else 2) + + +def preview(args: argparse.Namespace, invocation: list[str]) -> int: + receipt_path = transaction_sibling_path( + args.receipt, "simulation receipt", ("policy", "apt-config", "")) + _, origin = _transaction_context(args, invocation) + receipt, code = _simulation(args.package_spec, args.reinstall, receipt_path, origin) + return _emit(receipt_path, receipt, code) + + +def apply(args: argparse.Namespace, invocation: list[str]) -> int: + receipt_path = transaction_sibling_path(args.receipt, "receipt") + output = transaction_sibling_path( + args.output, "report", ("immediate-simulation.policy", + "immediate-simulation.apt-config", + "immediate-simulation", "")) + if output == receipt_path: + raise ValueError("APT apply report and reviewed receipt must be distinct sibling files") + plan, origin = _transaction_context(args, invocation) + reviewed_identity = file_identity(receipt_path, label="reviewed receipt") + receipt = read_json(receipt_path) + if not isinstance(receipt, dict) or receipt.get("kind") != "nvcodec-apt-simulation-receipt": + raise ValueError("the reviewed receipt is not an nvcodec APT simulation receipt") + if (receipt.get("package_spec") != args.package_spec + or receipt.get("reinstall") is not args.reinstall + or receipt.get("exit_code") != 0 or receipt.get("safe_to_apply") is not True): + raise ValueError("reviewed APT simulation receipt does not match this safe transaction") + repeated, code = _simulation(args.package_spec, args.reinstall, + output.with_name(output.name + ".immediate-simulation"), origin) + if code != 0: + raise ValueError("immediate pre-apply APT simulation is unsafe") + for name in ("package_spec", "reinstall", "simulation_argv", "transaction"): + if repeated[name] != receipt.get(name): + raise ValueError(f"APT transaction drifted after reviewed simulation: {name}") + if any(repeated["policy"][name] != receipt.get("policy", {}).get(name) + for name in POLICY_FACTS): + raise ValueError("APT candidate/origin/source policy drifted after review") + evidence = _run(_apt_argv("apply", args.package_spec, args.reinstall)) + transaction = _transaction_summary(evidence["stdout"], evidence["stderr"]) + return _emit(output, {"schema_version": APT_SCHEMA_VERSION, "kind": "nvcodec-apt-apply-report", + "package_spec": args.package_spec, "reinstall": args.reinstall, + "authorizing_plan_digest": plan["plan_digest"], + "reviewed_receipt": reviewed_identity, + "immediate_simulation": repeated, "transaction": transaction, + "safe": (safe := bool( + evidence["exit_code"] == 0 and not transaction["removed"] + and not transaction["downgrade_detected"])), + **_observed(evidence, output)}, 0 if safe else 2) + + +def validate_plan(data: Any) -> list[str]: + if not isinstance(data, dict): + return ["install plan must be an object"] + if data.get("schema_version") != SCHEMA_VERSION: + return [f"schema_version {SCHEMA_VERSION} is required; {data.get('schema_version')!r} is" + " historical-review-only and cannot validate for execution"] + components = data.get("components") if isinstance(data.get("components"), dict) else {} + renew = data.get("request_intent") != "setup-install" + states = [value.get("plan_status") for value in components.values() if isinstance(value, dict)] + errors = [message for ok, message in ( + (data.get("kind") == PLAN_KIND, f"kind must be {PLAN_KIND}"), + (data.get("request_intent") in REQUEST_INTENTS, + "request_intent must be plan-only or setup-install"), + (isinstance(data.get("components"), dict), "components must be an object"), + (len(states) == len(components), "every component must be an object"), + (not states or data.get("overall_status") == _overall(states), + f"overall_status should be {_overall(states)} for component states {states}"), + (canonical_json_sha256({key: value for key, value in data.items() + if key != "plan_digest"}) == data.get("plan_digest"), + "plan_digest does not cover its own content")) if not ok] + errors.extend(f"{name} is blocked but still contains executable commands" + for name, value in components.items() + if isinstance(value, dict) and value.get("plan_status") == "blocked" + and value.get("commands")) + for location, item in _plan_commands(data.get("preflight_commands"), components): + expected = "requires-confirmation" if renew else "preauthorized" + if (item.get("mutates") is True or item.get("requires_network") is True) and ( + item.get("authorization_status") != expected + or item.get("new_approval_required") is not renew): + errors.append(f"{location} must be {expected} with new_approval_required={renew}") + try: + _require_canonical_plan(data) + except (OSError, ValueError) as exc: + errors.append(f"canonical re-plan rejected this plan: {exc}") + return errors + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__, epilog="'validate PLAN' checks" + " a plan; refresh/preview/apply run APT beside the plan.") + parser.add_argument("environment", type=Path, + help=f"{ENVIRONMENT_KIND} {ENVIRONMENT_MAJOR}.x JSON from probe_nvcodec.py") + parser.add_argument("--component", action="append", choices=COMPONENTS, dest="components", + help="surface to plan; repeatable. Defaults to requested_runtime.") + parser.add_argument("--venv", type=Path, default=None, help="Isolated Python environment" + " target. Omitted, a complete observed environment is reused, else the" + " default new target. A different path needs --fresh-setup and a new one.") + parser.add_argument("--request-intent", choices=REQUEST_INTENTS, default="plan-only", + help="Use setup-install only when installation was explicitly requested") + parser.add_argument("--apt-refresh-completed", action="store_true", + help="Compatibility assertion; requires --apt-refresh-receipt") + parser.add_argument("--apt-refresh-receipt", type=Path, help="Successful nvcodec-apt-refresh" + "-receipt JSON; must be the sibling of the post-refresh probe artifact") + parser.add_argument("--fresh-setup", action="store_true", help="Require each surface's fresh" + " action, without uninstalling working base packages") + parser.add_argument("--profile", choices=VERIFICATION_PROFILES, default=SMOKE_PROFILE, + help="Official-sample verification profile the Python environment is" + " provisioned for. The default installs no Torch; full-samples adds" + " it for the sample routes that import it.") + parser.add_argument("--output", type=Path, + help="Also write the plan here; an existing path is never overwritten") + return parser + + +def plan_main(argv: list[str]) -> int: + args = build_parser().parse_args(argv) + if args.apt_refresh_completed and args.apt_refresh_receipt is None: + raise PlanContractError("--apt-refresh-completed requires --apt-refresh-receipt from the" + " successful refresh action and cannot stand alone") + plan = build_plan(args.environment, args.components, venv=args.venv, + request_intent=args.request_intent, fresh_setup=args.fresh_setup, + apt_refresh_receipt=args.apt_refresh_receipt, profile=args.profile) + if args.output: + write_new_json(args.output, plan) + print(_dumps(plan)) + return 2 if plan["overall_status"] == "blocked" else 0 + + +def validate_main(argv: list[str]) -> int: + parser = argparse.ArgumentParser( + prog=f"{Path(__file__).name} validate", + description=("Validate one reviewed nvcodec-install-plan. Non-mutating, but it re-plans" + " canonically, so the probe artifact named in plan_inputs must be readable.")) + parser.add_argument("plan", type=Path, help="nvcodec-install-plan JSON to validate") + data = read_json(parser.parse_args(argv).plan) + errors = validate_plan(data) + print(_dumps({"schema_version": VALIDATION_SCHEMA, "errors": errors, "valid": not errors, + "kind": "nvcodec-install-plan-validation", "overall_status": ( + data.get("overall_status") if isinstance(data, dict) else None)})) + return 0 if not errors else 2 + + +def transaction_main(argv: list[str]) -> int: + parser = argparse.ArgumentParser(prog=Path(__file__).name, + description="Execute one planned APT transaction.") + subparsers = parser.add_subparsers(dest="action", required=True) + subparsers.add_parser("refresh").add_argument("--receipt", type=Path, required=True) + for name in ("preview", "apply"): + sub = subparsers.add_parser(name) + sub.add_argument("--package-spec", required=True) + sub.add_argument("--reinstall", action="store_true") + sub.add_argument("--expected-origin-uri") + sub.add_argument("--expected-suite-component") + sub.add_argument("--source-sha256", action="append", default=[]) + sub.add_argument("--source-entry", action="append", default=[]) + sub.add_argument("--receipt", type=Path, required=True) + if name == "apply": + sub.add_argument("--output", type=Path, required=True) + args = parser.parse_args(argv) + if args.action == "refresh": + return refresh(args.receipt) + if not re.fullmatch(r"[^=\s]+=[^=\s]+", args.package_spec): + raise ValueError("--package-spec must be exact package=version") + if args.action == "preview": + return preview(args, argv) + return apply(args, argv) + + +def main() -> int: + action = sys.argv[1] if len(sys.argv) > 1 else "" + validating, status, code = action == "validate", "invalid", 2 + try: + require_isolated() + if validating: + return validate_main(sys.argv[2:]) + if action in TRANSACTION_ACTIONS: + return transaction_main(sys.argv[1:]) + return plan_main(sys.argv[1:]) + except PlanContractError as exc: + error = exc + except Exception as exc: # pylint: disable=broad-exception-caught + error, status, code = exc, "error", 3 + bare = action in TRANSACTION_ACTIONS # the APT actions keep exactly two keys, never a superset + if validating or bare: # both always terminate at rc 3; only planning reports invalid/rc 2 + status, code = "error", 3 + print(_dumps({"status": status, "error": str(error)} if bare else { + "schema_version": VALIDATION_SCHEMA if validating else SCHEMA_VERSION, "status": status, + "kind": f"{PLAN_KIND}-validation-error" if validating else f"{PLAN_KIND}-error", + "error": str(error)})) + return code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-setup/scripts/setup/probe_nvcodec.py b/skills/jetson-video-setup/scripts/setup/probe_nvcodec.py new file mode 100644 index 00000000..e936eefe --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/probe_nvcodec.py @@ -0,0 +1,998 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Emit the frozen read-only ``nvcodec-environment`` 1.2 artifact. Native/PyNv remain +independent; wheel RECORD attestation stays in verify_pynvc_sample.py.""" + +# pylint: disable=missing-function-docstring + +from __future__ import annotations + +import argparse +import importlib +import importlib.metadata +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +_SETUP_DIR = Path(__file__).resolve().parent +if str(_SETUP_DIR) not in sys.path: + sys.path.insert(0, str(_SETUP_DIR)) + +# pylint: disable=wrong-import-position +from setup_contract import ( # noqa: E402 + AUXILIARY_MODULE, AUXILIARY_ROLE, AUXILIARY_STEM, MINIMUM_JETSON_LINUX, + MINIMUM_VERSION_TEXT, PRIMARY_STEMS, PYNVC_PACKAGE, + associated_extensions, environment_contract_errors, extension_stem, + file_identity, + owned_extension, parse_apt_sources, parse_jetson_release, public_candidate_binding, + pycuda_bootstrap_packages, + query_pynvc_capabilities, read_json, require_isolated, root_evidence, run_command, sha256_bytes, + surface_contract_errors, system_executable, utc_now, write_new_json, +) +from verify_pynvc_sample import RegistryNotReady, load_pynvc_registry # noqa: E402 + +# pylint: enable=wrong-import-position + +KIND, SCHEMA_VERSION, SURFACES = "nvcodec-environment", "1.2", ("native", "pynvc") +ERROR_KIND = "nvcodec-environment-error" +VALIDATION_VERSION, VALIDATION_KIND = "1.0", "nvcodec-environment-validation" +VALIDATION_ERROR_KIND = "nvcodec-environment-validation-error" +NATIVE_PACKAGE, NATIVE_SDK_ROOT = "nvidia-video-codec-sdk", "/opt/nvidia/video-codec-sdk" +_TOOL_ALIASES = {"cxx": "g++", "pkg_config": "pkg-config"} +CMAKE_GENERATORS = (("ninja", "Ninja"), ("make", "Unix Makefiles")) +CODECS = ("h264", "hevc", "av1") +APPDEC_CONSUMER = "Samples/AppDecode/AppDec" +APPDEC_MODULES = ("libavcodec", "libavformat", "libavutil", "libswresample") +PYCUDA_VERSION, NUMPY_MINIMUM = "2026.1", (1, 24) +TORCH_VERSION, TORCH_CUDA_BUILD = "2.9.1+cu130", "13.0" + +_RELEASE_SOURCE = Path("/etc/nv_tegra_release") +_APT_SOURCES_LIST = Path("/etc/apt/sources.list") +_APT_SOURCES_DIR = Path("/etc/apt/sources.list.d") +_APT_BYPASS = re.compile( + r"(?:Acquire::Allow(?:Insecure|Weak|DowngradeToInsecure)Repositories" + r"|APT::Get::AllowUnauthenticated)\s+\"?(?:1|true|yes)\"?", re.I) +_POLICY_FIELD = re.compile(r"^\s*(Installed|Candidate):[ \t]*(\S+)[ \t]*$", re.M) +_POLICY_ORIGIN = re.compile(r"^\s+(\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+Packages\s*$") +_POLICY_VERSION = re.compile(r"^\s+(?:\*\*\*\s+)?(\S+)\s+\d+\s*$") +_NVCC_RELEASE = re.compile(r"\brelease\s+(\d+(?:\.\d+)+)") +_VERSION_TOKEN = re.compile(r"\d+(?:\.\d+)+") +_DIGITS = re.compile(r"\d+") +_CUDA_PREFIXES = {"PATH": ("bin",), "CPATH": ("include",), "LIBRARY_PATH": ("lib64", "lib")} +_COMMAND_EVIDENCE: list[dict[str, Any]] = [] +_DEPENDENCIES: dict[str, Any] = {} +_PROBE_WARNING = ("this probe executes no encode or decode operation; API capability fields" + " are not operational verification") +_PYTHON_PROBE = """ +import json, os, sysconfig +try: + import ensurepip, venv; venv.EnvBuilder(with_pip=True); usable = True +except BaseException: + usable = False +include = os.path.join(sysconfig.get_paths()["include"], "Python.h") +print(json.dumps({"venv": usable, "headers": os.path.isfile(include)})) +""" +_COMMAND_TIMEOUT = 30 +_DELEGATE_TIMEOUT = 900 +_MAX_ARTIFACT_BYTES = 4 * 1024 * 1024 + + +class _DuplicateKeyError(ValueError): + """A strict-JSON validation defect that keeps the legacy negative-result envelope.""" + + +def _run(argv: list[str], timeout: int = _COMMAND_TIMEOUT) -> dict[str, Any]: + """Run one bounded read-only command, recording argv and exit status as command evidence.""" + try: + done = run_command(argv, timeout=timeout) + except (OSError, ValueError, subprocess.SubprocessError) as exc: + result = {"status": "error", "returncode": None, "stdout": "", "stderr": str(exc)[:512]} + else: + result = {"status": "ok" if done.returncode == 0 else "error", + "returncode": done.returncode, + "stdout": done.stdout.decode("utf-8", "replace"), + "stderr": done.stderr.decode("utf-8", "replace")[:512]} + _COMMAND_EVIDENCE.append({"argv": list(argv), "status": result["status"], + "returncode": result["returncode"], "mutating": False}) + return result + + +def _tool(name: str) -> str | None: + """Return the one canonical system path for a tool, or None when there is not one.""" + try: + return system_executable(name) + except FileNotFoundError: + return None + + +def _identity(path: Any) -> dict[str, Any] | None: + """Return {path, size_bytes, sha256} for one real file, or None when unreadable.""" + if path is None: + return None + try: + return file_identity(os.path.realpath(os.fspath(path)), label="file") + except (OSError, TypeError, ValueError): + return None + + +def _read_text(path: Any, limit: int = 4096) -> str: + try: + return Path(path).read_text(encoding="utf-8", errors="replace")[:limit] + except OSError: + return "" + + +def _member(value: Any, key: str) -> dict[str, Any]: + """Read one nested object, tolerating an absent or malformed member.""" + item = value.get(key) if isinstance(value, dict) else None + return item if isinstance(item, dict) else {} + + +def _absolute(value: Any) -> bool: + return isinstance(value, str) and bool(value) and Path(value).is_absolute() + + +def _command_version(path: Any, pattern: re.Pattern[str] | None = None) -> str | None: + """Read one tool's own reported version; absent output yields None, never a guess.""" + if path is None: + return None + result = _run([str(path), "--version"], timeout=15) + output = result["stdout"] or result["stderr"] + if result["status"] != "ok" or not output: + return None + if pattern is not None: + match = pattern.search(output) + return match.group(1) if match else None + found = _VERSION_TOKEN.findall(output.splitlines()[0]) + return found[-1] if found else None + + +def _release_number(value: Any) -> list[int]: + return [int(part) for part in _DIGITS.findall(str(value))] + + +def _release_gate(record: dict[str, Any]) -> dict[str, Any]: + """Own the 38.5+ policy on the artifact that declares it: shared ``parse_jetson_release`` + returns structure only, so the decision is published once rather than re-derived.""" + major, minor = record.get("release_major"), record.get("release_minor") + if not isinstance(major, int) or not isinstance(minor, int): + status, reason = "unknown", (f"no parseable Jetson Linux release line in" + f" {_RELEASE_SOURCE}; the requirement was not evaluated") + elif (major, minor) >= MINIMUM_JETSON_LINUX: + status, reason = "compatible", f"Jetson Linux {record['version']} satisfies the minimum" + else: + status, reason = "unsupported", (f"Jetson Linux {record['version']} is older than the" + " documented minimum") + return {"status": status, "minimum_version": MINIMUM_VERSION_TEXT, + "reason": f"{reason} ({MINIMUM_VERSION_TEXT}+).", "revision": record.get("revision")} + + +def _nvidia_smi(gpu: int) -> dict[str, Any]: + """Observe the canonical nvidia-smi; every ``gpus`` row carries its own driver ``index``, so + a consumer selects the requested GPU by that field and never by list position.""" + executable = _tool("nvidia-smi") + record: dict[str, Any] = {"status": "unavailable", "executable": executable, "gpus": [], + "selected_gpu": gpu, "gpu_name": None, "driver_version": None, + "exit_code": None} + if executable is None: + return record + result = _run([executable, "--query-gpu=index,name,driver_version", + "--format=csv,noheader,nounits"]) + record["exit_code"] = result["returncode"] + record["status"] = "observed" if result["status"] == "ok" else "error" + for line in result["stdout"].splitlines(): + fields = [item.strip() for item in line.split(",")] + if len(fields) != 3 or not fields[0].isdigit(): + continue + row = {"index": int(fields[0]), "name": fields[1] or None, + "driver_version": fields[2] or None} + record["gpus"].append(row) + if row["index"] == gpu: + record["gpu_name"], record["driver_version"] = row["name"], row["driver_version"] + return record + + +def _platform_record() -> dict[str, Any]: + """Publish the 1.2 ``platform`` identity and release gate: ``jetson`` is the observed + presence of the release file and ``machine`` the kernel's own reported architecture.""" + record = parse_jetson_release(_read_text(_RELEASE_SOURCE, 1024)) + uname = os.uname() + return { + "system": uname.sysname, "release": uname.release, "machine": uname.machine, + "python": ".".join(str(part) for part in sys.version_info[:3]), + "python_abi": f"cp{sys.version_info.major}{sys.version_info.minor}", + "jetson": _RELEASE_SOURCE.exists(), + "jetson_linux": {"status": "observed" if record.get("release_line") else "unknown", + **record, "release_source": str(_RELEASE_SOURCE), + "compatibility": _release_gate(record)}, + } + + +def _apt_sources() -> list[dict[str, Any]]: + """Parse every configured binary APT source together with its exact file hash.""" + paths = [_APT_SOURCES_LIST] + if _APT_SOURCES_DIR.is_dir(): + paths.extend(sorted(path for path in _APT_SOURCES_DIR.iterdir() + if path.suffix in {".list", ".sources"})) + records: list[dict[str, Any]] = [] + for path in paths: + try: + raw = path.read_bytes() + except OSError: + continue + records.append({"path": str(path), "sha256": sha256_bytes(raw), + "entries": parse_apt_sources(path, raw.decode("utf-8", "replace"))}) + return records + + +def _signature_enforced(sources: list[dict[str, Any]]) -> bool: + """Require a readable apt configuration with no authentication bypass anywhere.""" + executable = _tool("apt-config") + if executable is None: + return False + result = _run([executable, "dump"]) + return bool(result["status"] == "ok" and _APT_BYPASS.search(result["stdout"]) is None + and not any(entry.get("trust_bypass") for item in sources + for entry in item["entries"])) + + +def _bind_origin(origin: dict[str, Any], sources: list[dict[str, Any]], + enforced: bool) -> dict[str, Any]: + """Bind one candidate origin to the configured source files that declare it. The digest list + is deliberately not deduplicated: the core requires digests and records to be one multiset.""" + suite, _, component = str(origin["suite_component"]).partition("/") + expected = (str(origin["uri"]), suite, component, False) + matches = [{"path": item["path"], "sha256": item["sha256"], "entry": entry} + for item in sources for entry in item["entries"] + if (entry.get("uri"), entry.get("suite"), entry.get("component"), + entry.get("trust_bypass")) == expected] + if enforced and matches: + origin["authentication"] = "apt-signature-chain" + origin["configured_source_sha256"] = sorted(item["sha256"] for item in matches) + origin["configured_sources"] = matches + return origin + + +def _policy_origins(stdout: str, candidate: str, sources: list[dict[str, Any]], + enforced: bool) -> list[dict[str, Any]]: + """Read the apt-cache version table and return only the candidate version's origins.""" + origins: list[dict[str, Any]] = [] + active = False + for line in stdout.splitlines(): + match = _POLICY_ORIGIN.match(line) + if match is None: + version = _POLICY_VERSION.match(line) + active = version is not None and version.group(1) == candidate + elif active: + origins.append(_bind_origin( + {"priority": int(match.group(1)), "uri": match.group(2).rstrip("/"), + "suite_component": match.group(3), "architecture": match.group(4), + "authentication": "unverified"}, sources, enforced)) + return origins + + +def _apt_candidate(apt_cache: str | None, package: str, sources: list[dict[str, Any]], + enforced: bool) -> dict[str, Any]: + """Observe one package's installed/candidate versions and its bound public origin; the exact + argv and exit code are recorded so the planner can re-check where the binding came from.""" + record: dict[str, Any] = {"package": package, "query_status": "unavailable", + "query_argv": None, "query_exit_code": None, "installed": None, + "candidate": None, "candidate_origins": []} + if apt_cache is not None: + argv = [apt_cache, "policy", package] + result = _run(argv) + found = {name: None if value == "(none)" else value + for name, value in _POLICY_FIELD.findall(result["stdout"])} + candidate = found.get("Candidate") + record.update({"query_status": result["status"], "query_argv": argv, + "query_exit_code": result["returncode"], + "installed": found.get("Installed"), "candidate": candidate, + "candidate_origins": _policy_origins( + result["stdout"], candidate, sources, enforced) if candidate else []}) + # The single APT trust decision lives in the shared core: an internal mirror, an + # unsigned or trust-bypassed source, or a suite outside exact rNN.N/main yields None. + record["public_origin"] = public_candidate_binding(record) + return record + + +def _apt_probe() -> dict[str, Any]: + """Observe local APT metadata only; the probe never refreshes an index or hits the network.""" + tag = f"python{sys.version_info.major}.{sys.version_info.minor}" + packages = (NATIVE_PACKAGE, "cuda-toolkit", "pkg-config", "python3-venv", f"{tag}-venv", + "python3-dev", f"{tag}-dev") + sources = _apt_sources() + enforced = _signature_enforced(sources) + apt_cache = _tool("apt-cache") + candidates = { + name: _apt_candidate(apt_cache, name, sources, enforced) for name in packages + } + for build_package in pycuda_bootstrap_packages(candidates["cuda-toolkit"].get("candidate")): + candidates[build_package] = _apt_candidate( + apt_cache, build_package, sources, enforced + ) + return { + "apt_cache": apt_cache, "signature_enforced": enforced, + "source_files": [item["path"] for item in sources], + "sources": [{"path": item["path"], "sha256": item["sha256"]} for item in sources], + "candidates": candidates, + } + + +def _dpkg_package(name: str) -> dict[str, Any]: + """Report one Debian package as {name, status, version} from dpkg-query alone.""" + executable = _tool("dpkg-query") + if executable is None: + return {"name": name, "status": "unknown", "version": None} + result = _run([executable, "-W", "-f=${db:Status-Abbrev}\t${Version}", name]) + fields = result["stdout"].split("\t") + installed = (result["status"] == "ok" and len(fields) == 2 + and fields[0].startswith("ii") and bool(fields[1].strip())) + return {"name": name, "status": "installed" if installed else "missing", + "version": fields[1].strip() if installed else None} + + +def _cuda_root() -> Path | None: + """Return the realpath of the selected CUDA root: the /usr/local/cuda link, else newest.""" + versioned = sorted(Path("/usr/local").glob("cuda-*"), + key=lambda path: _release_number(path.name), reverse=True) + for candidate in (Path("/usr/local/cuda"), *versioned): + if (candidate / "bin" / "nvcc").is_file(): + return Path(os.path.realpath(candidate)) + return None + + +def _cuda_marker_version(root: Path) -> str | None: + try: + data = read_json(root / "version.json") + except (OSError, ValueError): + data = None + entry = data.get("cuda") if isinstance(data, dict) else None + version = entry.get("version") if isinstance(entry, dict) else None + if isinstance(version, str) and version: + return version + match = _VERSION_TOKEN.search(_read_text(root / "version.txt", 256)) + return match.group(0) if match else None + + +def _nvcc_discovery(root: Any) -> dict[str, Any]: + return {"root": str(root) if root else None, + "nvcc": str(root / "bin" / "nvcc") if root else None, + "provenance": "the /usr/local/cuda link, else the newest /usr/local/cuda-* with nvcc"} + + +def _absent_cuda(root: Any = None, version: Any = None) -> dict[str, Any]: + return {"status": "absent", "version": version, "root": str(root) if root else None, + "nvcc_discovery": _nvcc_discovery(root), + "environment_prefixes": {name: [] for name in _CUDA_PREFIXES}} + + +def _cuda() -> dict[str, Any]: + """Publish the one CUDA observation; rule 9 requires status and version. The vocabulary is + installed/absent because every native consumer and the planner fail closed on anything else. + ``cuda_toolkit`` and the native surface carry this one observation, so they cannot disagree; + the prefixes are the plan-time PyCUDA source-build environment.""" + root = _cuda_root() + if root is None: + return _absent_cuda() + version = ( + _dpkg_package("cuda-toolkit")["version"] + or _cuda_marker_version(root) + or _command_version(root / "bin" / "nvcc", _NVCC_RELEASE) + ) + if not version: + return _absent_cuda(root) + return {"status": "installed", "version": version, "root": str(root), + "nvcc_discovery": _nvcc_discovery(root), + "environment_prefixes": { + name: [str(root / part) for part in parts if (root / part).is_dir()] + for name, parts in _CUDA_PREFIXES.items()}} + + +def _cuda_probe() -> dict[str, Any]: + """Observe CUDA exactly once; a failure here can never reach either surface.""" + try: + return _cuda() + except Exception: # pylint: disable=broad-exception-caught + return _absent_cuda() + + +def _tool_record(path: Any, pattern: re.Pattern[str] | None = None) -> dict[str, Any] | None: + identity = _identity(path) + version = _command_version(identity["path"] if identity else None, pattern) + if identity is None or version is None: + return None + return {"path": identity["path"], "version": version, "sha256": identity["sha256"]} + + +def _native_tools(cuda_root: Any) -> dict[str, Any]: + """Emit the five required native tools; `generator.name` is the CMake generator name (R4).""" + nvcc = str(Path(str(cuda_root)) / "bin" / "nvcc") if _absolute(cuda_root) else None + records: dict[str, Any] = { + "cmake": _tool_record(_tool("cmake")), "cxx": _tool_record(_tool("c++")), + "nvcc": _tool_record(nvcc, _NVCC_RELEASE), "generator": None, + "pkg_config": _tool_record(_tool("pkg-config"))} + for executable, name in CMAKE_GENERATORS: + record = _tool_record(_tool(executable)) + if record is not None: + records["generator"] = {**record, "name": name, "executable": executable} + break + return {name: record for name, record in records.items() if record is not None} + + +def _pkg_config_module(pkg_config: str | None, name: str) -> dict[str, Any]: + if pkg_config is None: + return {"status": "unknown", "version": None} + result = _run([pkg_config, "--modversion", name], timeout=15) + version = result["stdout"].strip().splitlines()[0].strip() if result["stdout"].strip() else "" + ready = result["status"] == "ok" and bool(version) + return {"status": "available" if ready else "missing", "version": version or None} + + +def _build_prerequisites() -> dict[str, Any]: + """Preflight the exact pkg-config modules that gate the official AppDec target.""" + pkg_config = _tool("pkg-config") + modules = {name: _pkg_config_module(pkg_config, name) for name in APPDEC_MODULES} + unresolved = [name for name in APPDEC_MODULES if modules[name]["status"] != "available"] + return {"status": "complete" if not unresolved else "incomplete", + "unresolved_modules": unresolved, "consumer": APPDEC_CONSUMER, + "required_modules": list(APPDEC_MODULES), "modules": modules} + + +def _native_surface(cuda: dict[str, Any]) -> dict[str, Any]: + """Produce and independently validate the native surface published as ``native_sdk``.""" + base = Path(NATIVE_SDK_ROOT) + roots: list[str] = [] + versioned: list[str] = [] + if base.is_dir() and not base.is_symlink(): + roots.append(os.path.realpath(base)) + for child in sorted(base.iterdir()): + if child.is_dir() and not child.is_symlink(): + resolved_child = os.path.realpath(child) + roots.append(resolved_child) + if re.fullmatch(r"13\.0(?:\.\d+)*", child.name): + versioned.append(resolved_child) + package = _dpkg_package(NATIVE_PACKAGE) + evidence = [root_evidence(root) for root in versioned] + complete_roots = [item["root"] for item in evidence if item["status"] == "complete"] + resolved = complete_roots[0] if len(versioned) == len(complete_roots) == 1 else None + complete = resolved is not None and package["status"] == "installed" + status = ("installed" if complete else "partial" + if package["status"] == "installed" or roots else "missing") + surface = { + "installed": False, "status": status, + "package": package, "sdk_root": resolved, + "roots": roots, "root_evidence": evidence, + "headers": sorted(name for item in evidence for name in item["required_files"][:3] + if os.path.isfile(name)), + "sample_roots": sorted(f"{root}/Samples" for root in roots + if Path(root, "Samples").is_dir()), + "complete_roots": complete_roots, + "reason": ("one coherent versioned SDK root carries every required member" if complete + else "no single complete versioned Video Codec SDK root was observed"), + "build_prerequisites": _build_prerequisites(), "cuda": cuda, + "tools": _native_tools(cuda.get("root")), + } + return _sealed(surface, surface_contract_errors("native", surface)) + + +def _dependency(name: str, ready: Any = None) -> dict[str, Any]: + try: + version: str | None = importlib.metadata.version(name) + except importlib.metadata.PackageNotFoundError: + version = None + satisfied = version is not None and (bool(ready(version)) if callable(ready) else True) + return {"status": "installed" if version is not None else "missing", "version": version, + "ready": satisfied, "requirement_satisfied": satisfied} + + +def _torch_dependency() -> dict[str, Any]: + """Report the CUDA-enabled PyTorch the official decode sample imports.""" + record = _dependency("torch") + if record["status"] != "installed": + return {**record, "cuda_build": None, "cuda_available": None, + "sample_readiness": "not_ready"} + try: + module = importlib.import_module("torch") + build = getattr(getattr(module, "version", None), "cuda", None) + available = bool(module.cuda.is_available()) + except Exception: # pylint: disable=broad-exception-caught + build, available = None, None + ready = record["version"] == TORCH_VERSION and build == TORCH_CUDA_BUILD and available is True + return {**record, "cuda_build": build, "cuda_available": available, "ready": ready, + "requirement_satisfied": ready, + "sample_readiness": "ready" if ready else "not_ready"} + + +def _dependencies() -> dict[str, Any]: + """Observe the official-sample Python dependencies once, memoized. The first call must follow + PyNvVideoCodec classification: importing PyTorch can rewrite LD_LIBRARY_PATH.""" + if not _DEPENDENCIES: + _DEPENDENCIES.update({ + "numpy": _dependency( + "numpy", lambda value: _release_number(value)[:2] >= list(NUMPY_MINIMUM)), + "pycuda": _dependency("pycuda", lambda value: value == PYCUDA_VERSION), + "torch": _torch_dependency(), + }) + return _DEPENDENCIES + + +def _owned_member(resolved: str, distribution: Any, + label: str) -> tuple[dict[str, Any] | None, list[str]]: + """Bind one owned extension to exactly one selected-distribution member and hash it, proving + membership only -- never that its wheel ``RECORD`` row still authenticates.""" + entries: list[str] = [] + for item in getattr(distribution, "files", None) or (): + try: + located = os.path.realpath(os.fspath(distribution.locate_file(item))) + except (OSError, TypeError, ValueError): + continue + if located == resolved: + entries.append(str(item)) + if len(entries) != 1: + return None, [f"{label} must be one member of the selected {PYNVC_PACKAGE} distribution;" + f" matched {sorted(entries)}"] + identity = _identity(resolved) + if identity is None: + return None, [f"{label} at {resolved} could not be hashed"] + return {"path": identity["path"], "sha256": identity["sha256"], "loaded_path": resolved, + "distribution_entry": entries[0]}, [] + + +def _selected_primary(module: Any, distribution: Any, package_dir: str, + prefix: str) -> tuple[dict[str, Any] | None, list[str]]: + """Select the one primary codec extension exclusively through the package binding: the only + admissible source is ``PyNvVideoCodec._PyNvVideoCodec.__file__``: the wheel ships both + binaries but executes one, so the primary is the package's own binding.""" + origin = getattr(getattr(module, "_PyNvVideoCodec", None), "__file__", None) + resolved = owned_extension(origin, package_dir, prefix) + if resolved is None: + return None, [f"{PYNVC_PACKAGE}._PyNvVideoCodec.__file__ must name a readable non-symlink" + f" extension directly in {package_dir} under {prefix}; saw {origin!r}"] + stem = extension_stem(resolved) + linked = PRIMARY_STEMS.get(stem or "") + if linked is None: + return None, [f"the selected primary stem {stem!r} is not an exact public stem" + f" {sorted(PRIMARY_STEMS)}"] + suffix = getattr(module, "module_suffix", None) + if suffix != linked[0]: + return None, [f"the selected primary stem {stem} disagrees with" + f" {PYNVC_PACKAGE}.module_suffix {suffix!r}"] + record, defects = _owned_member(resolved, distribution, "the selected primary extension") + if record is None: + return None, defects + return {**record, "stem": stem, "linked_nvenc_api": linked[1]}, [] + + +def _selected_auxiliary(associated: dict[str, list[str]], distribution: Any, package_dir: str, + prefix: str) -> tuple[dict[str, Any] | None, list[str]]: + """Permit zero or one auxiliary that is exactly ``PyNvVideoCodec.VersionCheck``, the helper + the wheel's ``__init__`` imports to pick ``module_suffix``. Zero is allowed for fixtures; + two distinct paths are ambiguous and fail.""" + candidates = [path for path, names in associated.items() if AUXILIARY_MODULE in names] + if not candidates: + return None, [] + if len(candidates) != 1: + return None, [f"{AUXILIARY_MODULE} must resolve to one loaded path; saw {candidates}"] + resolved = candidates[0] + if (extension_stem(resolved) != AUXILIARY_STEM + or owned_extension(resolved, package_dir, prefix) is None): + return None, [f"{AUXILIARY_MODULE} must be a readable {AUXILIARY_STEM} extension directly" + f" in {package_dir} under {prefix}; saw {resolved}"] + record, defects = _owned_member(resolved, distribution, AUXILIARY_MODULE) + if record is None: + return None, defects + return {**record, "role": AUXILIARY_ROLE, "module_names": associated[resolved]}, [] + + +def _classify_pynvc_extensions(module: Any, distribution: Any) -> tuple[ + dict[str, Any] | None, list[dict[str, Any]], list[str]]: + """Select one primary, name at most one auxiliary, reject every other association. This keeps + the second-or-foreign-primary check and only narrows its association rule.""" + origin = getattr(module, "__file__", None) + if not isinstance(origin, str) or not origin: + return None, [], [f"the imported {PYNVC_PACKAGE} package has no file location"] + package_dir = os.path.dirname(os.path.realpath(origin)) + prefix = os.path.realpath(sys.prefix) + associated, symlinked = associated_extensions(package_dir) + primary, defects = _selected_primary(module, distribution, package_dir, prefix) + auxiliary, notes = _selected_auxiliary(associated, distribution, package_dir, prefix) + defects.extend(notes) + if symlinked: + defects.append(f"a loaded {PYNVC_PACKAGE} extension came through a symlink: {symlinked}") + expected = {item["loaded_path"] for item in (primary, auxiliary) if item is not None} + if unexpected := sorted(set(associated) - expected): + defects.append(f"only the selected primary and one {AUXILIARY_MODULE} auxiliary may be" + f" loaded; also loaded {[(p, associated[p]) for p in unexpected]}") + if missing := sorted(expected - set(associated)): + defects.append(f"the selected {PYNVC_PACKAGE} extension is not registered in this" + f" interpreter's sys.modules: {missing}") + return primary, [auxiliary] if auxiliary is not None else [], defects + + +def _pynvc_surface() -> dict[str, Any]: + """Import PyNvVideoCodec and record the module and selected extension that actually loaded. + Rule 8: these observe the executed binary; wheel-``RECORD`` attestation is not claimed.""" + module = importlib.import_module(PYNVC_PACKAGE) + distribution = importlib.metadata.distribution(PYNVC_PACKAGE) + primary, auxiliaries, defects = _classify_pynvc_extensions(module, distribution) + selected = primary or {} + origin, version = getattr(module, "__file__", None), str(distribution.version) + suffix, linked = getattr(module, "module_suffix", None), selected.get("linked_nvenc_api") + surface = { + "installed": False, "imported": origin is not None, "module": PYNVC_PACKAGE, + "module_file": os.path.realpath(origin) if origin else None, + "module_version": str(getattr(module, "__version__", "") or "") or None, + "distribution_version": version, "version": version, + "linked_nvenc_api": {"status": "observed" if linked else "unknown", "value": linked, + "module_suffix": suffix, "provenance": "loaded module_suffix/stem", + "extension_file": selected.get("loaded_path")}, + "interpreter": sys.executable, "interpreter_identity": _identity(sys.executable), + "sys_prefix": str(Path(sys.prefix).resolve()), "auxiliary_extensions": auxiliaries, + "extension": {"path": selected.get("path"), "sha256": selected.get("sha256"), + "loaded_path": selected.get("loaded_path"), "module_suffix": suffix, + "linked_nvenc_api": linked, + "distribution_entry": selected.get("distribution_entry")}, + "dependencies": _dependencies(), + } + sealed = _sealed(surface, surface_contract_errors("pynvc", surface) + defects) + sealed["identity"] = _pynvc_identity(sealed, distribution) + sealed["errors"] = [{"module": PYNVC_PACKAGE, "error": item} + for item in sealed.get("defects") or []] + return sealed + + +def _pynvc_identity(surface: dict[str, Any], distribution: Any) -> dict[str, Any]: + """Emit the established identity: the verified member set, else exactly {status, reason}. + ``version``/``interpreter_identity`` are additive for the capability skill; no ``record`` is + claimed -- wheel-``RECORD`` attestation is verify_pynvc_sample.py's authority.""" + if surface["installed"] is not True: + return {"status": "not_ready", + "reason": "; ".join(surface.get("defects") or []) or "not observed"} + return { + "status": "verified", "interpreter": surface["interpreter"], + "sys_prefix": surface["sys_prefix"], "version": surface["distribution_version"], + "interpreter_identity": surface["interpreter_identity"], + "dist_info_path": next( + (os.path.dirname(os.path.realpath(os.fspath(distribution.locate_file(item)))) + for item in getattr(distribution, "files", None) or () + if str(item).endswith(".dist-info/RECORD")), None), + "distribution": {"name": PYNVC_PACKAGE, "version": surface["distribution_version"]}, + "module": {"name": PYNVC_PACKAGE, "version": surface["module_version"], + "path": surface["module_file"]}, + "extension": surface["extension"]} + + +def _sealed(surface: dict[str, Any], defects: list[str]) -> dict[str, Any]: + """Claim ``installed`` only for a surface that satisfies its own required subset.""" + surface["installed"] = not defects + if defects: + surface["defects"] = defects + return surface + + +def _blocked(name: str, reason: str) -> dict[str, Any]: + return {"installed": False, "defects": [f"the {name} surface is blocked: {reason}"]} + + +def _guarded(name: str, producer: Any) -> dict[str, Any]: + """Produce one surface in isolation: its failure can never reach the peer surface.""" + try: + return producer() + except Exception as exc: # pylint: disable=broad-exception-caught + return _blocked(name, f"{type(exc).__name__}: {exc}") + + +def _python_bootstrap() -> dict[str, Any]: + """Report whether the SYSTEM interpreter can bootstrap a venv and build C code: the plan-time + detection of a missing python3-venv or python3-dev. It reports ``missing`` rather than + raising, and blocks nothing.""" + record: dict[str, Any] = {"executable": None, "venv_module": "missing", + "development_headers": "missing"} + try: + executable = _tool("python3") + record["executable"] = executable + if executable is None: + return record + result = _run([executable, "-I", "-c", _PYTHON_PROBE]) + observed = read_json(result["stdout"]) if result["status"] == "ok" else None + except Exception: # pylint: disable=broad-exception-caught + return record + data = observed if isinstance(observed, dict) else {} + record["venv_module"] = "ok" if data.get("venv") is True else "missing" + record["development_headers"] = "ok" if data.get("headers") is True else "missing" + return record + + +def _readiness(runtime: str, surfaces: dict[str, Any]) -> dict[str, Any]: + """Publish each requested surface as its own readiness layer, never one merged claim. A + satisfied surface is ``partial``, not ``ready``: the probe proves no codec operation.""" + layers: dict[str, Any] = {} + reasons: list[str] = [] + for name, layer in (("native", "native_sdk"), ("pynvc", "pynvc")): + surface = surfaces.get(name) + if not isinstance(surface, dict): + continue + installed = surface.get("installed") is True + layers[layer] = {"status": "partial" if installed else "not_ready", + "installation": "ready" if installed else "not_ready", + "operation": "not_verified_by_probe"} + reasons.extend(surface.get("defects") or [ + f"the {name} surface satisfies its required subset; operational proof still" + " requires its own verification run"]) + states = [item["status"] for item in layers.values()] + if runtime != "both": + state = states[0] if states else "not_ready" + elif states and len(set(states)) == 1: + state = states[0] + else: + state = "partial" + return {"state": state, "layers": layers, "reasons": reasons} + + +def _requested(surfaces: dict[str, Any], name: str) -> dict[str, Any]: + """Return one normalized surface, or an explicitly not-requested one in its exact shape.""" + surface = surfaces.get(name) + if not isinstance(surface, dict): + surface = _blocked(name, "it was not requested by this probe") + if name != "pynvc" or "identity" in surface: + return surface + reason = "; ".join(surface.get("defects") or []) or "no PyNvVideoCodec import was attempted" + return {**surface, "imported": False, "module": None, "module_file": None, + "module_version": None, "distribution_version": None, + "errors": [{"module": PYNVC_PACKAGE, "error": reason}], + "linked_nvenc_api": {"status": "not_ready", "value": None, "reason": reason}, + "identity": {"status": "not_ready", "reason": reason}} + + +def _capabilities(gpu: int, runtime: str, + pynvc: dict[str, Any]) -> tuple[dict[str, Any], list[str]]: + """Publish the frozen native-unavailable/PyNv API-query authority split.""" + if runtime == "native": + return ({"authority": "native-adapter-unavailable", "gpu": gpu, "status": "unknown", + "reason": "No native capability-query adapter is bundled with this skill.", + "encode": {}, "decode": {}}, []) + module = sys.modules.get(PYNVC_PACKAGE) if pynvc.get("installed") is True else None + if module is None: + result = {"authority": "none", "gpu": gpu, "status": "not_ready", + "reason": "PyNvVideoCodec was not ready; no capability query ran.", + "encode": {}, "decode": {}} + return result, [] + result, warnings = query_pynvc_capabilities( + module, gpu, _member(pynvc, "linked_nvenc_api")) + if runtime == "both": + result["native"] = {"authority": "native-adapter-unavailable", "status": "unknown", + "reason": "Native inventory is reported independently."} + result.update({"status": "partial", "reason": + "PyNvVideoCodec was queried; native query authority is absent."}) + return result, warnings + + +def _installation(cuda: dict[str, Any], native: dict[str, Any]) -> dict[str, Any]: + """Publish the 1.2 ``installation`` inventory from this run's observations. + ``python.executable`` is the interpreter that produced this artifact after any validated- + venv delegation, because that is what ``@python-from`` re-enters; never a realpath. The + venv/header state beside it belongs to ``system_executable``.""" + bootstrap, running = _python_bootstrap(), _identity(sys.executable) + tools = _member(native, "tools") + return { + "status": "observed", "mutation_performed": False, "network_probed": False, + "python": {"executable": sys.executable, "identity": running, + "interpreter_identities": {"running": running}, + "sys_prefix": str(Path(sys.prefix).resolve()), "packages": _dependencies(), + "version": ".".join(str(part) for part in sys.version_info[:3]), + "abi": f"cp{sys.version_info.major}{sys.version_info.minor}", + "system_executable": bootstrap["executable"], + "venv_module": bootstrap["venv_module"], + "development_headers": bootstrap["development_headers"]}, + "apt": _apt_probe(), "cuda_toolkit": cuda, "native_sdk": native, + "native_build_prerequisites": native.get("build_prerequisites"), + "build_tools": {name: record.get("path") for name, record in tools.items()}, + "build_tool_identities": {(str(record.get("executable")) if name == "generator" + else _TOOL_ALIASES.get(name, name)): record + for name, record in tools.items()}, + } + + +def _environment(gpu: int, runtime: str, cuda: dict[str, Any], + surfaces: dict[str, Any]) -> dict[str, Any]: + """Assemble exactly the frozen closed 16-key 1.2 document and nothing else. ``libraries`` is + empty because this probe runs no platform-loader query; the internal normalized mapping ends + here, republished as ``pynvc`` and ``installation.native_sdk``.""" + pynvc = _requested(surfaces, "pynvc") + linked = _member(pynvc, "linked_nvenc_api").get("value") + capabilities, capability_warnings = _capabilities(gpu, runtime, pynvc) + return { + "schema_version": SCHEMA_VERSION, "kind": KIND, "generated_at": utc_now(), + "mode": "live", "requested_runtime": runtime, "platform": _platform_record(), + "selected_gpu": gpu, "nvidia_smi": _nvidia_smi(gpu), "libraries": {}, + "driver_nvenc_api": {"status": "observed" if linked else "unknown", "max_supported": None, + "linked_nvenc_api": linked, + "provenance": "selected PyNvVideoCodec primary module_suffix", + "reason": "the probe never calls NvEncodeAPIGetMaxSupportedVersion;" + " the ceiling is proven by the runtime helper"}, + "pynvc": pynvc, "installation": _installation(cuda, _requested(surfaces, "native")), + "capabilities": capabilities, + "warnings": [_PROBE_WARNING, *capability_warnings], + "readiness": _readiness(runtime, surfaces), + "command_evidence": list(_COMMAND_EVIDENCE), + } + + +def _registry_selection(requested: tuple[str, ...], + setup_candidate: bool) -> dict[str, Any] | None: + """Bind the fixed validated PyNv venv before any import, delegating once when needed. Returns + None to probe under the current interpreter, or a blocked ``pynvc`` surface when the registry + is absent or stale -- never a scan. An unregistered interpreter re-executes through the + registered lexical one; that child does not re-delegate.""" + if "pynvc" not in requested or setup_candidate: + return None + try: + registry = load_pynvc_registry() + except RegistryNotReady as exc: + return _blocked("pynvc", f"the validated PyNvVideoCodec venv registry is not ready" + f" ({exc.reason}); next action: {exc.next_action}") + registered = str(registry["interpreter"]) + if os.path.abspath(os.path.expanduser(sys.executable)) == os.path.abspath(registered): + return None + try: + completed = run_command([registered, "-I", os.path.abspath(__file__), *sys.argv[1:]], + timeout=_DELEGATE_TIMEOUT) + except (OSError, ValueError, subprocess.SubprocessError) as exc: + return _blocked("pynvc", f"the registered interpreter could not be launched: {exc}") + stdout = completed.stdout.decode("utf-8", "replace") + sys.stderr.write(completed.stderr.decode("utf-8", "replace")) + # Never forward a SUCCESS this probe cannot recognize. A child that answered with some + # other document kind or major has not produced the artifact the caller asked for, and + # relaying it at rc 0 would launder an unknown envelope into a trusted one. + if completed.returncode == 0 and not _own_envelope(stdout): + print(_render({"schema_version": SCHEMA_VERSION, "kind": ERROR_KIND, "status": "error", + "error": "the registered interpreter returned an unrecognized document" + f" (expected {KIND} {SCHEMA_VERSION}); refusing to forward it"})) + raise SystemExit(3) + sys.stdout.write(stdout) + raise SystemExit(completed.returncode) + + +def _own_envelope(stdout: str) -> bool: + """Recognize only this probe's own artifact envelope: exact kind AND exact major.""" + try: + value = json.loads(stdout) + except ValueError: + return False + return (isinstance(value, dict) and value.get("kind") == KIND + and value.get("schema_version") == SCHEMA_VERSION) + + +def _render(value: Any) -> str: + """Serialize strictly for stdout; the artifact itself is written by write_new_json.""" + return json.dumps(value, allow_nan=False, indent=1, sort_keys=True) + + +def _no_duplicate_keys(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + """Reject a repeated key instead of letting json.loads silently keep the last one.""" + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise _DuplicateKeyError(f"duplicate key in environment artifact: {key!r}") + result[key] = value + return result + + +def _reject_constant(constant: str) -> Any: + raise ValueError(f"non-RFC JSON numeric constant is forbidden: {constant}") + + +def _read_artifact(path: Any) -> Any: + """Read one artifact as size-bounded, symlink-free, unambiguous strict JSON. Oversize input + is malformed; a repeated key remains a strict validation defect instead of being silently + replaced by json.loads.""" + resolved = Path(os.path.abspath(os.fspath(Path(path).expanduser()))) + if resolved.stat().st_size > _MAX_ARTIFACT_BYTES: + raise ValueError(f"environment artifact exceeds the {_MAX_ARTIFACT_BYTES}-byte bound") + descriptor = os.open(resolved, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as handle: + raw = handle.read(_MAX_ARTIFACT_BYTES + 1) + if len(raw) > _MAX_ARTIFACT_BYTES: + raise ValueError(f"environment artifact exceeds the {_MAX_ARTIFACT_BYTES}-byte bound") + return json.loads(raw, object_pairs_hook=_no_duplicate_keys, parse_constant=_reject_constant) + + +def _reauthenticate_main(path: Path) -> int: + """Re-validate the 1.2 shape and the identities on disk now: rc 0 valid, rc 2 invalid. A + malformed document raises and the caller answers rc 3 on the validation-error envelope.""" + try: + data = _read_artifact(path) + except _DuplicateKeyError as exc: + data, defects = {}, [str(exc)] + else: + if not isinstance(data, dict): + raise ValueError("environment artifact must be a JSON object") + defects = environment_contract_errors(data, live=True) + print(_render({"schema_version": VALIDATION_VERSION, "kind": VALIDATION_KIND, + "valid": not defects, "readiness": _member(data, "readiness").get("state"), + "errors": defects})) + return 0 if not defects else 2 + + +def _mock_main(path: Path, gpu: int, runtime: str, output: Path | None) -> int: + """Emit an admitted offline fixture as explicit mock evidence.""" + environment = _read_artifact(path) + if not isinstance(environment, dict): + raise ValueError("mock fixture must be a JSON object") + environment.setdefault("schema_version", SCHEMA_VERSION) + environment.setdefault("kind", KIND) + environment.setdefault("generated_at", utc_now()) + environment["mode"], environment["requested_runtime"] = "mock", runtime + environment.setdefault("selected_gpu", gpu) + environment.setdefault("warnings", []).append( + "Mock fixture: no live hardware or package query was performed.") + if output is not None: + write_new_json(output, environment) + print(_render(environment)) + return 0 + + +def _probe_main(gpu: int, runtime: str, output: Path, setup_candidate: bool) -> int: + """Emit one fresh artifact; an existing --output is refused, never overwritten.""" + if os.path.lexists(output): + raise FileExistsError(f"refusing to overwrite an existing probe artifact: {output}") + requested = SURFACES if runtime == "both" else (runtime,) + selection = _registry_selection(requested, setup_candidate) + cuda = _cuda_probe() + surfaces: dict[str, Any] = {} + if "native" in requested: + surfaces["native"] = _guarded("native", lambda: _native_surface(cuda)) + if "pynvc" in requested: + surfaces["pynvc"] = selection or _guarded("pynvc", _pynvc_surface) + environment = _environment(gpu, runtime, cuda, surfaces) + write_new_json(output, environment) + print(_render(environment)) + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--gpu", type=int, default=0, help="GPU index (default: 0)") + parser.add_argument("--runtime", choices=("pynvc", "native", "both"), + default="pynvc", help="surface selection (default: pynvc)") + parser.add_argument("--mock-fixture", type=Path, + help="offline fixture; never treat it as live evidence") + parser.add_argument("--output", type=Path, + help="write the nvcodec-environment 1.2 artifact here; never overwritten") + parser.add_argument("--setup-candidate", action="store_true", + help=("authorized bootstrap/final-setup probe: run under the current" + " interpreter without validated-venv registry redirection")) + parser.add_argument("--reauthenticate", type=Path, metavar="ENVIRONMENT", + help=("re-validate an existing nvcodec-environment artifact against the" + " identities present on disk now instead of probing")) + args = parser.parse_args() + if args.gpu < 0: + parser.error("--gpu must be non-negative") + if args.reauthenticate is not None and args.mock_fixture is not None: + parser.error("--reauthenticate and --mock-fixture are mutually exclusive") + if args.reauthenticate is None and args.mock_fixture is None and args.output is None: + parser.error("a live probe requires --output") + validating = args.reauthenticate is not None + try: + require_isolated() + if validating: + return _reauthenticate_main(args.reauthenticate) + if args.mock_fixture is not None: + return _mock_main(args.mock_fixture, args.gpu, args.runtime, args.output) + return _probe_main(args.gpu, args.runtime, args.output, args.setup_candidate) + except Exception as exc: # pylint: disable=broad-exception-caught + print(_render({ + "schema_version": VALIDATION_VERSION if validating else SCHEMA_VERSION, + "kind": VALIDATION_ERROR_KIND if validating else ERROR_KIND, + "status": "error", "error": str(exc)})) + return 3 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-setup/scripts/setup/setup_contract.py b/skills/jetson-video-setup/scripts/setup/setup_contract.py new file mode 100644 index 00000000..914ebd6a --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/setup_contract.py @@ -0,0 +1,1003 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Minimal shared core for the Jetson video setup helpers. + +This private module owns isolation, strict artifacts, bounded subprocesses, public-APT binding, +and the small environment/surface schema shared by probe and planner. Installation policy, build +operations, wheel attestation, and the validated-PyNv-venv registry stay with the public helper +that owns them. The compact PyNvVideoCodec API-query adapter is shared only with the environment +probe that publishes the frozen schema-1.2 capability block. +""" + +# pylint: disable=missing-function-docstring,too-many-boolean-expressions,too-many-locals +# pylint: disable=too-many-lines + +from __future__ import annotations + +import errno +import hashlib +import importlib.machinery +import json +import os +import re +import shutil +import signal +import stat +import subprocess +import sys +from datetime import datetime, timezone +from pathlib import Path +from typing import Any +from urllib.parse import urlparse + +# Ambient variables preserved for subprocesses; captured URL credentials are redacted. +_ENV_ALLOWLIST = ( + "HOME", "TMPDIR", "TMP", "TEMP", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", + "http_proxy", "https_proxy", "no_proxy", "SSL_CERT_FILE", "SSL_CERT_DIR", + "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", +) +_ENV_OVERRIDES = {"PATH": "/usr/sbin:/usr/bin:/sbin:/bin", "LC_ALL": "C", "LANG": "C", + "LANGUAGE": "C", "PYTHONNOUSERSITE": "1"} +# Import-injection variables; run_command sets PYTHONNOUSERSITE itself. +_INJECTION_VARIABLES = ( + "PYTHONPATH", "PYTHONHOME", "PYTHONSTARTUP", "PYTHONEXECUTABLE", "PYTHONUSERBASE", + "PYTHONWARNINGS", "PYTHONOPTIMIZE", "LD_PRELOAD", "LD_AUDIT", + "LD_LIBRARY_PATH", "LIBRARY_PATH", "DYLD_INSERT_LIBRARIES", "DYLD_LIBRARY_PATH", + "DYLD_FRAMEWORK_PATH", +) +# Stock signed public Jetson origins only; internal mirrors remain ineligible. +_PUBLIC_APT_LOCATIONS = frozenset( + ("https", "repo.download.nvidia.com", f"/jetson/{part}") for part in ("common", "som")) +_PUBLIC_SUITE_COMPONENT = re.compile(r"r\d+\.\d+/main") +_APT_BYPASS = ("trusted", "allow-insecure", "allow-weak", "allow-downgrade-to-insecure") +_SHA256 = re.compile(r"[0-9a-f]{64}") +_VERSION_TEXT = re.compile(r"\d") +_URL_USERINFO = re.compile(rb"(?i)\b([a-z][a-z0-9+.-]*://)[^/\s@]+@") +_ENVIRONMENT_KIND = "nvcodec-environment" +# Pinned to probe_nvcodec.SCHEMA_VERSION to avoid a circular import. The 1.2 envelope is closed; +# unknown top-level keys fail validation. +_PYNVC_ENVIRONMENT_SCHEMA_VERSION = "1.2" +_ENVIRONMENT_KEYS = frozenset(( + "schema_version", "kind", "generated_at", "mode", "requested_runtime", "platform", + "selected_gpu", "nvidia_smi", "libraries", "driver_nvenc_api", "pynvc", "installation", + "capabilities", "warnings", "readiness", "command_evidence")) +_ENVIRONMENT_REQUIRED = ("schema_version", "kind", "platform", "selected_gpu", + "requested_runtime", "pynvc", "installation", "capabilities", + "readiness") +_ENVIRONMENT_MODES = ("live", "mock") +_READINESS_KEYS = frozenset(("state", "layers", "reasons")) +_READINESS_STATES = ("ready", "partial", "not_ready") +_RUNTIMES = ("pynvc", "native", "both") +_JETSON_MACHINES = re.compile(r"aarch64|arm64") +_SURFACES = ("native", "pynvc") +_NATIVE_PACKAGE = "nvidia-video-codec-sdk" +_NATIVE_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +_GENERATORS = ("Ninja", "Unix Makefiles") +_PY_DEPENDENCIES = ("numpy", "pycuda", "torch") +# --- Official-sample verification profiles (public) -------------------------- +# Smoke uses torch-free basic encode/decode_perf; full advanced decode imports Torch. +SMOKE_PROFILE, FULL_PROFILE = "pynvc-smoke", "full-samples" +VERIFICATION_PROFILES = (SMOKE_PROFILE, FULL_PROFILE) +PROFILE_DEPENDENCIES = {SMOKE_PROFILE: ("numpy", "pycuda"), FULL_PROFILE: _PY_DEPENDENCIES} +# --- Shared extension-classification contract (public) ----------------------- +# One predicate binds loaded compiled modules to the selected distribution; never scan sys.path. +PYNVC_PACKAGE = "PyNvVideoCodec" +# Public 2.1.0 primary stems are bound to module_suffix and linked NVENC API. +PRIMARY_STEMS = {"PyNvVideoCodec_121": ("_121", "12.1"), "PyNvVideoCodec_130": ("_130", "13.0")} +# The one legitimate compiled helper the wheel loads beside the primary: __init__.py imports +# .VersionCheck.DriverWrapper to pick module_suffix. Auxiliary evidence, never a primary. +AUXILIARY_MODULE, AUXILIARY_STEM = f"{PYNVC_PACKAGE}.VersionCheck", "VersionCheck" +AUXILIARY_ROLE = "driver_version_check" +EXTENSION_SUFFIXES = tuple(importlib.machinery.EXTENSION_SUFFIXES) +RECOGNIZED_STEMS = frozenset({*PRIMARY_STEMS, AUXILIARY_STEM}) +_LONGEST_SUFFIXES = tuple(sorted(EXTENSION_SUFFIXES, key=len, reverse=True)) +# R3: the Jetson Linux minimum has one owner; the producer decides it and consumers read it. +MINIMUM_JETSON_LINUX, MINIMUM_VERSION_TEXT = (38, 5), "38.5" +# The exact members an installed native SDK root must carry to count as complete. +NATIVE_REQUIRED_FILES = ( + "Interface/nvEncodeAPI.h", "Interface/nvcuvid.h", "Interface/cuviddec.h", + "Samples/CMakeLists.txt", "Samples/AppEncode/AppEncCuda/CMakeLists.txt", + "Samples/AppEncode/AppEncCuda/AppEncCuda.cpp", + "Samples/AppDecode/AppDec/CMakeLists.txt", "Samples/AppDecode/AppDec/AppDec.cpp") + + +def _check(condition: Any, message: str) -> None: + if not condition: + raise ValueError(message) + + +def require_isolated() -> None: + """Fail closed unless this interpreter runs isolated with no import-injection variables.""" + present = [name for name in _INJECTION_VARIABLES if os.environ.get(name)] + _check(not present, "import injection variables must be absent: " + ", ".join(present)) + _check(sys.flags.ignore_environment and sys.flags.no_user_site, + "setup helpers must run isolated with environment/user-site disabled (python -I)") + + +def utc_now() -> str: + return datetime.now(timezone.utc).isoformat() + + +# --- Hashing, strict JSON, identity, and exclusive writers ------------------- + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def _dumps(value: Any, **kwargs: Any) -> str: + """Serialize strictly: allow_nan=False rejects non-finite floats at the source.""" + return json.dumps(value, allow_nan=False, **kwargs) + + +def canonical_json_sha256(value: Any) -> str: + """Return sha256(strict JSON, sort_keys=True, separators=(",", ":")) -- the frozen form + plan_install's plan digest and every re-plan comparison depend on.""" + return sha256_bytes(_dumps(value, sort_keys=True, separators=(",", ":")).encode("utf-8")) + + +def _symlink_free(path: Any, *, label: str) -> Path: + """Return the absolute path, rejecting a symlink leaf or any symlinked parent component.""" + requested = Path(os.path.abspath(os.fspath(Path(path).expanduser()))) + current = Path(requested.anchor) + for part in requested.parent.parts[1:]: + current /= part + _check(not current.is_symlink(), f"{label} parent contains a symlink component: {current}") + _check(not requested.is_symlink(), f"{label} must not be a symlink: {requested}") + return requested + + +def _read_regular(path: Any, *, label: str) -> tuple[bytes, int, Path]: + """Read one regular file whole, following no symlink at any component of its path.""" + requested = _symlink_free(path, label=label) + descriptor = os.open(requested, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)) + with os.fdopen(descriptor, "rb") as handle: + details = os.fstat(handle.fileno()) + _check(stat.S_ISREG(details.st_mode), f"{label} must be a regular file: {requested}") + raw = handle.read() + return raw, details.st_size, requested.parent.resolve(strict=True) / requested.name + + +def file_identity(path: Any, *, label: str = "file") -> dict[str, Any]: + """Return the one canonical identity of a regular file: resolved path, size, and sha256.""" + raw, size_bytes, resolved = _read_regular(path, label=label) + return {"path": str(resolved), "size_bytes": size_bytes, "sha256": sha256_bytes(raw)} + + +def _absolute(value: Any) -> bool: + return isinstance(value, str) and bool(value) and Path(value).is_absolute() + + +def _version(value: Any) -> bool: + return isinstance(value, str) and _VERSION_TEXT.match(value) is not None + + +def _member(value: Any, name: str) -> dict[str, Any]: + member = value.get(name) if isinstance(value, dict) else None + return member if isinstance(member, dict) else {} + + +def _identity_errors(record: Any, label: str, live: bool) -> list[str]: + if not isinstance(record, dict) or not _absolute(record.get("path")): + return [f"{label} must record an absolute path and SHA-256"] + if not isinstance(record.get("sha256"), str) or not _SHA256.fullmatch(record["sha256"]): + return [f"{label} must record an absolute path and SHA-256"] + if not live: + return [] + try: + observed = file_identity(record["path"], label=label) + except (OSError, ValueError) as exc: + return [f"{label} is no longer readable: {exc}"] + return [] if ( + observed["path"] == record["path"] and observed["sha256"] == record["sha256"] + ) else [f"{label} changed after the environment probe"] + + +def extension_stem(path: str) -> str | None: + """Strip exactly one running-interpreter extension suffix, longest first, from a basename. + Never ``Path.stem`` or a bare ``.so`` rule: only EXTENSION_SUFFIXES decides the real stem.""" + name = os.path.basename(path) + for suffix in _LONGEST_SUFFIXES: + if name.endswith(suffix) and len(name) > len(suffix): + return name[:-len(suffix)] + return None + + +def owned_extension(path: Any, package_dir: str, prefix: str) -> str | None: + """Resolve one loaded extension that is a readable direct child of the package under prefix.""" + if not isinstance(path, str) or not path or os.path.islink(path): + return None + resolved = os.path.realpath(path) + if not os.path.isfile(resolved) or not os.access(resolved, os.R_OK): + return None + if os.path.dirname(resolved) != package_dir: + return None + return resolved if resolved.startswith(prefix + os.sep) else None + + +def associated_extensions(package_dir: str) -> tuple[dict[str, list[str]], list[str]]: + """Map every loaded compiled module associated with the package to its names. Association: a + registration below ``PyNvVideoCodec``, a recognized primary/VersionCheck stem, or a resolved + parent that is the package directory. Only this interpreter's ``sys.modules`` is read.""" + associated: dict[str, list[str]] = {} + symlinked: set[str] = set() + for name, value in list(sys.modules.items()): + origin = getattr(value, "__file__", None) + if not isinstance(origin, str) or not origin.endswith(EXTENSION_SUFFIXES): + continue + resolved = os.path.realpath(origin) + if (name == PYNVC_PACKAGE or name.startswith(f"{PYNVC_PACKAGE}.") + or extension_stem(resolved) in RECOGNIZED_STEMS + or os.path.dirname(resolved) == package_dir): + associated.setdefault(resolved, []).append(name) + if os.path.islink(origin): + symlinked.add(origin) + return ({path: sorted(names) for path, names in sorted(associated.items())}, + sorted(symlinked)) + + +def root_evidence(root: str) -> dict[str, Any]: + """Enumerate the required SDK members under one root, rejecting symlinked or escaped files.""" + base, missing, rejected = root.rstrip("/"), [], [] + for relative in NATIVE_REQUIRED_FILES: + candidate, target = Path(root) / relative, f"{base}/{relative}" + if not candidate.is_file(): + missing.append(target) + elif candidate.is_symlink() or os.path.realpath(candidate) != target: + rejected.append(target) + return {"root": base, "status": "complete" if not missing and not rejected else "incomplete", + "required_files": [f"{base}/{name}" for name in NATIVE_REQUIRED_FILES], + "missing_required_files": missing, "rejected_required_files": rejected} + + +def surface_contract_errors(name: str, surface: Any, *, live: bool = False) -> list[str]: + """Validate one installed surface without reading or blocking its independent peer.""" + if name not in _SURFACES or not isinstance(surface, dict): + return [f"surfaces.{name} must be an object"] + if name == "native": + package, cuda = _member(surface, "package"), _member(surface, "cuda") + tools, build = _member(surface, "tools"), _member(surface, "build_prerequisites") + errors = [] if ( + package.get("name") == _NATIVE_PACKAGE and package.get("status") == "installed" + and _version(package.get("version")) + ) else [f"{_NATIVE_PACKAGE} is not installed with a package version"] + if not _absolute(surface.get("sdk_root")): + errors.append("no canonical native SDK root was observed") + if cuda.get("status") != "installed" or not _version(cuda.get("version")): + errors.append("no installed CUDA toolkit version was observed") + for tool in _NATIVE_TOOLS: + record = tools.get(tool) + errors.extend(_identity_errors(record, f"surfaces.native.tools.{tool}", live)) + if (not isinstance(record, dict) or not _version(record.get("version")) + or (tool == "generator" and record.get("name") not in _GENERATORS)): + errors.append(f"surfaces.native.tools.{tool} is incomplete") + if build.get("status") != "complete" or build.get("unresolved_modules") != []: + errors.append("AppDec pkg-config prerequisites are incomplete") + return errors + # Established 1.2 spellings: module_file / module_version / distribution_version. The + # interpreter binding is read from the top level, where the probe also publishes it beside + # the established pynvc.identity that carries the same two values. + extension = _member(surface, "extension") + interpreter = _member(surface, "interpreter_identity") + errors = [] if _version(surface.get("distribution_version")) else [ + "no PyNvVideoCodec distribution version was observed"] + if not _absolute(surface.get("interpreter")) or not _absolute(surface.get("sys_prefix")): + errors.append("the lexical interpreter or sys_prefix is missing") + errors.extend(_identity_errors(interpreter, "pynvc.interpreter_identity", live)) + errors.extend(_identity_errors(extension, "pynvc.extension", live)) + if extension.get("loaded_path") != extension.get("path"): + errors.append("the loaded extension is not the extension hashed on disk") + if (not _absolute(surface.get("module_file")) + or surface.get("module_version") != surface.get("distribution_version")): + errors.append("the imported module does not match the distribution") + dependencies = surface.get("dependencies") + if not isinstance(dependencies, dict): + errors.append("surfaces.pynvc.dependencies must be an object") + else: + errors.extend( + f"surfaces.pynvc.dependencies.{name} must carry a boolean ready" + for name in _PY_DEPENDENCIES + if not isinstance(dependencies.get(name), dict) + or not isinstance(dependencies[name].get("ready"), bool)) + return errors + _auxiliary_errors(surface, live) + + +def _platform_errors(platform: Any) -> list[str]: + """Validate only the platform subset the evaluator and the corpora actually consume.""" + if not isinstance(platform, dict): + return ["platform must be an object"] + machine = platform.get("machine") + errors = [] if platform.get("jetson") is True else [ + "platform.jetson must be true: this contract describes a Jetson target"] + if not isinstance(machine, str) or _JETSON_MACHINES.fullmatch(machine) is None: + errors.append(f"platform.machine must be aarch64 or arm64, saw {machine!r}") + compatibility = _member(_member(platform, "jetson_linux"), "compatibility") + if not isinstance(compatibility.get("status"), str) or not compatibility["status"]: + errors.append("platform.jetson_linux.compatibility.status is required") + return errors + + +def _readiness_errors(readiness: Any) -> list[str]: + """Require a state and at least one per-surface layer; the layers are reported, not merged.""" + if not isinstance(readiness, dict): + return ["readiness must be an object"] + errors = [f"unknown readiness field: {key}" for key in sorted(set(readiness) - _READINESS_KEYS)] + if readiness.get("state") not in _READINESS_STATES: + errors.append(f"readiness.state must be one of {list(_READINESS_STATES)}") + layers = readiness.get("layers") + if not isinstance(layers, dict) or not layers: + errors.append("readiness.layers must record at least one surface layer") + elif any(not isinstance(item, dict) or not isinstance(item.get("status"), str) + for item in layers.values()): + errors.append("every readiness layer must record its own status") + if not isinstance(readiness.get("reasons", []), list): + errors.append("readiness.reasons must be an array") + return errors + + +def _installation_errors(installation: Any, live: bool) -> list[str]: + """Require the interpreter binding that ``@python-from`` and every helper re-enter. + + The executable must be absolute in every mode; its existence and executability are checked + only for a live re-authentication, where the recorded artifact is being replayed on disk. + """ + if not isinstance(installation, dict): + return ["installation must be an object"] + python = _member(installation, "python") + executable = python.get("executable") + if not _absolute(executable): + return ["installation.python.executable must be an absolute interpreter path"] + errors = [] if not live or ( + os.path.isfile(executable) and os.access(executable, os.X_OK) + ) else [f"installation.python.executable is not an executable file now: {executable}"] + if live: + errors.extend(_identity_errors(python.get("identity"), "installation.python.identity", + True)) + return errors + + +def _auxiliary_errors(surface: Any, live: bool) -> list[str]: + """Permit zero or one recorded auxiliary extension and re-authenticate the one present.""" + auxiliaries = surface.get("auxiliary_extensions", []) if isinstance(surface, dict) else None + if not isinstance(auxiliaries, list) or len(auxiliaries) > 1: + return ["pynvc.auxiliary_extensions must be an array of at most one auxiliary"] + errors: list[str] = [] + for item in auxiliaries: + if not isinstance(item, dict) or item.get("role") != AUXILIARY_ROLE: + errors.append(f"a pynvc auxiliary must record role {AUXILIARY_ROLE!r}") + continue + errors.extend(_identity_errors(item, "pynvc.auxiliary_extensions", live)) + return errors + + +def environment_contract_errors(data: Any, *, live: bool = False, + include_surface_values: bool = True) -> list[str]: + """Validate the closed 16-key ``nvcodec-environment`` 1.2 envelope by required subset. + + Closed only at the top level: an unknown top-level key is an error, but no nested structure + is compared for exact-set equality, so additive evidence beneath a member never invalidates + an otherwise conforming artifact. The two independent surfaces -- top-level ``pynvc`` and + ``installation.native_sdk`` -- are each validated against their own required subset only + when they claim to be installed, so one can never block its peer. + """ + if not isinstance(data, dict): + return ["environment must be a JSON object"] + errors = [f"unknown top-level environment field: {key}" + for key in sorted(set(data) - _ENVIRONMENT_KEYS)] + errors.extend(f"missing required field: {key}" + for key in _ENVIRONMENT_REQUIRED if key not in data) + errors.extend(f"{key} must be exactly {expected!r}" for key, expected in + (("kind", _ENVIRONMENT_KIND), + ("schema_version", _PYNVC_ENVIRONMENT_SCHEMA_VERSION)) + if data.get(key) != expected) + if data.get("mode") not in _ENVIRONMENT_MODES: + errors.append(f"mode must be one of {list(_ENVIRONMENT_MODES)}") + if data.get("requested_runtime") not in _RUNTIMES: + errors.append(f"requested_runtime must be the string {list(_RUNTIMES)}, never a list") + gpu = data.get("selected_gpu") + if not isinstance(gpu, int) or isinstance(gpu, bool) or gpu < 0: + errors.append("selected_gpu must be a non-negative integer") + errors.extend(_platform_errors(data.get("platform"))) + errors.extend(_readiness_errors(data.get("readiness"))) + errors.extend(_installation_errors(data.get("installation"), live)) + errors.extend(f"{key} must be an object" for key in ("pynvc", "capabilities", "nvidia_smi", + "libraries", "driver_nvenc_api") + if key in data and not isinstance(data[key], dict)) + errors.extend(f"{key} must be an array" for key in ("warnings", "command_evidence") + if key in data and not isinstance(data[key], list)) + if not include_surface_values: + return errors + for name, surface in (("pynvc", data.get("pynvc")), + ("native", _member(data.get("installation"), "native_sdk"))): + if isinstance(surface, dict) and surface.get("installed") is True: + errors.extend(f"{name}: {item}" + for item in surface_contract_errors(name, surface, live=live)) + return errors + + +def require_selected_surfaces_live(data: Any, components: list[str]) -> None: + """Reauthenticate only installed surfaces selected by one install plan.""" + records = { + "native-sdk": ("native", _member(_member(data, "installation"), "native_sdk")), + "pynvc": ("pynvc", _member(data, "pynvc")), + } + errors: list[str] = [] + for component in components: + _check(component in records, f"unknown setup component: {component!r}") + name, surface = records[component] + planner_ready = (surface.get("status") == "installed" if name == "native" else + surface.get("imported") is True + and _member(surface, "identity").get("status") == "verified") + if surface.get("installed") is True or planner_ready: + errors.extend(f"{name}: {item}" + for item in surface_contract_errors(name, surface, live=True)) + _check(not errors, "selected surface failed live authentication: " + "; ".join(errors)) + + +# --- 1.2 environment -> internal normalized surface -------------------------- +# Required-subset normalization; additive keys are ignored and no 2.0 spelling is required. +_CUDA_PRESENT = {"installed", "available"} +_PACKAGE_EXTRA = {"numpy": (), "pycuda": (), "torch": (("cuda_build", "13.0"), + ("cuda_available", True), ("sample_readiness", "ready"))} + + +def profile_dependencies(profile: Any) -> tuple[str, ...]: + """Return the Python dependencies one verification profile genuinely requires. + + An absent or unrecognized profile resolves to ``full-samples``. Evidence recorded before the + profile split carries no profile field, and a legacy artifact must never silently relax the + Torch requirement its own proof depended on. + """ + return PROFILE_DEPENDENCIES.get(profile, PROFILE_DEPENDENCIES[FULL_PROFILE]) + + +def stable_debian_version(value: Any) -> str: + """Strip an optional epoch and reject prerelease or Debian ``really`` versions.""" + text = str(value or "").strip().split(":", 1)[-1] + return "" if "~" in text or "really" in text.lower() else text + + +def cuda_release(value: Any) -> tuple[int, int] | None: + """Return a stable CUDA major/minor pair from one Debian candidate.""" + match = re.match(r"^(\d+)\.(\d+)(?:\.|$)", stable_debian_version(value)) + return (int(match.group(1)), int(match.group(2))) if match else None + + +def cuda_at_least_13(value: Any) -> bool: + """Whether a candidate is a stable CUDA 13.0-or-newer release.""" + return (release := cuda_release(value)) is not None and release >= (13, 0) + + +def native_sdk_release(value: Any) -> bool: + """Whether a Debian candidate belongs to the supported native SDK 13.0 family.""" + return re.fullmatch(r"13\.0(?:\.\d+)*(?:\+[0-9A-Za-z.]+)?(?:-[0-9A-Za-z.+]+)?", + stable_debian_version(value)) is not None + + +def cuda_build_package(candidate: Any) -> str | None: + """Map a stable CUDA 13+ candidate to its matching minimal-build meta-package.""" + release = cuda_release(candidate) + if release is None or release < (13, 0): + return None + return f"cuda-minimal-build-{release[0]}-{release[1]}" + + +def pycuda_bootstrap_packages(candidate: Any) -> tuple[str, ...]: + """Return the minimal exact package family needed by PyCUDA's default CURAND build.""" + build = cuda_build_package(candidate) + if build is None: + return () + suffix = build.removeprefix("cuda-minimal-build-") + return (f"libcurand-dev-{suffix}", build) + + +def _nested(value: Any, path: tuple[str, ...]) -> Any: + """Walk one nested path, yielding None -- never {} -- at the first non-object component.""" + for name in path: + if not isinstance(value, dict): + return None + value = value.get(name) + return value + + +def normalized_cuda_evidence(environment: Any) -> dict[str, Any]: + """Map installation.cuda_toolkit onto the installed/absent vocabulary consumers act on.""" + cuda = _nested(environment, ("installation", "cuda_toolkit")) + cuda = cuda if isinstance(cuda, dict) else {} + if cuda.get("status") not in _CUDA_PRESENT: + return {"status": "absent", "version": None, "root": None} + return {"status": "installed", "version": cuda.get("version"), + "root": cuda.get("root") or _nested(cuda, ("nvcc_discovery", "root")), + "environment_prefixes": cuda.get("environment_prefixes")} + + +def durable_venv_path(value: Path) -> Path | None: + """Resolve an absolute durable venv target outside cwd and transient roots.""" + requested = value.expanduser() + if not requested.is_absolute(): + return None + resolved, cwd = requested.resolve(strict=False), Path.cwd().resolve() + home = Path.home().resolve() + # These are rejection roots, never locations used to create files. + transient = any(resolved == root or root in resolved.parents + for root in map(Path, ("/tmp", "/var/tmp", "/run"))) # nosec B108 + under_worktree = resolved == cwd or (cwd not in (home, Path("/")) and cwd in resolved.parents) + return None if under_worktree or transient else resolved + + +def normalized_native_surface(environment: Any) -> tuple[dict[str, Any], list[str]]: + """Normalize the established installation.* native evidence into the internal surface.""" + native = _nested(environment, ("installation", "native_sdk")) + native = native if isinstance(native, dict) else {} + roots = [item for item in (native.get("complete_roots") or native.get("roots") or []) + if _absolute(item)] + build = _nested(environment, ("installation", "native_build_prerequisites")) + build = build if isinstance(build, dict) else {} + pkg_config = _nested(environment, ("installation", "build_tools", "pkg-config")) + producer_ready = native.get("installed") + installed = (native.get("status") == "installed" and producer_ready is True + and build.get("status") == "complete") + surface = {"installed": installed, "package": native.get("package"), + "sdk_root": roots[0] if roots else None, + "cuda": normalized_cuda_evidence(environment), + "build_prerequisites": build, + "tools": {"pkg_config": {"path": pkg_config}} if pkg_config else {}} + defects = [] if isinstance(native.get("status"), str) else [ + "installation.native_sdk.status is required"] + if surface["installed"] and not (roots and _nested(surface, ("package", "version"))): + defects.append("an installed native SDK requires a complete root and a package version") + return surface, defects + + +def normalized_pynvc_surface(environment: Any) -> tuple[dict[str, Any], list[str]]: + """Normalize the established pynvc + installation.python.packages evidence.""" + pynvc = environment.get("pynvc") if isinstance(environment, dict) else None + pynvc = pynvc if isinstance(pynvc, dict) else {} + identity = _nested(pynvc, ("identity",)) + identity = identity if isinstance(identity, dict) else {} + packages = _nested(environment, ("installation", "python", "packages")) + dependencies: dict[str, Any] = {} + for name, extra in _PACKAGE_EXTRA.items(): + row = _nested(packages, (name,)) + row = row if isinstance(row, dict) else {} + dependencies[name] = {"version": row.get("version"), "ready": bool( + row.get("status") == "installed" and row.get("requirement_satisfied") is True + and all(row.get(key) == value for key, value in extra))} + surface = {"installed": pynvc.get("imported") is True and identity.get("status") == "verified", + "version": pynvc.get("distribution_version"), "dependencies": dependencies, + "interpreter": identity.get("interpreter"), "sys_prefix": identity.get("sys_prefix")} + defects = [] if isinstance(pynvc.get("imported"), bool) else ["pynvc.imported is required"] + if surface["installed"] and not (_absolute(surface["interpreter"]) + and _absolute(surface["sys_prefix"])): + defects.append("a verified pynvc identity requires an absolute interpreter and sys_prefix") + return surface, defects + + +def normalized_python_bootstrap(environment: Any) -> dict[str, Any]: + """Map installation.python venv/header evidence onto the ok/missing vocabulary.""" + python = _nested(environment, ("installation", "python")) + python = python if isinstance(python, dict) else {} + return { + "venv_module": "ok" if python.get("venv_module") == "ok" else "missing", + "development_headers": "ok" if python.get("development_headers") == "ok" + else "missing"} + + +def read_json(source: Any) -> Any: + """Parse strict JSON, rejecting NaN/Infinity. A Path is read as a symlink-free regular + file; str/bytes are parsed as the document itself.""" + def reject(constant: str) -> None: + raise ValueError(f"non-RFC JSON numeric constant is forbidden: {constant}") + if isinstance(source, (str, bytes, bytearray)): + return json.loads(source, parse_constant=reject) + return json.loads(_read_regular(source, label="json document")[0], parse_constant=reject) + + +def write_new_bytes(path: Any, content: bytes, *, mode: int = 0o600) -> Path: + """Exclusively create one fresh regular file, never following or replacing an existing path.""" + requested = _symlink_free(path, label="output") + _check(requested.parent.is_dir(), f"output parent does not exist: {requested.parent}") + resolved = requested.parent.resolve(strict=True) / requested.name + descriptor = os.open( + resolved, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), mode) + try: + with os.fdopen(descriptor, "wb") as handle: + handle.write(content) + handle.flush() + os.fsync(handle.fileno()) + except BaseException: + resolved.unlink(missing_ok=True) + raise + return resolved + + +def write_new_json(path: Any, value: Any, *, mode: int = 0o600) -> Path: + """Exclusively create one fresh strict-JSON document with a trailing newline.""" + return write_new_bytes(path, (_dumps(value, sort_keys=True, indent=1) + "\n").encode("utf-8"), + mode=mode) + + +def transaction_sibling_path(path: Path, label: str, + observed_stems: tuple[str, ...] = ()) -> Path: + """Resolve one plan-sibling path and preflight every declared fresh output.""" + requested = path.expanduser().absolute() + if requested.is_symlink(): + raise ValueError(f"APT {label} path must not be a symlink") + if requested.parent.resolve() != Path.cwd().resolve(): + raise ValueError(f"APT {label} must be written beside the reviewed plan") + resolved = requested.parent.resolve() / requested.name + outputs = [resolved] + [resolved.with_name(f"{resolved.name}{'.' + stem if stem else ''}" + f".{stream}.log") + for stem in observed_stems for stream in ("stdout", "stderr")] + existing = next((item for item in outputs if os.path.lexists(item)), None) + if observed_stems and existing is not None: + raise FileExistsError(f"APT {label} output already exists: {existing}") + return resolved + + +# --- Bounded setup subprocesses ---------------------------------------------- + +def system_executable(name: str) -> str: + """Return the one canonical system path for ``name``. + + Raises FileNotFoundError -- never ValueError -- when zero or several distinct executables + answer to the name; callers branch on the absent-tool case by that exception type. + """ + matches = set() + for directory in ("/usr/bin", "/usr/sbin", "/bin", "/sbin"): + try: + resolved = (Path(directory) / name).resolve(strict=True) + except OSError: + continue + if resolved.is_file() and os.access(resolved, os.X_OK): + matches.add(str(resolved)) + unique = sorted(matches) + if len(unique) != 1: + raise FileNotFoundError( + f"expected one canonical system executable for {name}, found {unique}") + return unique[0] + + +def _terminate_group(process: Any) -> None: + """SIGTERM then unconditionally SIGKILL the child's whole group: a grandchild that ignores + SIGTERM would otherwise hold the pipes and make communicate() wait forever.""" + for number, wait in ((signal.SIGTERM, True), (signal.SIGKILL, False)): + try: + os.killpg(process.pid, number) + except (ProcessLookupError, PermissionError): + pass + if wait: + try: + process.wait(timeout=0.25) + except subprocess.TimeoutExpired: + pass + + +def _redact_child_output(value: bytes) -> bytes: + return _URL_USERINFO.sub(rb"\1***@", value) + + +def run_command(argv: list[str], *, cwd: Any = None, timeout: float | None = None, + extra_env: Any = None) -> "subprocess.CompletedProcess[bytes]": + """Run one no-shell command in its own session under the sanitized environment. + + The executable is resolved against the pinned PATH and the child gets its own session so the + whole process GROUP can be signalled. On timeout the group is terminated and reaped and + subprocess.TimeoutExpired carries the partial byte output. + """ + command = list(argv) if isinstance(argv, list) else [] + _check(command and all(isinstance(token, str) and "\x00" not in token for token in command) + and command[0], "command argv must be a non-empty list of NUL-free strings") + env = {name: value for name in _ENV_ALLOWLIST if (value := os.environ.get(name)) is not None} + env.update(_ENV_OVERRIDES) + for name, value in dict(extra_env or {}).items(): + _check(isinstance(name, str) and name and "=" not in name and "\x00" not in name + and isinstance(value, str) and "\x00" not in value, + "extra_env names and values must be NUL-free strings") + env[name] = value + executable = shutil.which(command[0], path=env["PATH"]) + if executable is None: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), command[0]) + with subprocess.Popen( # pylint: disable=consider-using-with + command, executable=executable, cwd=str(cwd) if cwd is not None else None, env=env, + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False, + start_new_session=True, + ) as process: + try: + stdout, stderr = process.communicate(timeout=timeout) + except subprocess.TimeoutExpired as exc: + _terminate_group(process) + stdout, stderr = process.communicate() + raise subprocess.TimeoutExpired( + command, timeout, _redact_child_output(stdout), _redact_child_output(stderr) + ) from exc + return subprocess.CompletedProcess( + command, process.returncode, _redact_child_output(stdout), _redact_child_output(stderr) + ) + + +# --- Jetson Linux release evidence ------------------------------------------- + +def parse_jetson_release(release_text: str) -> dict[str, Any]: + """Parse the first physical line of nv_tegra_release into structured release fields. + + Returns the literal line plus release_major/release_minor/revision/version; an unparseable + line yields all-None rather than a guess. Minimum-version policy is the caller's. + """ + line = release_text.splitlines()[0] if release_text else "" + release = re.search(r"\bR(\d+)(?:\.(\d+))?", line) + if not release or int(release.group(1)) <= 0: + return {"release_line": line, "release_major": None, "release_minor": None, + "revision": None, "version": None} + revision = re.search(r"\bREVISION\s*:\s*(\d+)\.(\d+)(?![\w.])", line, re.I) + major = int(release.group(1)) + minor = (int(release.group(2)) if release.group(2) is not None + else int(revision.group(1)) if revision else None) + return {"release_line": line, "release_major": major, "release_minor": minor, + "revision": f"{revision.group(1)}.{revision.group(2)}" if revision else None, + "version": f"{major}.{minor}" if minor is not None else None} + + +# --- Configured APT sources and the stock signed public origin --------------- + +def parse_apt_sources(path: Any, text_value: str) -> list[dict[str, Any]]: + """Parse active binary APT source tuples and explicit authentication-bypass options. + + Handles the one-line ``.list`` format and deb822 stanzas alike, returning one + ``{format, uri, suite, component, trust_bypass}`` record per enabled binary tuple. + """ + entries: list[dict[str, Any]] = [] + source = Path(path) + if source.suffix == ".list" or source.name == "sources.list": + line_pattern = re.compile( + r"^\s*deb\s+(?:\[([^\]]*)\]\s+)?(\S+)\s+(\S+)\s+([^#]+?)(?:\s+#.*)?$", re.I) + bypass_pattern = re.compile( + rf"(?:^|\s)(?:{'|'.join(_APT_BYPASS)})\s*=\s*(?:yes|true|1)(?:\s|$)", re.I) + for line in text_value.splitlines(): + skip = not line.strip() or line.lstrip().startswith("#") + match = None if skip else line_pattern.match(line) + if match: + options, uri, suite, components = match.groups() + entries.extend( + {"format": "list", "uri": uri.rstrip("/"), "suite": suite, + "component": component, + "trust_bypass": bool(bypass_pattern.search(options or ""))} + for component in components.split()) + return entries + fields: dict[str, str] = {} + key: str | None = None + for line in [*text_value.splitlines(), ""]: + if not line.strip(): + if (fields.get("enabled", "yes").strip().lower() not in {"no", "false", "0"} + and "deb" in fields.get("types", "").split()): + bypass = any(fields.get(name, "").strip().lower() in {"yes", "true", "1"} + for name in _APT_BYPASS) + entries.extend( + {"format": "deb822", "uri": uri.rstrip("/"), "suite": suite, + "component": component, "trust_bypass": bypass} + for uri in fields.get("uris", "").split() + for suite in fields.get("suites", "").split() + for component in fields.get("components", "").split()) + fields, key = {}, None + elif line.lstrip().startswith("#"): + continue + elif line[:1].isspace() and key: + fields[key] = fields[key] + " " + line.strip() + else: + name, separator, value = line.partition(":") + key = name.strip().lower() if separator else None + if key is not None: + fields[key] = value.strip() + return entries + + +def _sources_bind_origin(origin: dict[str, Any]) -> bool: + """Require every configured source file behind the origin to be hash-bound and bypass-free.""" + hashes = origin.get("configured_source_sha256") + sources = origin.get("configured_sources") + if not isinstance(hashes, list) or not isinstance(sources, list) or not hashes or not sources: + return False + if not all(isinstance(value, str) and _SHA256.fullmatch(value) for value in hashes): + return False + suite, separator, component = str(origin.get("suite_component", "")).partition("/") + expected = {"uri": str(origin.get("uri", "")).rstrip("/"), "suite": suite, + "component": component, "trust_bypass": False} + for item in sources: + entry = item.get("entry") if isinstance(item, dict) else None + if (not separator or not isinstance(entry, dict) + or not isinstance(item.get("path"), str) or not Path(item["path"]).is_absolute() + or not isinstance(item.get("sha256"), str) + or not _SHA256.fullmatch(item["sha256"]) + or entry.get("format") not in {"list", "deb822"} + or {name: entry.get(name) for name in expected} != expected): + return False + return sorted(hashes) == sorted(item["sha256"] for item in sources) + + +def _authenticated_origin(origin: Any) -> bool: + """Recognize one configured APT origin protected by the package-signature chain.""" + if not isinstance(origin, dict) or origin.get("authentication") != "apt-signature-chain": + return False + uri, suite_component = origin.get("uri"), origin.get("suite_component") + if not isinstance(uri, str) or not isinstance(suite_component, str): + return False + try: + parsed = urlparse(uri) + rejected = (parsed.username, parsed.password, parsed.port) != (None, None, None) + except ValueError: + return False + if rejected or parsed.params or parsed.query or parsed.fragment: + return False + return (parsed.scheme in {"http", "https"} and bool(parsed.hostname) + and re.fullmatch(r"[^/\s]+/[^/\s]+", suite_component) is not None + and _sources_bind_origin(origin)) + + +def _public_origin(origin: Any) -> bool: + """Recognize only the signature-authenticated stock public NVIDIA Jetson origin.""" + if not _authenticated_origin(origin): + return False + parsed = urlparse(origin["uri"]) + path = parsed.path[:-1] if parsed.path.endswith("/") else parsed.path + return ((parsed.scheme, parsed.hostname, path) in _PUBLIC_APT_LOCATIONS + and _PUBLIC_SUITE_COMPONENT.fullmatch(origin["suite_component"]) is not None) + + +def authenticated_candidate_binding(record: Any) -> dict[str, Any] | None: + """Return a signature-authenticated configured origin for a base package, else None.""" + if (not isinstance(record, dict) or record.get("query_status") not in {"ok", "success"} + or not isinstance(record.get("candidate"), str) or not record["candidate"]): + return None + origins = record.get("candidate_origins") + approved = [item for item in origins if _authenticated_origin(item)] if isinstance( + origins, list) else [] + return approved[0] if approved else None + + +def public_candidate_binding(record: Any) -> dict[str, Any] | None: + """Return the stock signed public origin backing an apt-cache candidate, else None. + + Fails closed for internal mirrors, unsigned or trust-bypassed sources, a suite/component + outside exact rNN.N/main, and any origin whose configured source files are not hash-bound to + it. This is the native-SDK release-origin decision; never widen it. + """ + if (not isinstance(record, dict) or record.get("query_status") not in {"ok", "success"} + or not isinstance(record.get("candidate"), str) or not record["candidate"]): + return None + origins = record.get("candidate_origins") + approved = [item for item in origins if _public_origin(item)] if isinstance( + origins, list) else [] + return approved[0] if approved else None + + +# --- PyNvVideoCodec capability adapter --------------------------------------- + +_ENCODER_CODECS = ("h264", "hevc", "av1") +_DECODER_CODECS = { + "mpeg1": ("MPEG1", "MPEG_1"), "mpeg2": ("MPEG2", "MPEG_2"), + "mpeg4": ("MPEG4", "MPEG_4"), "vc1": ("VC1", "VC_1"), + "h264": ("H264", "H_264"), "hevc": ("HEVC", "H265", "H_265"), + "vp8": ("VP8", "VP_8"), "vp9": ("VP9", "VP_9"), "av1": ("AV1",), + "jpeg": ("JPEG",), +} +_API_13_CAPS = ("support_yuv422_encode", "support_mvhevc_encode", + "support_temporal_filter", "support_lookahead_level", + "support_unidirectional_b") + + +def _scalar_attributes(value: Any) -> dict[str, Any]: + """Copy only JSON-scalar fields from a mapping or a PyBind capability object.""" + names = value if isinstance(value, dict) else ( + name for name in dir(value) if not name.startswith("_")) + result: dict[str, Any] = {} + for name in names: + try: + item = value[name] if isinstance(value, dict) else getattr(value, name) + except Exception: # pylint: disable=broad-exception-caught + continue + if item is None or isinstance(item, (str, int, float, bool)): + result[str(name)] = item + return result + + +def _enum_member(module: Any, containers: tuple[str, ...], names: tuple[str, ...]) -> Any: + for container_name in containers: + container = getattr(module, container_name, None) + for name in names: + if container is not None and hasattr(container, name): + return getattr(container, name) + return next((getattr(module, name) for name in names if hasattr(module, name)), None) + + +def query_pynvc_capabilities( + module: Any, gpu: int, linked_api: dict[str, Any] +) -> tuple[dict[str, Any], list[str]]: + """Query the established schema-1.2 PyNv encoder and baseline decoder capability fields.""" + classification = { + role: {"evidence_source_type": "api_query_helper", "official_sample": False, + "api_symbol": f"PyNvVideoCodec.Get{role.title()}rCaps"} + for role in ("encode", "decode") + } + result: dict[str, Any] = { + "authority": "pynvc", "gpu": gpu, "evidence_classification": classification, + "linked_nvenc_api": linked_api, "encode": {}, "decode": {}, + } + if gpu != 0: + reason = ("PyNvVideoCodec capability helpers route both encoder and decoder queries to" + " GPU 0; nonzero-GPU capability support remains unknown.") + result.update({"status": "unknown", "reason": reason}) + return result, [reason] + encoder, decoder = getattr(module, "GetEncoderCaps", None), getattr( + module, "GetDecoderCaps", None) + for codec in _ENCODER_CODECS: + try: + values = _scalar_attributes(encoder(gpu, codec)) if callable(encoder) else {} + except Exception as exc: # pylint: disable=broad-exception-caught + result["encode"][codec] = {"status": "unknown", "query_status": "error", + "supported": None, "error": str(exc)} + continue + if not values: + result["encode"][codec] = { + "status": "unknown", "supported": None, + "reason": ("GetEncoderCaps returned no fields" if callable(encoder) + else "GetEncoderCaps unavailable")} + continue + unavailable = ({ + key: {"status": "unknown", + "reason": "capability is absent from the linked NVENC 12.1 header"} + for key in _API_13_CAPS if key not in values + } if linked_api.get("value") == "12.1" else {}) + result["encode"][codec] = { + "status": "capability_reported", "supported": None, + "session_status": "opened_by_GetEncoderCaps", + "operation_status": "not_verified_by_probe", + "interpretation": ("GetEncoderCaps returned capability fields but no explicit codec" + " support result; an official encode operation is still required"), + "values": values, "unavailable_fields": unavailable, + } + chroma = _enum_member(module, ("cudaVideoChromaFormat", "ChromaFormat"), + ("420", "YUV420")) + for codec, aliases in _DECODER_CODECS.items(): + codec_enum = _enum_member(module, ("cudaVideoCodec", "Codec", "VideoCodec"), aliases) + try: + values = (_scalar_attributes(decoder(gpu, codec_enum, chroma, 8)) + if callable(decoder) and codec_enum is not None and chroma is not None + else {}) + except Exception as exc: # pylint: disable=broad-exception-caught + result["decode"][codec] = {"status": "unknown", "query_status": "error", + "supported": None, "error": str(exc)} + continue + if not values: + reason = ("GetDecoderCaps unavailable" if not callable(decoder) else + "required codec or 4:2:0 decoder enum was not found" + if codec_enum is None or chroma is None else + "GetDecoderCaps returned no fields") + result["decode"][codec] = { + "status": "unknown", "supported": None, "reason": reason} + continue + supported_value = values.get("supported") + supported = (False if supported_value in (0, False) else True + if isinstance(supported_value, (int, bool)) and int(supported_value) > 0 + else None) + status = "unsupported" if supported is False else ( + "capability_reported" if supported is True else "unknown") + interpretation = ( + "bIsSupported=0; remaining zeroed OUT fields are inapplicable" + if supported is False else "bIsSupported=1; returned limits are applicable" + if supported is True else "bIsSupported was absent or not interpretable") + result["decode"][codec] = { + "status": status, "supported": supported, + "query": {"chroma": "420", "bit_depth": 8}, + "values": values, "interpretation": interpretation, + } + records = [*result["encode"].values(), *result["decode"].values()] + states = {record.get("status") for record in records} + successful = {"capability_reported", "unsupported"} + result["status"] = ("complete" if records and states <= successful else + "partial" if states & successful else "unknown") + if result["status"] != "complete": + result["reason"] = ("Some capability queries succeeded while others are unknown or failed" + if result["status"] == "partial" + else "No capability query completed successfully") + return result, [] diff --git a/skills/jetson-video-setup/scripts/setup/verify_native.py b/skills/jetson-video-setup/scripts/setup/verify_native.py new file mode 100644 index 00000000..d22ab956 --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/verify_native.py @@ -0,0 +1,785 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Build and run the package-installed NVIDIA Video Codec SDK samples as an operation proof. + +This helper is the *operation* authority for the native surface: it builds the official +``AppEncCuda`` and ``AppDec`` samples from the package-owned ``Samples`` tree, encodes one fresh +NV12 frame, and -- only after AppEncCuda's own success marker has been proven by this run -- +decodes that fresh bitstream with AppDec. An API query is never a substitute for either step. + +Only ``installation.native_sdk`` of the ``nvcodec-environment`` 1.2 artifact is read, and only +its required subset is validated, so a malformed or absent ``pynvc`` block can neither block nor +be blocked by this proof. Invoked directly as ``python3 -I .../verify_native.py``: it owns its +own CLI, never activates a virtual environment, and never imports another skill. +""" + +# pylint: disable=missing-function-docstring,too-many-locals + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path +from typing import Any + +_SETUP_DIR = Path(__file__).resolve().parent +if str(_SETUP_DIR) not in sys.path: + sys.path.insert(0, str(_SETUP_DIR)) + +# pylint: disable=wrong-import-position +from setup_contract import ( # noqa: E402 + canonical_json_sha256, file_identity, parse_jetson_release, read_json, require_isolated, + run_command, sha256_bytes, system_executable, utc_now, write_new_bytes, write_new_json, +) + +# pylint: enable=wrong-import-position + +SCHEMA_VERSION = "1.5" +ENVIRONMENT_KIND = "nvcodec-environment" +ENVIRONMENT_MAJOR = "1" +NATIVE_PACKAGE = "nvidia-video-codec-sdk" + +# The official, non-negotiable operation markers. Each is anchored to a whole physical line so +# no embedded or trailing text can satisfy a count: "...encoded: 11" and "...encoded: 1x" are +# both rejected as one encoded frame. +_ENCODED_RE = re.compile(r"^[^\S\r\n]*Total frames encoded:[^\S\r\n]*(\d+)[^\S\r\n]*\r?$", + re.MULTILINE) +_BITSTREAM_RE = re.compile(r"^Bitstream saved in file\s+(.+?)\s*$", re.MULTILINE) +_DECODED_RE = re.compile(r"^[^\S\r\n]*Total frame decoded:[^\S\r\n]*(\d+)[^\S\r\n]*\r?$", + re.MULTILINE) +# The smoke is one fixed 640x360 NV12 frame. --width/--height carry the requested geometry and +# default to it; 345,600 stays the authoritative payload size for the fixture and the decoded +# output, so a request outside the fixed smoke geometry is refused rather than silently resized. +_SMOKE_GEOMETRY = (640, 360) +_SMOKE_NV12_BYTES = 345600 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") +_VERSION_RE = re.compile(r"\d+(?:\.\d+)*(?:[.+~-][0-9A-Za-z.+~-]*)?") +# Private replacement for the removed public-release-family predicate: stable Debian revisions +# of the public 13.0.x family only, never prerelease or "really" family redirects. +_PUBLIC_SDK_FAMILY = "13.0" +_PUBLIC_SDK_RE = re.compile(r"13\.0(?:\.\d+)*(?:\+[0-9A-Za-z.]+)?(?:-[0-9A-Za-z.+]+)?") +_FAILURE_PATTERNS = ( + ("nvenc_error", re.compile(r"(?im)^[^\S\r\n]*(?:\[[^]\r\n]*\][^\S\r\n]*)*" + r"(?:NV_ENC_ERR_[A-Z0-9_]+\b|NVENC error\b)")), + ("cuda_error", re.compile(r"(?im)^[^\S\r\n]*(?:\[[^]\r\n]*\][^\S\r\n]*)*" + r"(?:CUDA_ERROR_[A-Z0-9_]+\b|CudaCheckError\(\) failed\b|" + r"CUDA (?:driver |runtime )?(?:API )?error\b)")), + ("error", re.compile(r"(?im)^\s*(?:Traceback \(most recent call last\)|\[(?:FATAL|ERROR)\s*\]|" + r"(?:uncaught )?exception\b|error\b|failed\b|failure\b|" + r"[a-z_][a-z0-9_.:~]*(?:\([^\r\n]*\))?\s+returned error\b)")), +) +# installation.native_sdk.tools carries exactly these five entries; tools.generator.name is the +# CMake generator name and nothing outside _GENERATOR_NAMES is accepted. +_REQUIRED_TOOLS = ("cmake", "cxx", "nvcc", "pkg_config", "generator") +_GENERATOR_NAMES = ("Ninja", "Unix Makefiles") +_SAMPLES = {"encoder": ("AppEncode", "AppEncCuda"), "decoder": ("AppDecode", "AppDec")} +_DRIVER_LIBRARIES = {"encoder": ("libcuda.so.1", "libnvidia-encode.so.1"), + "decoder": ("libcuda.so.1", "libnvcuvid.so.1")} +# Static AppDec module -> Debian package repair table: a rendering aid for the probe's own +# unresolved_modules, never a live query. The deterministic repair hint is the payload. +_APPDEC_PACKAGES = {"libavcodec": "libavcodec-dev", "libavformat": "libavformat-dev", + "libavutil": "libavutil-dev", "libswresample": "libswresample-dev"} +# The two official aggregate reports nvcodec-native-verification 1.5 publishes beside the +# operation proof. They are RAW sample-reported rows: never an API support claim, never operation +# evidence, and never a readiness gate. Each runs between two identity checks of its own binary. +_AGGREGATE_ROUTES = {"encoder": ("AppEncCuda", "-ec"), "decoder": ("AppDec", "-dc")} +_ENCODER_CAPS_RE = re.compile(r"(?m)^Encoder Capability Summary[ \t]*\r?$") +_CAPS_GPU_RE = re.compile(r"(?m)^GPU\s+(\d+)\s+-\s+(.+?)[ \t]*\r?$") +_CAPS_CODEC_RE = re.compile(r"(?m)^[ \t]*(H264|HEVC|AV1):[ \t]*(yes|no)[ \t]*\r?$") +_ENCODER_CAPS_HINT = "For detailed information about all capabilities, use -ec-detail" +_DECODER_CAPS_RE = re.compile(r"(?m)^GPU Decoder Capabilities[ \t]*\r?$") +_DECODER_SUMMARY_RE = re.compile(r"(?m)^Codec Support Summary[ \t]*\r?$") +_DECODER_CAPS_HINT = "For detailed information about all capabilities, use -dc-detail" +_ENCODER_SECTIONS = ("=== CODEC SUPPORT SUMMARY ===", "=== ENCODER CAPABILITIES SUMMARY TABLE ===") +_AGGREGATE_SCOPE = { + "encoder": ("raw sample-reported values only; nvEncGetEncodeCaps status is unchecked by the" + " sample, so rows are neither authoritative unsupported nor operation evidence"), + "decoder": ("raw sample-reported values only; this report is neither authoritative" + " cuvidGetDecoderCaps evidence nor operation evidence")} + + +def _require(condition: Any, message: str) -> None: + if not condition: + raise ValueError(message) + + +def _absolute(value: Any, *, label: str) -> Path: + _require(isinstance(value, str) and value and Path(value).is_absolute(), + f"{label} must be an absolute path, got {value!r}") + return Path(value) + + +def _sha256_path(path: Path) -> str: + """Hash content through any symlink; file_identity deliberately refuses a symlinked leaf.""" + return sha256_bytes(path.read_bytes()) + + +def _identity(path: Path, label: str) -> dict[str, Any] | None: + return file_identity(path, label=label) if path.is_file() and not path.is_symlink() else None + + +def _fresh_directory(path: Path, *, label: str) -> Path: + requested = Path(os.path.abspath(os.fspath(Path(path).expanduser()))) + _require(all(31 < ord(character) != 127 for character in str(requested)), + f"{label} must not contain control characters") + _require(requested.parent.is_dir(), f"{label} parent does not exist: {requested.parent}") + resolved = requested.parent.resolve(strict=True) / requested.name + _require(not os.path.lexists(requested) and not os.path.lexists(resolved), + f"{label} must not already exist: {resolved}") + resolved.mkdir(mode=0o700) + return resolved + + +def _run(argv: list[str], *, cwd: Path | None = None, timeout: int = 300) -> dict[str, Any]: + """Run one bounded child; a timeout becomes failure evidence rather than an exception.""" + try: + done = run_command(argv, cwd=cwd, timeout=timeout) + code, out, err, expired = done.returncode, done.stdout, done.stderr, False + except subprocess.TimeoutExpired as exc: + code, out, err, expired = None, exc.stdout, exc.stderr, True + return {"argv": list(argv), "cwd": str(cwd) if cwd else str(Path.cwd()), "exit_code": code, + "timed_out": expired, "stdout": bytes(out or b"").decode("utf-8", errors="replace"), + "stderr": bytes(err or b"").decode("utf-8", errors="replace")} + + +def _failure_markers(evidence: dict[str, Any]) -> list[str]: + """Return stable labels for explicit sample or runtime failures in either stream.""" + combined = evidence["stdout"] + "\n" + evidence["stderr"] + return [label for label, pattern in _FAILURE_PATTERNS if pattern.search(combined)] + + +def _tail(evidence: dict[str, Any], limit: int = 4000) -> dict[str, Any]: + return {"argv": evidence["argv"], "exit_code": evidence["exit_code"], + "timed_out": evidence["timed_out"], "stdout_tail": evidence["stdout"][-limit:], + "stderr_tail": evidence["stderr"][-limit:]} + + +# --- nvcodec-environment 1.2: required subset of installation.native_sdk only - + +def _version_token(value: Any, *, label: str) -> str: + """Validate -- never merely read -- a version token.""" + text = value.strip() if isinstance(value, str) else "" + _require(text and _VERSION_RE.fullmatch(text), f"{label} must be a version token: {value!r}") + return text + + +def _public_sdk_release(value: Any) -> bool: + normalized = value.strip().split(":", 1)[-1] if isinstance(value, str) else "" + if not normalized or "~" in normalized or "really" in normalized.lower(): + return False + return _PUBLIC_SDK_RE.fullmatch(normalized) is not None + + +def _tool(tools: Any, name: str) -> dict[str, Any]: + """Validate one required tool entry and re-authenticate it live against its recorded hash. + + An *absent* tool -- no entry, or a path that is not an executable regular file -- returns an + ``unavailable`` record naming the defect so the run still emits the complete ready=false + result document. A present-but-corrupt entry is a malformed artifact and still raises. + """ + label = f"installation.native_sdk.tools.{name}" + record = tools.get(name) if isinstance(tools, dict) else None + if not isinstance(record, dict): + return {"status": "unavailable", "name": name, + "reason": f"required build tool {name} is absent: {label} is missing"} + path = _absolute(record.get("path"), label=f"{label}.path") + digest = record.get("sha256") + _require(isinstance(digest, str) and _SHA256_RE.fullmatch(digest), + f"{label}.sha256 must be a sha-256 digest") + entry = {"status": "verified", "path": str(path), "sha256": digest, + "version": _version_token(record.get("version"), label=f"{label}.version")} + if name == "generator": + _require(record.get("name") in _GENERATOR_NAMES, + f"{label}.name must be the CMake generator name, one of " + f"{list(_GENERATOR_NAMES)}, got {record.get('name')!r}") + entry["name"] = record["name"] + if not (path.is_file() and os.access(path, os.X_OK)): + return {"status": "unavailable", "name": name, + "reason": f"required build tool {name} is absent: {path} is not an executable" + " regular file"} + _require(_sha256_path(path) == digest, + f"{label} changed on disk after the environment was produced") + return entry + + +def _prerequisite_reason(build: Any) -> str: + """Render deterministic repair guidance for incomplete AppDec build prerequisites.""" + unresolved = build.get("unresolved_modules") if isinstance(build, dict) else None + modules = sorted(str(item) for item in unresolved) if isinstance(unresolved, list) else [] + packages = [_APPDEC_PACKAGES[name] for name in modules if name in _APPDEC_PACKAGES] + repair = (f"install {', '.join(packages)} before CMake" if len(packages) == len(modules) + else "read the CMake configure stderr tail for the unmapped module(s)") + return ("installation.native_sdk.build_prerequisites are not complete" + + (f": AppDec needs {', '.join(modules)}; {repair}." if modules else "")) + + +def _native_surface(data: dict[str, Any]) -> dict[str, Any]: + """Validate the required subset of ``installation.native_sdk``. + + Additive keys are ignored and no set equality is applied. ``pynvc`` is never read. + """ + installation = data.get("installation") + native = installation.get("native_sdk") if isinstance(installation, dict) else None + _require(isinstance(native, dict), "environment has no installation.native_sdk block") + _require(native.get("installed") is True, "installation.native_sdk.installed is not true") + package = native.get("package") if isinstance(native.get("package"), dict) else {} + _require(package.get("name") == NATIVE_PACKAGE and package.get("status") == "installed", + f"installation.native_sdk.package does not show {NATIVE_PACKAGE} installed") + version = _version_token(package.get("version"), + label="installation.native_sdk.package.version") + _require(_public_sdk_release(version), f"installation.native_sdk.package.version {version!r}" + f" is outside the public {_PUBLIC_SDK_FAMILY}.x Video Codec SDK release family") + cuda = native.get("cuda") if isinstance(native.get("cuda"), dict) else {} + _require(cuda.get("status") == "installed", "installation.native_sdk.cuda.status must be" + f" 'installed', got {cuda.get('status')!r}") + build = native.get("build_prerequisites") + _require(isinstance(build, dict) and build.get("status") == "complete" + and build.get("unresolved_modules") == [], _prerequisite_reason(build)) + return {"package": {"name": NATIVE_PACKAGE, "status": "installed", "version": version}, + "sdk_root": str(_absolute(native.get("sdk_root"), + label="installation.native_sdk.sdk_root")), + "cuda": {"status": "installed", + "version": _version_token(cuda.get("version"), + label="installation.native_sdk.cuda.version"), + "root": str(_absolute(cuda.get("root"), + label="installation.native_sdk.cuda.root"))}, + "build_prerequisites": {"status": "complete", "unresolved_modules": []}, + "tools": {name: _tool(native.get("tools"), name) for name in _REQUIRED_TOOLS}} + + +def _platform(data: dict[str, Any]) -> dict[str, Any]: + """Cross-check platform release evidence. Minimum-version gating belongs to the probe.""" + block = data.get("platform") if isinstance(data.get("platform"), dict) else {} + jetson = block.get("jetson_linux") if isinstance(block.get("jetson_linux"), dict) else {} + line = jetson.get("release_line") + _require(isinstance(line, str) and line and "\n" not in line and "\r" not in line, + "platform.jetson_linux.release_line must be exactly one non-empty physical line") + parsed = parse_jetson_release(line) + _require(parsed["release_major"] is not None, + "platform.jetson_linux.release_line carries no Jetson Linux release token") + _require(parsed["version"] == jetson.get("version"), + "platform.jetson_linux.version disagrees with its own release_line") + return {"jetson_linux": parsed["version"], "machine": block.get("machine"), + "release": parsed} + + +def _load_environment(path: Path, gpu: int) -> dict[str, Any]: + identity = file_identity(path, label="nvcodec-environment") + raw = Path(identity["path"]).read_bytes() + _require(sha256_bytes(raw) == identity["sha256"], + "nvcodec-environment changed while it was being read") + data = read_json(raw) + _require(isinstance(data, dict) and data.get("kind") == ENVIRONMENT_KIND, + "input is not an nvcodec-environment artifact") + schema = data.get("schema_version") + _require(isinstance(schema, str) and schema, "environment schema_version is missing") + _require(schema.split(".", 1)[0] == ENVIRONMENT_MAJOR, + f"unsupported nvcodec-environment major version {schema!r}: native verification" + f" consumes schema {ENVIRONMENT_MAJOR}.x only") + _require(data.get("mode") == "live", "environment mode must be 'live'") + _require(data.get("selected_gpu") == gpu, + "environment selected_gpu does not match the requested --gpu") + return {"path": identity["path"], "identity": identity, "schema_version": schema, + "canonical_sha256": canonical_json_sha256(data), "selected_gpu": gpu, + "platform": _platform(data), "native": _native_surface(data)} + + +# --- Live package-ownership re-authentication -------------------------------- + +def _package_ownership(sdk_root: Path, declared: str, timeout: int) -> dict[str, Any]: + """Re-authenticate ownership of ``sdk_root`` live, never from a recorded root inventory. + + ``dpkg-query -W`` must report exactly the declared version, ``dpkg-query -L`` must own + ``sdk_root`` as a directory plus at least one regular file beneath it, and ``dpkg --verify`` + must be silent. + """ + query, dpkg = system_executable("dpkg-query"), system_executable("dpkg") + status = _run([query, "-W", "-f=${db:Status-Abbrev}\t${Version}", NATIVE_PACKAGE], + timeout=timeout) + fields = status["stdout"].strip().split("\t") + live = fields[1].strip() if len(fields) == 2 and fields[0].startswith("ii") else None + listing = _run([query, "-L", NATIVE_PACKAGE], timeout=timeout) + owned = [Path(item.strip()) for item in listing["stdout"].splitlines() + if item.strip().startswith("/")] + files = [item for item in owned if item != sdk_root and item.is_relative_to(sdk_root) + and item.is_file() and not item.is_symlink()] + integrity = _run([dpkg, "--verify", NATIVE_PACKAGE], timeout=timeout) + root_owned = bool(listing["exit_code"] == 0 and sdk_root in owned and sdk_root.is_dir() + and not sdk_root.is_symlink()) + silent = integrity["exit_code"] == 0 and not integrity["stdout"].strip() + reasons = [message for ok, message in ( + (status["exit_code"] == 0, + f"dpkg-query -W exited {status['exit_code']} for {NATIVE_PACKAGE}"), + (live == declared, + f"live dpkg version {live!r} is not the declared {NATIVE_PACKAGE} version {declared!r}"), + (_public_sdk_release(live), f"live dpkg version {live!r} is outside the public" + f" {_PUBLIC_SDK_FAMILY}.x release family"), + (root_owned, f"{sdk_root} is not a package-owned directory of {NATIVE_PACKAGE}"), + (bool(files), f"{NATIVE_PACKAGE} owns no regular file beneath {sdk_root}"), + (silent, f"dpkg --verify reported a modified {NATIVE_PACKAGE} payload")) if not ok] + return {"status": "verified" if not reasons else "failed", "package": NATIVE_PACKAGE, + "declared_version": declared, "live_version": live, "sdk_root": str(sdk_root), + "sdk_root_owned": root_owned, "owned_file_count": len(files), "verify_silent": silent, + "queries": [_tail(status, 2000), _tail(listing, 2000), _tail(integrity)], + "reasons": reasons} + + +# --- Configure and build the two official targets ---------------------------- + +def _configure_and_build(environment: dict[str, Any], build_dir: Path, timeout: int, + reasons: list[str]) -> tuple[Path, dict[str, Any]]: + """Configure and build only the official AppEncCuda and AppDec targets in a fresh tree.""" + native = environment["native"] + tools = native["tools"] + samples = Path(native["sdk_root"]) / "Samples" + _require(samples.is_dir() and not samples.is_symlink(), + f"package-owned Samples tree is missing: {samples}") + build_root = _fresh_directory(build_dir, label="native build directory") + arguments = ["-S", str(samples), "-B", str(build_root), "-G", tools["generator"]["name"], + f"-DCMAKE_MAKE_PROGRAM={tools['generator']['path']}", + "-DCMAKE_BUILD_TYPE=Release", + f"-DCMAKE_CXX_COMPILER={tools['cxx']['path']}", + f"-DCUDAToolkit_ROOT={native['cuda']['root']}", + f"-DCUDAToolkit_NVCC_EXECUTABLE={tools['nvcc']['path']}", + f"-DCMAKE_CUDA_COMPILER={tools['nvcc']['path']}", + f"-DPKG_CONFIG_EXECUTABLE={tools['pkg_config']['path']}"] + cmake = tools["cmake"]["path"] + commands = [_run([cmake, *arguments], timeout=timeout)] + if commands[0]["exit_code"] != 0: + reasons.append("official sample configure failed") + for target in ("AppEncCuda", "AppDec") if not reasons else (): + commands.append(_run([cmake, "--build", str(build_root), "--target", target, + "--parallel", "2"], timeout=timeout)) + if commands[-1]["exit_code"] != 0: + reasons.append(f"official {target} sample build failed") + return build_root, {"status": "failed" if reasons else "built", "root": str(build_root), + "generator": tools["generator"]["name"], "arguments": arguments, + "commands": [_tail(item) for item in commands]} + + +# --- Built binaries and real driver-library linkage -------------------------- + +def _sample_binary(build_root: Path | None, role: str) -> Path | None: + """Return only CMake's deterministic target output, confined to the fresh build tree.""" + parts = _SAMPLES[role] + path = build_root.joinpath(*parts, parts[-1]) if build_root is not None else None + if path is None or not path.is_file() or path.is_symlink(): + return None + resolved = path.resolve(strict=True) + return resolved if resolved.is_relative_to(build_root.resolve()) else None + + +def _resolved_library(line: str | None) -> tuple[str, str | None]: + """Classify one ldd row as a real resolution, a missing library, or an SDK link stub.""" + value = line.split("=>", 1)[1].strip().split(" ", 1)[0] if line and "=>" in line else "" + try: + real = str(Path(value).resolve(strict=True)) if value.startswith("/") else "" + except OSError: + real = "" + if not real or not Path(real).is_file(): + return "missing", value or None + return ("stub" if "/stubs/" in value or "/stubs/" in real else "resolved"), real + + +def _linkage(binary: Path, required: tuple[str, ...], timeout: int) -> dict[str, Any]: + """Reject stub-linked builds: every required driver library must resolve for real.""" + try: + evidence = _run([system_executable("ldd"), str(binary)], timeout=timeout) + except FileNotFoundError: + return {"status": "unknown", "reason": "ldd is unavailable", "required": list(required)} + rows = (evidence["stdout"] + "\n" + evidence["stderr"]).splitlines() + libraries: dict[str, Any] = {} + for library in required: + line = next((item.strip() for item in rows if item.strip().startswith(library)), None) + state, real = _resolved_library(line) + libraries[library] = {"status": state, "real_path": real, "line": line} + failures = sorted(f"{name} runtime resolution is {item['status']}" + for name, item in libraries.items() if item["status"] != "resolved") + return {"status": "verified" if not failures and evidence["exit_code"] == 0 else "failed", + "command": evidence["argv"], "exit_code": evidence["exit_code"], + "libraries": libraries, "failures": failures, + "stderr_tail": evidence["stderr"][-2000:]} + + +def _built_samples(build_root: Path | None, timeout: int, + reasons: list[str]) -> tuple[dict[str, Any], dict[str, Any]]: + """Locate, hash, and link-check both official sample binaries.""" + binaries = {role: _sample_binary(build_root, role) for role in _SAMPLES} + records: dict[str, Any] = {} + for role, path in binaries.items(): + label = _SAMPLES[role][-1] + if path is None: + reasons.append(f"official {label} binary is not built") + records[role] = {"binary": None, "identity": None, "linkage": { + "status": "unknown", "reason": f"{label} binary is not built"}} + continue + linkage = _linkage(path, _DRIVER_LIBRARIES[role], timeout) + if linkage["status"] != "verified": + reasons.append(f"official {label} runtime driver-library linkage is not verified") + records[role] = {"binary": str(path), "linkage": linkage, + "identity": file_identity(path, label=f"{label} binary")} + return binaries, records + + +# --- The official aggregate capability reports (schema 1.5) ------------------- + +def _classification(role: str) -> dict[str, Any]: + """The frozen official_sample_report / sample_reported classification for one route.""" + sample, option = _AGGREGATE_ROUTES[role] + return {"evidence_source_type": "official_sample_report", "official_sample": True, + "sample_name": sample, "report_option": option, "scope": "sample_reported"} + + +def _encoder_gpus(stdout: str) -> list[dict[str, Any]]: + """Split the AppEncCuda -ec report per GPU and keep its rows verbatim, never as a verdict.""" + matches = list(_CAPS_GPU_RE.finditer(stdout)) + records = [] + for index, match in enumerate(matches): + end = matches[index + 1].start() if index + 1 < len(matches) else len(stdout) + block = stdout[match.end():end] + rows = _CAPS_CODEC_RE.findall(block) + records.append({ + "gpu": int(match.group(1)), "name": match.group(2).strip(), + "sample_reported_codec_rows": [{"codec": codec.lower(), "value": value} + for codec, value in rows], + "required_sections_present": bool( + all(block.count(section) == 1 for section in _ENCODER_SECTIONS) + and sorted(codec for codec, _ in rows) == ["AV1", "H264", "HEVC"])}) + return records + + +def _aggregate(role: str, binary: Path, identity: dict[str, Any], + request: dict[str, int]) -> dict[str, Any]: + """Run one official aggregate report without promoting its rows to a support claim. + + The sample binary is re-authenticated immediately before and after the report, so a binary + substituted around the report window fails closed rather than producing quiet evidence. + """ + sample, option = _AGGREGATE_ROUTES[role] + initial = file_identity(binary, label=f"{sample} before {option}") + _require(initial == identity, f"official {sample} changed before its {option} report") + evidence = _run([str(binary), option], timeout=request["timeout"]) + terminal = file_identity(binary, label=f"{sample} after {option}") + _require(terminal == identity, f"official {sample} changed while its {option} report ran") + stdout, gpu = evidence["stdout"], request["gpu"] + records = (_encoder_gpus(stdout) if role == "encoder" else + [{"gpu": int(item.group(1)), "name": item.group(2).strip()} + for item in _CAPS_GPU_RE.finditer(stdout)]) + ordinals = [item["gpu"] for item in records] + failures = _failure_markers(evidence) + shape = (bool(len(_ENCODER_CAPS_RE.findall(stdout)) == 1 + and stdout.count(_ENCODER_CAPS_HINT) == 1 + and all(item["required_sections_present"] for item in records)) + if role == "encoder" else + bool(len(_DECODER_CAPS_RE.findall(stdout)) == len(ordinals) + and len(_DECODER_SUMMARY_RE.findall(stdout)) == len(ordinals) + and stdout.count(_DECODER_CAPS_HINT) == 1)) + valid = bool(evidence["exit_code"] == 0 and not evidence["timed_out"] and not failures + and ordinals and len(ordinals) == len(set(ordinals)) and gpu in ordinals and shape) + return {"status": "completed" if valid else "unknown", + "evidence_classification": _classification(role), + "authority": f"official {sample} aggregate report", + "claim_scope": _AGGREGATE_SCOPE[role], "command": evidence["argv"], + "cwd": evidence["cwd"], "exit_code": evidence["exit_code"], + "timed_out": evidence["timed_out"], "selected_gpu": gpu, + "selected_gpu_present": gpu in ordinals, "reported_gpus": records, + "failure_markers": failures, "stdout_tail": stdout[-8000:], + "stderr_tail": evidence["stderr"][-4000:], + "binary_integrity": {"initial": initial, "terminal": terminal}} + + +def _aggregates(binaries: dict[str, Any], samples: dict[str, Any], request: dict[str, int], + reasons: list[str]) -> dict[str, Any]: + """Capture both aggregate reports. They never add a reason: readiness is the operation proof. + + This CLI always requires --run-encode and --run-decode, so the baseline ``not_run`` state is + unreachable here; an unbuilt or stub-linked sample yields ``unknown`` and the run still emits + the complete artifact shape. + """ + records: dict[str, Any] = {} + for role, (sample, _) in _AGGREGATE_ROUTES.items(): + record = samples.get(role) or {} + if reasons or binaries.get(role) is None or record.get("identity") is None or ( + record.get("linkage") or {}).get("status") != "verified": + records[role] = { + "status": "unknown", "evidence_classification": _classification(role), + "reason": f"authenticated {sample} build and runtime linkage are unavailable"} + continue + records[role] = _aggregate(role, binaries[role], record["identity"], request) + return records + + +# --- The official encode -> independent decode operation proof --------------- + +def _operation(command: list[str], evidence: dict[str, Any], success: bool, + extra: dict[str, Any]) -> dict[str, Any]: + return {"status": "operation_verified" if success else "operation_failed", + "command": command, "cwd": evidence["cwd"], "exit_code": evidence["exit_code"], + "timed_out": evidence["timed_out"], "stdout_tail": evidence["stdout"][-4000:], + "stderr_tail": evidence["stderr"][-4000:], **extra} + + +def _reason(sample: str, evidence: dict[str, Any], failures: list[str], detail: str) -> str: + if evidence["timed_out"]: + return f"official {sample} timed out" + if failures: + return f"official {sample} emitted explicit failure marker(s): " + ", ".join(failures) + return detail + + +def _encode(binary: Path, work: Path, request: dict[str, int], + reasons: list[str]) -> dict[str, Any]: + """Encode exactly one fresh NV12 frame with the official AppEncCuda sample.""" + width, height = request["width"], request["height"] + source = work / f"native-smoke-{width}x{height}.nv12" + output = work / f"native-appenc-{width}x{height}.h264" + _require(not os.path.lexists(source) and not os.path.lexists(output), + "a native encode input or bitstream already exists; use a fresh work directory") + write_new_bytes(source, bytes([16]) * (width * height) + bytes([128]) * (width * height // 2)) + fixture = file_identity(source, label="native NV12 input") + _require(fixture["size_bytes"] == _SMOKE_NV12_BYTES, + f"the one-frame NV12 fixture must be exactly {_SMOKE_NV12_BYTES} bytes," + f" got {fixture['size_bytes']}") + command = [str(binary), "-i", str(source), "-s", f"{width}x{height}", "-if", "nv12", + "-gpu", str(request["gpu"]), "-codec", "h264", "-o", str(output)] + evidence = _run(command, cwd=work, timeout=request["timeout"]) + failures = _failure_markers(evidence) + frames = _ENCODED_RE.findall(evidence["stdout"]) + markers = [item.strip() for item in _BITSTREAM_RE.findall(evidence["stdout"])] + matched = bool(len(markers) == 1 + and Path(markers[0]).expanduser().resolve() == output.resolve()) + identity = _identity(output, "native H.264 bitstream") + success = bool(evidence["exit_code"] == 0 and not evidence["timed_out"] and not failures + and frames == ["1"] and matched and identity and identity["size_bytes"] > 0) + if not success: + reasons.append(_reason("AppEncCuda", evidence, failures, + "official AppEncCuda did not report exactly one encoded frame, its" + " exact requested output marker, and a non-empty bitstream")) + return _operation(command, evidence, success, { + "encoded_frames": int(frames[0]) if len(frames) == 1 else None, + "frame_marker_matches": len(frames), "failure_markers": failures, + "output_marker": markers[0] if len(markers) == 1 else None, + "output_marker_matches_requested": matched, "bitstream": identity, + "input": {"path": str(source), "format": "NV12", "width": width, "height": height, + "frames": 1, "sha256": fixture["sha256"], + "size_bytes": fixture["size_bytes"]}}) + + +def _encode_marker_accepted(encode: dict[str, Any]) -> bool: + """The encode-before-decode gate. + + AppDec is launched only after AppEncCuda's own success marker has been proven by this run: a + verified operation status, exactly one reported encoded frame, the exact requested + ``Bitstream saved in file`` marker, and a non-empty authenticated bitstream. This is an + ordering guarantee over a real operation; it must never be weakened to a capability query. + """ + bitstream = encode.get("bitstream") + return bool(encode.get("status") == "operation_verified" + and encode.get("encoded_frames") == 1 + and encode.get("output_marker_matches_requested") is True + and isinstance(bitstream, dict) and isinstance(bitstream.get("sha256"), str) + and isinstance(bitstream.get("size_bytes"), int) and bitstream["size_bytes"] > 0) + + +def _fresh_bitstream(encode: dict[str, Any]) -> dict[str, Any] | None: + """Re-authenticate the encoded bitstream on disk BEFORE AppDec is launched. + + Freshness and non-emptiness are preconditions of the decode launch, never postconditions. A + stale, replaced, truncated, or empty bitstream stops the run before the decoder starts, so + AppDec is never pointed at a file this run did not just produce. + """ + recorded = encode.get("bitstream") + if not isinstance(recorded, dict) or not isinstance(recorded.get("path"), str): + return None + try: + live = _identity(Path(recorded["path"]), "AppDec input bitstream") + except (OSError, ValueError): + return None + return live if live == recorded and live["size_bytes"] > 0 else None + + +def _decode(binary: Path, work: Path, bitstream: dict[str, Any], request: dict[str, int], + reasons: list[str]) -> dict[str, Any]: + """Decode the fresh AppEncCuda bitstream with the official AppDec sample.""" + width, height = request["width"], request["height"] + source, expected = Path(bitstream["path"]), _SMOKE_NV12_BYTES + output, legacy = work / f"native-appdec-{width}x{height}.nv12", work / "out.native" + _require(not os.path.lexists(output) and not os.path.lexists(legacy), + "native decoded output appeared before AppDec launch; use a fresh work directory") + command = [str(binary), "-i", str(source), "-o", str(output), "-gpu", str(request["gpu"])] + evidence = _run(command, cwd=work, timeout=request["timeout"]) + _require(file_identity(source, label="AppDec input bitstream") == bitstream, + "the fresh AppEncCuda bitstream changed while AppDec was running") + failures = _failure_markers(evidence) + frames = _DECODED_RE.findall(evidence["stdout"]) + stray = os.path.lexists(legacy) + identity = _identity(output, "decoded NV12 output") + success = bool(evidence["exit_code"] == 0 and not evidence["timed_out"] and not failures + and frames == ["1"] and not stray and identity + and identity["size_bytes"] == expected) + if not success: + reasons.append("official AppDec created an unexpected out.native output" if stray else + _reason("AppDec", evidence, failures, + "official AppDec did not report exactly one decoded frame and" + f" produce the exact {expected}-byte one-frame NV12 output")) + return _operation(command, evidence, success, { + "decoded_frames": int(frames[0]) if len(frames) == 1 else None, + "frame_marker_matches": len(frames), "failure_markers": failures, + "unexpected_out_native_appeared": stray, "input_bitstream": bitstream, + "decoded_output": {"path": str(output), "format": "NV12", "width": width, + "height": height, "frames": 1, "expected_size_bytes": expected, + **(identity or {})}}) + + +def _operations(binaries: dict[str, Any], work_dir: Path, request: dict[str, int], + reasons: list[str]) -> dict[str, Any]: + """Run the official encode, then -- only on a proven encode marker -- the official decode.""" + decode = {"status": "not_run", "reason": "official AppDec is launched only after a proven" + " AppEncCuda success marker from this run"} + if reasons or not binaries["encoder"] or not binaries["decoder"]: + return {"decode": decode, "encode": { + "status": "not_run", "reason": "official AppEncCuda was not launched: " + + ("; ".join(reasons) or "no authenticated binary")}} + work = _fresh_directory(work_dir, label="native verification work directory") + encode = _encode(binaries["encoder"], work, request, reasons) + if not _encode_marker_accepted(encode): + return {"encode": encode, "decode": decode} + bitstream = _fresh_bitstream(encode) + if bitstream is None: + reasons.append("the AppEncCuda bitstream was not fresh, unchanged, and non-empty at the" + " AppDec launch point") + return {"encode": encode, "decode": dict(decode, reason=reasons[-1])} + return {"encode": encode, + "decode": _decode(binaries["decoder"], work, bitstream, request, reasons)} + + +def _validate_request(request: dict[str, int]) -> None: + for name in ("gpu", "width", "height", "timeout"): + _require(isinstance(request[name], int) and not isinstance(request[name], bool), + f"--{name} must be an integer") + _require(request["gpu"] >= 0, "--gpu must be non-negative") + _require(request["timeout"] > 0, "--timeout must be positive") + _require((request["width"], request["height"]) == _SMOKE_GEOMETRY, + f"the native smoke is fixed at {_SMOKE_GEOMETRY[0]}x{_SMOKE_GEOMETRY[1]}, got" + f" {request['width']}x{request['height']}") + _require(request["width"] * request["height"] * 3 // 2 == _SMOKE_NV12_BYTES, + f"the one-frame NV12 payload must be exactly {_SMOKE_NV12_BYTES} bytes") + + +def _select_sdk_root(requested: Path, recorded: str) -> Path: + """Accept the recorded root or its unversioned parent with one 13.0.x child.""" + selected = Path(recorded) + root = requested.expanduser().absolute() + if root == selected: + return selected + _require(root.is_dir() and not root.is_symlink(), + f"--sdk-root {requested} is not a regular SDK directory") + candidates = sorted( + path for path in root.iterdir() + if path.is_dir() and not path.is_symlink() and _public_sdk_release(path.name) + ) + _require(candidates == [selected], + f"--sdk-root {requested} does not contain only the canonical" + f" installation.native_sdk.sdk_root {recorded}") + return selected + + +def verify(paths: dict[str, Path], request: dict[str, int]) -> dict[str, Any]: + """Run the full official native build plus encode -> decode proof and return its artifact.""" + _validate_request(request) + environment = _load_environment(paths["environment"], request["gpu"]) + native = environment["native"] + sdk_root = _select_sdk_root(paths["sdk_root"], native["sdk_root"]) + # An absent required build tool blocks the run but still yields the complete result shape. + reasons: list[str] = sorted(str(item["reason"]) for item in native["tools"].values() + if item.get("status") != "verified") + ownership = _package_ownership(sdk_root, native["package"]["version"], + min(request["timeout"], 60)) + reasons.extend(ownership["reasons"]) + build_root, build = ( + (None, {"status": "blocked", "root": None, "commands": [], + "reason": "native configure was blocked before CMake: " + "; ".join(reasons)}) + if reasons else + _configure_and_build(environment, paths["build_dir"], request["timeout"], reasons)) + binaries, samples = _built_samples(build_root, min(request["timeout"], 60), reasons) + aggregates = _aggregates(binaries, samples, request, reasons) + operations = _operations(binaries, paths["work_dir"], request, reasons) + states = [operations[name].get("status") for name in ("encode", "decode")] + ready = bool(states == ["operation_verified"] * 2 and not reasons) + return { + "schema_version": SCHEMA_VERSION, "kind": "nvcodec-native-verification", + "generated_at": utc_now(), "ready": ready, "software_fallback": False, + "status": ("operation_verified" if ready else + "operation_failed" if "operation_failed" in states else "unknown"), + "command_provenance": ("package-installed Video Codec SDK Samples/AppEncode/AppEncCuda ->" + " Samples/AppDecode/AppDec"), + "environment": environment, "package_ownership": ownership, + "native_build_prerequisites": native["build_prerequisites"], + "build": build, "samples": samples, + "aggregate_encoder_capabilities": aggregates["encoder"], + "aggregate_decoder_capabilities": aggregates["decoder"], + "hardware_operations": operations, "reasons": reasons, + } + + +def _emit_error(exc: BaseException, output: Path | None, code: int) -> int: + document = {"schema_version": SCHEMA_VERSION, "kind": "nvcodec-native-verification-error", + "status": "error", "generated_at": utc_now(), "error": str(exc)} + # allow_nan=False keeps stdout strict even on the error path. + print(json.dumps(document, allow_nan=False, indent=2, sort_keys=True)) + if output is not None: + try: + write_new_json(output, document) + except (OSError, ValueError): + pass + return code + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--environment", type=Path, required=True, + help="Validated live nvcodec-environment 1.2 JSON") + parser.add_argument("--sdk-root", type=Path, default=Path("/opt/nvidia/video-codec-sdk"), + help="Canonical SDK root or its unversioned parent") + parser.add_argument("--build-dir", type=Path, default=Path.cwd() / "nvcodec-native-build") + parser.add_argument("--work-dir", type=Path, default=Path.cwd() / "nvcodec-native-smoke") + for flag, note in (("build", "Configure and build official AppEncCuda and AppDec"), + ("run-encode", "Run the built official AppEncCuda"), + ("run-decode", "Run official AppDec on the fresh AppEncCuda bitstream")): + parser.add_argument(f"--{flag}", action="store_true", help=note) + for name, default in (("gpu", 0), ("width", _SMOKE_GEOMETRY[0]), + ("height", _SMOKE_GEOMETRY[1]), ("timeout", 300)): + parser.add_argument(f"--{name}", type=int, default=default) + parser.add_argument("--output", type=Path, + help="Write the JSON artifact here; an existing path is never overwritten") + args = parser.parse_args() + if not (args.build and args.run_encode and args.run_decode): + parser.error("native verification requires --build, --run-encode, and --run-decode") + try: + require_isolated() + if args.output is None: + raise ValueError("native verification requires --output") + if os.path.lexists(args.output.expanduser()): + raise FileExistsError(f"native verification output already exists: {args.output}") + result = verify({"environment": args.environment, "sdk_root": args.sdk_root, + "build_dir": args.build_dir, "work_dir": args.work_dir}, + {"gpu": args.gpu, "width": args.width, "height": args.height, + "timeout": args.timeout}) + print(write_new_json(args.output, result).read_bytes().decode("utf-8"), end="") + return 0 if result["ready"] else 2 + except subprocess.TimeoutExpired as exc: + return _emit_error(exc, args.output, 2) + except (FileNotFoundError, ValueError) as exc: + return _emit_error(exc, args.output, 3) + except Exception as exc: # pylint: disable=broad-exception-caught + return _emit_error(exc, args.output, 3) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-setup/scripts/setup/verify_pynvc_sample.py b/skills/jetson-video-setup/scripts/setup/verify_pynvc_sample.py new file mode 100644 index 00000000..c42864e1 --- /dev/null +++ b/skills/jetson-video-setup/scripts/setup/verify_pynvc_sample.py @@ -0,0 +1,1027 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Authenticate and run the wheel-owned NVIDIA PyNvVideoCodec encode/decode samples. + +This setup operation authority scores real sample output and authenticates every imported module, +native extension, sample, helper, and config against wheel ``RECORD``. It is not a capability +query, owns its command line, uses only the standard library, and imports no sibling skill. +""" + +# pylint: disable=missing-function-docstring,too-many-arguments,too-many-boolean-expressions +# pylint: disable=too-many-lines,too-many-locals + +from __future__ import annotations + +import argparse +import base64 +import csv +import importlib +import importlib.metadata +import io +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path +from typing import Any + +_SETUP_DIR = Path(__file__).resolve().parent +if str(_SETUP_DIR) not in sys.path: + sys.path.insert(0, str(_SETUP_DIR)) + +# pylint: disable=wrong-import-position +from setup_contract import ( # noqa: E402 + AUXILIARY_MODULE, AUXILIARY_ROLE, AUXILIARY_STEM, FULL_PROFILE, SMOKE_PROFILE, _read_regular, + _symlink_free, VERIFICATION_PROFILES, associated_extensions, canonical_json_sha256, read_json, + extension_stem, file_identity, parse_jetson_release, profile_dependencies, run_command, + require_isolated, sha256_bytes, utc_now, write_new_bytes, write_new_json, +) +# pylint: enable=wrong-import-position + +SCHEMA_VERSION = "1.3" +DIST_NAME = "PyNvVideoCodec" +EXPECTED_VERSION = "2.1.0" +RESULT_KIND = "nvcodec-pynvc-sample-verification" +ERROR_KIND = "nvcodec-pynvc-sample-verification-error" +ENVIRONMENT_KIND = "nvcodec-environment" +ENVIRONMENT_MAJOR = "1" +KNOWN_STALE_RECORD = "known_upstream_stale_record" +MINIMUM_JETSON_LINUX = (38, 5) +_REGISTRY_PATH = Path.home() / ".local/state/jetson-videosdk/current-pynvc.json" +_REGISTRY_SCHEMA = "jetson-videosdk/current-pynvc/1" +_PYNVC_FIELDS = ("interpreter", "interpreter_identity", "sys_prefix", "version", "extension") +# The bounded smoke proves this tuple; full-samples retains the caller's positive geometry/count. +SMOKE_WIDTH, SMOKE_HEIGHT, SMOKE_FRAMES = 640, 360, 1 +# Public PyNvVideoCodec 2.1.0 has one evidenced stale RECORD row for its native extension. +# No other member may miss its hash, and the exact distribution version must also match. +_NATIVE_EXTENSION_MEMBER = re.compile( + r"PyNvVideoCodec/_?PyNvVideoCodec(?:_(?:121|130))?\.[^/]*\.so") +_AUXILIARY_FIELDS = ("role", "path", "loaded_path", "sha256") +_DEPENDENCY_RULES: dict[str, dict[str, Any]] = { + "numpy": {"minimum": (1, 24)}, "pycuda": {"version": "2026.1"}, + "torch": {"version": "2.9.1+cu130", "cuda_build": "13.0", "cuda_available": True}} +# Every wheel-owned row the official samples read at run time; "config" is resolved per install. +# Both decode routes stay authenticated under either profile, but exactly one is executed. +SAMPLE_PATHS = {"encode": "basic/encode.py", "decode": "advanced/decode.py", + "decode_perf": "advanced/decode_perf.py", + # ``from utils. import`` executes this initializer, so it is authenticated code. + "__init__.py": "utils/__init__.py", + "Utils.py": "utils/Utils.py", "frame_utils.py": "utils/frame_utils.py", + "encode_parser.py": "utils/encode_parser.py", + "decode_parser.py": "utils/decode_parser.py"} +PROFILE_DECODE_SAMPLE = {SMOKE_PROFILE: "decode_perf", FULL_PROFILE: "decode"} +# One decoded-output invariant, owned by verify() and merely restated by the CLI. +_SMOKE_DECODED_CONFLICT = ("a decoded-output path is valid only with --profile full-samples; the" + " pynvc-smoke decode route (advanced/decode_perf.py) writes no raw" + " output") +_SMOKE_GEOMETRY_CONFLICT = ( + f"the pynvc-smoke readiness proof is fixed at {SMOKE_WIDTH}x{SMOKE_HEIGHT}" +) + + +def _validate_smoke_request(profile: str, width: int, height: int, frames: int, + decoded_path: Path | None) -> None: + if profile != SMOKE_PROFILE: + return + if frames != SMOKE_FRAMES: + raise ValueError(_SMOKE_FRAMES_CONFLICT) + if (width, height) != (SMOKE_WIDTH, SMOKE_HEIGHT): + raise ValueError(_SMOKE_GEOMETRY_CONFLICT) + if decoded_path is not None: + raise ValueError(_SMOKE_DECODED_CONFLICT) + + +_SMOKE_FRAMES_CONFLICT = ("--profile pynvc-smoke verifies exactly one bounded frame; use" + " --frames 1 or provision --profile full-samples for a multi-frame" + " official-sample proof") +ENCODE_SUCCESS_RE = re.compile(r"Completed encoding\s+(\d+)\s+frames using CPU buffers") +DECODE_SUCCESS_RE = re.compile( + r"^Successfully decoded requested\s+(\d+)\s+frames to\s+(.+?)\s*$", re.MULTILINE) +# The bounded one-frame decode_perf.py smoke. Public 2.1.0 emits, on the ``-m thread`` path this +# argv pins, "Thread : Successfully decoded requested frames" (line 124) and the +# unprefixed summary "Total frames decoded: " (line 274). Both are mandatory and each must +# appear exactly once: line 205 prints its worker error from inside an ``except`` block, so the +# child can exit 0 after a failed decode and a clean exit alone is never a proof. Each marker gets +# its own anchored pattern rather than one loose "Successfully decoded" pattern, because line 134 +# also emits the distinct "Successfully decoded all frames" that a loose pattern double-counts. +ENCODE_FAILURE_PATTERN = re.compile( + r"(?im)^\s*(?:CreateEncoder failure:|An unexpected error occurred:" + r"|Traceback \(most recent call last\)|Preset [^\r\n]* not found\. Using default\." + r"|Invalid RC mode given\. Using cbr as default" + r"|Can't parse bitrate string\. Using default value)") +_DECODE_FAILURES = ( + r"Operation or configuration not supported:|An unexpected error occurred:" + r"|CUDA error \(context/stream/memory\):|Traceback \(most recent call last\)" + r"|Decode Error occurred for picture\b" + r"|NvDecoder::~NvDecoder\(\): exception during cleanup \(suppressed\)") +DECODE_FAILURE_PATTERN = re.compile(rf"(?im)^\s*(?:{_DECODE_FAILURES})") +# The perf route additionally rejects every swallowed worker failure and every degraded-run +# warning, either of which can accompany a zero exit code: "Thread/Process error:" (line +# 205) and "Thread : Warning: Video ended before reaching requested frame count" (line 132). +# Match only the sample's exact degraded-run warning; unrelated library warnings are diagnostic. +PERF_FAILURE_PATTERN = re.compile( + rf"(?im)(?:^\s*(?:{_DECODE_FAILURES})|\b(?:Thread|Process)\b[^:\r\n]*\berror:" + r"|^\s*Thread\b[^:\r\n]*:\s*Warning:\s*Video ended before reaching requested frame count\b)") + + +class _NotReady(ValueError): + """A valid surface cannot run the official sample yet; a completed negative.""" + + +def _render(value: Any) -> str: + """Strict stdout serialization; allow_nan=False rejects non-finite floats at the source.""" + return json.dumps(value, allow_nan=False, indent=2, sort_keys=True) + + +def _result(kind: str, status: str, **fields: Any) -> dict[str, Any]: + return {"schema_version": SCHEMA_VERSION, "kind": kind, "status": status, **fields} + + +def _version_at_least(value: Any, minimum: tuple[int, ...]) -> bool: + match = re.fullmatch(r"\s*(\d+(?:\.\d+)*)\s*", value) if isinstance(value, str) else None + if not match: + return False + observed = tuple(int(part) for part in match.group(1).split(".")) + width = max(len(observed), len(minimum)) + return observed + (0,) * (width - len(observed)) >= minimum + (0,) * (width - len(minimum)) + + +def _require_fields(value: Any, fields: tuple[str, ...], *, label: str) -> dict[str, Any]: + """Required-subset validation: present and non-null. Unknown keys are always permitted.""" + if not isinstance(value, dict): + raise ValueError(f"{label} must be a JSON object") + missing = [name for name in fields if value.get(name) is None] + if missing: + raise ValueError(f"{label} is missing required field(s): {', '.join(missing)}") + return value + + +def _same_path(first: Any, second: Any) -> bool: + return Path(str(first)).resolve() == Path(str(second)).resolve() + + +# --- Wheel RECORD authentication of the files this process actually executes -- + +def _wheel_owned_identity(distribution: Any, path: Any, *, label: str) -> dict[str, Any]: + """Prove one executed file is a wheel-RECORD member inside sys.prefix and hash it locally. + + Ownership is the assertion: the executed path resolves to exactly one installed distribution + entry and exactly one ``RECORD`` row, both inside the running ``sys.prefix``. The recorded + digest is re-derived here and published as evidence rather than silently trusted. + """ + identity = file_identity(path, label=label) + resolved = Path(identity["path"]) + if not resolved.is_relative_to(Path(sys.prefix).resolve()): + raise ValueError(f"{DIST_NAME} {label} {resolved} is outside sys.prefix {sys.prefix}") + entries = [entry for entry in distribution.files or [] + if Path(distribution.locate_file(entry)).resolve() == resolved] + if len(entries) != 1: + raise ValueError( + f"expected one {DIST_NAME} distribution entry for {resolved}, found {len(entries)}") + text = distribution.read_text("RECORD") + if not text: + raise ValueError(f"installed {DIST_NAME} distribution has no readable RECORD") + member = str(entries[0]).replace("\\", "/") + rows = [row for row in csv.reader(io.StringIO(text)) + if row and row[0].replace("\\", "/") == member] + if len(rows) != 1: + raise ValueError(f"expected one wheel RECORD row for {member}, found {len(rows)}") + encoded_hash, recorded_size = (rows[0] + ["", ""])[1:3] + record: dict[str, Any] = { + "record_path": member, "hash": encoded_hash or None, + "size_bytes": int(recorded_size) if str(recorded_size).isdigit() else None} + if not encoded_hash or "=" not in encoded_hash: + record["consistency"] = "record_hash_absent" + else: + algorithm, expected = encoded_hash.split("=", 1) + if algorithm.lower() != "sha256": + raise ValueError(f"{label} wheel RECORD hash algorithm must be sha256, got {algorithm}") + digest = base64.urlsafe_b64encode( + bytes.fromhex(identity["sha256"])).rstrip(b"=").decode("ascii") + record["consistency"] = ( + "matches_record" if digest == expected + and record["size_bytes"] in (None, identity["size_bytes"]) else "upstream_mismatch") + return {**identity, "wheel_member": member, "record": record} + + +def _require_record_consistency(identity: dict[str, Any], *, label: str, version: str, + allow_native_stale: bool = False) -> None: + """Fail on a RECORD mismatch, permitting only the one evidenced native-extension row. + + Scoped by exact distribution version AND exact ``RECORD`` member: the public PyNvVideoCodec + 2.1.0 native extension and nothing else. Ownership of the executed file is still proven + independently, so the exception narrows to the recorded digest alone and is published.""" + record, member = identity["record"], identity["wheel_member"] + if record["consistency"] == "matches_record": + return + if (allow_native_stale and record["consistency"] == "upstream_mismatch" + and version == EXPECTED_VERSION + and _NATIVE_EXTENSION_MEMBER.fullmatch(member) is not None): + record["consistency"] = KNOWN_STALE_RECORD + record["exception"] = ( + f"known upstream {DIST_NAME} {EXPECTED_VERSION} stale RECORD row for its native" + " extension; wheel ownership of the executed file is still proven locally") + return + raise ValueError( + f"{DIST_NAME} {label} does not match its wheel RECORD ({record['consistency']}): {member}") + + +def _authenticate_auxiliaries(distribution: Any, module_identity: dict[str, Any], + extension_identity: dict[str, Any]) -> list[dict[str, Any]]: + """Authenticate zero or one exact ``PyNvVideoCodec.VersionCheck`` helper beside the primary. + + The helper is held to the ORDINARY wheel-RECORD rule: ``_require_record_consistency`` is + called with the default ``allow_native_stale=False``, so the primary's version- and + member-scoped public-2.1.0 stale-row exception can never widen to cover it. It must live in + the same package directory under the same ``sys.prefix``, and every other associated + compiled path -- a second helper, a foreign extension, an unexpected submodule -- is + rejected outright rather than ignored.""" + package_dir = os.path.dirname(module_identity["path"]) + prefix = str(Path(sys.prefix).resolve()) + associated, symlinked = associated_extensions(package_dir) + if symlinked: + raise ValueError(f"loaded PyNvVideoCodec extensions must not be symlinks: {symlinked}") + candidates = [path for path, names in associated.items() if AUXILIARY_MODULE in names] + if len(candidates) > 1: + raise ValueError(f"{AUXILIARY_MODULE} must resolve to one loaded path; saw {candidates}") + rows: list[dict[str, Any]] = [] + for resolved in candidates: + if (extension_stem(resolved) != AUXILIARY_STEM + or os.path.dirname(resolved) != package_dir + or not resolved.startswith(prefix + os.sep)): + raise ValueError(f"{AUXILIARY_MODULE} must be a readable {AUXILIARY_STEM} extension" + f" directly in {package_dir} under {prefix}; saw {resolved}") + identity = _wheel_owned_identity(distribution, resolved, label="VersionCheck auxiliary") + _require_record_consistency(identity, label="VersionCheck auxiliary", + version=str(distribution.version)) + rows.append({**identity, "role": AUXILIARY_ROLE, "loaded_path": resolved, + "module_names": associated[resolved]}) + expected = {extension_identity["path"], *candidates} + if unexpected := sorted(set(associated) - expected): + raise ValueError(f"only the selected primary and one {AUXILIARY_MODULE} auxiliary may be" + f" loaded; also loaded {[(p, associated[p]) for p in unexpected]}") + return rows + + +def _authenticated_pynvc(module: Any, distribution: Any) -> dict[str, Any]: + """Authenticate the imported module and the extension this interpreter actually loaded. + + ``loaded_path`` comes from the imported ``_PyNvVideoCodec.__file__``, never from metadata, so + it proves which binary really ran rather than which one merely exists on disk.""" + distribution_version = str(distribution.version) + module_version = str(getattr(module, "__version__", "") or "") + if distribution_version != EXPECTED_VERSION or module_version != EXPECTED_VERSION: + raise _NotReady( + f"{DIST_NAME} distribution and imported module must both be exact public" + f" {EXPECTED_VERSION}; observed {distribution_version!r} and {module_version!r}") + module_file = getattr(module, "__file__", None) + extension_file = getattr(getattr(module, "_PyNvVideoCodec", None), "__file__", None) + if not module_file or not extension_file: + raise ValueError(f"imported {DIST_NAME} package/extension paths are unavailable") + module_identity = _wheel_owned_identity(distribution, module_file, label="module") + _require_record_consistency(module_identity, label="module", version=distribution_version) + extension_identity = _wheel_owned_identity(distribution, extension_file, label="extension") + _require_record_consistency(extension_identity, label="extension", + version=distribution_version, allow_native_stale=True) + auxiliaries = _authenticate_auxiliaries(distribution, module_identity, extension_identity) + return { + "status": "verified", "authentication": "local_wheel_record_ownership", + # Lexical: exactly what was invoked. Resolved identity: what actually ran. A venv's + # bin/python is a symlink, so the identity is always taken from the resolved target. + "interpreter": sys.executable, "sys_prefix": str(Path(sys.prefix).resolve()), + "interpreter_identity": file_identity(Path(sys.executable).resolve(strict=True), + label="verification interpreter"), + "version": distribution_version, + "module": {**module_identity, "loaded_path": str(module_file), "version": module_version}, + "extension": {**extension_identity, "loaded_path": str(extension_file)}, + "auxiliary_extensions": auxiliaries} + + +# --- nvcodec-environment 1.2 consumption (required subset only) --------------- + +def _require_dependencies(dependencies: Any, profile: str) -> None: + """Require exactly the recorded dependency rows the selected profile's samples need. + + The smoke pair -- basic/encode.py in ``-m cpu`` mode and advanced/decode_perf.py -- never + reaches the unguarded module-level ``import torch`` in samples/utils/Utils.py, so a missing + Torch is a complete ready state under ``pynvc-smoke`` and stays a hard block under + ``full-samples``, whose advanced/decode.py route imports that helper directly.""" + if not isinstance(dependencies, dict): + raise ValueError("pynvc.dependencies must be a JSON object") + for name in profile_dependencies(profile): + rule, row = _DEPENDENCY_RULES[name], dependencies.get(name) + if (not isinstance(row, dict) or row.get("status") != "installed" + or row.get("ready") is not True): + raise _NotReady(f"environment does not show {profile} official-sample dependency" + f" {name} installed and ready") + for field, expected in rule.items(): + if field == "minimum" and not _version_at_least(row.get("version"), expected): + raise _NotReady(f"official sample requires {name} >= " + + ".".join(str(part) for part in expected)) + if field != "minimum" and row.get(field) != expected: + raise _NotReady(f"official sample requires {name} {field}={expected!r}," + f" environment reports {row.get(field)!r}") + + +def _platform_compatibility(jetson_linux: Any) -> dict[str, Any]: + """Recompute the documented Jetson Linux minimum from the environment release line.""" + fields = _require_fields(jetson_linux, ("version", "release_line"), + label="platform.jetson_linux") + minimum = ".".join(str(part) for part in MINIMUM_JETSON_LINUX) + release = parse_jetson_release(str(fields["release_line"])) + if not _version_at_least(release["version"], MINIMUM_JETSON_LINUX): + raise _NotReady( + f"official {DIST_NAME} sample verification requires Jetson Linux {minimum} or newer;" + f" the environment release line resolves to {release['version']!r}") + return {"minimum_version": minimum, "observed_version": release["version"], "supported": True, + "release_line": release["release_line"], "jetson_linux": str(fields["version"]), + "evidence": "recomputed_from_environment_release_line"} + + +def _validate_environment(data: Any, *, gpu: int, profile: str = FULL_PROFILE) -> dict[str, Any]: + """Validate a live schema-1.x environment and normalize its established PyNv identity. + + Only the pynvc surface is inspected: a malformed or absent native surface can never block this + one, because the two surfaces remain structurally independent.""" + if not isinstance(data, dict) or data.get("kind") != ENVIRONMENT_KIND: + raise ValueError(f"a {ENVIRONMENT_KIND} artifact is required") + schema_version = str(data.get("schema_version") or "") + if schema_version.partition(".")[0] != ENVIRONMENT_MAJOR: + raise ValueError(f"unsupported {ENVIRONMENT_KIND} schema major {schema_version!r}; this" + f" verifier consumes schema {ENVIRONMENT_MAJOR}.x only") + if data.get("mode") != "live": + raise ValueError(f"a live {ENVIRONMENT_KIND} artifact is required") + if data.get("selected_gpu") != gpu: + raise ValueError("environment selected_gpu does not match the requested smoke GPU") + pynvc = data.get("pynvc") + if not isinstance(pynvc, dict) or pynvc.get("imported") is not True: + raise _NotReady(f"environment does not show an imported {DIST_NAME} surface") + identity = _require_fields( + pynvc.get("identity"), + ("status", "interpreter", "sys_prefix", "module", "extension"), + label="pynvc.identity") + if identity["status"] != "verified": + raise _NotReady("environment does not carry a verified PyNvVideoCodec identity") + module = _require_fields(identity["module"], ("version", "path"), + label="pynvc.identity.module") + extension = _require_fields(identity["extension"], ("path",), + label="pynvc.identity.extension") + record = extension.get("record") if isinstance(extension.get("record"), dict) else {} + extension = {**extension, "sha256": extension.get("sha256") or record.get("sha256"), + "loaded_path": extension.get("loaded_path") or extension.get("path")} + _require_fields(extension, ("path", "sha256", "loaded_path"), + label="pynvc.identity.extension") + python = data.get("installation", {}).get("python", {}) + identities = python.get("interpreter_identities", {}) if isinstance(python, dict) else {} + interpreter_identity = identity.get("interpreter_identity") + if not isinstance(interpreter_identity, dict): + interpreter_identity = identities.get("running") if isinstance(identities, dict) else None + _require_fields(interpreter_identity, ("path", "sha256"), + label="installation.python.interpreter_identities.running") + packages = python.get("packages", {}) if isinstance(python, dict) else {} + dependencies = {} + for name in profile_dependencies(profile): + row = packages.get(name, {}) if isinstance(packages, dict) else {} + dependencies[name] = {**row, "ready": bool( + row.get("status") == "installed" and row.get("requirement_satisfied") is True)} + _require_dependencies(dependencies, profile) + return { + "version": identity.get("version") or pynvc.get("distribution_version"), + "interpreter": identity["interpreter"], "sys_prefix": identity["sys_prefix"], + "interpreter_identity": interpreter_identity, "module": module, "extension": extension, + "auxiliary_extensions": pynvc.get("auxiliary_extensions"), + "dependencies": dependencies} + + +def _auxiliary_rows(value: Any, *, label: str) -> list[dict[str, Any]]: + """Normalize a recorded auxiliary list. Absent means "none observed", which keeps controlled + fixtures usable; an explicitly present value must be a list carrying at most one complete + ``driver_version_check`` row, so a helper can never be smuggled in as an unnamed role.""" + if value is None: + return [] + if not isinstance(value, list) or len(value) > 1: + raise ValueError(f"{label} must be a list with at most one {AUXILIARY_ROLE} row") + rows = [_require_fields(row, _AUXILIARY_FIELDS, label=f"{label}[0]") for row in value] + if any(row["role"] != AUXILIARY_ROLE for row in rows): + raise ValueError(f"{label} may carry only the {AUXILIARY_ROLE} role") + return rows + + +def _require_surface_binding(surface: dict[str, Any], live: dict[str, Any], + *, require_running: bool = True) -> None: + """Require exactly one loaded extension matching ``extension.loaded_path`` and + ``module.{version,path}`` matching the imported module. ``loaded_path != path`` fails closed. + + The auxiliary lists are compared the same way, and a count disagreement fails: if either the + environment or this process observed a compiled helper the other did not, neither side's + evidence describes what actually ran.""" + extension, module = surface["extension"], surface["module"] + recorded, interpreter = surface["interpreter_identity"], live["interpreter_identity"] + recorded_aux = surface.get("auxiliary_extensions") + aux = _auxiliary_rows(recorded_aux, label="pynvc.auxiliary_extensions") + live_aux = live["auxiliary_extensions"] + auxiliary_checks = ((len(aux) == len(live_aux), + "auxiliary_extensions disagree: one side observed a compiled helper the" + " other did not"), + (all(row["role"] == item["role"] and row["sha256"] == item["sha256"] + and _same_path(row["path"], item["path"]) + and _same_path(row["loaded_path"], item["loaded_path"]) + for row, item in zip(aux, live_aux)), + "auxiliary_extensions do not match the helper this interpreter actually" + " loaded")) if recorded_aux is not None else () + running_checks = (((_same_path(surface["interpreter"], sys.executable), + "interpreter is not the interpreter running this verification"),) + if require_running else ()) + failed = [message for passed, message in ( + *auxiliary_checks, + (_same_path(extension["loaded_path"], extension["path"]), + "extension.loaded_path does not equal extension.path"), + (_same_path(extension["loaded_path"], live["extension"]["loaded_path"]), + "extension.loaded_path is not the extension this interpreter loaded"), + (extension["sha256"] == live["extension"]["sha256"], + "extension.sha256 does not match the loaded extension"), + (str(module["version"]) == live["module"]["version"], + "module.version does not match the imported module"), + (_same_path(module["path"], live["module"]["loaded_path"]), + "module.path does not match the imported module"), + (str(surface["version"]) == live["version"], + "version does not match the installed distribution"), + (str(surface["sys_prefix"]) == live["sys_prefix"], "sys_prefix does not match"), + *running_checks, + (recorded.get("path") == interpreter["path"] + and recorded.get("sha256") == interpreter["sha256"], + "interpreter_identity does not match this interpreter on disk")) if not passed] + if failed: + raise ValueError("environment pynvc mismatch: " + "; ".join(failed)) + + +# --- Validated PyNvVideoCodec venv registry ----------------------------------- + +class RegistryNotReady(Exception): + """The validated-venv registry is absent, stale, or fails to re-authenticate.""" + + def __init__(self, reason: str, next_action: str) -> None: + super().__init__(reason) + self.reason = reason + self.next_action = next_action + + +def _json_snapshot(path: Any, *, label: str) -> tuple[dict[str, Any], Any]: + """Parse the exact bytes hashed as identity, then require the path to retain that identity.""" + raw, size_bytes, resolved = _read_regular(path, label=label) + identity = {"path": str(resolved), "size_bytes": size_bytes, "sha256": sha256_bytes(raw)} + data = read_json(raw) + if file_identity(resolved, label=label) != identity: + raise ValueError(f"{label} changed while it was authenticated") + return identity, data + + +def _recorded_profile(data: Any, *, label: str) -> str: + """Read one recorded verification profile, treating absent evidence as ``full-samples``. + + Evidence written before the profile split records no profile at all, and reading that silence + as the Torch-requiring full profile is the fail-closed direction: it can only refuse a + registry a stricter proof would have accepted, never admit a weaker one.""" + profile = str(data.get("verification_profile") or FULL_PROFILE + ) if isinstance(data, dict) else FULL_PROFILE + if profile not in VERIFICATION_PROFILES: + raise RegistryNotReady("profile_unrecognized", + f"{label} records verification profile {profile!r}, which this" + f" helper does not know; re-run setup and re-register") + return profile + + +def _verified_pynvc(path: Path, *, expected: Any = None) -> tuple[ + dict[str, Any], dict[str, Any], str]: + """Re-authenticate verification -> environment -> PyNv surface before registry use.""" + try: + identity, data = _json_snapshot(path, label="pynvc verification artifact") + except (OSError, ValueError) as exc: + raise RegistryNotReady("artifact_unreadable", f"re-run setup: {exc}") from exc + if expected is not None and identity != expected: + raise RegistryNotReady("artifact_identity_changed", "re-run and re-register setup") + profile = _recorded_profile(data, label="the pynvc verification artifact") + try: + surface = _require_fields(data.get("pynvc") if isinstance(data, dict) else None, + _PYNVC_FIELDS, label="pynvc verification surface") + operations = _require_fields(data.get("hardware_operations"), + ("encode", "decode"), label="hardware_operations") + if (data.get("schema_version") != SCHEMA_VERSION or data.get("kind") != RESULT_KIND + or data.get("ready") is not True or data.get("status") != "operation_verified" + or data.get("software_fallback") is not False + or any(not isinstance(operations[name], dict) + or operations[name].get("status") != "operation_verified" + for name in ("encode", "decode"))): + raise ValueError("verification result is not a live no-fallback encode/decode proof") + bound = _require_fields( + data.get("environment"), + ("path", "sha256", "canonical_sha256", "kind", "schema_version", "mode", + "selected_gpu", "interpreter", "official_sample_dependencies"), + label="bound environment") + if not Path(str(bound["path"])).is_absolute(): + raise ValueError("bound environment path must be absolute") + environment_identity, environment = _json_snapshot( + bound["path"], label="bound environment") + if not isinstance(environment, dict): + raise ValueError("bound environment must be a JSON object") + if (environment_identity["path"] != bound["path"] + or environment_identity["sha256"] != bound["sha256"] + or canonical_json_sha256(environment) != bound["canonical_sha256"]): + raise ValueError("bound environment file identity or canonical hash changed") + gpu = environment.get("selected_gpu") if isinstance(environment, dict) else None + if (bound["kind"] != ENVIRONMENT_KIND or environment.get("kind") != ENVIRONMENT_KIND + or bound["schema_version"] != "1.2" + or environment.get("schema_version") != "1.2" + or bound["mode"] != "live" or environment.get("mode") != "live" + or not isinstance(gpu, int) or isinstance(gpu, bool) or gpu < 0 + or bound["selected_gpu"] != gpu): + raise ValueError("bound environment kind/schema/mode/GPU is not the verified live run") + if _recorded_profile(bound, label="the bound environment record") != profile: + raise RegistryNotReady( + "profile_binding_mismatch", + "the verification artifact and its bound environment record different" + f" verification profiles; re-run verify_pynvc_sample.py --profile {profile}" + " --register-current against a freshly probed environment") + environment_surface = _validate_environment(environment, gpu=gpu, profile=profile) + _require_surface_binding(environment_surface, surface, require_running=False) + python = _require_fields( + environment.get("installation", {}).get("python"), + ("executable", "interpreter_identities"), label="environment Python installation") + identities = _require_fields( + python["interpreter_identities"], ("running",), + label="environment Python interpreter identities") + legacy_venv = python.get("venv") if isinstance(python.get("venv"), dict) else {} + prefix = python.get("sys_prefix") or legacy_venv.get("prefix") + python_path, prefix_path = Path(str(python["executable"])), Path(str(prefix)) + # The recorded dependency set is compared inside the profile that selected it, and its own + # failure is separated out: an artifact proven under one profile is not defective, it is + # simply not the environment another profile asks for, so the caller is told which venv to + # provision instead of a generic "not verified". + if bound["official_sample_dependencies"] != environment_surface["dependencies"]: + raise RegistryNotReady( + "profile_dependency_mismatch", + f"the registered environment does not carry the {profile} dependency set" + f" {sorted(profile_dependencies(profile))}; provision a new {profile} venv" + " (plan_install.py --profile) and re-run verify_pynvc_sample.py" + f" --profile {profile} --register-current") + if (not isinstance(prefix, str) or not Path(prefix).is_absolute() + or not python_path.is_absolute() or python_path.parent.parent != prefix_path + or Path(str(environment_surface["interpreter"])).parent.parent != prefix_path + or not _same_path(python["executable"], environment_surface["interpreter"]) + or not _same_path(prefix, environment_surface["sys_prefix"]) + or identities["running"] != environment_surface["interpreter_identity"]): + raise ValueError("environment Python/PyNv binding disagrees with verification") + if (not Path(str(bound["interpreter"])).is_absolute() + or not _same_path(bound["interpreter"], surface["interpreter"])): + raise ValueError("bound environment interpreter disagrees with verification") + except (KeyError, OSError, TypeError, ValueError) as exc: + raise RegistryNotReady( + "artifact_not_verified", "re-run verification until encode and decode both pass" + ) from exc + _live_interpreter(surface) + return identity, surface, profile + + +def _live_interpreter(surface: dict[str, Any]) -> None: + """Re-hash the exact interpreter bound inside its recorded environment prefix.""" + recorded = surface["interpreter_identity"] + try: + interpreter, prefix = Path(str(surface["interpreter"])), Path(str(surface["sys_prefix"])) + lexical = interpreter.resolve(strict=True) + live = file_identity(recorded["path"], label="registered interpreter") + except (KeyError, OSError, TypeError, ValueError) as exc: + raise RegistryNotReady("interpreter_missing", f"re-run setup: {exc}") from exc + if (not isinstance(recorded, dict) or not interpreter.is_absolute() + or not prefix.is_absolute() or interpreter.parent.parent != prefix + or str(lexical) != recorded.get("path") or live != recorded): + raise RegistryNotReady("interpreter_changed", "re-run and re-register setup") + + +def publish_pynvc_registry(verification_artifact: Any) -> Path: + """Atomically bind one fully re-authenticated operation proof to the fixed registry.""" + identity, surface, profile = _verified_pynvc(Path(verification_artifact)) + directory = _symlink_free(_REGISTRY_PATH.parent, label="pynvc registry directory") + directory.mkdir(parents=True, exist_ok=True) + os.chmod(directory, 0o700) + staged, target = (directory / f".tmp-current-pynvc-{os.getpid()}.json", + directory / _REGISTRY_PATH.name) + staged.unlink(missing_ok=True) + try: + os.replace(write_new_json(staged, { + "schema": _REGISTRY_SCHEMA, "verification_artifact": identity, "pynvc": surface, + "verification_profile": profile, "published_utc": utc_now()}), target) + except BaseException: + staged.unlink(missing_ok=True) + raise + return target + + +def load_pynvc_registry() -> dict[str, Any]: + """Re-authenticate registry, proof, environment, surface and interpreter; never scan.""" + try: + _identity, registry = _json_snapshot(_REGISTRY_PATH, label="pynvc registry") + except FileNotFoundError as exc: + raise RegistryNotReady("registry_absent", "run setup and register the validated venv" + " (verify_pynvc_sample.py --register-current)") from exc + except (OSError, ValueError) as exc: + raise RegistryNotReady("registry_unreadable", f"re-register setup: {exc}") from exc + bound = registry.get("verification_artifact") if isinstance(registry, dict) else None + if (not isinstance(bound, dict) or registry.get("schema") != _REGISTRY_SCHEMA + or not isinstance(bound.get("path"), str) or not Path(bound["path"]).is_absolute()): + raise RegistryNotReady("registry_binding_missing", "re-register setup") + identity, surface, profile = _verified_pynvc(Path(bound["path"]), expected=bound) + recorded = surface["interpreter_identity"] + if (surface != registry.get("pynvc") or not isinstance(recorded, dict) + or _recorded_profile(registry, label="the pynvc registry") != profile + or not isinstance(recorded.get("path"), str)): + raise RegistryNotReady("registry_surface_stale", "re-run and re-register setup") + # The published profile travels with the surface so a consumer routes on the proof that was + # actually run: the smoke pair proves frame production, not the byte-exact raw decode. + return {"verification_artifact": identity, "verification_profile": profile, **surface} + + +# --- Wheel-owned official sample payload -------------------------------------- + +def _sample_files(distribution: Any) -> tuple[Path, dict[str, Path]]: + """Locate the official encode/decode pair, its helpers and its config in this install.""" + prefix = Path(sys.prefix).resolve() + for entry in distribution.files or []: + if not str(entry).replace("\\", "/").endswith("samples/basic/encode.py"): + continue + located = Path(distribution.locate_file(entry)) + if located.is_symlink() or not located.is_file(): + continue + root = located.resolve().parents[1] + if not root.is_relative_to(prefix) or not all( + (root / relative).is_file() for relative in SAMPLE_PATHS.values()): + continue + files = {name: (root / relative).resolve() for name, relative in SAMPLE_PATHS.items()} + for parent in ("basic", "advanced"): + config = root / parent / "encode_config.json" + if config.is_file() and not config.is_symlink(): + return root, files | {"config": config.resolve()} + raise FileNotFoundError("the installed sample payload has no encode_config.json") + raise FileNotFoundError( + f"installed {DIST_NAME} distribution does not contain the official encode/decode samples") + + +def _authenticated_samples(distribution: Any, files: dict[str, Path]) -> dict[str, Any]: + """Every sample, helper and config a child reads needs its own verified RECORD row. + + ``allow_native_stale`` is deliberately absent: the stale-row exception covers the native + extension only and can never excuse a sample payload mismatch.""" + records = {} + for name, path in sorted(files.items()): + identity = _wheel_owned_identity(distribution, path, label=f"official sample {name}") + _require_record_consistency(identity, label=f"official sample {name}", + version=str(distribution.version)) + records[name] = identity + return records + + +def _run_child(argv: list[str], *, cwd: Path, timeout: int, prefix: Path) -> dict[str, Any]: + started_at, monotonic_start = utc_now(), time.monotonic() + try: + completed = run_command(argv, cwd=cwd, timeout=timeout) + streams = (completed.stdout, completed.stderr) + exit_code, timed_out = completed.returncode, False + except subprocess.TimeoutExpired as exc: + streams, exit_code, timed_out = (exc.stdout, exc.stderr), None, True + raw = [value if isinstance(value, bytes) else str(value or "").encode() for value in streams] + return { + "argv": argv, "cwd": str(cwd), "exit_code": exit_code, "timed_out": timed_out, + "stdout": raw[0].decode("utf-8", errors="replace"), + "stderr": raw[1].decode("utf-8", errors="replace"), + "started_at": started_at, "ended_at": utc_now(), + "duration_seconds": time.monotonic() - monotonic_start, + "streams": {name: {"path": str(write_new_bytes( + prefix.with_name(f"{prefix.name}.{name}.log"), data)), "size_bytes": len(data), + "sha256": sha256_bytes(data)} for name, data in zip(("stdout", "stderr"), raw)}} + + +def _score(child: dict[str, Any], *, label: str, frames: int, timeout: int, matches: list[Any], + reported_frames: int | None, failure_pattern: re.Pattern[str], + checks: list[tuple[bool, str]], reasons: list[str]) -> tuple[dict[str, Any], bool]: + """Score one official operation. A clean child exit alone is explicitly insufficient: the exact + frame marker, the operation-specific proofs and the absence of any known failure marker are + each mandatory, and every failed proof is reported verbatim.""" + marker = failure_pattern.search(child["stdout"] + "\n" + child["stderr"]) + proofs = [(len(matches) == 1 and reported_frames == frames, + f"official {label} sample did not report exactly {frames} {label}d frames"), + *checks, + (marker is None, "official {} sample reported failure marker: {}".format( + label, marker.group(0).strip() if marker else None))] + if child["timed_out"]: + reasons.append(f"official {label} sample timed out after {timeout} seconds") + elif child["exit_code"] != 0: + reasons.append(f"official {label} sample exited {child['exit_code']}") + reasons.extend(reason for passed, reason in proofs if not passed) + success = not child["timed_out"] and child["exit_code"] == 0 and all(p for p, _ in proofs) + return {"status": "operation_verified" if success else "operation_failed", + "command": child["argv"], "cwd": child["cwd"], "exit_code": child["exit_code"], + "timed_out": child["timed_out"], "reported_frames": reported_frames, + "frame_marker_matches": len(matches), "stdout_tail": child["stdout"][-4000:], + "stderr_tail": child["stderr"][-4000:], "streams": child["streams"]}, success + + +def _output_identity(path: Path, *, label: str) -> dict[str, Any] | None: + return file_identity(path, label=label) if path.is_file() and not path.is_symlink() else None + + +def _perf_decode_argv(sample: Path, bitstream: Path, *, gpu: int, frames: int) -> list[str]: + """Exactly the public 2.1.0 decode_perf.py options its own ``--help`` publishes, and no other. + + ``-d 1`` pins the wheel's current ``use_device_memory`` default explicitly so the smoke stays + deterministic if upstream changes it, and ``-n 1``/``-m thread`` pin the single in-process + worker the marker contract is written against. The sample has no ``-o``: it writes no raw + output at all, so none is passed.""" + return [sys.executable, "-I", str(sample), "-i", str(bitstream), "-d", "1", + "-f", str(frames), "-n", "1", "-m", "thread", "-g", str(gpu)] + + +def _perf_markers(frames: int) -> tuple[re.Pattern[str], re.Pattern[str]]: + """Bind both anchored decode_perf.py markers to the exact requested frame count. + + The count comes from the ``-f`` operand rather than being any positive integer, and the + per-worker marker matches only the "decoded requested" literal.""" + return (re.compile(rf"^(?:Thread|Process) .*: Successfully decoded requested {frames}" + r" frames\s*$", re.MULTILINE), + re.compile(rf"^Total frames decoded:\s*{frames}\s*$", re.MULTILINE)) + + +def _perf_decode_evidence(child: dict[str, Any], *, frames: int, timeout: int, + reasons: list[str]) -> tuple[dict[str, Any], bool]: + """Score the bounded decode_perf.py smoke: frame production, proven by two exact markers. + + This route proves FRAME PRODUCTION ONLY. decode_perf.py writes no raw frames, so no decoded + artifact, size or path proof is claimed here; the byte-exact raw-output proof stays with + advanced/decode.py under full-samples. Reported FPS and elapsed time are deliberately not + asserted: the sample subtracts session-initialization time, so both can go negative here.""" + worker_re, total_re = _perf_markers(frames) + text = child["stdout"] + "\n" + child["stderr"] + workers, totals = worker_re.findall(text), total_re.findall(child["stdout"]) + operation, success = _score( + child, label="decode", frames=frames, timeout=timeout, matches=totals, + reported_frames=frames if len(totals) == 1 else None, + failure_pattern=PERF_FAILURE_PATTERN, reasons=reasons, + checks=[(len(workers) == 1, "official decode_perf sample did not report exactly one" + f" worker that successfully decoded the requested {frames} frames")]) + operation.update({"sample": SAMPLE_PATHS["decode_perf"], "worker_marker_matches": len(workers), + "decoded_output": None, "proves": "frame_production_only", + "raw_output_proof": "not_applicable: decode_perf.py writes no raw frames"}) + return operation, success + + +def _encode_evidence(child: dict[str, Any], *, frames: int, bitstream: Path, timeout: int, + reasons: list[str]) -> tuple[dict[str, Any], bool, dict[str, Any] | None]: + matches = ENCODE_SUCCESS_RE.findall(child["stdout"]) + identity = _output_identity(bitstream, label="H.264 bitstream output") + size = identity["size_bytes"] if identity else 0 + operation, success = _score( + child, label="encode", frames=frames, timeout=timeout, matches=matches, + reported_frames=int(matches[0]) if len(matches) == 1 else None, + failure_pattern=ENCODE_FAILURE_PATTERN, reasons=reasons, + checks=[(size > 0, "official encode sample did not produce a non-empty H.264 bitstream")]) + operation["bitstream"] = {"path": str(bitstream), "size_bytes": size, + "sha256": identity["sha256"] if identity else None} + return operation, success, identity + + +def _decode_evidence(child: dict[str, Any], *, frames: int, decoded: Path, expected_size: int, + timeout: int, reasons: list[str]) -> tuple[dict[str, Any], bool]: + matches = DECODE_SUCCESS_RE.findall(child["stdout"]) + marker_path = matches[0][1] if len(matches) == 1 else None + named = bool(marker_path) and decoded.is_file() and _same_path(marker_path, decoded) + identity = _output_identity(decoded, label="decoded NV12 output") + size = identity["size_bytes"] if identity else 0 + operation, success = _score( + child, label="decode", frames=frames, timeout=timeout, matches=matches, + reported_frames=int(matches[0][0]) if len(matches) == 1 else None, + failure_pattern=DECODE_FAILURE_PATTERN, reasons=reasons, + checks=[(named, "official decode success marker did not name the requested output"), + (size == expected_size, + f"official decode sample produced {size} bytes; expected {expected_size}")]) + operation.update({"reported_output_path": marker_path, "reported_output_matches": named, + "decoded_output": {"path": str(decoded), "format": "NV12", "size_bytes": size, + "expected_size_bytes": expected_size, + "sha256": identity["sha256"] if identity else None}}) + return operation, success + + +def _fresh_path(path: Any, *, label: str) -> Path: + """Resolve one attempt-owned path; a pre-existing path is refused so stale output can't pass.""" + requested = Path(path).expanduser().absolute() + if any(ord(character) < 32 or ord(character) == 127 for character in str(requested)): + raise ValueError(f"{label} path must not contain control characters") + if not requested.parent.is_dir(): + raise FileNotFoundError(f"{label} parent directory does not exist: {requested.parent}") + if os.path.lexists(requested): + raise FileExistsError( + f"{label} already exists; every attempt and retry requires a fresh path: {requested}") + return requested.parent.resolve(strict=True) / requested.name + + +def _prepare_paths(*, work_dir: Path, input_path: Path | None, bitstream_path: Path | None, + decoded_path: Path | None, gpu: int, width: int, height: int, frames: int, + timeout: int) -> dict[str, Any]: + """Validate the request tuple, then create the fresh directory and paths this attempt owns.""" + for name, value in (("GPU index", gpu), ("frame count", frames), ("timeout", timeout)): + if not isinstance(value, int) or isinstance(value, bool) or value < 0: + raise ValueError(f"{name} must be a non-negative integer") + if frames <= 0 or timeout <= 0: + raise ValueError("frame count and timeout must be positive") + if width <= 0 or height <= 0 or width % 2 or height % 2: + raise ValueError("NV12 smoke dimensions must be positive and even") + work = _fresh_path(work_dir, label="verification work directory") + work.mkdir(mode=0o700) + source = (write_new_bytes(work / f"pynvc-smoke-{width}x{height}.nv12", + (bytes([16]) * (width * height) + + bytes([128]) * (width * height // 2)) * frames) + if input_path is None else Path(input_path).expanduser().absolute()) + return {"work": work, "source": source, "generated_input": input_path is None, + "frame_bytes": width * height * 3 // 2, + "bitstream": _fresh_path(bitstream_path or work / f"pynvc-sample-{width}x{height}.h264", + label="bitstream output"), + "decoded": _fresh_path(decoded_path or work / f"pynvc-decode-{width}x{height}.nv12", + label="decoded output")} + + +def verify(*, environment_path: Path, work_dir: Path, gpu: int, width: int, height: int, + frames: int, input_path: Path | None, bitstream_path: Path | None, + decoded_path: Path | None, timeout: int, + profile: str = SMOKE_PROFILE) -> dict[str, Any]: + """Run the official encode/decode pair the selected profile owns and return its result. + + The encode half is the same wheel-owned basic/encode.py ``-m cpu`` proof in either profile; + the decode half is the profile's own route -- the bounded one-frame advanced/decode_perf.py + frame-production smoke, or advanced/decode.py's byte-exact raw-output proof.""" + if profile not in VERIFICATION_PROFILES: + raise ValueError(f"verification profile must be one of {list(VERIFICATION_PROFILES)}") + _validate_smoke_request(profile, width, height, frames, decoded_path) + require_isolated() + distribution = importlib.metadata.distribution(DIST_NAME) + live = _authenticated_pynvc(importlib.import_module(DIST_NAME), distribution) + identity = file_identity(environment_path, label="environment artifact") + # read_json() parses a str/bytes source as the document itself, so the resolved path this was + # just hashed from must be handed over as a Path to be read as a file. + environment = read_json(Path(identity["path"])) + surface = _validate_environment(environment, gpu=gpu, profile=profile) + _require_surface_binding(surface, live) + compatibility = _platform_compatibility(_require_fields( + environment.get("platform"), ("jetson_linux",), label="platform")["jetson_linux"]) + samples, files = _sample_files(distribution) + records = _authenticated_samples(distribution, files) + paths = _prepare_paths(work_dir=work_dir, input_path=input_path, bitstream_path=bitstream_path, + decoded_path=decoded_path, gpu=gpu, width=width, height=height, + frames=frames, timeout=timeout) + expected_size = paths["frame_bytes"] * frames + source = file_identity(paths["source"], label="raw NV12 input") + if source["size_bytes"] < expected_size: + raise ValueError("raw NV12 input does not contain the requested number of complete frames") + + reasons: list[str] = [] + encode_operation, encode_success, bitstream_identity = _encode_evidence( + _run_child([sys.executable, "-I", str(files["encode"]), "-i", source["path"], "-o", + str(paths["bitstream"]), "-s", f"{width}x{height}", "-m", "cpu", "-if", "NV12", + "-c", "h264", "-f", str(frames), "-g", str(gpu), "-json", str(files["config"])], + cwd=samples, timeout=timeout, prefix=paths["work"] / "official-encode"), + frames=frames, bitstream=paths["bitstream"], timeout=timeout, reasons=reasons) + decode_operation: dict[str, Any] = {"status": "not_run", + "reason": "official encode operation did not verify"} + decode_success = False + decode_sample = files[PROFILE_DECODE_SAMPLE[profile]] + if encode_success and profile == SMOKE_PROFILE: + decode_operation, decode_success = _perf_decode_evidence( + _run_child(_perf_decode_argv(decode_sample, paths["bitstream"], gpu=gpu, + frames=frames), + cwd=samples, timeout=timeout, prefix=paths["work"] / "official-decode"), + frames=frames, timeout=timeout, reasons=reasons) + elif encode_success: + decode_operation, decode_success = _decode_evidence( + _run_child([sys.executable, "-I", str(decode_sample), "-i", str(paths["bitstream"]), + "-o", str(paths["decoded"]), "-d", "1", "-g", str(gpu), "-f", str(frames)], + cwd=samples, timeout=timeout, prefix=paths["work"] / "official-decode"), + frames=frames, decoded=paths["decoded"], expected_size=expected_size, timeout=timeout, + reasons=reasons) + + # Terminal re-authentication covering both child launches: a sample payload, raw input, + # environment, wheel or bitstream substitution anywhere in the window fails closed. + for label, before, after in ( + ("official sample payload", records, _authenticated_samples(distribution, files)), + ("raw NV12 input", source, file_identity(paths["source"], label="raw NV12 input")), + ("environment artifact", identity, + file_identity(identity["path"], label="environment artifact")), + (f"{DIST_NAME} wheel identity", live, _authenticated_pynvc( + sys.modules[DIST_NAME], importlib.metadata.distribution(DIST_NAME))), + ("H.264 bitstream output", bitstream_identity, + _output_identity(paths["bitstream"], label="H.264 bitstream output") + if encode_success else bitstream_identity)): + if before != after: + raise ValueError(f"{label} changed during authenticated sample execution") + + success = encode_success and decode_success + return _result( + RESULT_KIND, "operation_verified" if success else "operation_failed", + generated_at=utc_now(), ready=success, software_fallback=False, pynvc=live, + platform_compatibility=compatibility, verification_profile=profile, + distribution={"name": DIST_NAME, "version": live["version"], "python": sys.executable}, + environment={"path": identity["path"], "sha256": identity["sha256"], + "canonical_sha256": canonical_json_sha256(environment), + "kind": environment.get("kind"), "mode": environment.get("mode"), + "schema_version": environment.get("schema_version"), "selected_gpu": gpu, + "interpreter": sys.executable, "verification_profile": profile, + "official_sample_dependencies": { + name: surface["dependencies"].get(name) + for name in profile_dependencies(profile)}}, + official_samples={"root": str(samples), "rows": records, "executed_decode": + SAMPLE_PATHS[PROFILE_DECODE_SAMPLE[profile]], + "provenance": f"{DIST_NAME} {EXPECTED_VERSION} wheel-owned" + f" {profile} encode/decode samples"}, + input={"path": source["path"], "generated_fixture": paths["generated_input"], + "format": "NV12", "width": width, "height": height, "frames": frames, + "size_bytes": source["size_bytes"], "sha256": source["sha256"]}, + hardware_operation=encode_operation, + hardware_operations={"encode": encode_operation, "decode": decode_operation}, + reasons=reasons) + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Run version-matched NVIDIA PyNvVideoCodec encode and decode samples.") + parser.add_argument("--environment", type=Path, required=True, + help="Validated live nvcodec-environment 1.2 JSON") + parser.add_argument("--work-dir", type=Path, default=Path.cwd() / "nvcodec-pynvc-smoke", + help="Fresh, nonexistent operation directory; every retry needs a new one") + for name, default in (("gpu", 0), ("width", SMOKE_WIDTH), ("height", SMOKE_HEIGHT), + ("frames", SMOKE_FRAMES), ("timeout", 120)): + parser.add_argument(f"--{name}", type=int, default=default) + for name, description in (("input", "Optional existing raw NV12 input"), + ("bitstream", "H.264 bitstream output path"), + ("decoded", "Decoded NV12 output path"), + ("output", "Fresh, nonexistent JSON report path; a failed attempt" + " consumes it and every retry requires a new one")): + parser.add_argument(f"--{name}", type=Path, help=description) + parser.add_argument("--profile", choices=VERIFICATION_PROFILES, default=SMOKE_PROFILE, + help="Official-sample verification profile. The default proves" + " basic/encode.py -m cpu plus a bounded one-frame" + " advanced/decode_perf.py and needs no Torch; full-samples runs" + " advanced/decode.py's byte-exact raw proof and does need Torch.") + parser.add_argument("--register-current", action="store_true", + help="After a safely written, operation_verified live result, atomically" + " publish the fixed validated-venv registry" + " ($HOME/.local/state/jetson-videosdk/current-pynvc.json). Requires" + " --output. A non-ready result leaves any prior registry unchanged.") + args = parser.parse_args() + if args.register_current and args.output is None: + parser.error("--register-current requires --output") + try: + _validate_smoke_request( + args.profile, args.width, args.height, args.frames, args.decoded + ) + except ValueError as exc: + parser.error(str(exc)) + if args.output is not None and os.path.lexists(args.output): + print(_render(_result(ERROR_KIND, "error", error=( + "verification report output already exists; every attempt and retry requires a fresh" + f" --output path: {args.output}")))) + return 3 + + try: + result = verify(environment_path=args.environment, work_dir=args.work_dir, gpu=args.gpu, + width=args.width, height=args.height, frames=args.frames, + input_path=args.input, bitstream_path=args.bitstream, + decoded_path=args.decoded, timeout=args.timeout, profile=args.profile) + exit_code = 0 if result["ready"] else 2 + # A valid-but-not-yet-ready surface blocks only pynvc and leaves native actionable. Import + # failure and bad input are helper errors at rc 3, preserving the established result contract. + except (_NotReady, subprocess.TimeoutExpired) as exc: + result, exit_code = _result(RESULT_KIND, "unknown", ready=False, software_fallback=False, + reasons=[str(exc)]), 2 + except Exception as exc: # pylint: disable=broad-exception-caught + result, exit_code = _result(ERROR_KIND, "error", error=str(exc)), 3 + + if args.output is not None: + try: + write_new_json(args.output, result) + except (OSError, ValueError) as exc: + print(_render(_result(ERROR_KIND, "error", error=( + f"verification report output could not be written safely: {exc}")))) + return 3 + print(_render(result)) + if args.register_current and result.get("ready") is True: + # Publication is a separate step AFTER the verification evidence is safely written. Its + # failure preserves the written evidence and leaves any prior registry unchanged. + try: + publish_pynvc_registry(args.output) + except Exception as exc: # pylint: disable=broad-exception-caught + print(f"registration_failed: {exc}", file=sys.stderr) + return 3 + return exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/jetson-video-setup/skill-card.md b/skills/jetson-video-setup/skill-card.md new file mode 100644 index 00000000..f821cd61 --- /dev/null +++ b/skills/jetson-video-setup/skill-card.md @@ -0,0 +1,84 @@ +## Description:
+Use when installing, repairing, probing, or verifying native NVIDIA Video Codec SDK or PyNvVideoCodec on Jetson with official encode-to-decode samples, including registered-environment recovery.
+ +This skill is ready for commercial/non-commercial use.
+ +## Owner +NVIDIA
+ +### License/Terms of Use:
+Apache-2.0
+## Use Case:
+Developers and engineers installing, verifying, or repairing NVIDIA Video Codec SDK and PyNvVideoCodec surfaces on Jetson devices.
+ +### Deployment Geography for Use:
+Global
+ +## Requirements / Dependencies:
+**Requires API Key or External Credential:** [Not Specified]
+**Credential Type(s):** [None identified]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ +## Known Risks and Mitigations:
+Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
+Mitigation: Review and scan skill before deployment.
+ +## Reference(s):
+- [setup-workflow.md](references/setup-workflow.md)
+- [setup-install.md](references/setup-install.md)
+- [setup-output-contract.md](references/setup-output-contract.md)
+- [video-content.md](references/video-content.md)
+ + +## Skill Output:
+**Output Type(s):** [Shell commands, Configuration instructions, Analysis]
+**Output Format:** [Markdown with inline bash code blocks]
+**Output Parameters:** [1D]
+**Other Properties Related to Output:** [None]
+ +## Evaluation Agents Used:
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
+ + + +## Evaluation Tasks:
+Evaluated against 4 evaluation tasks (4 positive) in isolated k8s-sandbox pods.
+ +## Evaluation Metrics Used:
+Reported benchmark dimensions:
+- Security: Whether the skill is safe to use, checking for unsafe operations, secret leakage, and unauthorized access.
+- Correctness: Whether the skill produces correct answers against the reference.
+- Discoverability: Whether the right skill was found and executed when needed.
+- Effectiveness: Whether the skill helped complete the user's goal and expected workflow.
+- Efficiency: Whether the skill avoided wasted tool or skill usage.
+ +Underlying evaluation signals used in this run:
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
+ + + +## Evaluation Results:
+| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 36% → 81% (+45 points) | 24% → 83% (+58 points) | +| Security | 50% → 100% (+50 points) | 0% → 100% (+100 points) | +| Correctness | 35% → 75% (+40 points) | 50% → 95% (+45 points) | +| Discoverability | 38% → 98% (+61 points) | 33% → 84% (+52 points) | +| Effectiveness | 24% → 46% (+22 points) | 13% → 46% (+32 points) | +| Efficiency | 32% → 85% (+53 points) | 26% → 90% (+63 points) | + +## Skill Version(s):
+e61c045 (source: git SHA, committed 2026-08-10)
+ +## Ethical Considerations:
+NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
+ +(For Release on NVIDIA Platforms Only)
+Please report quality, risk, security vulnerabilities or NVIDIA AI Concerns [here](https://app.intigriti.com/programs/nvidia/nvidiavdp/detail).
diff --git a/skills/jetson-video-setup/skill.oms.sig b/skills/jetson-video-setup/skill.oms.sig new file mode 100644 index 00000000..8d849fe9 --- /dev/null +++ b/skills/jetson-video-setup/skill.oms.sig @@ -0,0 +1 @@ +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAiamV0c29uLXZpZGVvLXNldHVwIiwKICAgICAgImRpZ2VzdCI6IHsKICAgICAgICAic2hhMjU2IjogIjlhZTJjZmZjOWNlNzRmOGVhYTkzMGQ4OGZhNmE4NmUwNzRmYjExNjVhMGZlNGU1OTQ3MzA1OTM5ZDU5NWFkMWMiCiAgICAgIH0KICAgIH0KICBdLAogICJwcmVkaWNhdGVUeXBlIjogImh0dHBzOi8vbW9kZWxfc2lnbmluZy9zaWduYXR1cmUvdjEuMCIsCiAgInByZWRpY2F0ZSI6IHsKICAgICJzZXJpYWxpemF0aW9uIjogewogICAgICAiYWxsb3dfc3ltbGlua3MiOiBmYWxzZSwKICAgICAgImhhc2hfdHlwZSI6ICJzaGEyNTYiLAogICAgICAibWV0aG9kIjogImZpbGVzIiwKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdCIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0aWdub3JlIiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0KICAgIH0sCiAgICAicmVzb3VyY2VzIjogWwogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzZTM2NjM4ZWEyMjJjN2Y4NDBiNGJjMWQ3ZTBjMDYyNWRkNDYyZWRmM2ExZTY4ZGQ4NGEzMmY2NzdkNzQ3NWM0IiwKICAgICAgICAibmFtZSI6ICJCRU5DSE1BUksubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkYjk0N2ZkMjUwYjQxMmQzYzc1Njg4ODA3YzMzNGY5NjQ1MTM0ZDk3MmI0NjEyODkzMGM2Mjc5MTJhMTJjM2ZmIiwKICAgICAgICAibmFtZSI6ICJTS0lMTC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogImM2ZGQ4ZWZjNzhmMWVkMWM5MDE3YjRhNDZkNDJkMzRkNDUzNGY1YzFkMDNmNDQ2ODE3ZmRkZGZmMWE1ZjNhMzQiLAogICAgICAgICJuYW1lIjogImV2YWxzL2V2YWxzLmpzb24iLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJjOGI2NTViMzM3ZWIwYTQ3ZTk3ZTgzNTA0YjRkYmY3ZmVlZmIxY2I2OWE4ZGNhYzc1MThmMGJkNmY2ZGI1ZDVjIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3NldHVwLWluc3RhbGwubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4ZmU4N2ZlNDk5MzQ0NTM4NzNjNTgyMGI3NzllOWQ1NWI1OWVjMmU0MTRmODIxYTk1MWQ1ZTE5ZGUxNTFjMGNjIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3NldHVwLW91dHB1dC1jb250cmFjdC5tZCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjE5ZmNhYmZlNTkyNDE4MDc2Yzc2NDhhODAzYzdiZmJiNmEzYTY4N2RhZWU1Y2I0NWMyNDZmMTlhYTg2ZWQzYjciLAogICAgICAgICJuYW1lIjogInJlZmVyZW5jZXMvc2V0dXAtd29ya2Zsb3cubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJiYzdhYWY1MTIyOWE1ODgwZmVhNGFkODg5YjFiYmNkODg1YWY1NzY0ZjRjNTY0Y2Q4YzBiMjFlMjNkNjcxMjBmIiwKICAgICAgICAibmFtZSI6ICJyZWZlcmVuY2VzL3ZpZGVvLWNvbnRlbnQubWQiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMGU4MWY4YzAwNGNiYjNiZjMzY2JkNDkzZjcyZjExMmQxYTA5YjVjOTk4Mjg5NzUxNzZjNGUxOGMzMDExNGEzIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3NldHVwL2xvY2tfcGlwX3JlcG9ydHMucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI1Njk5NDI0MWY4ZGIwNmU4ZDkxOWYzNGU5ZDk5NmUzOGY4MTZlYjQxYzA4MGUzYTlmZjZhZjE0YjEzMTRmZDc1IiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3NldHVwL3BsYW5faW5zdGFsbC5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjU0NDY1MGYwZTVkOTA3NTgwOWM0Mjk3Y2ViMzg0YTQxN2YyZTk4Yjg1ODBjODlkYjlmYTc0YThiMWQ3NWJiOTUiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvc2V0dXAvcHJvYmVfbnZjb2RlYy5weSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiCiAgICAgIH0sCiAgICAgIHsKICAgICAgICAiZGlnZXN0IjogIjVlMmE0OGRlZDFjNWE5NGFhNGY3MTE0NzA3NTQ4ZWRkNmVkMmJjNGQ0NzE2MjU4ZjZiMjFkY2RiM2NhMmYzMmUiLAogICAgICAgICJuYW1lIjogInNjcmlwdHMvc2V0dXAvc2V0dXBfY29udHJhY3QucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkNGY4MGYyMzNiY2M4NjI2NTM4MmY2N2JmNGNlZWRiOWE1ZmIyZGJmYzQ0MDYzYjlhNjVkMWE5N2ZhYTcxOGRhIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3NldHVwL3ZlcmlmeV9uYXRpdmUucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICI4MmU0MWJjYjVhMDUwMDdhZGVjMTI5MjhiMzViNWMzNmNlOTBlOTUxMmY1OTIyNmMzYjRmNmIyNjBlMDZhNjBkIiwKICAgICAgICAibmFtZSI6ICJzY3JpcHRzL3NldHVwL3ZlcmlmeV9weW52Y19zYW1wbGUucHkiLAogICAgICAgICJhbGdvcml0aG0iOiAic2hhMjU2IgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIzNDczYjZlNDVkZWVkZTk3N2Y1OTlmMTcyZmQ2ZjI2ZDk2MGQ1ODgwNzliZmI5ZTlkZWYzNWIzZDkwNjI1ODVkIiwKICAgICAgICAibmFtZSI6ICJza2lsbC1jYXJkLm1kIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIKICAgICAgfQogICAgXQogIH0KfQ==","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDObBUsJKOsdMfqtVdm96YFQ27hF80uGi1q9WCfo2x8mDMHP5ljnUMgX4tZVlYuB+YCMQDo0OyLTrhbNS/Hc5/Y6/mVB2Jgr7viO9N6QomProkF/oW9Th/uPa6ABrrEUb3Py3s=","keyid":""}]}} \ No newline at end of file diff --git a/skills/nemo-mbridge-perf-activation-recompute/BENCHMARK.md b/skills/nemo-mbridge-perf-activation-recompute/BENCHMARK.md index f342dcb4..ff85e946 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/BENCHMARK.md +++ b/skills/nemo-mbridge-perf-activation-recompute/BENCHMARK.md @@ -1,82 +1,101 @@ -# Evaluation Report +# Skill Benchmark: nemo-mbridge-perf-activation-recompute -Evaluation of the `nemo-mbridge-perf-activation-recompute` skill before publication through NVSkills-Eval. +> ✅ **Overall verdict: PASS — Recommended for publication** -This benchmark summarizes 3-Tier Evaluation from NVSkills-Eval results for the skill. The goal is to document whether the skill is safe, discoverable, effective, and useful for agents before it is published for broader workflow use. +## Publication Recommendation + +Recommended for publication based on the completed evaluation evidence in this report. -## Evaluation Summary +## Evaluation Metadata - Skill: `nemo-mbridge-perf-activation-recompute` -- Evaluation date: 2026-06-15 -- NVSkills-Eval profile: `external` -- Environment: `astra-sandbox` -- Dataset: 1 evaluation tasks +- Evaluation date: 2026-08-16 +- Evaluator version: `1.2.7` +- Agents: Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`), Codex (`openai/openai/gpt-5.5`) +- Tasks: 3 evaluation tasks (3 positive) +- Dataset digest: `sha256:b55bb3c51b83cb2245c0d7a8969bbac7066177e342bd9b9c316ac7cf26212ef6` (skill-evaluator-dataset-snapshot/1) - Attempts per task: 1 -- Pass threshold: 50% -- Overall verdict: PASS +- Environment: `local` +- Tier 3 evidence: required for publication -## Agents Used +Tasks ran on the trusted local host; local mode is not sandboxed. -- `claude-code` -- `codex` +## What This Report Answers -## Metrics Used +The three-tier evaluation checks whether the skill: -Reported benchmark dimensions: +- is safe to use; +- produces correct answers; +- is discovered and activated when needed; +- helps the agent complete the user's goal and expected workflow; and +- avoids wasted skill and tool usage. -- Security: checks whether skill-assisted execution avoids unsafe behavior such as secret leakage, destructive commands, or unauthorized access. -- Correctness: checks whether the agent follows the expected workflow and produces the correct final output. -- Discoverability: checks whether the agent loads the skill when relevant and avoids using it when irrelevant. -- Effectiveness: checks whether the agent performs measurably better with the skill than without it. -- Efficiency: checks whether the agent uses fewer tokens and avoids redundant work. +## Results at a Glance -Underlying evaluation signals used in this run: +| Measure | Claude Code (Baseline → Skill Uplift) | Codex (Baseline → Skill Uplift) | +|---|---:|---:| +| Overall | 65% → 98% (+34 points) | 58% → 98% (+39 points) | +| Security | 100% → 100% (±0 points) | 100% → 100% (±0 points) | +| Correctness | 13% → 100% (+87 points) | 73% → 100% (+27 points) | +| Discoverability | 100% → 100% (±0 points) | 42% → 94% (+52 points) | +| Effectiveness | 13% → 91% (+78 points) | 45% → 95% (+50 points) | +| Efficiency | 97% → 100% (+3 points) | 32% → 100% (+68 points) | -- `security` (Security): checks for unsafe operations, secret leakage, and unauthorized access. -- `skill_execution` (Skill Execution): verifies that the agent loaded the expected skill and workflow. -- `skill_efficiency` (Efficiency): checks routing quality, decoy avoidance, and redundant tool usage. -- `accuracy` (Accuracy): grades final-answer correctness against the reference answer. -- `goal_accuracy` (Goal Accuracy): checks whether the overall user task completed successfully. -- `behavior_check` (Behavior Check): verifies expected behavior steps, including safety expectations. -- `token_efficiency` (Token Efficiency): compares token usage with and without the skill. +**How to read this table:** baseline is the same task attempted without the target skill. Uplift is `skill score - baseline score`, shown in percentage points. -## Test Tasks +Example: `47% → 92% (+45 points)` means the skill-assisted run scored 92%, 45 percentage points above its 47% no-skill baseline. -The benchmark dataset contained 1 evaluation tasks: +## Tier Status -- Positive tasks: 1 tasks where the skill was expected to activate. -- Negative tasks: 0 tasks where no skill was expected. -- Unlabeled tasks: 0 tasks where positive/negative intent could not be inferred. +| Tier | Purpose | Status | Evidence | +|---|---|---|---| +| Tier 1 | Static validation | **PASSED WITH OBSERVATIONS** | 1 validator(s); 4 finding(s) | +| Tier 2 | Semantic deduplication | **NOT RUN** | No result was recorded | +| Tier 3 | Live agent evaluation | **PASS** | 2 agent(s); 3 task(s) | -Task composition is derived from the evaluation dataset when possible. Entries with `expected_skill` set are treated as positive skill-activation cases, while entries with `expected_skill: null` are treated as negative activation cases. +## Findings and Observations -## Results +
+Show detailed findings and successful checks -| Dimension | Num | `claude-code` | `codex` | -|---|---:|---:|---:| -| Security | 1 | 100% (+0%) | 100% (+0%) | -| Correctness | 1 | 100% (+100%) | 87% (+40%) | -| Discoverability | 1 | 100% (+100%) | 97% (+0%) | -| Effectiveness | 1 | 96% (+80%) | 80% (+54%) | -| Efficiency | 1 | 94% (+67%) | 96% (-0%) | +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) +- **MEDIUM** SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) +- **MEDIUM** SCHEMA/author_missing: Author not specified in metadata (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) +- **LOW** SCHEMA/unexpected_file: Unexpected 'card.yaml' in skill root (`skills/nemo-mbridge-perf-activation-recompute/card.yaml`) -Score values show skill-assisted performance. Values in parentheses show uplift versus the no-skill baseline when baseline data is available. +
-## Tier 1: Static Validation Summary +## Scoring Methodology -Tier 1 validation passed with observations. NVSkills-Eval ran 1 checks and found 4 total findings. +
+Show dimension definitions, source signals, and thresholds -Top findings: +| Dimension | Question | Scored signals | +|---|---|---| +| Security | Is it safe to use? | `security` (100%) | +| Correctness | Is the answer correct? | `accuracy` (100%) | +| Discoverability | Was the right skill loaded when needed? | `skill_execution` (100%) | +| Effectiveness | Did the skill help complete the task? | `goal_accuracy` (50%) + `behavior_check` (50%) | +| Efficiency | Did it avoid wasted tool or skill usage? | `skill_efficiency` (100%) | -- MEDIUM SCHEMA/body_recommended_section: Missing recommended section: '## Instructions' (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) -- MEDIUM SCHEMA/body_recommended_section: Missing recommended section: '## Examples' (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) -- MEDIUM SCHEMA/author_missing: Author not specified in metadata (`skills/nemo-mbridge-perf-activation-recompute/SKILL.md`) -- LOW SCHEMA/unexpected_file: Unexpected 'card.yaml' in skill root (`skills/nemo-mbridge-perf-activation-recompute/card.yaml`) +- Dimension bands: PASS at 50% or above; NEUTRAL from 40% to below 50%; FAIL below 40%. +- Overall Tier 3 lift: PASS at +5 points or more; FAIL at -10 points or less; values between those bands are NEUTRAL. +- Overall verdict: PASS only when every configured dimension passes for at least one supported agent. Lift is reported as diagnostic evidence and does not override this gate. +- The 50% attempt pass threshold is a separate per-task gate; it is not the dimension pass threshold. +- Effectiveness is the equal-weight mean of goal completion (`goal_accuracy`) and expected workflow adherence (`behavior_check`). +- Token efficiency is a separate report-only signal. It does not change a dimension score or the overall verdict. -## Tier 2: Deduplication Summary +Signals present in this run: -This tier was not run or did not produce findings in this report. +- `security` (Security): unsafe operations, secret leakage, and unauthorized access. +- `skill_execution` (Skill Execution): whether the expected skill was found and executed. +- `skill_efficiency` (Efficiency): routing quality, workspace-aware skill reads, and productive tool use. +- `accuracy` (Accuracy): final-answer correctness against the reference answer. +- `goal_accuracy` (Goal Accuracy): whether the user's goal was achieved. +- `behavior_check` (Behavior Check): whether the expected workflow behavior was followed. -## Publication Recommendation +
+ +## Freshness -The skill is suitable to proceed toward NVSkills-Eval publication based on this benchmark. Skill owners should keep this file with the skill and refresh it when the evaluation dataset, skill behavior, or target agents materially change. +Regenerate this benchmark when the skill, evaluation dataset, target agent/model, evaluator version, environment, or scoring policy changes. diff --git a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md index b50bcb74..d29fe4f6 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/SKILL.md +++ b/skills/nemo-mbridge-perf-activation-recompute/SKILL.md @@ -1,8 +1,12 @@ --- name: nemo-mbridge-perf-activation-recompute -description: Validate and use selective and full activation recompute in Megatron Bridge to reduce GPU memory usage at the cost of extra compute. +description: >- + Validate and use selective and full activation recompute in Megatron Bridge + to reduce GPU memory usage at the cost of extra compute. Use for activation + memory OOMs or regressions involving recompute_granularity, + recompute_num_layers, recompute_modules, recompute_method, selective + recompute, full recompute, or activation checkpointing. license: Apache-2.0 -when_to_use: Reducing GPU memory via activation recompute, or investigating a commit that changed recompute settings and caused OOM or a regression; 'recompute_granularity', 'recompute_num_layers', 'recompute_modules', 'recompute_method', 'selective recompute', 'full recompute', 'activation memory OOM'. --- # Activation Recompute @@ -10,41 +14,24 @@ when_to_use: Reducing GPU memory via activation recompute, or investigating a co Stable docs: @docs/training/activation-recomputation.md Card: @skills/nemo-mbridge-perf-activation-recompute/card.yaml - + -## What It Is +Activation recompute (activation checkpointing) trades additional forward work during backward for lower retained-activation memory. The useful checkpoint boundary depends on the model architecture, attention backend, parallelism, and the tensor that actually drives the per-rank peak. -Activation recompute trades GPU compute for memory by discarding intermediate -activations during the forward pass and recomputing them during backward. -Megatron Bridge supports two granularities: +## Quick Decision Guide -| Granularity | What you specify | What gets recomputed | Memory savings | Compute cost | -|---|---|---|---|---| -| `selective` | `recompute_modules` list (e.g. `core_attn`, `mlp`) | specific submodules within each layer | moderate (module-dependent) | low to high | -| `full` | `recompute_num_layers` + `recompute_method` | entire transformer layers (N layers) | strongest | highest | +1. Confirm the pressure is real allocation, not allocator fragmentation. Compare `max_memory_allocated()` with `max_memory_reserved()` on every rank. +2. Keep an explicit no-recompute control when the workload fits. Under selective granularity, `recompute_modules=[]` is valid and useful for this comparison. +3. Select the first boundary from the architecture and observed peak: + - **Standard attention:** `core_attn` is the common first candidate. It is strongest when unfused attention materializes score/probability tensors. With Transformer Engine fused or Flash Attention, compare it against `[]` because those backends already rematerialize attention internals. + - **Multi-Latent Attention (MLA):** start with `mla_up_proj` when expanded Q/K/V projections dominate. Add `core_attn` only when the attention-core state still matters. + - **Grouped MoE:** start with `moe_act` when the expert intermediate activation dominates; add `layernorm` when norm outputs are material. Use whole `moe` recompute only after accounting for the extra expert compute and communication it replays. + - **Dense FFN:** `mlp` can save the whole dense-MLP activation region, but it usually costs more compute than a narrow output-discard boundary. +4. Change one label at a time. Record per-rank allocated/reserved peaks plus steady-state step time or throughput; do not infer a global module ranking from one recipe. +5. Use full-layer recompute only when targeted selective boundaries do not make the workload fit. Full recompute has the broadest memory effect and the largest replay cost. +6. Treat CUDA graphs, FP8, context-parallel communication, and overlap features as compatibility constraints, not afterthoughts. -Note: MCore names these "selective" (submodule-level) vs "full" (layer-level). -"Full" means recomputing full layers, not the full model — you still choose -how many layers via `recompute_num_layers`. - -## Quick Decision - -1. Rule out allocator fragmentation first with - `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True`; see - @skills/nemo-mbridge-perf-memory-tuning/SKILL.md. -2. For activation pressure, start with selective recompute: - `recompute_granularity="selective"` and `recompute_modules=["core_attn"]`. -3. Add modules by cost: `"layernorm"` is cheap but saves little, while `"mlp"` - saves much more memory at a clear throughput cost. -4. Use full-layer recompute only when selective recompute does not fit, and set - all required fields: `recompute_granularity="full"`, `recompute_method`, and - `recompute_num_layers`. -5. With FP8 or TE-scoped CUDA graphs, avoid full-layer recompute unless graph - scope is `full_iteration`; otherwise use selective recompute or disable TE - graph capture. - -CPU offloading (`cpu_offloading=True`) is an alternative that avoids recompute -cost entirely, but it is **incompatible with PP > 1**. +Megatron Core's `cpu_offloading=True` is an alternative when PCIe/NVLink transfer overhead is preferable to replayed compute. It cannot be combined with activation recompute and is not compatible with pipeline parallelism greater than one. ## Enablement @@ -52,157 +39,229 @@ cost entirely, but it is **incompatible with PP > 1**. ```python cfg.model.recompute_granularity = "selective" -cfg.model.recompute_modules = ["core_attn"] # add "layernorm", "mlp", or other valid modules as needed +cfg.model.recompute_modules = ["core_attn"] # Common standard-attention candidate, not a universal default. ``` +Use the decision table below to replace or extend that list for MLA, MoE, dense-MLP, or GDN workloads. + ### Full-layer recompute ```python cfg.model.recompute_granularity = "full" cfg.model.recompute_method = "uniform" -cfg.model.recompute_num_layers = 4 +cfg.model.recompute_num_layers = 1 ``` -### Available recompute_modules +- `uniform`: checkpoint fixed groups of `recompute_num_layers` transformer layers. +- `block`: checkpoint the first `recompute_num_layers` layers on each pipeline stage, with virtual-pipeline-aware distribution. -| Module | What it recomputes | Compute cost | Memory savings | -|---|---|---|---| -| `core_attn` | attention softmax/dropout/QKV dot product | low (Flash Attention already recomputes internally) | moderate | -| `layernorm` | layer normalization | negligible (~0%) | negligible | -| `mlp` | full FFN block | high (~16% on Llama3 70B, hidden=28672) | ~3 GB | -| `moe` | MoE expert dispatch | varies | varies | -| `moe_act` | MoE activation functions | low | small | -| `shared_experts` | shared expert layers | moderate | moderate | -| `mla_up_proj` | Multi-Latent Attention up projection | moderate | moderate | - -### Performance harness CLI - -```bash -uv run python scripts/performance/run_script.py \ - -m llama \ - -mr llama3_8b \ - --task pretrain \ - -g h100 \ - -c bf16 \ - -ng 8 \ - --recompute_modules core_attn,layernorm \ - ... -``` +## Selective Module Decision Table -## Compatibility and Constraints - -- `recompute_granularity=selective` requires a non-empty `recompute_modules` list -- `recompute_granularity=full` requires `recompute_method` and `recompute_num_layers` -- **Layer-level recompute (`recompute_granularity="full"` + - `recompute_num_layers`) is incompatible with TE-scoped CUDA graphs.** - MCore calls this "full" granularity — the name refers to recomputing - full transformer layers, not the full model. Even though you're selecting - how many layers to recompute, MCore treats it differently from submodule - recompute. Any TE-scoped scope (`attn`, `mlp`, `moe_router`, etc.) will - assert. This commonly hits FP8 configs that enable TE-scoped graphs by - default (e.g. `LLAMA3_70B_SFT_CONFIG_H100_FP8_CS_V1` sets - `cuda_graph_impl="transformer_engine"`, `cuda_graph_scope="mlp"`). Options: - - use submodule recompute (`recompute_granularity="selective"` + - `recompute_modules`) — compatible with TE-scoped graphs - - disable CUDA graphs (`cuda_graph_impl="none"`) and use layer-level recompute - - switch to `cuda_graph_impl="local"`, `cuda_graph_scope="full_iteration"` -- `distribute_saved_activations=True` cannot be combined with `sequence_parallel=True` -- Combining `mlp` + `core_attn` recompute is slightly worse than `mlp` alone - due to double recompute overhead - -## Measured Results - -Llama3 70B SFT on 32x H100 80GB, FP8 (Current Scaling): -- Baseline: TP=4, PP=4, VPP=5, DP=2, MBS=1, GBS=32, seq_len=4096 -- Golden GPU utilization: 709.93 TFLOP/s/GPU -- Regression threshold: 5% - -| Experiment | recompute_modules | TFLOP/s/GPU | vs Golden | Peak Mem (GB) | Result | -|---|---|---|---|---|---| -| Baseline | [core_attn] | ~704 | -0.8% | 58.8 (OOM rank0) | OOM | -| Exp 1 | [mlp] | 593.6 | -16.4% | 55.6 | Perf regression | -| Exp 2 | [mlp, core_attn] | 586.8 | -17.3% | 55.6 | Perf regression | -| Exp 3 | [core_attn, layernorm] | ~702 | -1.1% | 59.6 (OOM rank0) | OOM | - -Key takeaways: - -- `layernorm` recompute is nearly free compute-wise but saves negligible memory -- `mlp` recompute saves ~3 GB peak but costs ~16% because the Llama3 70B FFN - (hidden=28672) is expensive to recompute -- Combining `mlp` + `core_attn` is slightly worse than `mlp` alone -- For this workload, the actual OOM fix was `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` - (memory fragmentation, not capacity). See @skills/nemo-mbridge-perf-memory-tuning/SKILL.md. +The currently pinned Megatron Core accepts these labels. A development branch can add model-specific labels, so validate against the exact target revision rather than copying a list across branches. -## Code Anchors - -### Recompute modules enum and selective checkpoint logic - -```python -# 3rdparty/Megatron-LM/megatron/core/transformer/transformer_block.py -# _checkpointed_forward() applies selective recompute based on recompute_modules -``` +| Module | Checkpoint boundary | When to test it | Main cost or caveat | +|---|---|---|---| +| `core_attn` | Core attention | Standard attention, especially an unfused backend retaining attention intermediates | Replays attention. Incremental savings can be small with TE fused/Flash Attention; context parallelism can replay attention communication. | +| `mla_up_proj` | MLA Q/KV up-projection plus RoPE region | MLA models retaining expanded Q/K/V tensors | Replays the MLA expansion path. It is a distinct, potentially additive boundary from `core_attn`. | +| `layernorm` | Input and pre-MLP normalization outputs | Norm outputs contribute materially to the peak, often alongside MoE or MLA boundaries | Usually narrow, but savings depend on hidden size, sequence length, and which graph paths are active. | +| `moe_act` | Activation output between grouped expert FC1 and FC2 | Grouped MoE expert-intermediate activations dominate | Narrow output-discard checkpoint. It does not replay dispatch, FC1, or FC2, but has FP8 delayed-scaling restrictions. | +| `mlp` | Whole dense MLP | Dense layers dominate after narrower boundaries are exhausted | Replays the complete dense MLP. It has no effect on layers whose MLP is MoE. | +| `moe` | Whole MoE forward | A broad MoE region must be discarded to make the workload fit | Replays routing, dispatch/combine communication, experts, and shared-expert work. It is incompatible with expert-parallel overlap. | +| `shared_experts` | Non-overlapped shared-expert MLP | Shared experts are a distinct material peak | Replays the shared-expert MLP and is invalid with shared-expert overlap. Outer `moe` already removes its original-forward saves, but nesting can still change the transient backward-replay peak. | +| `gdn_norm_out` | GDN gated-normalization output | GDN/hybrid models retain this output | Replays the normalization and its HP-to-CP all-to-all path. | + +For example, DeepSeek V4 configurations can use the model-specific `mhc` +label only with their required Megatron Core development branch. It is not a +portable label for the pinned revision and therefore is not included in the +table above. + +Common performance configurations consequently fall into several patterns rather than one universal list: + +- standard transformer recipes often use `core_attn`; +- MLA recipes often use `mla_up_proj`, sometimes with `mlp`; +- grouped-MoE recipes often use `moe_act` or `layernorm` plus `moe_act`; +- higher-pressure MoE recipes sometimes use broader combinations such as `moe` plus `layernorm`. + +These are candidate patterns, not an ordering guarantee. Peak attribution and matched measurements decide the final list. + +## Measurement Contract + +For every candidate, capture: + +- exact Bridge and Megatron Core revisions; +- model, sequence length, micro/global batch sizes, precision, attention backend, and parallelism; +- the exact `recompute_granularity`, module list, method, and layer count; +- per-rank `max_memory_allocated()` and `max_memory_reserved()`; +- steady-state step time or throughput after warmup; +- a short convergence or numerical-sanity check appropriate to the task. + +Use a matched no-recompute control and change one recompute choice at a time. Peak memory from different jobs, backends, or parallel layouts is not a module-ranking benchmark. + +Do not call a candidate successful merely because it advances farther than the +control. Run through optimizer-state initialization and multiple steady-state +steps: selective recompute can move the memory wall from forward into gradient +synchronization or the optimizer without making the workload viable. + +## Matched H100 Evidence: Moonlight 16B + +A 2026-08-12 short-run study used the exact Bridge revision +`600d069b824dd5ce50367a311a5a3244478faf22` and Megatron Core revision +`24bad8e677d22625d86ef2a54c9506b6e4992c93`. The Moonlight 16B BF16 +pretraining recipe ran on 8 H100 80GB GPUs with sequence length 4096, MBS=1, +GBS=4, TP=2, PP=1, CP=1, EP=8, mock data, and 20 steps. This model mixes one +dense layer with 26 MLA+MoE layers. Each row changed only +`recompute_modules`; all 20 losses were finite with zero skipped or NaN +iterations. + +Peak allocated memory is the maximum post-optimizer value reported after +iteration 2. Time and throughput are means over iterations 11--20. + +| Selective modules | Peak allocated (GB) | Step time (ms) | TFLOP/s/GPU | Allocated vs `[]` | Time vs `[]` | +|---|---:|---:|---:|---:|---:| +| `[]` | 36.618 | 457.18 | 77.50 | control | control | +| `core_attn` | 36.614 | 474.80 | 74.44 | -0.01% | +3.85% | +| `mla_up_proj` | 35.902 | 480.89 | 73.72 | -1.96% | +5.19% | +| `mla_up_proj`, `mlp` | 35.917 | 496.73 | 71.50 | -1.91% | +8.65% | +| `moe_act` | 35.941 | 466.26 | 75.50 | -1.85% | +1.99% | +| `layernorm`, `moe_act` | 35.949 | 506.53 | 70.27 | -1.83% | +10.79% | + +For this exact workload, `moe_act` is the best first boundary: it recovered +nearly as much allocated memory as `mla_up_proj` for less replay cost. +`mla_up_proj` is the next candidate if its roughly 39 MB additional reduction +matters. Adding `mlp` to `mla_up_proj` or `layernorm` to `moe_act` did not +improve the observed peak and made steps slower. Explicit `core_attn` added +cost without material memory benefit under fused attention. + +Maximum reserved memory stayed near 40 GB and did not fall monotonically. +That is allocator caching, not contrary evidence: boundary selection in this +study is based on allocated memory and successful end-to-end steps. + +## Matched H100 Evidence: Nemotron 3 Nano + +The same 2026-08-12 study used the native 16-H100 BF16 performance recipe for +the 52-layer hybrid Mamba/fused-attention MoE model. The matched short-run +configuration used sequence length 8192, MBS=1, GBS=16, TP=1, PP=1, CP=1, +EP=8, DP=16, expert-DP=2, HybridEP, grouped GEMM, TE CUDA graphs for attention and Mamba, +mock data, and 12 steps. Each row changed only `recompute_modules`. + +| Selective modules | Outcome | Rank-0 measured peak | Failure or steady-state evidence | +|---|---|---:|---| +| `[]` | OOM after iteration 1 | 66.297 GB after iteration 1 | Iteration-2 MoE router allocation failed; hot ranks had about 72.9 GiB allocated. | +| `core_attn` | OOM in iteration 1 | not comparable | Grouped-expert linear allocation failed; explicit attention recompute did not make the fused-attention workload fit. | +| `moe_act` | OOM after iteration 1 | 62.103 GB after iteration 1 | 4.194 GB (6.33%) below the control at the matched checkpoint, but the iteration-2 output projection still needed 2 GiB. | +| `layernorm`, `moe_act` | OOM in iteration 1 | not comparable | Output projection still needed 2 GiB; CUDA-graph private pools were material. | +| `moe` | completed 12 steps | 64.653 GB after iteration 2 | 657.42 ms and 277.72 TFLOP/s/GPU over iterations 7--12. | +| `moe`, `layernorm` | completed 12 steps | 63.639 GB after iteration 2 | 677.62 ms and 270.62 TFLOP/s/GPU over iterations 7--12. | + +Both successful rows had finite losses and zero skipped or NaN iterations. +For this exact capacity-limited recipe, whole-`moe` recompute is the smallest +tested passing boundary. Adding `layernorm` recovered another 1.014 GB (1.57%) +of rank-0 peak at 3.07% higher step time, so the recipe's broader combination +is justified when that headroom is required. Narrow `moe_act` produced real +activation relief but did not make the whole training step viable. + +An exploratory native 8-H100 layout failed during FP32 optimizer-state +initialization even at sequence length 4096. That is optimizer capacity, not a +selective-boundary throughput baseline; no timing comparison from those runs +is used here. + +### Cross-model conclusion -### Recompute config validation +These measurements do not define one ranking. Moonlight fit with an empty +control and favored narrow `moe_act`; Nemotron required broad whole-`moe` +recompute; historical dense Llama evidence found whole-`mlp` replay costly and +lacked an empty control. The correct first candidate is therefore the narrowest +boundary implicated by the architecture and peak, followed by broader replay +only when the narrow choice does not pass the complete step. -```python -# 3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py -# Validates recompute_granularity, recompute_method, recompute_num_layers -``` +## Compatibility and Validation -### Llama3 recipe defaults +### Configuration semantics + +- `recompute_granularity="selective"` uses `recompute_modules`; an empty list is accepted as an explicit control. +- `recompute_granularity="full"` uses `recompute_method` and `recompute_num_layers`; selective labels do not apply. +- Full granularity supersedes selective module choices rather than composing with them. +- Unknown labels fail Megatron Core validation. Labels may differ on development branches, so use the exact revision's `TransformerConfig` validator as the source of truth. -```99:103:src/megatron/bridge/recipes/llama/llama3.py - # Memory saving (recompute & offloading) - cfg.model.recompute_granularity = None - cfg.model.recompute_modules = None - cfg.model.fine_grained_activation_offloading = False - cfg.model.offload_modules = None -``` +### Attention backend and context parallelism + +- TE fused and Flash Attention already use internal rematerialization. Explicit `core_attn` may still change retained inputs/outputs, but it must earn its place in a matched `[]` comparison. +- Under context parallelism, an attention checkpoint can replay communication as well as compute. Include CP size and topology in the measurement record. + +### MoE restrictions + +- Whole-`moe` recompute is incompatible with expert-parallel overlap because backward replay would repeat the overlapped routing/communication region. +- `shared_experts` recompute is incompatible with shared-expert overlap. +- `moe_act` applies to grouped-GEMM experts and is the narrower choice when only the expert activation needs to be discarded. +- `mlp` targets dense MLPs and is a no-op on MoE layers; mixed dense/MoE models can still benefit on their dense layers. -### Full recompute + CUDA graph assertion (MCore) +### FP8 restrictions + +- `moe_act` and `layernorm` recompute are not supported with FP8 delayed scaling and require a compatible Transformer Engine version. +- Absorbed MLA paths have additional FP8/FP4 restrictions. Validate the exact model/provider path before selecting `mla_up_proj`. + +### CUDA graphs + +- Selective recompute is valid only when a checkpointed module lies wholly inside or wholly outside the selected graph scope. A checkpoint boundary that straddles a graph boundary is invalid. +- Capture/warmup can bypass checkpoint wrappers, so verify the final graph scope and replay path rather than assuming eager behavior carries over. +- Full recompute with CUDA graphs requires `cuda_graph_impl="full_iteration"` in the pinned Megatron Core. Otherwise disable CUDA graphs; scoped/local graph capture is not a substitute for full-iteration capture here. + +## Historical Measurement: Context, Not a Module Ranking + +Historical H100 measurements from Bridge PR #3107 used Llama 3 70B SFT on 32 H100 80GB GPUs with FP8 current scaling, sequence length 4096, micro-batch size 1, global batch size 32, TP=4, PP=4, VPP=5, and DP=2: + +| Configuration | TFLOP/s/GPU | Peak memory | +|---|---:|---:| +| `core_attn` baseline in that run | ~704 | 58.8 GB (OOM on rank 0) | +| `mlp` | 593.6 | 55.6 GB | +| `mlp` + `core_attn` | 586.8 | 55.6 GB | +| `core_attn` + `layernorm` | ~702 | 59.6 GB (OOM on rank 0) | +| Golden throughput recorded in the PR context | 709.93 | Not a paired memory measurement | + +Limitations of this evidence: -```2001:2005:3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py - if self.recompute_granularity: - if self.recompute_granularity != "selective": - assert self.cuda_graph_scope == [ - CudaGraphScope.full_iteration - ], "full recompute is only supported with full iteration CUDA graph." -``` +- it did not include a matched no-recompute row; +- the golden row was not a paired module-only comparison; +- the measurements cover one dense Llama workload, not MLA or MoE; +- the table supports the local memory/throughput tradeoff only and must not be used to rank all recompute labels. -### CPU offloading PP incompatibility (MCore) +## Code Anchors -```1303:1306:3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py - if self.cpu_offloading and self.pipeline_model_parallel_size > 1: - raise ValueError( - "Currently there is no support for Pipeline parallelism with CPU offloading" - ) -``` +- Selective-label validation and cross-feature checks: `3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py` +- Checkpoint implementations: `3rdparty/Megatron-LM/megatron/core/tensor_parallel/random.py` +- Standard-attention checkpoint boundary: `3rdparty/Megatron-LM/megatron/core/transformer/attention.py` +- MLA up-projection boundary: `3rdparty/Megatron-LM/megatron/core/transformer/multi_latent_attention.py` +- Layernorm, dense-MLP, and outer-MoE placement: `3rdparty/Megatron-LM/megatron/core/transformer/transformer_layer.py` +- Grouped expert activation boundary: `3rdparty/Megatron-LM/megatron/core/transformer/moe/experts.py` +- Shared-expert and whole-MoE paths: `3rdparty/Megatron-LM/megatron/core/transformer/moe/moe_layer.py` +- GDN normalization boundary: `3rdparty/Megatron-LM/megatron/core/ssm/gated_delta_net/gdn.py` ## Failure Diagnosis -| Symptom | Cause | Confirm | Fix | -|---|---|---|---| -| >15% GPU utilization drop | `mlp` recompute on a large FFN | check whether `recompute_modules` includes `mlp` | remove `mlp`, lower micro batch size, or use CPU offload if PP=1 | -| Still OOM after adding layernorm | layernorm activations are too small to move the peak materially | compare peak memory before/after | switch to a higher-impact module or full-layer recompute | -| `AssertionError: full recompute is only supported with full iteration CUDA graph` | layer-level recompute with TE-scoped graph capture | check `cuda_graph_impl` and `cuda_graph_scope` | use `selective`, set `cuda_graph_impl=none`, or use `local` + `full_iteration` | -| ValueError: PP + CPU offloading | `cpu_offloading=True` with `pipeline_model_parallel_size > 1` | check PP config | disable CPU offloading or set PP=1 | -| mlp+core_attn worse than mlp alone | double recompute overhead | compare Exp 1 vs Exp 2 | use mlp alone | +| Symptom | Likely cause | Next action | +|---|---|---| +| `core_attn` gives little or no peak reduction | Fused/Flash attention already rematerializes the expensive internals, or the peak is elsewhere | Compare with `[]`, attribute the peak, then test the architecture-specific boundary such as `mla_up_proj` or `moe_act`. | +| MLA still OOMs after `core_attn` | Expanded Q/K/V projection tensors, not attention-core tensors, dominate | Test `mla_up_proj`; add `core_attn` only if matched evidence supports it. | +| MoE peak remains high | Expert intermediate or norm outputs dominate | Test `moe_act`, then `layernorm`; reserve whole `moe` for broader pressure. | +| Expert-overlap validation fails | Whole-`moe` or `shared_experts` recompute conflicts with overlap | Keep overlap and use a compatible inner boundary, or disable overlap and remeasure the entire configuration. | +| A selected label has no measurable effect | That module is absent or inactive on the measured layers, or graph capture bypassed the wrapper | Inspect the provider/layer mix and final graph scope; for example, `mlp` is ineffective on pure-MoE layers. | +| Full recompute plus CUDA graphs asserts | Graph implementation is not full-iteration | Set `cuda_graph_impl="full_iteration"` or disable CUDA graphs. | +| Reserved memory is high but allocated memory is stable | Allocator fragmentation or caching | Try `PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True` before adding recompute. | +| OOM moves to a different rank after enabling recompute | Pipeline/virtual-pipeline layer distribution changed the bottleneck | Compare per-rank peaks and tune full block/uniform placement or selective boundaries for the actual hot stage. | +| A candidate gets farther but still OOMs | Recompute moved the peak into gradient synchronization or optimizer-state initialization | Record the changed failure stage as diagnostic evidence, but require optimizer initialization and multiple steady steps before calling it a pass. | ## Known Limitations -- Per-module memory savings vary significantly by model architecture and hidden - dimension -- No automatic module selection — users must choose which modules to recompute -- `layernorm` recompute is almost never worth it as a standalone fix -- CPU offloading (the zero-compute-cost alternative) is blocked when PP > 1 +- A module list is not portable across model families, attention backends, parallel layouts, or Megatron Core revisions. +- Memory savings are nonlinear when boundaries overlap or nest; additive arithmetic is unreliable. +- Full recompute changes RNG execution paths; dropout workloads need a numerical/convergence check. +- Activation recompute does not address parameter, optimizer-state, or allocator-fragmentation pressure. +- The correct result is the smallest measured replay cost that satisfies the per-rank memory target, not the longest module list. -## Verification - -```bash -uv run python -m pytest \ - tests/unit_tests/training/test_config.py -k "recompute" -q -``` +## Further Reading -Success criteria: -- Unit tests pass for recompute config validation -- No assertion errors from config validation +- `docs/performance-guide.md` +- `skills/nemo-mbridge-perf-memory-tuning/SKILL.md` +- `skills/nemo-mbridge-perf-cuda-graphs/SKILL.md` +- `skills/nemo-mbridge-perf-cpu-offloading/SKILL.md` +- Megatron Core activation recomputation guide: diff --git a/skills/nemo-mbridge-perf-activation-recompute/card.yaml b/skills/nemo-mbridge-perf-activation-recompute/card.yaml index f5df9060..fc133ae1 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/card.yaml +++ b/skills/nemo-mbridge-perf-activation-recompute/card.yaml @@ -1,87 +1,112 @@ title: activation_recompute -validated_on: "2026-04-02" +validated_on: "2026-08-12" summary: > - Selective activation recompute trades GPU compute for memory by recomputing - specific module outputs during backward instead of storing them. Megatron - Bridge exposes recompute_modules (core_attn, mlp, layernorm, moe, moe_act, - shared_experts, mla_up_proj) for fine-grained control. Measured on Llama3 - 70B SFT (32x H100, FP8 CS): mlp recompute saves ~3 GB peak memory but - costs ~16% GPU utilization; layernorm recompute is nearly free (~0% cost) - but saves negligible memory; core_attn recompute is the default and is - cheap when Flash Attention is active. Full-layer recompute - (recompute_granularity=full) gives the strongest memory reduction but the - highest compute overhead. + Selective activation recompute trades replayed forward work for lower retained + activation memory. The correct checkpoint boundary is architecture- and + backend-specific: core_attn is common for standard attention, mla_up_proj is + often more relevant for MLA, and moe_act or layernorm plus moe_act are common + narrow choices for grouped MoE. The pinned Megatron Core also accepts mlp, + moe, shared_experts, and gdn_norm_out. Full-layer recompute has the broadest + memory effect and replay cost. Development branches may expose additional + model-specific labels, so the exact target revision is authoritative. validation_status: selective_recompute_config: - - code_verified # transformer_config.py, transformer_block.py + - code_verified # transformer_config.py and module-local checkpoint sites full_layer_recompute_config: - - code_verified # transformer_config.py + - code_verified # transformer_config.py and transformer_block.py recompute_modules_enum: - - code_verified # transformer_block.py + - code_verified # transformer_config.py llama3_70b_sft_fp8_cs_experiment: - - measured # PR #3107, 32x H100, TP4 PP4 VPP5 DP2 + - measured # historical PR #3107; contextual, not a cross-module ranking + moonlight_16b_bf16_experiment: + - measured # matched 8-H100 module sweep with an empty-list control + nemotron_3_nano_bf16_experiment: + - measured # matched 16-H100 capacity sweep through complete training steps training_dimensions: speed: - effect: "~0% (core_attn/layernorm) to ~16% slower (mlp)" - confidence: high + effect: workload-dependent replay overhead + confidence: medium rationale: > - Measured on Llama3 70B SFT 32x H100: core_attn recompute costs ~0.8% - (704 vs 710 TFLOP/s/GPU); layernorm recompute adds ~0.3% on top; - mlp recompute costs ~16% (594 TFLOP/s/GPU) because the FFN - (hidden=28672) is expensive to recompute. + Narrow output-discard boundaries such as moe_act can replay much less work + than whole mlp or moe boundaries. Attention cost also depends on the + backend and context-parallel communication. The historical Llama 3 70B + run measured mlp only relative to a core_attn baseline, not to a matched + no-recompute control. The matched Moonlight run found 1.99% overhead for + moe_act, 5.19% for mla_up_proj, and no useful core_attn memory reduction + in that exact fused-attention recipe. In the matched Nemotron run, + whole-moe recompute was the smallest tested passing boundary; adding + layernorm cost 3.07% more step time for 1.014 GB more rank-0 headroom. memory: - effect: "~0 GB (layernorm) to ~3 GB (mlp) peak memory reduction" - confidence: high + effect: workload-dependent reduction in retained activations + confidence: medium rationale: > - Measured on Llama3 70B SFT 32x H100: mlp recompute reduced peak from - 58.8 to 55.6 GB. Layernorm recompute did not measurably reduce peak. - core_attn+layernorm together still OOMed at 59.6 GB. + Savings depend on which tensors drive the per-rank peak. Standard + attention, MLA expansions, dense FFNs, grouped expert activations, and + norm outputs require different boundaries; nested boundaries are not + additive. The matched Moonlight sweep directly confirmed comparable + savings from moe_act and mla_up_proj, no material core_attn reduction, + and no incremental peak benefit from the tested nested additions. The + Nemotron sweep showed that a narrow boundary can materially reduce an + intermediate peak yet remain insufficient for the complete step. scale: - effect: "neutral" - confidence: medium + effect: can replay communication at some boundaries + confidence: high rationale: > - Recompute is per-GPU and does not change communication patterns. + core_attn under context parallelism, whole moe, and gdn_norm_out can + replay communication during backward. Scale and topology therefore form + part of the measurement contract. convergence: - effect: "no change expected (numerically identical forward)" - confidence: high + effect: no semantic change expected when RNG replay is correct; verify + confidence: medium rationale: > - Recompute replays the same forward computation — no numerical change. + Recomputation should reproduce the forward region during backward, but + dropout, graph capture, precision, and branch-specific behavior warrant a + numerical or short convergence check. stability: - effect: "neutral" + effect: conditional on feature compatibility confidence: high rationale: > - No additional failure modes beyond the existing config constraints. + Megatron Core validates restrictions involving CUDA graphs, FP8 modes, + overlap features, and the selected module labels. enable_when: - - training OOMs or is close to the GPU memory limit - - memory savings from lighter modules (core_attn, layernorm) are enough - - throughput loss from mlp recompute is acceptable for the use case - - full-layer recompute is needed as a last resort for very tight memory + - allocated activation memory, rather than allocator fragmentation, drives OOM or inadequate headroom + - a matched architecture-specific checkpoint boundary reduces the hot rank's peak at acceptable throughput + - full-layer recompute is needed after narrower selective boundaries are exhausted avoid_when: - - the model already fits with acceptable headroom - - mlp recompute cost (~16%) is too high and VPP or parallelism tuning can fix OOM instead - - TE-scoped CUDA graphs are enabled with recompute_granularity=full (incompatible) - - CPU offloading is available and cheaper (PP=1 only) + - the model fits with acceptable headroom and recompute does not improve another measured objective + - max reserved memory is high but max allocated memory does not support an activation-capacity diagnosis + - the selected module is absent or inactive for the model's measured layers + - a required CUDA-graph, FP8, or overlap feature is incompatible with the boundary interactions: required: [] conditional: - - recompute_granularity=full is incompatible with TE-scoped CUDA graphs - - recompute_granularity=full with uniform method requires recompute_num_layers - - recompute_granularity=selective requires recompute_modules list - - mlp recompute combined with core_attn is slightly worse than mlp alone due to double recompute overhead + - selective granularity accepts an empty recompute_modules list as a matched no-recompute control + - full granularity requires recompute_method and recompute_num_layers + - full recompute with CUDA graphs requires cuda_graph_impl=full_iteration + - selective graph capture requires each checkpoint boundary to lie wholly inside or wholly outside the graph scope + - core_attn can replay context-parallel communication and may add little with fused or Flash Attention + - moe_act and layernorm have FP8 delayed-scaling restrictions + - whole moe recompute is incompatible with expert-parallel overlap + - shared_experts recompute is incompatible with shared-expert overlap incompatible: - - recompute_granularity=full with cuda_graph_impl=transformer_engine + - full recompute with scoped or local non-full-iteration CUDA graphs + - whole moe recompute with expert-parallel overlap + - shared_experts recompute with shared-expert overlap feature_meaning: recompute_granularity: > - Controls scope: null (no recompute), selective (per-module), full - (entire transformer layer). + Controls scope: null (no recompute), selective (per-module), or full + (complete transformer-layer regions). recompute_modules: > - List of modules to selectively recompute: core_attn, mlp, layernorm, - moe, moe_act, shared_experts, mla_up_proj. + Pinned-revision labels are core_attn, mla_up_proj, layernorm, moe_act, mlp, + moe, shared_experts, and gdn_norm_out. Development revisions can add labels; + for example, DeepSeek V4's mhc label requires its development MCore branch. recompute_method: > - For full granularity: uniform (divide layers evenly into blocks) or - block (recompute a fixed number of layers per PP stage). + For full granularity: uniform groups fixed-size layer blocks; block selects + a fixed number of layers per pipeline stage with virtual-pipeline-aware + distribution. recompute_num_layers: > - For full granularity: number of layers per recomputation block. + For full granularity: the layer group size or number of selected layers, + depending on recompute_method. config_keys: - model.recompute_granularity - model.recompute_modules @@ -89,22 +114,140 @@ config_keys: - model.recompute_num_layers - model.distribute_saved_activations recommended_path: - first_try: "recompute_granularity=selective, recompute_modules=[core_attn]" - if_still_oom: "add layernorm (cheap) or mlp (expensive but saves ~3 GB)" - last_resort: "recompute_granularity=full, recompute_method=uniform" - alternative: "see skills/nemo-mbridge-perf-memory-tuning/ for VPP tuning and other memory strategies" + first_step: compare per-rank allocated and reserved peaks and keep a matched no-recompute control + standard_attention: test core_attn, but compare it with an empty list under fused or Flash Attention + mla: test mla_up_proj first when expanded Q/K/V tensors dominate; add core_attn only with evidence + grouped_moe: test moe_act and then layernorm when their outputs drive the peak + dense_ffn: test mlp only when its broader replay cost is acceptable + broader_moe: reserve whole moe for cases where narrower MoE boundaries do not fit + last_resort: use full recompute with recompute_method and recompute_num_layers + alternative: see skills/nemo-mbridge-perf-memory-tuning for allocator, parallelism, and offloading options expected_metric_change: - metric: peak_memory direction: down - magnitude: "~0 GB (layernorm) to ~3 GB (mlp) on Llama3 70B SFT" - conditions: Llama3 70B, TP4 PP4 VPP5 DP2, 32x H100 80GB, FP8 CS - evidence: measured_pr_3107 - - metric: gpu_utilization - direction: down - magnitude: "~0% (core_attn/layernorm) to ~16% (mlp)" - conditions: same as peak_memory - evidence: measured_pr_3107 + magnitude: must be measured for the exact boundary and hot rank + conditions: matched model, backend, parallelism, precision, batch, and sequence configuration + evidence: module checkpoint implementations, matched Moonlight and Nemotron studies, and historical measured_pr_3107 + - metric: steady_state_step_time + direction: up + magnitude: depends on replayed compute and communication + conditions: same matched configuration and warmup + evidence: module checkpoint implementations, matched Moonlight and Nemotron studies, and historical measured_pr_3107 measured_results: + - model: Moonlight 16B + task: pretrain_short_run + revisions: + bridge: 600d069b824dd5ce50367a311a5a3244478faf22 + megatron_core: 24bad8e677d22625d86ef2a54c9506b6e4992c93 + parallelism: TP2_PP1_CP1_EP8_DP4 + gpus: 8 + gpu: H100_80GB + precision: BF16 + seq_length: 4096 + mbs: 1 + gbs: 4 + steps: 20 + measurement: peak allocated after iteration 2; mean time and throughput over iterations 11-20 + numerical_sanity: all losses finite; zero skipped and NaN iterations + experiments: + - name: empty_control + recompute_modules: [] + peak_mem_gb: 36.618 + step_time_ms: 457.18 + tflops_per_gpu: 77.50 + - name: core_attn + recompute_modules: ["core_attn"] + peak_mem_gb: 36.614 + peak_mem_vs_control_pct: -0.01 + step_time_ms: 474.80 + step_time_vs_control_pct: 3.85 + - name: mla_up_proj + recompute_modules: ["mla_up_proj"] + peak_mem_gb: 35.902 + peak_mem_vs_control_pct: -1.96 + step_time_ms: 480.89 + step_time_vs_control_pct: 5.19 + - name: mla_up_proj_plus_mlp + recompute_modules: ["mla_up_proj", "mlp"] + peak_mem_gb: 35.917 + peak_mem_vs_control_pct: -1.91 + step_time_ms: 496.73 + step_time_vs_control_pct: 8.65 + - name: moe_act + recompute_modules: ["moe_act"] + peak_mem_gb: 35.941 + peak_mem_vs_control_pct: -1.85 + step_time_ms: 466.26 + step_time_vs_control_pct: 1.99 + - name: layernorm_plus_moe_act + recompute_modules: ["layernorm", "moe_act"] + peak_mem_gb: 35.949 + peak_mem_vs_control_pct: -1.83 + step_time_ms: 506.53 + step_time_vs_control_pct: 10.79 + conclusion: > + For this exact mixed dense/MLA/MoE recipe, moe_act is the efficient first + boundary and mla_up_proj is the next candidate when its roughly 39 MB + additional saving matters. Adding mlp or layernorm did not reduce the + observed peak, and core_attn added cost without material memory benefit. + - model: Nemotron 3 Nano + task: pretrain_capacity_short_run + revisions: + bridge: 600d069b824dd5ce50367a311a5a3244478faf22 + megatron_core: 24bad8e677d22625d86ef2a54c9506b6e4992c93 + architecture: 52-layer hybrid Mamba/fused-attention MoE + parallelism: TP1_PP1_CP1_EP8_DP16 + expert_data_parallel_size: 2 + gpus: 16 + gpu: H100_80GB + precision: BF16 + seq_length: 8192 + mbs: 1 + gbs: 16 + steps: 12 + execution: HybridEP, grouped GEMM, TE CUDA graphs for attention and Mamba + measurement: rank-0 reported peak; mean time and throughput over iterations 7-12 for passing rows + experiments: + - name: empty_control + recompute_modules: [] + peak_mem_gb_after_iteration_1: 66.297 + status: OOM_iteration_2_moe_router + - name: core_attn + recompute_modules: ["core_attn"] + status: OOM_iteration_1_grouped_expert_linear + - name: moe_act + recompute_modules: ["moe_act"] + peak_mem_gb_after_iteration_1: 62.103 + peak_mem_vs_control_pct_at_iteration_1: -6.33 + status: OOM_iteration_2_output_projection + - name: layernorm_plus_moe_act + recompute_modules: ["layernorm", "moe_act"] + status: OOM_iteration_1_output_projection + - name: moe + recompute_modules: ["moe"] + peak_mem_gb: 64.653 + step_time_ms: 657.42 + tflops_per_gpu: 277.72 + status: completed_12_steps + - name: moe_plus_layernorm + recompute_modules: ["moe", "layernorm"] + peak_mem_gb: 63.639 + peak_mem_vs_moe_pct: -1.57 + step_time_ms: 677.62 + step_time_vs_moe_pct: 3.07 + tflops_per_gpu: 270.62 + status: completed_12_steps + numerical_sanity: both passing rows had finite losses and zero skipped or NaN iterations + comparison_limitations: > + Failed rows do not provide steady-state timing, and peaks from different + failure phases are not directly rankable. A native 8-H100 exploration + hit FP32 optimizer-state capacity and is excluded from throughput claims. + conclusion: > + Whole-moe recompute was the smallest tested boundary that completed the + exact capacity-limited workload. Adding layernorm bought 1.014 GB more + rank-0 headroom at 3.07% higher step time. Moe-act provided real relief + but did not make the complete step viable; core-attn did not make the + workload fit and yielded no positive memory evidence. - model: Llama3 70B task: sft parallelism: TP4_PP4_VPP5_DP2 @@ -115,8 +258,11 @@ measured_results: mbs: 1 gbs: 32 golden_tflops: 709.93 + comparison_limitations: > + No matched no-recompute row was recorded; golden throughput was not a + paired module-only comparison; results do not cover MLA or MoE. experiments: - - name: baseline + - name: core_attn_baseline recompute_modules: ["core_attn"] tflops: 704 vs_golden_pct: -0.8 @@ -141,34 +287,51 @@ measured_results: peak_mem_gb: 59.6 status: OOM_on_rank0 failure_modes: - - name: mlp_recompute_too_expensive - symptom: ">15% GPU utilization drop" - likely_cause: FFN hidden dimension is large (e.g. 28672 for Llama3 70B) - fix: use VPP tuning or parallelism changes instead - - name: layernorm_insufficient_savings - symptom: still OOM after adding layernorm recompute - likely_cause: layernorm activations are small relative to total peak - fix: add mlp recompute or switch to VPP tuning - - name: full_recompute_with_te_cuda_graphs + - name: core_attn_has_little_incremental_effect + symptom: no material peak reduction against the empty-list control + likely_cause: fused attention already rematerializes the expensive internals or another architecture region sets the peak + fix: attribute the peak and test mla_up_proj, moe_act, layernorm, or another applicable boundary + - name: module_is_inactive + symptom: selected recompute label causes no measurable change + likely_cause: the module is absent on the measured layers, such as mlp on pure-MoE layers, or graph capture bypasses the wrapper + fix: inspect the provider and final graph scope before choosing another boundary + - name: overlap_validation_failure + symptom: Megatron Core rejects whole-moe or shared-expert recompute + likely_cause: expert-parallel or shared-expert overlap conflicts with backward replay + fix: retain overlap and use a compatible inner boundary, or disable overlap and remeasure the complete configuration + - name: later_failure_is_misclassified_as_success + symptom: recompute advances beyond the control's failure point but still OOMs + likely_cause: the boundary moved the peak into a later forward region, gradient synchronization, or optimizer initialization + fix: record the changed failure stage, but require optimizer initialization and multiple steady-state steps before calling it a pass + - name: full_recompute_with_scoped_cuda_graphs symptom: "AssertionError: full recompute is only supported with full iteration CUDA graph" - likely_cause: recompute_granularity=full with any TE-scoped CUDA graph (attn, mlp, moe_router, etc.). Common on FP8 CS configs that default to cuda_graph_impl=transformer_engine + scope=mlp (e.g. LLAMA3_70B_SFT_CONFIG_H100_FP8_CS_V1). Enforced in MCore transformer_config.py:2001-2005. - fix: use recompute_granularity=selective with recompute_modules, or set cuda_graph_impl=none, or switch to cuda_graph_impl=local + cuda_graph_scope=full_iteration + likely_cause: full granularity is combined with a non-full-iteration graph implementation + fix: use selective recompute, set cuda_graph_impl=none, or use cuda_graph_impl=full_iteration known_constraints: - - recompute_granularity=selective requires a non-empty recompute_modules list + - recompute_granularity=selective accepts an empty recompute_modules list - recompute_granularity=full requires recompute_method and recompute_num_layers - distribute_saved_activations cannot be used with sequence_parallel=True - - combining mlp+core_attn is slightly worse than mlp alone due to double overhead + - valid module labels are revision-specific; use the exact target Megatron Core validator + - module boundaries can overlap or nest, so memory and replay costs are not additive known_limitations: - - per-module memory savings vary significantly by model architecture - - no automatic selection of optimal recompute_modules - - memory savings from layernorm are negligible on most architectures + - no automatic selection of the optimal recompute_modules list + - no single module ordering applies across standard attention, MLA, MoE, and GDN models + - matched Moonlight evidence covers one short BF16 MLA+MoE configuration and is not a universal ranking + - matched Nemotron evidence covers one short BF16 hybrid MoE capacity configuration and is not a universal ranking + - historical Llama evidence lacks a matched empty-list control evidence: - docs/training/activation-recomputation.md - - "PR #3107 (Llama3 70B SFT OOM fix experiment)" - - src/megatron/bridge/recipes/llama/llama3.py + - "2026-08-12 matched H100 Moonlight 16B activation-recompute study" + - "2026-08-12 matched H100 Nemotron 3 Nano activation-recompute study" + - "PR #3107 (historical Llama3 70B SFT experiment)" - 3rdparty/Megatron-LM/megatron/core/transformer/transformer_config.py - - 3rdparty/Megatron-LM/megatron/core/transformer/transformer_block.py + - 3rdparty/Megatron-LM/megatron/core/transformer/attention.py + - 3rdparty/Megatron-LM/megatron/core/transformer/multi_latent_attention.py + - 3rdparty/Megatron-LM/megatron/core/transformer/transformer_layer.py + - 3rdparty/Megatron-LM/megatron/core/transformer/moe/experts.py + - 3rdparty/Megatron-LM/megatron/core/transformer/moe/moe_layer.py + - 3rdparty/Megatron-LM/megatron/core/ssm/gated_delta_net/gdn.py follow_up_validation: - - Measure selective recompute impact on MoE models (moe, moe_act modules). - - Measure mla_up_proj recompute impact on DeepSeek-style MLA models. - - Test selective recompute + TE-scoped CUDA graphs combined perf impact. + - Add matched GDN measurements. + - Measure context-parallel communication replay for attention and GDN boundaries. + - Validate representative selective boundaries under the supported CUDA-graph scopes and FP8 recipes. diff --git a/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json b/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json index 9f4b4d33..0023d0a1 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json +++ b/skills/nemo-mbridge-perf-activation-recompute/evals/evals.json @@ -1,16 +1,47 @@ [ { - "id": "activation-recompute-positive-memory-smoke", - "question": "Use the nemo-mbridge-perf-activation-recompute skill. My Megatron Bridge model is close to OOM and an FP8 config already uses TE-scoped CUDA graphs. Give a concise checklist with the first environment fix, the exact selective-to-full recompute order, the required full-recompute config fields, and the CUDA-graph assertion workaround.", + "id": "activation-recompute-architecture-aware-memory-smoke", + "question": "Use the nemo-mbridge-perf-activation-recompute skill. I have three Megatron Bridge jobs near OOM: one uses standard attention with Transformer Engine fused attention, one uses MLA, and one uses grouped MoE. Give a concise first-candidate recompute choice for each, explain why core_attn is not universal, state the required full-recompute fields, and cover the CUDA-graph workaround.", "expected_skill": "nemo-mbridge-perf-activation-recompute", "expected_script": null, - "ground_truth": "The answer should use the activation recompute skill. It should say to try PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True first, then start with recompute_granularity=\"selective\" and recompute_modules=[\"core_attn\"], optionally add layernorm, and use full recompute only if selective still does not fit. It should state full recompute requires recompute_method and recompute_num_layers, and that full/layer-level recompute is incompatible with TE-scoped CUDA graph scopes such as attn, mlp, or moe_router. It should give valid workarounds: use selective recompute, disable CUDA graphs with cuda_graph_impl=\"none\", or switch to cuda_graph_impl=\"local\" with cuda_graph_scope=\"full_iteration\".", + "ground_truth": "The answer should first distinguish allocated activation pressure from reserved-memory fragmentation and keep a matched no-recompute control. For standard attention, core_attn is a common candidate, but with TE fused or Flash Attention it must be compared with an empty selective module list because the backend already rematerializes attention internals. For MLA, mla_up_proj is the first candidate when expanded Q/K/V tensors dominate; core_attn can be additive when the attention core remains material. For grouped MoE, moe_act is the narrow first candidate and layernorm can be added when norm outputs matter; whole moe is broader and replays routing, experts, and communication. Full recompute requires recompute_method and recompute_num_layers. Full recompute with CUDA graphs requires cuda_graph_impl=full_iteration; otherwise use selective recompute or cuda_graph_impl=none.", "expected_behavior": [ "Read the nemo-mbridge-perf-activation-recompute skill before answering.", - "Recommend PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True before recompute changes.", - "Prefer selective recompute with core_attn and optionally layernorm before full recompute.", + "Use a matched no-recompute control and distinguish allocated memory from allocator fragmentation.", + "Describe core_attn as common for standard attention but not a universal first choice.", + "Recommend mla_up_proj for MLA and moe_act, optionally layernorm, for grouped MoE based on peak attribution.", "State that full recompute requires recompute_method and recompute_num_layers.", - "Explain the TE-scoped CUDA graph incompatibility and list a valid workaround." + "Explain that full recompute with CUDA graphs requires full_iteration, with selective recompute or disabled graphs as alternatives." + ] + }, + { + "id": "activation-recompute-moonlight-evidence", + "question": "Use the nemo-mbridge-perf-activation-recompute skill. For the exact Moonlight 16B H100 configuration measured by the skill, which selective boundary should I try first? Explain the tradeoff versus mla_up_proj, core_attn, and adding layernorm or mlp, and state whether the conclusion is portable to other recipes.", + "expected_skill": "nemo-mbridge-perf-activation-recompute", + "expected_script": null, + "ground_truth": "For the exact measured Moonlight workload, recommend moe_act first: it lowered peak allocated memory 1.85% with 1.99% step-time overhead. mla_up_proj saved 1.96%, only about 39 MB more, with 5.19% overhead, so it is the next choice only if the small additional saving matters. core_attn provided no material allocated-memory reduction and cost 3.85% under fused attention. Adding mlp to mla_up_proj or layernorm to moe_act did not improve the observed peak and increased step time. The answer must limit this conclusion to the exact measured architecture, backend, topology, precision, and shape rather than claim a universal label ranking.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-activation-recompute skill before answering.", + "Recommend moe_act first for the exact measured Moonlight workload.", + "Compare allocated-memory and step-time deltas against the matched empty-list control.", + "Explain that mla_up_proj provides only a small additional saving at higher cost in this run.", + "Reject core_attn and the broader tested additions as first choices for this exact run.", + "Do not extrapolate the measured ordering into a universal recommendation." + ] + }, + { + "id": "activation-recompute-nemotron-capacity-evidence", + "question": "Use the nemo-mbridge-perf-activation-recompute skill. In the measured Nemotron 3 Nano 16-H100 workload, moe_act lowered an iteration-1 peak but the job still OOMed. What is the smallest tested passing choice, what does layernorm add, and how should I classify the moe_act result?", + "expected_skill": "nemo-mbridge-perf-activation-recompute", + "expected_script": null, + "ground_truth": "Whole moe recompute was the smallest tested choice that completed all 12 steps. Adding layernorm lowered the rank-0 post-iteration-2 peak from 64.653 GB to 63.639 GB, another 1.014 GB or 1.57%, but raised steady-state step time from 657.42 ms to 677.62 ms, or 3.07%. Moe_act lowered the matched iteration-1 rank-0 peak by 4.194 GB or 6.33% relative to the empty control, but then OOMed in the iteration-2 output projection. It is diagnostic evidence of real activation relief, not a pass. A pass requires optimizer initialization and multiple steady-state steps.", + "expected_behavior": [ + "Read the nemo-mbridge-perf-activation-recompute skill before answering.", + "Identify whole moe as the smallest tested passing boundary for the exact Nemotron workload.", + "Quantify the extra memory headroom and step-time cost from adding layernorm.", + "Classify moe_act as useful failure-stage evidence rather than a successful configuration.", + "Require optimizer initialization and multiple steady-state steps for a pass.", + "Limit the conclusion to the measured Nemotron configuration." ] } ] diff --git a/skills/nemo-mbridge-perf-activation-recompute/skill-card.md b/skills/nemo-mbridge-perf-activation-recompute/skill-card.md index 30448497..8667b469 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/skill-card.md +++ b/skills/nemo-mbridge-perf-activation-recompute/skill-card.md @@ -9,65 +9,77 @@ NVIDIA
### License/Terms of Use:
Apache 2.0
## Use Case:
-Developers and engineers reducing GPU memory pressure via activation recompute in Megatron Bridge training workloads, or investigating commits that changed recompute settings and caused OOM errors or performance regressions.
+Developers and engineers diagnosing activation-memory OOMs and configuring selective or full activation recompute boundaries in Megatron Bridge training recipes.
### Deployment Geography for Use:
Global
+## Requirements / Dependencies:
+**Requires API Key or External Credential:** [No]
+**Credential Type(s):** [None]
+ +Do not include secrets in prompts/logs/output; use least-privilege credentials; rotate keys as appropriate.
+ ## Known Risks and Mitigations:
Risk: Review before execution as proposals could introduce incorrect or misleading guidance into skills.
Mitigation: Review and scan skill before deployment.
## Reference(s):
-- [Megatron Bridge Performance Tuning Guide](docs/performance-guide.md)
-- [Megatron Bridge Documentation](https://docs.nvidia.com/nemo/megatron-bridge/latest/)
+- [Activation Recomputation (Megatron Bridge docs)](docs/training/activation-recomputation.md)
+- [Megatron Core API Guide](https://docs.nvidia.com/megatron-core/developer-guide/latest/api-guide/index.html)
+- [Performance Tuning Guide](docs/performance-guide.md)
## Skill Output:
-**Output Type(s):** [Configuration instructions, Shell commands, Analysis]
-**Output Format:** [Markdown with inline Python and bash code blocks]
+**Output Type(s):** [Configuration instructions, Analysis]
+**Output Format:** [Markdown with inline Python code blocks]
**Output Parameters:** [1D]
**Other Properties Related to Output:** [None]
## Evaluation Agents Used:
-- Claude Code (`claude-code`)
-- Codex (`codex`)
+- Claude Code (`aws/anthropic/bedrock-claude-opus-4-8`)
+- Codex (`openai/openai/gpt-5.5`)
## Evaluation Tasks:
-Evaluated against 1 internal skill-activation task in the NVSkills-Eval external profile.
+Evaluated against 3 positive evaluation tasks covering architecture-aware memory diagnosis, matched Moonlight 16B evidence interpretation, and matched Nemotron 3 Nano capacity evidence interpretation.
## Evaluation Metrics Used:
Reported benchmark dimensions:
-- Security: Checks whether skill-assisted execution avoids unsafe behavior such as secret leakage, destructive commands, or unauthorized access.
-- Correctness: Checks whether the agent follows the expected workflow and produces the correct final output.
-- Discoverability: Checks whether the agent loads the skill when relevant and avoids using it when irrelevant.
-- Effectiveness: Checks whether the agent performs measurably better with the skill than without it.
-- Efficiency: Checks whether the agent uses fewer tokens and avoids redundant work.
+- Security: Whether the skill is safe to use (no unsafe operations, secret leakage, or unauthorized access).
+- Correctness: Whether the skill produces correct answers against reference ground truth.
+- Discoverability: Whether the right skill is loaded and activated when needed.
+- Effectiveness: Whether the skill helps the agent complete the user's goal and expected workflow.
+- Efficiency: Whether the skill avoids wasted tool or skill usage.
Underlying evaluation signals used in this run:
-- `security`: Checks for unsafe operations, secret leakage, and unauthorized access.
-- `skill_execution`: Verifies that the agent loaded the expected skill and workflow.
-- `skill_efficiency`: Checks routing quality, decoy avoidance, and redundant tool usage.
-- `accuracy`: Grades final-answer correctness against the reference answer.
-- `goal_accuracy`: Checks whether the overall user task completed successfully.
-- `behavior_check`: Verifies expected behavior steps, including safety expectations.
-- `token_efficiency`: Compares token usage with and without the skill.
+- `security`: Unsafe operations, secret leakage, and unauthorized access.
+- `skill_execution`: Whether the expected skill was found and executed.
+- `skill_efficiency`: Routing quality, workspace-aware skill reads, and productive tool use.
+- `accuracy`: Final-answer correctness against the reference answer.
+- `goal_accuracy`: Whether the user's goal was achieved.
+- `behavior_check`: Whether the expected workflow behavior was followed.
## Evaluation Results:
-| Dimension | Num | `claude-code` | `codex` | -|---|---:|---:|---:| -| Security | 1 | 100% (+0%) | 100% (+0%) | -| Correctness | 1 | 100% (+100%) | 87% (+40%) | -| Discoverability | 1 | 100% (+100%) | 97% (+0%) | -| Effectiveness | 1 | 96% (+80%) | 80% (+54%) | -| Efficiency | 1 | 94% (+67%) | 96% (-0%) | +| Measure | Claude Code (Baseline → Skill) | Codex (Baseline → Skill) | +|---|---:|---:| +| Overall | 65% → 98% (+34 pts) | 58% → 98% (+39 pts) | +| Security | 100% → 100% (±0) | 100% → 100% (±0) | +| Correctness | 13% → 100% (+87 pts) | 73% → 100% (+27 pts) | +| Discoverability | 100% → 100% (±0) | 42% → 94% (+52 pts) | +| Effectiveness | 13% → 91% (+78 pts) | 45% → 95% (+50 pts) | +| Efficiency | 97% → 100% (+3 pts) | 32% → 100% (+68 pts) | + +## Testing Completed:
+**[x] Agent Red-Teaming**
+**[ ] Network Security**
+**[ ] Product Security**
## Skill Version(s):
-v0.2.0rc6-1622-g853062e4 (source: git describe)
+1.0.0+b7643bd (source: pyproject.toml)
## Ethical Considerations:
NVIDIA believes Trustworthy AI is a shared responsibility and we have established policies and practices to enable development for a wide array of AI applications. When downloaded or used in accordance with our terms of service, developers should work with their internal team to ensure this skill meets requirements for the relevant industry and use case and addresses unforeseen product misuse.
diff --git a/skills/nemo-mbridge-perf-activation-recompute/skill.oms.sig b/skills/nemo-mbridge-perf-activation-recompute/skill.oms.sig index e495d156..9b1738cf 100644 --- a/skills/nemo-mbridge-perf-activation-recompute/skill.oms.sig +++ b/skills/nemo-mbridge-perf-activation-recompute/skill.oms.sig @@ -1 +1 @@ -{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1tYnJpZGdlLXBlcmYtYWN0aXZhdGlvbi1yZWNvbXB1dGUiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiYmZlMWEzOTFiMDI1NWM3Yjc3NTcyMmMxM2NjZjg1M2IwN2IxMTMzOTc4NjI5NDQ5ZTZjNTE4NzFmODMzNzIyYyIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzcyZDc3YmQ2OTBmZjVkYjE5NmIxNDBkNDA0Zjg5MWY2YWI0ZTMzMzNjMGJjNDFmOWRlNWJlZjIwZWMyZGY4OSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiZTQyNDQ1MjYyMzEyMjBiOGE2NzQ1ZTM2MmQyOWRlZTVmZTAyMzkyMzljNWYzYjQ2YjQwNjAzMTZmYTg2YTk4MiIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIwZTQzMmQ0YWI3M2Q0MDA5M2FkMmE0OGZlMTJkNGU1YWVjZmU5Njk3YWFkNGRlYjBlOGExZmJiY2MwMTgzYWVjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiY2FyZC55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIyYzQ1YjUxZjAxNjM2YWM5YTk1YjE5NmY4ZTEzNjc0MzRlODFmN2ExMjAwOGIxMmEyYTAwOWVlMDI1N2FhNDZlIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMWViNWU2M2RkNTIxMGZkZTZiM2U1NGY4YzFjNDg2ZjBjNTFkNGIwZDFiNmFmMjQxMGFkZWFiZmQwMmU5NjcwMyIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImFsbG93X3N5bWxpbmtzIjogZmFsc2UsCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJpZ25vcmVfcGF0aHMiOiBbCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0YXR0cmlidXRlcyIsCiAgICAgICAgIi5naXRpZ25vcmUiLAogICAgICAgICIuZ2l0IgogICAgICBdCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGYCMQDMMUH4NhW8T+N6ofif5IeQf4Kj+gIVvm66aVhnvPWQgIN1hQHvRdT4OxLaCKDkfMQCMQCiWP1N8r4wajKJwYU4P4lhcigCAo36DHCbsVp05Xj4ChQ1Qy91/JE/pceT3IsH8qQ=","keyid":""}]}} \ No newline at end of file +{"mediaType":"application/vnd.dev.sigstore.bundle.v0.3+json","verificationMaterial":{"x509CertificateChain":{"certificates":[{"rawBytes":"MIICgzCCAgmgAwIBAgIUKIyS7SxNteQIiWzK1dWj85E6520wCgYIKoZIzj0EAwMwVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwHhcNMjYwNDAxMDAwMDAwWhcNMjgwNDIyMTUzMzA5WjBUMQswCQYDVQQGEwJVUzEbMBkGA1UECgwSTlZJRElBIENvcnBvcmF0aW9uMSgwJgYDVQQDDB9OVklESUEgQWdlbnQgU2tpbGxzIFNpZ25pbmcgMDAxMHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEYoRM9bQl/dGlwSRNi6bTpIJUXH8Nv9GciP6LSflJYYMLCc296kpyuTSsk5ddbAWiDcFX3C/ydX3jwc+qCLYP6uHy9XphyLjOQ27Yb2J6rBLVtRBS1mgGco/Gr7fL6ODco4GaMIGXMB0GA1UdDgQWBBRQ/5ZW3nJ6lmo9SVk7I15o7UGmpTAfBgNVHSMEGDAWgBRPGpILxMBBleJSsBGjrMKsby1CgjAMBgNVHRMBAf8EAjAAMA4GA1UdDwEB/wQEAwIHgDA3BggrBgEFBQcBAQQrMCkwJwYIKwYBBQUHMAGGG2h0dHA6Ly9vY3NwLm5kaXMubnZpZGlhLmNvbTAKBggqhkjOPQQDAwNoADBlAjAUygu/GiOCIXrgGr4SmLgeEVDcEitfFUv7ALbvLVGVyMysB3mxmO/uInZfXzWcJZsCMQDxuoxj4ZmO30jhkPIcCxGFCOvnUsnfU3TfGcouYm4M6iRpbKvtVnHPiy4bi6pcKf0="},{"rawBytes":"MIICiDCCAg6gAwIBAgIUZsIuSv9NkpJCNqtYEfCouVv5BzowCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowVTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjEpMCcGA1UEAwwgTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBJQ0EgMDEwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAASI72cR3ctKGg4VWnB3bNja6g1Z2PnOmFEopkPof+QeIcPk9rT+g9MjJnq51EQXL93a7C2GJ9J985G4o2V85VD7wJ1RaXhluHW2rf3y8bQGeAYaKMr5s/hUgn+M3/9WlWejgaAwgZ0wHQYDVR0OBBYEFE8akgvEwEGV4lKwEaOswqxvLUKCMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMBIGA1UdEwEB/wQIMAYBAf8CAQAwDgYDVR0PAQH/BAQDAgEGMDcGCCsGAQUFBwEBBCswKTAnBggrBgEFBQcwAYYbaHR0cDovL29jc3AubmRpcy5udmlkaWEuY29tMAoGCCqGSM49BAMDA2gAMGUCMQCeIMMfAbyzPDacw2MxG+Yt1cikrJX/DVxiGfXuHmkkXn6VgSzE79+lkqDErpVO2gYCMCNEColOyvUvkzZGUEI1hQ3PfMgi3FIo9tHoBKMw4/wGBLFpu/0ubtmbBXM6/UMOEw=="},{"rawBytes":"MIICRTCCAcygAwIBAgIUeJdY3rV86EdvFmG7L8LJBsyQFYkwCgYIKoZIzj0EAwMwUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTAgFw0yNjA0MDEwMDAwMDBaGA85OTk5MTIzMTIzNTk1OVowUTELMAkGA1UEBhMCVVMxGzAZBgNVBAoMEk5WSURJQSBDb3Jwb3JhdGlvbjElMCMGA1UEAwwcTlZJRElBIEFnZW50IENhcGFiaWxpdGllcyBDQTB2MBAGByqGSM49AgEGBSuBBAAiA2IABAYpiXCDjJ9NT2eSDhyHJVSw1Tbze18cGG2F/578oWvHxg23eQAhNRYdq88i1iOshZSO6C29doKui5Xpmo/7Ctw9Sx4PP2RzOmIuOLCuTdNtKcTRwi4GEsd5BAFvWj42M6NjMGEwHQYDVR0OBBYEFItnoAjjfuCEUvzyvWyI2vOGvwPjMB8GA1UdIwQYMBaAFItnoAjjfuCEUvzyvWyI2vOGvwPjMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMAoGCCqGSM49BAMDA2cAMGQCMCwtAjWLaNwgGWNCgdyNoTyvNhqWRECRJV2r3+7w8g0PL6NHLOsbkgE09BH95h8XlgIwTaQmbbUh2ChAJ5TA1wRiVDnCcvbzHlZl2jM2FcwQQZlk19LOAbyGMRixbu2Ww/rj"}]},"tlogEntries":[]},"dsseEnvelope":{"payload":"ewogICJfdHlwZSI6ICJodHRwczovL2luLXRvdG8uaW8vU3RhdGVtZW50L3YxIiwKICAic3ViamVjdCI6IFsKICAgIHsKICAgICAgIm5hbWUiOiAibmVtby1tYnJpZGdlLXBlcmYtYWN0aXZhdGlvbi1yZWNvbXB1dGUiLAogICAgICAiZGlnZXN0IjogewogICAgICAgICJzaGEyNTYiOiAiMTIxYmIzYjg3MjE0MjBhODg5MzQ3ZmYyODNhMjNkM2RlYTJkMTNiM2ZmMWEyZmQ2MjM3ZWY5OWU5MGEzYjgzMSIKICAgICAgfQogICAgfQogIF0sCiAgInByZWRpY2F0ZVR5cGUiOiAiaHR0cHM6Ly9tb2RlbF9zaWduaW5nL3NpZ25hdHVyZS92MS4wIiwKICAicHJlZGljYXRlIjogewogICAgInJlc291cmNlcyI6IFsKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiYzkyN2U0NzIxODNjODMxMWI3YjhkNTM1ZWMyOWNkM2NlNzUxNTI4YjJlNjFiMDRjNDUxY2YzOWJiMDMzMjMwMCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIkJFTkNITUFSSy5tZCIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiMDMzNzk3MjMxNDNkZmNkM2NjZTQzMzhjMTQ4ZmVjYzJhYTUxNTY0OWFiODVkMzYyNGU0M2EyOWI4MGYwM2U0ZSIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogIlNLSUxMLm1kIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICJkMDhmYjYyNzgwMTkzNmNjYTRiODE5ZTE5NDI3NDk0OTk4YjdiMjM2MzRlMzFhNThjMTYxYzQ0ZDk5ZmNjZWRkIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiY2FyZC55YW1sIgogICAgICB9LAogICAgICB7CiAgICAgICAgImRpZ2VzdCI6ICIwZjMxMDRmYTc3N2I1Nzg1NzNhZDYyNmY2MjJlNDFjODA3MDEzNWQyMWJmNTdkMTU0YjdlOTU5YTRkOTFiMjRjIiwKICAgICAgICAiYWxnb3JpdGhtIjogInNoYTI1NiIsCiAgICAgICAgIm5hbWUiOiAiZXZhbHMvZXZhbHMuanNvbiIKICAgICAgfSwKICAgICAgewogICAgICAgICJkaWdlc3QiOiAiOGZmMDA1NDA3OGU3ZTU3M2E1ZGM4YWUzMGQ5Mjg5MGVhNGI4Yzc1ZjkzZWJjYjdkMmJjYjIxZmRmMDRkOTkyMCIsCiAgICAgICAgImFsZ29yaXRobSI6ICJzaGEyNTYiLAogICAgICAgICJuYW1lIjogInNraWxsLWNhcmQubWQiCiAgICAgIH0KICAgIF0sCiAgICAic2VyaWFsaXphdGlvbiI6IHsKICAgICAgImlnbm9yZV9wYXRocyI6IFsKICAgICAgICAiLmdpdGlnbm9yZSIsCiAgICAgICAgIi5naXRodWIiLAogICAgICAgICIuZ2l0IiwKICAgICAgICAiLmdpdGF0dHJpYnV0ZXMiCiAgICAgIF0sCiAgICAgICJtZXRob2QiOiAiZmlsZXMiLAogICAgICAiaGFzaF90eXBlIjogInNoYTI1NiIsCiAgICAgICJhbGxvd19zeW1saW5rcyI6IGZhbHNlCiAgICB9CiAgfQp9","payloadType":"application/vnd.in-toto+json","signatures":[{"sig":"MGUCMDXkzce8SSH69QqG58syv7rgtAQeRjXfSDAjYa+h+BggNUgGkFhA9RTq01bTfSB5yAIxAJA9BZZ7I+ZexbefHfraLElUJ0gvAC8eh81xSoRq8glH/rI4SaRsoI6MJ17n+SFFMQ==","keyid":""}]}} \ No newline at end of file