diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 4f84c3dac..63de236d4 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -68,6 +68,37 @@ jobs: PR_BODY: ${{ github.event.pull_request.body }} run: python3 tools/check_pr_references.py + benchmark-artifact-controls: + # Software correctness only: no model, secrets, device or performance gates. + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Install isolated artifact test dependencies + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends bubblewrap + sudo /usr/bin/python3 -m pip install "numpy>=1.24" + /usr/bin/python3 -m pip install "pytest>=7.4" "PyYAML>=6.0" + /usr/bin/python3 -I -c "import numpy; print(numpy.__version__)" + - name: Real NumPy sandbox and authored two-arm software controls + env: + INFERRSI_REQUIRE_BENCHMARK_SANDBOX: "1" + run: | + /usr/bin/python3 -m pytest tests/test_skill_eval_benchmark.py tests/test_skill_eval_benchmark_compare.py --junitxml=benchmark-controls.xml + - name: Retain software-control results including failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-artifact-controls-${{ github.run_id }}-${{ github.run_attempt }} + path: benchmark-controls.xml + if-no-files-found: error + retention-days: 14 + kv-task-qualification: # Bounded software-control qualification; not a branch-protection or efficacy gate. if: github.event.pull_request.draft == false diff --git a/docs/architecture.md b/docs/architecture.md index 6589edcc9..9377407d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,13 @@ # Modular architecture +The [shared engineering benchmark adapter](engineering-benchmarks.md) lives in +`harness/benchmarks`, with task data in `data/engineering_benchmarks`. It binds +task-owned oracles and workloads to the existing Workbench runner and offline +SkillEval verifier contract. It adds neither a search controller nor another +execution platform. The current device pilot runs trusted affine RMSNorm on +Ascend through Workbench; the formal artifact sandbox remains CPU NumPy only. +Each task's actual qualification and limits are recorded separately. + The local experiment path now has concrete Harness, Model and Infra modules. The CLI, RunSpec, knowledge retrieval and existing command contracts remain stable. The diagrams below describe implemented code, with the historical compatibility diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md new file mode 100644 index 000000000..8c95f6553 --- /dev/null +++ b/docs/engineering-benchmarks.md @@ -0,0 +1,147 @@ +# Shared engineering benchmarks + +The `benchmarks` commands check and measure an implementation against a versioned +task package. Packages define semantics, inputs, correctness rules and timing; +the existing Workbench runs paired comparisons and retains evidence. Agent +execution, knowledge selection and optimization search remain caller-owned. + +| Task | Candidate entry point | Execution boundary | +| --- | --- | --- | +| [Ascend affine RMSNorm](../src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md) | `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | +| [CPU Softmax](../src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md) | `compute(x, *, eps)` | Reviewed NumPy source; a separate SkillEval artifact sandbox also supports this task | + +The task READMEs retain source attribution, licenses and workload limits. To +optimize an implementation, copy the task's `baseline.py` to a separate +`implementation.py`. The packaged `candidate.py` is a reference control, not an +Agent-produced optimization. Keep task, oracle and baseline fixed while changing +the implementation. Review source before using the trusted worker: content +hashes detect drift, not malicious code. + +## Run an NPU implementation + +Discovery imports neither torch nor torch_npu. Execution requires the target's +compatible installed torch/torch_npu/CANN stack; InferRSI does not install or +replace that runtime. Enter a caller-managed single-NPU allocation before any +device execution. The worker requires one `ASCEND_RT_VISIBLE_DEVICES` entry, +uses logical device 0 and never falls back to CPU. + +```bash +inferrsi benchmarks list +inferrsi benchmarks show rmsnorm-npu +inferrsi benchmarks check rmsnorm-npu --candidate --output-dir --timeout-seconds +inferrsi benchmarks compare rmsnorm-npu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +``` + +RMSNorm uses contiguous 2D float32 inputs and nonuniform weights. Input and +weight generation, scalar double/math.fsum oracle computation and transfers +occur outside timing. The clock covers eager dispatch and completion +synchronization after an untimed pre-synchronization. It measures synchronized +eager-call latency, not pure kernel time. Shape, dtype, device, numerical values +and input/weight preservation are checked outside the timer for every call. + +Profiling is optional and separate from measurement: + +```bash +inferrsi benchmarks profile rmsnorm-npu --candidate --output-dir --timeout-seconds +``` + +This retains real device events in `npu-trace.json` and a path-free kernel +summary, with no performance verdict. Distinguish verification copies from +compute kernels. Review raw traces for private paths/identifiers before sharing. + +## Use the CPU software control + +Install the development checkout with `pip install -e ".[dev,benchmarks]"`. +Discovery and planning do not import NumPy; CPU execution requires it. + +```bash +inferrsi benchmarks show softmax-cpu +inferrsi benchmarks check softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks measure softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks grade softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks plan softmax-cpu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +``` + +The Softmax contract reduces axis 1 and includes strided and boundary inputs. +The independent oracle uses scalar double precision and `math.fsum`. `check` +evaluates the base cases and the same varying-input sequence used by measurement. +Every timed input has distinct retained objects and bytes, so an earlier call +cannot prime an identity/content memo for a later sample. This public suite does +not prevent hostile implementations or establish held-out generalization. + +Input generation, oracle evaluation, validation, startup and warmups are outside +timing; allocations inside `compute` are included. The score sums per-case +medians for the task's selected workloads. Inspect individual cases as well as +the aggregate. The zero-input case checks correctness but contributes no timing +score. CPU `profile` retains `profile.txt`, not a device timeline or speed score. + +## Comparison and retained results + +Replace `plan` with `compare` to execute through Workbench. `plan` prints a +RunSpec; `compare` constructs its own RunSpec rather than consuming a previously +printed one. Freeze the actual inputs and gates before candidate search using +the [acceptance procedure](contributing.md#acceptance-checklist). All placeholders +above come from that experiment, not repository defaults. Single-action output +directories must be new. + +`TaskPackage` freezes task and grader identities. Commands also bind candidate +bytes and runtime fingerprints, checked before and after worker execution. The +Git checkout supplied by `--repo` identifies source context; task and candidate +files have additional hashes. Runtime fingerprints alone do not establish an +uncontended host, equivalent device/CANN installations or native dependencies. + +Results retain the planned `input_sequence`, actual `executed_calls` and sample +input hashes. Failed candidates may execute only a prefix. Wrong outputs produce +no usable performance metric. Preserve unsuccessful trials and inspect +`execution.json`, per-trial `result.json`, `evaluation.json`, `status.json` and +source-integrity records. + +`compare` returns `run_dir` and the retained status. Its exit codes are PASS = 0, +FAIL = 1 and INCONCLUSIVE = 2. A complete comparison missing its gain gate is +FAIL. A wrong candidate preventing measurement leaves the comparison +INCONCLUSIVE, while independent artifact correctness can be FAIL. Exceptions, +missing dependencies, drift and timeouts must not become successful comparisons. +Workbench statistics are descriptive; these results do not establish service +gains, statistical significance or Agent/knowledge efficacy. + +`grade` returns artifact-correctness PASS/FAIL/INCONCLUSIVE with SkillEval's +`passed`/`assertions` structure. The lower-level +`benchmarks.skill_verifier.task_descriptor` binds an already reviewed artifact +for `skill_evaluation.run_trial`; its answer supplies `candidate_sha256`. +For actual CPU file handoff through `tools.skill_eval_compare`, see +[engineering task artifacts](engineering-task-artifacts.md). That formal adapter +checks correctness only and rejects NPU tasks. It does not automatically send +generated NPU code to this trusted worker. Provider generation mode and the +offline harness-improvement controller do not yet support this task adapter. + +## Qualify the tools with authored controls + +The NPU utility exercises correct, wrong, replay, same-code and reference +implementations through the worker and Workbench: + +```bash +python -m tools.npu_benchmark_validate --repo --output-dir --repetitions --min-improvement-pct --max-aa-difference-pct --max-minutes --timeout-seconds +``` + +It freezes policy and implementation identities before execution, retains all +samples and checks actual device/CANN/runtime consistency. The utility includes +profiling as a tool control; this does not make profiling mandatory for ordinary +optimization. `--timeout-seconds` caps its standalone actions, while paired +Workbench trials use the comparison deadline. It does not enforce a separate +whole-utility budget or the standalone cap on each paired trial. Establish the +required outer allocation/process deadlines and verify cleanup before relying +on those budget boundaries. + +The CPU integration utility dynamically selects the packaged NumPy task: + +```bash +python -m tools.engineering_benchmark_validate --repo --output-dir --repetitions +``` + +It retains profiles and AB/BA samples and runs the existing SkillEval trial path +with correct and wrong authored controls. Neither utility generates Agent +candidates. Their reports separate software controls from campaign acceptance. +Preserve experiment plans and original results with their tested revisions using +the [evidence conventions](contributing.md#evidence-and-claims); review/redact +copies of raw logs before publication. A successful control run does not finish +the Agent-to-NPU handoff or prove an optimization benefit. diff --git a/docs/engineering-task-artifacts.md b/docs/engineering-task-artifacts.md new file mode 100644 index 000000000..75f66063c --- /dev/null +++ b/docs/engineering-task-artifacts.md @@ -0,0 +1,115 @@ +# Engineering task artifacts in SkillEval + +`engineering-benchmark-v1` connects the packaged `softmax-cpu` task to +`tools.skill_eval_compare` in **update mode**. It freezes two skill versions, +retains each producer's actual `implementation.py`, and grades correctness with +the shared task oracle. It does not measure speed. Reviewed implementations can +later enter a separate [Workbench comparison](engineering-benchmarks.md). + +## Prepare the task and comparison + +Run from a development checkout. Generate a canonical case rather than editing +its contract or relying on a same-named directory in the working tree: + +```bash +python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('softmax-cpu')])" +``` + +Create `config.json` beside `cases.json`: + +```json +{ + "adapter": "engineering-benchmark-v1", + "mode": "update", + "skill": "", + "skill_root": "", + "cases": "cases.json", + "snapshots": { + "baseline": {"git_revision": ""}, + "candidate": {"git_revision": ""} + }, + "model": {"name": "", "effort": ""}, + "limits": { + "repeats": 1, + "max_task_calls": 2, + "timeout_seconds": 60, + "campaign_seconds": 180, + "token_stop_new_trials": 10000 + } +} +``` + +The example limits are illustrative; choose and freeze the actual bounds before +execution. Use full commits available in the local repository. Do not add a +provider client, custom schema or grader. See [comparison configuration](skill-eval-comparison.md) +for Git prerequisites, snapshot rules and budget semantics. + +```bash +python -m tools.skill_eval_compare prepare --config config.json --repo --output +``` + +Preparation freezes task, starter, schema, verifier/controllers and skill +bundles. Formal loading always uses canonical packaged resources. Retained task +copies must match that descriptor. The supplied CPU starter is already correct; +leaving it unchanged can pass, so correctness alone cannot identify optimization. + +## Hand off and retain actual files + +Stage a fresh workspace outside the evidence tree for one arm: + +```bash +python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Stage does not launch an Agent. Run the producer under the caller's established +permissions and budget. Only `implementation.py` may change; keep task, +instructions and assigned knowledge frozen. The file implements +`compute(x, *, eps)` with self-contained NumPy/math helpers. Its final answer is: + +```json +{"artifact": "implementation.py", "summary": "Describe the actual change"} +``` + +Stop the producer before capture. Retain `stdout.log`, `stderr.log` and +`execution.json` in a separate submission directory, following the exact fields +in [author-run handoff](local-skill-evaluation.md#author-run-handoff). +Model, access, resources and usage in these files are author-reported, not +authenticated observations. Missing usage stays unknown. + +```bash +python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Repeat stage/produce/capture for `--condition candidate` with distinct workspace +and log directories, then run: + +```bash +python -m tools.skill_eval_compare report --output +``` + +The plan hash binds this attempt; it does not authorize model spending. Changed +inputs require a new plan. Capture preserves actual source bytes and hashes; +report rechecks retained artifacts without rerunning the producer. Failed or +incomplete attempts remain recorded and cannot be overwritten as clean results. + +## Execution boundary and interpretation + +The trusted parent owns inputs, the scalar oracle and assertions. Submitted +source executes in a Linux bubblewrap child with system Python, matching NumPy +and a bounded runner; the repository, grader and other arm are not mounted. +Every returned array is checked by the parent. The public suite does not prove +hidden-test generalization, hostile-protocol resistance or independent tracing +of every candidate operation. Producer permissions are a separate boundary. + +Windows, missing bubblewrap/namespace support or an incompatible NumPy runtime +produce `artifact_sandbox_unavailable` / INCONCLUSIVE without host fallback. +NPU tasks are explicitly rejected. CPU replay qualification does not establish +NPU isolation, and the default unrestricted Agent client is refused for automatic +engineering runs. Provider generation and the offline improvement controller do +not support this adapter. + +A PASS means correctness on the public input sequence, not faster code. A local +handoff report showing `artifact_improved` establishes a difference in retained +artifact checks, not a causal skill benefit or verified total cost. Performance, +Agent efficacy and service claims need their own controlled evidence. Profiling +is optional and is not invoked by this adapter. diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..ac0c7c0bc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,5 +1,10 @@ # Operation guide +Shared engineering tasks use `inferrsi benchmarks list|show|plan` for read-only +inspection and `check|measure|profile|grade|compare` for explicit local execution. +See the [engineering benchmark contract](engineering-benchmarks.md) for trusted +source requirements, task identities, Ascend/CPU coverage and offline SkillEval reuse. + One concise map from an agent's intent to the supported entry point. This guide is navigation only: detailed contracts stay with their owning documents, and it is not a third primary matrix. The two authoritative matrices remain the diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index 41b0aa4c9..3736bce86 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -10,6 +10,15 @@ authorization and retained-result publication around these interfaces. The [offline improvement MVP](harness-improvement.md) reuses the same interfaces for bounded INV/DOM probes, successor rerun and a one-shot held-out gate. +Shared Softmax CPU artifacts use the `engineering-benchmark-v1` adapter +in **update mode**, with the same prepare/execute/report and local stage/capture +interfaces. The trusted parent grades output from a Linux bubblewrap NumPy +worker; unsupported sandbox/runtime leaves INCONCLUSIVE without host fallback. +This is correctness-only artifact evaluation, not timing or skill-efficacy +qualification. See the [task contract and handoff example](engineering-task-artifacts.md). +The provider generation mode and offline improvement controller do not support +this adapter yet. + ```mermaid flowchart LR A["New: small JSON config"] --> B["New: fixed Git snapshots"] diff --git a/pyproject.toml b/pyproject.toml index e44a7a806..166f438c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/engineering_benchmarks/**/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" @@ -35,8 +35,9 @@ prerelease = true [project.optional-dependencies] skills = ["PyYAML>=6.0"] -dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0"] +dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0", "numpy>=1.24"] image = ["numpy>=1.24", "Pillow>=10", "scikit-image>=0.21"] +benchmarks = ["numpy>=1.24"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 46b5e9336..6d7aece10 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -39,6 +39,9 @@ def parser() -> argparse.ArgumentParser: ) root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + from .harness.benchmarks.cli import add_parser + + add_parser(commands) doctor = commands.add_parser( "doctor", help="Inspect a prepared local target without installing dependencies" ) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 349e3df34..87c369c90 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -116,6 +116,20 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "benchmarks": + from .harness.benchmarks.cli import dispatch + + result = dispatch(args) + _print(result) + if args.benchmark_action == "compare": + verdict = result["status"].get("verdict") + return 0 if verdict == "PASS" else (1 if verdict == "FAIL" else 2) + if isinstance(result, dict): + if "verdict" in result: + return 0 if result["verdict"] == "PASS" else 1 + if "status" in result: + return 0 if result["status"] == "completed" else 1 + return 0 if args.command == "skills": # Optional dependencies load only for the skills command; the core # CLI stays independent of them. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE new file mode 100644 index 000000000..f74af3c84 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE @@ -0,0 +1,41 @@ +rom PyTorch: + +Copyright (c) 2020, Huawei Technologies Co., Ltd +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md new file mode 100644 index 000000000..6fb0d2a68 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md @@ -0,0 +1,50 @@ +# Affine RMSNorm on Ascend NPU + +Implement `compute(x, weight, *, eps) -> y` for contiguous float32 NPU tensors +`x[rows, features]` and `weight[features]`: +`y = x / sqrt(mean(x*x, axis=-1, keepdims=True) + eps) * weight`. +Preserve input and weight bytes, shape, dtype and device. The public, seeded +suite includes nonuniform signed weights, ordinary, zero, large and tiny inputs. +Only the decode/prefill shapes are timed. The independent CPU oracle uses scalar +double arithmetic and `math.fsum`; CPU reference computation is never a timed +candidate or a device fallback. + +The baseline composes eager PyTorch NPU operations. The example candidate uses +the existing fused `torch_npu.npu_rms_norm` implementation; it is a reference +control, not a newly written kernel or evidence of Agent efficacy. + +## Execution boundary + +Run reviewed source only, inside a caller-owned single-device allocation. The +worker requires exactly one `ASCEND_RT_VISIBLE_DEVICES` entry and uses logical +device 0 without changing the scheduler's mapping. Discovery needs no PyTorch +import; execution requires an installed compatible torch/torch_npu/CANN runtime. +The existing formal SkillEval CPU sandbox explicitly rejects this adapter. + +Input/weight generation, transfer, oracle preparation and output validation are +outside timing. Each timed call has distinct retained input objects and bytes. +The clock surrounds the eager call and its completion synchronization, following +an untimed pre-synchronization. It includes Python dispatch and synchronization +cost, so **synchronized eager-call latency is not pure device kernel time**. +Output copies and checks happen after the clock stops. Every measured output is +checked; incorrect output has no usable performance metric. + +`profile` uses the Ascend PyTorch CPU/NPU profiler after warmup, separately from +measurement. Its raw local trace includes verification copies and must be +reviewed before publication; it is diagnostic, not a speed score. An empty +device-kernel trace is an error rather than successful profiling. + +## Sources and limits + +API/shape semantics follow the [official op-plugin API](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/docs/en/custom_APIs/torch_npu/%28beta%29torch_npu-npu_rms_norm.md), +[implementation](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) +and [tests](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py). +The source BSD-3-Clause license is retained in `LICENSE`. Workloads, scalar +oracle and adapters here are independently authored; upstream performance +numbers are not imported. + +First qualification is limited to the recorded device and installed runtime, +float32, contiguous 2D forward execution and this public suite. It establishes +neither BF16/FP16, gradients, graph replay, hostile-producer isolation, arbitrary +NPU qualification nor service/skill/search benefit. A/A and reference controls +and issue-scoped predeclared gates are needed before choosing by speed. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py new file mode 100644 index 000000000..af36f4051 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py @@ -0,0 +1,7 @@ +"""Eager affine RMSNorm baseline; all arithmetic executes on the input NPU.""" + +import torch + + +def compute(x, weight, *, eps): + return x * torch.rsqrt(torch.mean(x * x, dim=-1, keepdim=True) + eps) * weight diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py new file mode 100644 index 000000000..7b492f940 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py @@ -0,0 +1,7 @@ +"""Authored reference optimization, not an Agent/skill efficacy result.""" + +import torch_npu + + +def compute(x, weight, *, eps): + return torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0] diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json new file mode 100644 index 000000000..3ea4889d5 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "id": "rmsnorm-npu", + "adapter": "torch-npu-rmsnorm-v1", + "description": "Optimize affine RMSNorm on contiguous 2D float32 NPU tensors; preserve input and nonuniform weight.", + "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; explicit Ascend workload qualification", + "source": { + "url": "https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py", + "revision": "9f23f00b1ebe54cca612b0537bc582dba409a0af", + "license": "BSD-3-Clause", + "adaptation": "Official affine RMSNorm semantics; independently authored smaller public workloads, scalar double oracle and trusted Workbench adapter. Baseline composes eager NPU operations, reference candidate calls existing fused operator. No upstream scores imported." + }, + "environment": { + "backend": "ascend", + "runtime": "torch_npu", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "rmsnorm", + "axis": 1, + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-06 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "device-synchronized-eager-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 10, + "samples": 25, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "decode", + "prefill" + ] + }, + "workloads": [ + { + "id": "decode", + "shape": [ + 1, + 4096 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "prefill", + "shape": [ + 64, + 4096 + ], + "seed": 253, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "zeros", + "shape": [ + 2, + 4096 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 4, + 4096 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 4, + 4096 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads; not holdout or Agent/skill/search efficacy", + "Single allocated Ascend NPU, contiguous float32 2D forward only; device/runtime qualification recorded separately", + "No BF16/FP16, gradients, graph replay, strided inputs, communication, capacity or service/E2E claim", + "Trusted reviewed source only; formal SkillEval NPU isolation not implemented", + "Distinct retained per-call inputs and weights; CPU oracle and transfers excluded from synchronized eager-call latency; profiling separate" + ] +} diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE new file mode 100644 index 000000000..eaae6b11e --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2023 Anne Ouyang, Simon Guo, Azalia Mirhoseini +Scaling Intelligence Lab, Stanford University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md new file mode 100644 index 000000000..91cef765a --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md @@ -0,0 +1,31 @@ +# softmax-cpu + +CPU adaptation of [KernelBench 23_Softmax.py](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py). +Source license is retained in LICENSE. No upstream scoring result transfers here. + +Implement `compute(x, *, eps)` in a self-contained `implementation.py`. Return a +float32 NumPy array of the same shape. Preserve input values. Compute stable softmax along axis 1. +Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; +the evaluator is not a security sandbox for model-produced arbitrary code. + +The task JSON owns six base correctness workloads, seeds, layouts, numerical +tolerances and the v2 timing policy. Five varying workloads have a frozen +per-call input sequence; invariant zero inputs are correctness-only. Baseline and example candidate are separate from the scalar +double-precision oracle. The oracle uses math.fsum and is not editable by a +candidate. The task package and grader are content-frozen per comparison. + +Each measured call actually runs compute and checks its output and input +preservation outside the timed region. Input generation, reference computation, +warmups, validation, process startup and profiling are excluded. Output allocation +inside compute is included. Samples use perf_counter_ns and report milliseconds. +Sum of per-case medians is only this task's fixed-suite objective, not a universal +score. Retain every per-case sample and input hash to inspect regressions. Inputs +are generated before timing and kept alive; each precheck, warmup and sampled +input has distinct bytes. Baseline and candidate consume the same sequence. +This prevents ordinary identity/content memo hits, not hostile-code execution. +Small CPU timings still need repeatability qualification; no cold-start serving +or device synchronization is claimed. + +All tests are public. This small subset cannot demonstrate production inference +speedup, memory savings, or GPU/NPU support. A fresh task version and device +qualification are needed for changed shapes, dtype, backend or timing. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py new file mode 100644 index 000000000..c6c630b0f --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py @@ -0,0 +1,8 @@ +"""NumPy CPU adaptation of the attributed KernelBench task; see LICENSE.""" + +import numpy as np + + +def compute(x, *, eps): + values = np.exp(x - np.max(x, axis=1, keepdims=True)) + return values / np.sum(values, axis=1, keepdims=True) diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py new file mode 100644 index 000000000..f19af8c23 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py @@ -0,0 +1,10 @@ +"""Reviewed example using temporary buffers; not an agent-generated result.""" + +import numpy as np + + +def compute(x, *, eps): + output = np.subtract(x, np.max(x, axis=1, keepdims=True)) + np.exp(output, out=output) + output /= np.sum(output, axis=1, keepdims=True) + return output diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json new file mode 100644 index 000000000..e25252d59 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json @@ -0,0 +1,117 @@ +{ + "schema_version": 1, + "id": "softmax-cpu", + "adapter": "numpy-axis1-v2", + "description": "Optimize softmax on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", + "searchspace": "docs/searchspace-matrix.md \u2014 Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "source": { + "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py", + "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", + "license": "MIT", + "adaptation": "Adapted mathematical task, not upstream scores. PyTorch/CUDA replaced by NumPy CPU; reduced shapes, public stress distributions and strided inputs added. Axis 1 retained. Baseline is a NumPy implementation; scalar math.fsum supplies independent reference." + }, + "environment": { + "backend": "cpu", + "runtime": "numpy", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "softmax", + "axis": 1, + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-07 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "synchronous-compute-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 2, + "samples": 5, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ] + }, + "workloads": [ + { + "id": "normal", + "shape": [ + 16, + 257 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "strided", + "shape": [ + 16, + 257 + ], + "seed": 253, + "distribution": "normal", + "layout": "strided" + }, + { + "id": "zeros", + "shape": [ + 3, + 7 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 3, + 31 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 2, + 17 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + }, + { + "id": "singleton", + "shape": [ + 2, + 1 + ], + "seed": 257, + "distribution": "normal", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads, not held-out generalization", + "CPU float32 finite inputs only; no float16/bfloat16, NaN/Inf inputs or gradients", + "No CUDA/ROCm/Ascend execution or accelerator timing qualification", + "No service/E2E, communication, memory-capacity or skill/search benefit claim", + "Trusted self-contained reviewed source only; no hostile-agent isolation", + "Distinct frozen per-call inputs; zeros correctness-only; allocations inside compute included, setup/oracle/profile excluded from timing" + ] +} diff --git a/src/inferrsi/harness/benchmarks/__init__.py b/src/inferrsi/harness/benchmarks/__init__.py new file mode 100644 index 000000000..21a95e916 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/__init__.py @@ -0,0 +1,5 @@ +"""Shared engineering task contracts; execution belongs to the existing harness.""" + +from .contract import TaskPackage, builtin_tasks, load_builtin_task, load_task + +__all__ = ["TaskPackage", "builtin_tasks", "load_builtin_task", "load_task"] diff --git a/src/inferrsi/harness/benchmarks/cli.py b/src/inferrsi/harness/benchmarks/cli.py new file mode 100644 index 000000000..490a74961 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/cli.py @@ -0,0 +1,84 @@ +"""CLI-first engineering benchmark operations; all execution is explicit.""" + +import json +import math +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..runner import run_experiment +from .consumers import grade_artifact, run_action +from .contract import builtin_tasks, load_task + + +def add_parser(commands): + root = commands.add_parser("benchmarks", help="Shared engineering tasks (trusted sources)") + actions = root.add_subparsers(dest="benchmark_action", required=True) + actions.add_parser("list") + show = actions.add_parser("show") + show.add_argument("task") + for name in ("check", "measure", "profile", "grade", "plan", "compare"): + child = actions.add_parser(name) + child.add_argument("task", help="Built-in task ID or local task package directory") + child.add_argument( + "--candidate", + type=Path, + required=True, + help="Reviewed self-contained Python source; no security sandbox", + ) + child.add_argument("--output-dir", type=Path, required=True) + if name in {"plan", "compare"}: + child.add_argument("--repo", type=Path, required=True) + child.add_argument("--repetitions", type=int, required=True) + child.add_argument("--max-minutes", type=float, required=True) + child.add_argument("--min-improvement-pct", type=float, required=True) + else: + child.add_argument("--timeout-seconds", type=float, required=True) + + +def dispatch(args) -> dict | list: + action = args.benchmark_action + if action == "list": + return builtin_tasks() + task = load_task(args.task) + if action == "show": + return { + "manifest": task.manifest, + "task_sha256": task.task_sha256, + "grader_sha256": task.grader_sha256, + } + if action in {"plan", "compare"}: + spec = task.runspec( + args.candidate, + repo=args.repo, + output_dir=args.output_dir, + repetitions=args.repetitions, + max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct, + ) + if action == "plan": + return spec.to_dict() + identity = task.identity(args.candidate) + run_dir = run_experiment( + spec, + { + "benchmark": identity, + "backend": task.manifest["environment"]["backend"], + "qualification": "public development workload; see task coverage and run evidence", + }, + ) + atomic_write_json(run_dir / "task.lock.json", task.manifest) + return { + "run_dir": str(run_dir), + "status": json.loads((run_dir / "status.json").read_text(encoding="utf-8")), + "scope": "local-artifact-comparison", + "campaign_acceptance": "not established", + } + if not math.isfinite(args.timeout_seconds) or args.timeout_seconds <= 0: + raise ValueError("Timeout must be positive and finite") + if action == "grade": + return grade_artifact( + task, args.candidate, args.output_dir, timeout_seconds=args.timeout_seconds + ) + return run_action( + task, args.candidate, action, args.output_dir, timeout_seconds=args.timeout_seconds + ) diff --git a/src/inferrsi/harness/benchmarks/consumers.py b/src/inferrsi/harness/benchmarks/consumers.py new file mode 100644 index 000000000..8d2334c75 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/consumers.py @@ -0,0 +1,69 @@ +"""Thin consumers; no alternate runner, agent client or result truth.""" + +import json +import math +import time +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..scenarios.external import execute_trial +from ..skill_evaluation import valid_checks + + +def run_action(task, candidate: Path, action: str, output: Path, *, timeout_seconds: float) -> dict: + """Execute trusted local source. This API is not an untrusted-code sandbox.""" + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or not 0 < timeout_seconds <= 7 * 24 * 3600 + ): + raise ValueError("Timeout must be positive, finite and at most seven days") + command = task.command(action, candidate) + return execute_trial( + command, + candidate.resolve().parent, + output.resolve(), + time.monotonic() + timeout_seconds, + task.manifest["measurement"]["key"] if action == "measure" else None, + ) + + +def grade_artifact(task, candidate: Path, output: Path, *, timeout_seconds: float) -> dict: + """Independent correctness for offline engineering evaluation. + + Checks have SkillEval's existing shape. An agent/skill comparison must still + establish its own task delivery, trace, budget and matched conditions. + """ + identity = task.identity(candidate) + execution = run_action(task, candidate, "check", output, timeout_seconds=timeout_seconds) + result, checks = None, None + try: + result = json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + task.assert_unchanged() + if ( + task.identity(candidate) == identity + and result.get("identity") == identity + and result.get("action") == "check" + and result.get("status") == "completed" + ): + proposed = {key: result.get(key) for key in ("passed", "assertions")} + if ( + valid_checks(proposed) + and execution["status"] in {"completed", "failed"} + and execution["returncode"] == (0 if proposed["passed"] else 1) + ): + checks = proposed + except (OSError, ValueError, TypeError): + pass + verdict = "INCONCLUSIVE" if checks is None else "PASS" if checks["passed"] else "FAIL" + record = { + "scope": "artifact-correctness-only", + "identity": identity, + "verdict": verdict, + "checks": checks, + "execution_status": execution["status"], + "agent_benefit": "not measured", + } + atomic_write_json(output / "artifact-evaluation.json", record) + return record diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py new file mode 100644 index 000000000..4acac0f3a --- /dev/null +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -0,0 +1,326 @@ +"""Versioned task truth and bindings to existing Workbench/SkillEval contracts.""" + +import hashlib +import json +import math +import os +import platform +import re +import sys +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from importlib.resources import files +from pathlib import Path + +from ...runspec import RunSpec + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_digest(root: Path) -> str: + entries = {} + for path in sorted(root.rglob("*")): + if "__pycache__" in path.parts or path.suffix == ".pyc": + continue + if path.is_symlink(): + raise ValueError("Task packages must not contain symlinks") + if path.is_file(): + entries[path.relative_to(root).as_posix()] = digest(path) + return hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() + + +def grader_digest() -> str: + return tree_digest(Path(__file__).parent) + + +def runtime_digest(environment=None) -> str: + try: + numpy_version = version("numpy") + except PackageNotFoundError: + numpy_version = "unavailable" + facts = [platform.python_version(), platform.system(), platform.machine(), numpy_version] + if environment and environment["backend"] == "ascend": + for package in ("torch", "torch-npu"): + try: + facts.append(version(package)) + except PackageNotFoundError: + facts.append("unavailable") + facts.append(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "unallocated")) + return hashlib.sha256(json.dumps(facts).encode()).hexdigest() + + +def builtin_tasks() -> list[str]: + root = files("inferrsi").joinpath("data/engineering_benchmarks") + return sorted(p.name for p in root.iterdir() if p.joinpath("task.json").is_file()) + + +def _text(value, field): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a nonempty string") + + +def _positive(value, field, *, integer=False): + if ( + isinstance(value, bool) + or not isinstance(value, int if integer else (int, float)) + or not math.isfinite(value) + or value <= 0 + ): + raise ValueError(f"{field} must be positive and finite") + + +@dataclass(frozen=True) +class TaskPackage: + root: Path + frozen_manifest: str + task_sha256: str + grader_sha256: str + + @property + def manifest(self) -> dict: + # Callers cannot mutate the task by changing a returned dictionary. + return json.loads(self.frozen_manifest) + + def assert_unchanged(self) -> None: + if tree_digest(self.root) != self.task_sha256 or grader_digest() != self.grader_sha256: + raise ValueError("Frozen task or grader changed") + + def identity(self, candidate: Path) -> dict: + self.assert_unchanged() + candidate = candidate.absolute() + if candidate.is_symlink() or not candidate.is_file(): + raise ValueError("Candidate must be a regular source file") + return { + "task_id": self.manifest["id"], + "task_sha256": self.task_sha256, + "grader_sha256": self.grader_sha256, + "candidate_sha256": digest(candidate), + "runtime_sha256": runtime_digest(self.manifest["environment"]), + } + + def command(self, action: str, candidate: Path) -> tuple[str, ...]: + if action not in {"check", "measure", "profile"}: + raise ValueError("Unknown task action") + identity = self.identity(candidate) + return ( + sys.executable, + "-B", + "-m", + "inferrsi.harness.benchmarks.worker", + action, + "--package", + str(self.root), + "--candidate", + str(candidate.resolve()), + "--task-sha256", + identity["task_sha256"], + "--grader-sha256", + identity["grader_sha256"], + "--candidate-sha256", + identity["candidate_sha256"], + "--runtime-sha256", + identity["runtime_sha256"], + ) + + def runspec( + self, + candidate: Path, + *, + repo: Path, + output_dir: Path, + repetitions: int, + max_minutes: float, + min_improvement_pct: float, + ) -> RunSpec: + """Bind one task to AB/BA; the caller owns repetition/budget/gain gates.""" + metric = self.manifest["measurement"] + spec = RunSpec( + repo=str(repo.resolve()), + repo_type="engineering_task", + backend=self.manifest["environment"]["backend"], + baseline_command=self.command("measure", self.root / "baseline.py"), + candidate_command=self.command("measure", candidate), + validation_command=self.command("check", candidate), + metric_key=metric["key"], + metric_unit=metric["unit"], + comparison_scope=self.manifest["id"] + ":" + metric["boundary"], + direction="minimize", + repetitions=repetitions, + max_minutes=max_minutes, + min_improvement_pct=min_improvement_pct, + workload_id=self.task_sha256, + output_dir=str(output_dir.resolve()), + mode="code", + ) + spec.validate() + return spec + + +def load_task(name: str | Path) -> TaskPackage: + """Load a reviewed path or a packaged name for the trusted Workbench API.""" + root = Path(name) + if not root.is_dir(): + return load_builtin_task(str(name)) + return _load_task_root(root) + + +def load_builtin_task(name: str) -> TaskPackage: + """Resolve only packaged task resources, never a same-named cwd directory.""" + if not isinstance(name, str) or name not in builtin_tasks(): + raise ValueError(f"Unknown built-in task package: {name}") + root = Path(str(files("inferrsi").joinpath("data/engineering_benchmarks", name))) + package = _load_task_root(root) + if package.manifest["id"] != name: + raise ValueError("Built-in task identity does not match its resource name") + return package + + +def _load_task_root(root: Path) -> TaskPackage: + if root.is_symlink(): + raise ValueError("Task root must not be a symlink") + root = root.resolve() + data = json.loads((root / "task.json").read_text(encoding="utf-8")) + required = { + "schema_version", + "id", + "description", + "searchspace", + "source", + "environment", + "candidate", + "oracle", + "measurement", + "workloads", + "coverage", + "adapter", + } + if ( + not isinstance(data, dict) + or set(data) != required + or type(data["schema_version"]) is not int + ): + raise ValueError("Invalid task fields") + if data["schema_version"] != 1 or data["adapter"] not in { + "numpy-axis1-v2", "torch-npu-rmsnorm-v1" + }: + raise ValueError("Unsupported task schema or adapter") + npu = data["adapter"] == "torch-npu-rmsnorm-v1" + if not isinstance(data["id"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", data["id"]): + raise ValueError("Invalid task id") + for field in ("description", "searchspace"): + _text(data[field], field) + source = data["source"] + if not isinstance(source, dict) or set(source) != {"url", "revision", "license", "adaptation"}: + raise ValueError("Source provenance is required") + for key, value in source.items(): + _text(value, "source." + key) + if not re.fullmatch(r"[0-9a-f]{40}", source["revision"]): + raise ValueError("Source revision must be a full commit") + env = data["environment"] + expected_env = ( + {"backend": "ascend", "runtime": "torch_npu", "dtype": "float32"} if npu + else {"backend": "cpu", "runtime": "numpy", "dtype": "float32"} + ) + if env != expected_env: + raise ValueError("Unsupported environment; no implicit device fallback") + if data["candidate"] != {"entrypoint": "compute", "editable": ["implementation.py"]}: + raise ValueError("Unsupported candidate contract") + oracle = data["oracle"] + if ( + not isinstance(oracle, dict) + or set(oracle) != {"operation", "axis", "eps", "rtol", "atol"} + or oracle["operation"] not in {"softmax", "rmsnorm"} + or type(oracle["axis"]) is not int + or oracle["axis"] != 1 + ): + raise ValueError("Unsupported oracle") + for key in ("eps", "rtol", "atol"): + _positive(oracle[key], "oracle." + key) + if npu and oracle["operation"] != "rmsnorm": + raise ValueError("NPU adapter only supports affine RMSNorm") + metric = data["measurement"] + if ( + not isinstance(metric, dict) + or set(metric) + != { + "key", + "unit", + "boundary", + "aggregation", + "warmups", + "samples", + "input_sequence", + "case_ids", + } + or metric["key"] != "metrics.suite_latency_ms" + or metric["unit"] != "ms" + or metric["boundary"] != ( + "device-synchronized-eager-call" if npu else "synchronous-compute-call" + ) + or metric["aggregation"] != "sum-of-per-case-medians" + or metric["input_sequence"] != "seeded-distinct-per-call-v1" + ): + raise ValueError("Unsupported measurement contract") + for key in ("warmups", "samples"): + _positive(metric[key], "measurement." + key, integer=True) + if ( + not isinstance(metric["case_ids"], list) + or not metric["case_ids"] + or any(not isinstance(name, str) for name in metric["case_ids"]) + or len(set(metric["case_ids"])) != len(metric["case_ids"]) + ): + raise ValueError("Distinct measured case IDs are required") + if not isinstance(data["workloads"], list) or not data["workloads"]: + raise ValueError("Workloads are required") + names = set() + for case in data["workloads"]: + if not isinstance(case, dict) or set(case) != { + "id", + "shape", + "seed", + "distribution", + "layout", + }: + raise ValueError("Invalid workload fields") + _text(case["id"], "workload id") + if case["id"] in names: + raise ValueError("Duplicate workload id") + names.add(case["id"]) + if ( + not isinstance(case["shape"], list) + or not 2 <= len(case["shape"]) <= 4 + or any(type(n) is not int or n < 1 for n in case["shape"]) + or math.prod(case["shape"]) > 1048576 + ): + raise ValueError("Invalid or oversized workload") + if npu and (len(case["shape"]) != 2 or case["layout"] != "contiguous"): + raise ValueError("NPU task requires contiguous 2D inputs") + if type(case["seed"]) is not int or case["seed"] < 0: + raise ValueError("Seed must be a nonnegative integer") + if case["distribution"] not in {"normal", "zeros", "large", "tiny"}: + raise ValueError("Unknown distribution") + if case["layout"] not in {"contiguous", "strided"}: + raise ValueError("Unknown layout") + measured_cases = [case for case in data["workloads"] if case["id"] in metric["case_ids"]] + if [case["id"] for case in measured_cases] != metric["case_ids"]: + raise ValueError("Measured case IDs must follow workload order") + if any(case["distribution"] == "zeros" for case in measured_cases): + raise ValueError("Invariant zero workloads are correctness-only") + prepared_elements = sum(math.prod(case["shape"]) for case in data["workloads"]) + prepared_elements += (metric["warmups"] + metric["samples"]) * sum( + math.prod(case["shape"]) for case in measured_cases + ) + if prepared_elements > (16777216 if npu else 1048576): + raise ValueError("Prepared input sequence exceeds the workload bound") + if ( + not isinstance(data["coverage"], list) + or not data["coverage"] + or any(not isinstance(v, str) or not v.strip() for v in data["coverage"]) + ): + raise ValueError("Coverage limits are required") + for path in ("baseline.py", "candidate.py", "LICENSE", "README.md"): + if not (root / path).is_file(): + raise ValueError(f"Task package missing {path}") + return TaskPackage(root, json.dumps(data, sort_keys=True), tree_digest(root), grader_digest()) diff --git a/src/inferrsi/harness/benchmarks/npu_task.py b/src/inferrsi/harness/benchmarks/npu_task.py new file mode 100644 index 000000000..2303c534b --- /dev/null +++ b/src/inferrsi/harness/benchmarks/npu_task.py @@ -0,0 +1,211 @@ +"""Trusted affine RMSNorm workload on an explicitly allocated Ascend NPU. + +The independent CPU oracle and host/device transfers are outside timing. Timing +is synchronized eager-call latency, not pure kernel time. No CPU fallback and no +untrusted-code sandbox are provided by this Workbench adapter. +""" + +import hashlib +import json +import math +import os +import platform +import statistics +import time +from pathlib import Path + +from .numpy_task import input_record, inputs + + +def reference(x, weight, eps, np): + """Scalar double arithmetic, independent of the device reduction/fused op.""" + expected = np.empty_like(x) + for index, row in enumerate(x): + values = [float(value) for value in row] + scale = math.sqrt(math.fsum(value * value for value in values) / len(values) + eps) + expected[index] = [value / scale * float(w) for value, w in zip(values, weight)] + return expected + + +def environment(torch, torch_npu, np): + from torch_npu.npu.utils import get_cann_version + + return { + "backend": "ascend", + "device": torch.npu.get_device_name(0), + "logical_device": 0, + "python": platform.python_version(), + "torch": torch.__version__, + "torch_npu": torch_npu.__version__, + "numpy": np.__version__, + "cann": str(get_cann_version()), + "os": platform.system(), + "machine": platform.machine(), + "allocation": "caller-owned; one visible device", + } + + +def check_output(output, x, weight, original, original_weight, expected, oracle, torch, np): + tensor = isinstance(output, torch.Tensor) + shape_dtype_device = ( + tensor and tuple(output.shape) == original.shape + and output.dtype == torch.float32 and output.device == torch.device("npu:0") + ) + def preserved(tensor, original): + return (tuple(tensor.shape) == original.shape and tensor.dtype == torch.float32 + and tensor.device == torch.device("npu:0") and tensor.is_contiguous() + and tensor.cpu().numpy().tobytes() == original.tobytes()) + + # Reading results and checking preservation happens after device synchronization. + values = output.detach().cpu().numpy() if shape_dtype_device else None + return { + "shape_dtype_device": shape_dtype_device, + "finite": values is not None and bool(np.isfinite(values).all()), + "input_preserved": preserved(x, original), + "weight_preserved": preserved(weight, original_weight), + "values": values is not None + and bool(np.allclose(values, expected, rtol=oracle["rtol"], atol=oracle["atol"])), + } + + +def device_kernels(document): + """Read actual Ascend hardware records, not CPU launches/flow arrows.""" + events = document.get("traceEvents", []) if isinstance(document, dict) else document + hardware = {event["pid"] for event in events + if event.get("name") == "process_name" + and event.get("args", {}).get("name") == "Ascend Hardware"} + return [event for event in events + if event.get("ph") == "X" and event.get("pid") in hardware + and str(event.get("args", {}).get("Task Type", "")).startswith("KERNEL_")] + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # The caller owns resource admission. This adapter does not reserve devices + # or reinterpret the scheduler's selected physical device IDs. + allocated = os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "") + if not allocated or len(allocated.split(",")) != 1: + raise RuntimeError("One explicitly allocated visible NPU is required") + import numpy as np + import torch + import torch_npu + + if not torch.npu.is_available() or torch.npu.device_count() != 1: + raise RuntimeError("One NPU must be available; there is no CPU fallback") + torch.npu.set_device(0) + manifest = task.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(candidate.read_bytes(), "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, weight, *, eps)") + + prepared, records, seen = [], [], set() + for case in manifest["workloads"]: + count = 1 + if case["id"] in measurement["case_ids"]: + count += measurement["warmups"] + measurement["samples"] + for index in range(count): + original = inputs(case, np, sequence=index) + rng = np.random.default_rng(np.random.SeedSequence([case["seed"], index, 7])) + original_weight = rng.uniform(-1.5, 1.5, case["shape"][-1]).astype(np.float32) + expected = reference(original, original_weight, oracle["eps"], np) + record = input_record(case, original) + record["weight_sha256"] = hashlib.sha256(original_weight.tobytes()).hexdigest() + record["call_index"] = index + record["phase"] = ( + "precheck" if index == 0 + else "warmup" if index <= measurement["warmups"] else "measurement" + ) + # x itself varies: a weight-only change cannot evade x-output replay checks. + if record["input_sha256"] in seen: + raise ValueError("Repeated input bytes in NPU sequence") + seen.add(record["input_sha256"]) + x = torch.from_numpy(original.copy()).to("npu:0") + weight = torch.from_numpy(original_weight.copy()).to("npu:0") + prepared.append((case, x, weight, original, original_weight, expected, record)) + records.append(record) + torch.npu.synchronize() + checks, executed, samples = {}, [], {key: [] for key in measurement["case_ids"]} + result = { + "status": "completed", "action": action, "passed": False, + "assertions": checks, "environment": environment(torch, torch_npu, np), + "coverage": manifest["coverage"], "input_sequence": records, "executed_calls": executed, + } + + def call(entry, *, timed=False): + case, x, weight, original, original_weight, expected, record = entry + torch.npu.synchronize() + started = time.perf_counter_ns() if timed else None + with torch.inference_mode(): + output = compute(x, weight, eps=oracle["eps"]) + torch.npu.synchronize() + elapsed = (time.perf_counter_ns() - started) / 1e6 if timed else None + assertions = check_output( + output, x, weight, original, original_weight, expected, oracle, torch, np + ) + checks.update({f"{case['id']}.call-{record['call_index']}.{k}": v + for k, v in assertions.items()}) + executed.append(record) + if not all(assertions.values()): + result["failed_call"] = {**record, "usable_measurement": False} + return False + if timed: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve positive latency") + samples[case["id"]].append({"elapsed_ms": elapsed, **record}) + return True + + # All correctness cases are checked before any timed input. + for entry in prepared: + if entry[-1]["phase"] == "precheck" and not call(entry): + return result + + if action == "profile": + # Warm all measured shapes before attaching the profiler. Profile one + # fresh input per shape; ordinary measurement never enables instrumentation. + for entry in prepared: + if entry[-1]["phase"] == "warmup" and not call(entry): + return result + selected = [entry for entry in prepared if entry[-1]["phase"] == "measurement" + and entry[-1]["call_index"] == measurement["warmups"] + 1] + with torch_npu.profiler.profile( + activities=[torch_npu.profiler.ProfilerActivity.CPU, + torch_npu.profiler.ProfilerActivity.NPU], + record_shapes=True, profile_memory=False, with_stack=False, + ) as profiler: + for entry in selected: + with torch.profiler.record_function("rmsnorm-candidate-call"): + if not call(entry): + return result + trace = result_dir / "npu-trace.json" + profiler.export_chrome_trace(str(trace)) + # Raw profiler output may contain local identifiers: retain locally; + # publication should use a reviewed, path-free kernel summary. + document = json.loads(trace.read_text(encoding="utf-8")) + kernels = device_kernels(document) + if not kernels: + raise RuntimeError("NPU profiler produced no device kernel events") + result["diagnostic"] = { + "kind": "ascend-torch-profiler", "artifact": "npu-trace.json", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + "performance_verdict": None, + "kernels": [{"name": event["name"], "duration_us": event.get("dur")} + for event in kernels], + "limitation": "Instrumented diagnostic; includes verification copies; not a score", + } + else: + for entry in prepared: + if entry[-1]["phase"] == "precheck": + continue + if not call(entry, timed=action == "measure" and entry[-1]["phase"] == "measurement"): + return result + if action == "measure": + cases = [{"id": key, "samples_ms": [v["elapsed_ms"] for v in values], + "sample_input_sha256": [v["input_sha256"] for v in values], + "median_ms": statistics.median(v["elapsed_ms"] for v in values)} + for key, values in samples.items()] + result.update(cases=cases, measurement=measurement, + metrics={"suite_latency_ms": sum(case["median_ms"] for case in cases)}) + result["passed"] = all(checks.values()) + return result diff --git a/src/inferrsi/harness/benchmarks/numpy_task.py b/src/inferrsi/harness/benchmarks/numpy_task.py new file mode 100644 index 000000000..883c7f559 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/numpy_task.py @@ -0,0 +1,211 @@ +"""CPU task payload; never schedules processes or makes performance verdicts. + +Only reviewed, trusted implementations may run here. Hash checks detect drift, +not malicious code. The public input suite is not a hidden evaluation set. +""" + +import cProfile +import hashlib +import math +import platform +import pstats +import statistics +import time +from pathlib import Path + + +def inputs(case, np, *, sequence=0): + seed = case["seed"] if sequence == 0 else np.random.SeedSequence([case["seed"], sequence]) + rng = np.random.default_rng(seed) + x = rng.standard_normal(case["shape"]).astype(np.float32) + if case["distribution"] == "zeros": + x.fill(0) + elif case["distribution"] == "large": + x = x * np.float32(100) + np.float32(1000) + elif case["distribution"] == "tiny": + x *= np.float32(1e-7) + if case["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + return x + + +def reference(x, oracle, np): + """Scalar double-precision oracle, independent of NumPy reduction kernels.""" + rows = np.moveaxis(x, 1, -1) + output = np.empty_like(rows) + for index in np.ndindex(rows.shape[:-1]): + values = [float(v) for v in rows[index]] + if oracle["operation"] == "softmax": + maximum = max(values) + exps = [math.exp(v - maximum) for v in values] + total = math.fsum(exps) + result = [v / total for v in exps] + else: + scale = math.sqrt(math.fsum(v * v for v in values) / len(values) + oracle["eps"]) + result = [v / scale for v in values] + output[index] = result + return np.moveaxis(output, -1, 1) + + +def check_output(output, x, original, expected, oracle, np): + return { + "shape_dtype": isinstance(output, np.ndarray) + and output.shape == x.shape + and output.dtype == x.dtype, + "finite": isinstance(output, np.ndarray) and bool(np.isfinite(output).all()), + "input_preserved": x.tobytes() == original.tobytes(), + "values": isinstance(output, np.ndarray) + and output.shape == expected.shape + and bool( + np.allclose(output, expected, rtol=oracle["rtol"], atol=oracle["atol"], equal_nan=False) + ), + } + + +def input_record(case, x): + return { + "id": case["id"], + "input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), + "shape": list(x.shape), + "dtype": str(x.dtype), + "strides": list(x.strides), + } + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # Optional dependency is loaded only for execution, never task discovery. + import numpy as np + + manifest = task.manifest + source = candidate.read_bytes() + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(source, "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, *, eps)") + oracle = manifest["oracle"] + measurement = manifest["measurement"] + checks, cases, workload_records, sequence_records, executed = {}, [], [], [], [] + prepared, sequences = [], [] + # Prepare all inputs/oracles before candidate calls. Keep every input alive: + # a previous call must not prime a later call's object/content memo entry. + for case in manifest["workloads"]: + x = inputs(case, np) + original = x.copy() + expected = reference(x, oracle, np) + workload_records.append(input_record(case, x)) + prepared.append((case, x, original, expected)) + seen = {record["input_sha256"] for record in workload_records} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + calls = [] + for index in range(measurement["warmups"] + measurement["samples"]): + x = inputs(case, np, sequence=index + 1) + record = input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier input bytes") + seen.add(record["input_sha256"]) + record.update( + call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement", + ) + sequence_records.append(record) + calls.append((x, x.copy(), reference(x, oracle, np), record)) + sequences.append((case, calls)) + for (case, x, original, expected), record in zip(prepared, workload_records, strict=True): + executed.append({**record, "phase": "precheck"}) + output = compute(x, eps=oracle["eps"]) + assertions = check_output(output, x, original, expected, oracle, np) + checks.update({case["id"] + "." + k: v for k, v in assertions.items()}) + result = { + "status": "completed", + "action": action, + "passed": all(checks.values()), + "assertions": checks, + "environment": { + "backend": "cpu", + "python": platform.python_version(), + "numpy": np.__version__, + "os": platform.system(), + "machine": platform.machine(), + }, + "coverage": manifest["coverage"], + "workloads": workload_records, + "input_sequence": sequence_records, + "executed_calls": executed, + } + if not result["passed"]: + return result + + # Correctness and profiling do not masquerade as a paired measurement. + profiler = cProfile.Profile() if action == "profile" else None + for case, calls in sequences: + samples = [] + for index, (x, original, expected, record) in enumerate(calls): + executed.append(record) + elapsed = None + if action == "check": + output = compute(x, eps=oracle["eps"]) + else: + if profiler: + profiler.enable() + started = time.perf_counter_ns() + output = compute(x, eps=oracle["eps"]) + elapsed = (time.perf_counter_ns() - started) / 1e6 + if profiler: + profiler.disable() + assertions = check_output(output, x, original, expected, oracle, np) + checks.update({f"{case['id']}.call-{index}.{k}": v for k, v in assertions.items()}) + if not all(assertions.values()): + result.update( + passed=False, + cases=cases, + failed_call={ + "case": case["id"], + "call_index": index, + "phase": "warmup" if index < measurement["warmups"] else "measurement", + "elapsed_ms": elapsed, + "prior_samples_ms": samples, + "usable_measurement": False, + }, + ) + return result + if action != "check" and index >= measurement["warmups"]: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve a positive duration") + samples.append(elapsed) + if action != "check": + cases.append( + { + "id": case["id"], + "samples_ms": samples, + "median_ms": statistics.median(samples), + "sample_input_sha256": [ + record["input_sha256"] + for _, _, _, record in calls[measurement["warmups"] :] + ], + } + ) + if action == "check": + return result + if profiler: + # Strip local installation paths from the textual profile before capture. + with (result_dir / "profile.txt").open("w", encoding="utf-8") as stream: + pstats.Stats(profiler, stream=stream).strip_dirs().sort_stats( + "cumulative" + ).print_stats() + result["diagnostic"] = { + "kind": "cpu-cprofile", + "artifact": "profile.txt", + "performance_verdict": None, + "limitation": "Instrumented CPU calls; no accelerator timeline", + } + else: + result["cases"] = cases + result["measurement"] = measurement + result["metrics"] = {"suite_latency_ms": sum(c["median_ms"] for c in cases)} + return result diff --git a/src/inferrsi/harness/benchmarks/skill_verifier.py b/src/inferrsi/harness/benchmarks/skill_verifier.py new file mode 100644 index 000000000..2ae3e3a74 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/skill_verifier.py @@ -0,0 +1,55 @@ +"""SkillEval verifier for a controller-selected, reviewed retained artifact. + +This is an offline handoff after artifact review, not live generated-code +execution. It deliberately leaves the production engineering sandbox unchanged. +""" + +import json +import os +import sys +from pathlib import Path + +from inferrsi.harness.benchmarks.consumers import grade_artifact +from inferrsi.harness.benchmarks.contract import load_task +from inferrsi.infra.artifacts import atomic_write_json + + +def task_descriptor(task, candidate: Path) -> dict: + return { + "id": task.manifest["id"], + "split": "development", + "request": "Submit the SHA-256 of the controller-reviewed retained implementation.", + "provenance": task.manifest["source"]["url"], + "benchmark": { + "package": str(task.root), + "candidate": str(candidate.resolve()), + "identity": task.identity(candidate), + }, + } + + +def verify(task: dict, answer: dict, output_dir: Path) -> dict: + binding = task["benchmark"] + package = load_task(binding["package"]) + candidate = Path(binding["candidate"]) + if package.identity(candidate) != binding["identity"]: + raise ValueError("Retained artifact or task identity changed") + if answer != {"candidate_sha256": binding["identity"]["candidate_sha256"]}: + return {"passed": False, "assertions": {"artifact_identity": False}} + record = grade_artifact(package, candidate, output_dir, timeout_seconds=25) + if record["checks"] is None: + raise ValueError("Artifact verification did not complete") + return record["checks"] + + +def main(): + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + answer = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, answer, destination.parent / "artifact") + atomic_write_json(destination, result) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/harness/benchmarks/worker.py b/src/inferrsi/harness/benchmarks/worker.py new file mode 100644 index 000000000..d1aca05bf --- /dev/null +++ b/src/inferrsi/harness/benchmarks/worker.py @@ -0,0 +1,63 @@ +"""Execute one frozen task action under an existing InferRSI executor.""" + +import argparse +import json +import os +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from .contract import load_task + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("check", "measure", "profile")) + for field in ( + "package", + "candidate", + "task-sha256", + "grader-sha256", + "candidate-sha256", + "runtime-sha256", + ): + parser.add_argument("--" + field, required=True) + args = parser.parse_args() + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + result = {"status": "error", "action": args.action, "passed": False} + try: + task = load_task(args.package) + candidate = Path(args.candidate) + expected = { + "task_id": task.manifest["id"], + "task_sha256": args.task_sha256, + "grader_sha256": args.grader_sha256, + "candidate_sha256": args.candidate_sha256, + "runtime_sha256": args.runtime_sha256, + } + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed") + if task.manifest["adapter"] == "torch-npu-rmsnorm-v1": + from .npu_task import evaluate + else: + from .numpy_task import evaluate + + result = evaluate(task, candidate, args.action, destination.parent) + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed during execution") + result["identity"] = expected + except Exception as error: + # Errors are incomplete evidence, never numerical correctness failures. + # Do not capture arbitrary exception text that may expose private paths. + result = { + "status": "error", + "action": args.action, + "passed": False, + "error_type": type(error).__name__, + } + atomic_write_json(destination, result) + print(json.dumps({k: result[k] for k in ("status", "action", "passed")})) + return 0 if result["status"] == "completed" and result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/harness/skill_eval_benchmark.py b/src/inferrsi/harness/skill_eval_benchmark.py new file mode 100644 index 000000000..224a6c7e4 --- /dev/null +++ b/src/inferrsi/harness/skill_eval_benchmark.py @@ -0,0 +1,333 @@ +"""Shared CPU tasks for SkillEval's captured-artifact path. + +The trusted parent owns inputs and the scalar oracle. Submitted source runs only +inside the existing Linux bubblewrap boundary, without the repository or grader. +This adapter grades correctness, never child-reported timing or performance. +""" + +import base64 +import binascii +import hashlib +import json +import os +import signal +import subprocess +import tempfile +from pathlib import Path + +from . import skill_eval_engineering as engineering +from .benchmarks import numpy_task +from .benchmarks.contract import TaskPackage, load_builtin_task + +ADAPTER = "engineering-benchmark-v1" +ARTIFACT = "implementation.py" +MAX_SOURCE_BYTES = engineering.MAX_SOURCE_BYTES +MAX_OUTPUT_BYTES = 32 * 1024 * 1024 +SandboxUnavailable = engineering.SandboxUnavailable +read_regular = engineering.read_regular +SCHEMA = { + "type": "object", "additionalProperties": False, + "required": ["artifact", "summary"], + "properties": { + "artifact": {"type": "string", "enum": [ARTIFACT]}, + "summary": {"type": "string"}, + }, +} +PROMPT = ( + "Read task.json and the assigned knowledge. Change only implementation.py, " + "implementing compute(x, *, eps) under the public shared CPU task contract. " + "Use NumPy, math and self-contained helpers; preserve input contents. " + "Do not change frozen inputs or create other files. No network, installs, " + "other agents, commits or publication. Return artifact='implementation.py' " + "and a concise summary, not a verdict or claimed speedup. The controller " + "retains the actual file and independently checks its behavior in isolation." +) + + +def task_package(name: str) -> TaskPackage: + """The artifact sandbox supports only the packaged NumPy CPU contract.""" + package = load_builtin_task(name) + manifest = package.manifest + if (manifest["adapter"] != "numpy-axis1-v2" + or manifest["environment"] != { + "backend": "cpu", "runtime": "numpy", "dtype": "float32"}): + raise ValueError("Formal benchmark artifacts require the NumPy CPU adapter") + return package + + +def task_descriptor(name: str) -> dict: + package = task_package(name) + manifest = package.manifest + return { + "id": name, + "split": "development", + "request": ( + manifest["description"] + " Implement compute(x, *, eps) in implementation.py. " + "Inputs and output are NumPy float32 arrays; preserve shape and input bytes. " + "Reduce axis 1; RMSNorm workloads include four-dimensional inputs. Use eps for " + "RMSNorm. Public finite inputs include strided layouts, zeros, large and tiny " + "values and singleton reductions. The shared task's base and distinct-per-call " + "input sequence is checked. This SkillEval adapter grades correctness only; " + "no latency, hardware, holdout generalization or skill-benefit claim follows." + ), + "provenance": manifest["source"]["url"], + "contract": manifest, + "benchmark": { + "task_id": name, + "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + }, + } + + +def validate_task(task): + if not isinstance(task, dict) or not isinstance(task.get("benchmark"), dict): + raise ValueError("Shared benchmark task requires a canonical descriptor") + name = task["benchmark"].get("task_id") + if not isinstance(name, str) or task != task_descriptor(name): + raise ValueError("Shared benchmark descriptor or frozen task identity changed") + + +def starter(task) -> bytes: + validate_task(task) + return (task_package(task["benchmark"]["task_id"]).root / "candidate.py").read_bytes() + + +def input_hashes(workspace: Path) -> dict: + return engineering.input_hashes(workspace, editable=ARTIFACT) + + +def _numpy(): + try: + import numpy as np + except (ImportError, OSError): + raise SandboxUnavailable("Shared benchmark verification requires NumPy") from None + return np + + +# Expected outputs, task packages, and the grader are never mounted in the child. +# All arrays remain alive and compute is loaded once for the complete sequence. +RUNNER = '''import base64, json, platform, resource, sys +resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) +resource.setrlimit(resource.RLIMIT_AS, (1073741824, 1073741824)) +resource.setrlimit(resource.RLIMIT_FSIZE, (33554432, 33554432)) +resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) +resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +sys.path.insert(0, "/runtime") +import numpy as np +if sys.argv[1:] == ["probe"]: + print(json.dumps({"ready": True, "numpy": np.__version__, + "python": platform.python_version(), "machine": platform.machine()})) +else: + print("INFERRSI_BENCHMARK_READY", file=sys.stderr, flush=True) + request = json.loads(sys.stdin.read(16777217)) + arrays = [] + for record in request["inputs"]: + x = np.frombuffer(base64.b64decode(record["data"], validate=True), + dtype=np.float32).reshape(record["shape"]).copy() + if record["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + arrays.append(x) + namespace = {"__name__": "benchmark_candidate", "__file__": "/work/implementation.py"} + with open("/work/implementation.py", encoding="utf-8") as source: + exec(compile(source.read(), "implementation.py", "exec"), namespace) + compute = namespace["compute"] + outputs = [] + for x in arrays: + y = compute(x, eps=request["eps"]) + output = None + if isinstance(y, np.ndarray) and y.shape == x.shape and y.dtype == np.float32: + output = {"shape": list(y.shape), "dtype": str(y.dtype), + "data": base64.b64encode(y.tobytes()).decode("ascii")} + outputs.append({"output": output, + "input_after": base64.b64encode(x.tobytes()).decode("ascii")}) + print(json.dumps({"outputs": outputs}, allow_nan=False)) +''' + + +def sandbox_command(work: Path, *, probe=False): + # Reuse the established system-Python namespaces and read-only mount policy. + command = engineering.sandbox_command(work, probe=probe) + np = _numpy() + + query = ( + "import json,numpy,pathlib; " + "print(json.dumps({'version':numpy.__version__," + "'root':str(pathlib.Path(numpy.__file__).resolve().parent)}))" + ) + try: + runtime = subprocess.run( + ["/usr/bin/python3", "-I", "-c", query], check=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=5, env={"PATH": "/usr/bin:/bin", "LANG": "C"}, + ) + info = json.loads(runtime.stdout) + root = Path(info["root"]) + if (info["version"] != np.__version__ or root.name != "numpy" + or not root.is_absolute() or not root.is_dir()): + raise ValueError("Unsupported NumPy runtime") + except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError): + raise SandboxUnavailable("Matching system Python NumPy runtime is required") from None + mounts = ["--ro-bind", str(root), "/runtime/numpy"] + libraries = root.parent / "numpy.libs" + if libraries.is_dir(): + mounts += ["--ro-bind", str(libraries.resolve()), "/runtime/numpy.libs"] + mounts += ["--setenv", "OPENBLAS_NUM_THREADS", "1", "--setenv", "OMP_NUM_THREADS", "1"] + # Create the narrow mount destinations before the final read-only root remount. + position = command.index("--remount-ro") + return command[:position] + mounts + command[position:] + + +def _invoke(work, request=None, *, probe=False): + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + try: + process = subprocess.Popen( + sandbox_command(work, probe=probe), stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + env={"PATH": "/usr/bin:/bin", "LANG": "C"}, start_new_session=True, + ) + except OSError: + raise SandboxUnavailable("Cannot start benchmark artifact sandbox") from None + timed_out = False + try: + process.communicate(json.dumps(request).encode(), timeout=15) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + timed_out = True + if not probe: + stderr.seek(0) + if not stderr.read(128).startswith(b"INFERRSI_BENCHMARK_READY\n"): + raise SandboxUnavailable("Benchmark worker did not establish its trusted boundary") + if timed_out: + return None + stdout.seek(0) + raw = stdout.read(MAX_OUTPUT_BYTES + 1) + if process.returncode or len(raw) > MAX_OUTPUT_BYTES: + return None + try: + return json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + return None + + +def _prepare(package, np): + manifest = package.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + prepared, base, sequence = [], [], [] + for case in manifest["workloads"]: + x = numpy_task.inputs(case, np) + record = numpy_task.input_record(case, x) + base.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), + {**record, "phase": "precheck"}, case["id"])) + seen = {record["input_sha256"] for record in base} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + for index in range(measurement["warmups"] + measurement["samples"]): + x = numpy_task.inputs(case, np, sequence=index + 1) + record = numpy_task.input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier bytes") + seen.add(record["input_sha256"]) + record.update(call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement") + sequence.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), record, + f"{case['id']}.call-{index}")) + return prepared, base, sequence + + +def _array(value, original, np): + if (not isinstance(value, dict) or set(value) != {"shape", "dtype", "data"} + or value["shape"] != list(original.shape) or value["dtype"] != "float32" + or not isinstance(value["data"], str)): + return None + try: + raw = base64.b64decode(value["data"], validate=True) + except (ValueError, binascii.Error): + return None + if len(raw) != original.nbytes: + return None + return np.frombuffer(raw, dtype=np.float32).reshape(original.shape) + + +def verify(task, output): + validate_task(task) + valid = (isinstance(output, dict) + and set(output) == {"artifact", "summary", "artifact_source"} + and output["artifact"] == ARTIFACT + and isinstance(output["summary"], str) + and isinstance(output["artifact_source"], str)) + if not valid: + return {"passed": False, "assertions": {"artifact_schema": False}} + try: + source = output["artifact_source"].encode("utf-8") + except UnicodeError: + source = b"" + if not 0 < len(source) <= MAX_SOURCE_BYTES: + return {"passed": False, "assertions": {"artifact_size": False}} + np = _numpy() + + package = task_package(task["benchmark"]["task_id"]) + prepared, base, sequence = _prepare(package, np) + request = { + "eps": package.manifest["oracle"]["eps"], + "inputs": [{"shape": list(x.shape), "layout": case["layout"], + "data": base64.b64encode(x.tobytes()).decode("ascii")} + for case, x, _, _, _ in prepared], + } + with tempfile.TemporaryDirectory(prefix="skill-benchmark-") as temporary: + work = Path(temporary) + (work / "runner.py").write_text(RUNNER, encoding="utf-8") + (work / ARTIFACT).write_bytes(source) + runtime = _invoke(work, probe=True) + if (not isinstance(runtime, dict) + or set(runtime) != {"ready", "numpy", "python", "machine"} + or runtime["ready"] is not True or runtime["numpy"] != np.__version__ + or any(not isinstance(runtime[k], str) or not runtime[k] + for k in ("python", "machine"))): + raise SandboxUnavailable("Benchmark sandbox preflight failed; no host fallback") + actual = _invoke(work, request) + package.assert_unchanged() + receipt = {"identity": task["benchmark"], "workloads": base, "input_sequence": sequence, + "artifact_sha256": hashlib.sha256(source).hexdigest(), + "runtime": {k: runtime[k] for k in ("numpy", "python", "machine")}, + "returned_calls": [], "scope": "isolated-artifact-correctness-only", + "timing": "not measured", "public_inputs": True, + "coverage": ( + "returned_calls indexes well-formed response rows, not independently " + "observed invocations; a crash may leave no rows. Isolation protects " + "the host and keeps the oracle outside, but does not prevent runtime " + "protocol spoofing by submitted source or public-suite overfitting." + )} + if (not isinstance(actual, dict) or set(actual) != {"outputs"} + or not isinstance(actual["outputs"], list) + or len(actual["outputs"]) != len(prepared)): + return {"passed": False, "assertions": {"artifact_response": False}, "benchmark": receipt} + checks = {} + for (case, original, expected, record, prefix), result in zip( + prepared, actual["outputs"], strict=True + ): + if not isinstance(result, dict) or set(result) != {"output", "input_after"}: + checks[prefix + ".response"] = False + continue + received = _array(result["output"], original, np) + after = _array({"shape": list(original.shape), "dtype": "float32", + "data": result["input_after"]}, original, np) + assertions = numpy_task.check_output( + received, original if after is None else after, original, expected, + package.manifest["oracle"], np, + ) + assertions["input_preserved"] = after is not None and assertions["input_preserved"] + checks.update({prefix + "." + key: value for key, value in assertions.items()}) + receipt["returned_calls"].append(record) + return {"passed": all(checks.values()), "assertions": checks, "benchmark": receipt} diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py index 3fdb79904..218075ab6 100644 --- a/src/inferrsi/harness/skill_eval_engineering.py +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -171,7 +171,7 @@ def read_regular(path: Path, limit: int) -> bytes: return data -def input_hashes(workspace: Path) -> dict: +def input_hashes(workspace: Path, *, editable: str = "parser.py") -> dict: """Frozen engineering inputs only; special nodes are never opened.""" if any(path.is_symlink() for path in (workspace.absolute(), *workspace.absolute().parents)): raise ValueError("Workspace paths must not contain symlinks") @@ -181,7 +181,7 @@ def input_hashes(workspace: Path) -> dict: mode = path.lstat().st_mode if stat.S_ISLNK(mode) or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): raise ValueError("Workspace contains a symlink or special node") - if stat.S_ISREG(mode) and name != "parser.py": + if stat.S_ISREG(mode) and name != editable: hashes[name] = hashlib.sha256(read_regular(path, 16 * 1024 * 1024)).hexdigest() return hashes diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 27ce1ed3e..7ca4da9f3 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -190,12 +190,16 @@ def run_trial( timeout_seconds: float | None = None, engineering_artifact: bool = False, generation_only: bool = False, + artifact_adapter=None, ) -> dict: - if generation_only and not engineering_artifact: + adapter = artifact_adapter if artifact_adapter is not None else engineering + is_artifact = engineering_artifact or artifact_adapter is not None + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") + if generation_only and (not is_artifact or adapter is not engineering): raise ValueError("Generation-only requires the engineering artifact adapter") trial_dir.mkdir(parents=True) final = trial_dir / "answer.json" - before = engineering.input_hashes(workspace) if engineering_artifact else { + before = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -238,7 +242,7 @@ def run_trial( try: answer_limit = 1048576 if generation_only else 65536 answer_bytes = (engineering.read_regular(final, answer_limit) - if engineering_artifact + if is_artifact else final.read_bytes()) output = json.loads(answer_bytes.decode("utf-8")) except (ValueError, UnicodeError): @@ -248,25 +252,25 @@ def run_trial( output = engineering.materialize_generated_artifact(workspace, output) except (ValueError, OSError): evaluation_error = "invalid_generated_artifact" - if evaluation_error is None and engineering_artifact: - artifact = workspace / "parser.py" + if evaluation_error is None and is_artifact: + artifact = workspace / artifact_name if not isinstance(output, dict) or set(output) != {"artifact", "summary"}: evaluation_error = "invalid_artifact_answer" else: try: - artifact_bytes = engineering.read_regular(artifact, engineering.MAX_SOURCE_BYTES) + artifact_bytes = engineering.read_regular(artifact, adapter.MAX_SOURCE_BYTES) source = artifact_bytes.decode("utf-8") if any(path.is_symlink() for path in (trial_dir, *trial_dir.absolute().parents)): raise ValueError("Retained directory contains a symlink") if any((trial_dir / name).exists() or (trial_dir / name).is_symlink() - for name in ("agent-answer.json", "parser.py", "verification")): + for name in ("agent-answer.json", artifact_name, "verification")): raise ValueError("Agent created reserved evidence paths") except (ValueError, UnicodeError): evaluation_error = "missing_or_invalid_artifact" else: # A single bounded snapshot supplies both retained bytes and grading. (trial_dir / "agent-answer.json").write_bytes(answer_bytes) - (trial_dir / "parser.py").write_bytes(artifact_bytes) + (trial_dir / artifact_name).write_bytes(artifact_bytes) output = {**output, "artifact_source": source} atomic_write_json(final, output) verification_dir = trial_dir / "verification" @@ -279,14 +283,19 @@ def run_trial( evaluation_error = "artifact_verification_deferred" else: try: - checks = engineering.verify(task, output) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, output) + except adapter.SandboxUnavailable: evaluation_error = "artifact_sandbox_unavailable" else: - atomic_write_json(verification_dir / "result.json", checks) - verification = {"status": "completed" if checks["passed"] else "failed", - "returncode": 0 if checks["passed"] else 1} - if evaluation_error is None and not engineering_artifact: + atomic_write_json(verification_dir / "result.json", raw_checks) + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if not valid_checks(proposed): + evaluation_error = "invalid_verifier_result" + else: + checks = proposed + verification = {"status": "completed" if checks["passed"] else "failed", + "returncode": 0 if checks["passed"] else 1} + if evaluation_error is None and not is_artifact: verifier_environment = getattr(client, "verifier_environment", None) options = {"environment": verifier_environment()} if verifier_environment else {} verification = execute_command( @@ -315,7 +324,7 @@ def run_trial( if verification["returncode"] != (0 if checks["passed"] else 1): evaluation_error = "verifier_exit" try: - after = engineering.input_hashes(workspace) if engineering_artifact else { + after = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -343,11 +352,11 @@ def run_trial( "input_hashes": before, "output_sha256": sha256(final) if final.is_file() and not final.is_symlink() else None, } - if engineering_artifact: - retained = trial_dir / "parser.py" + if is_artifact: + retained = trial_dir / artifact_name record["artifact_sha256"] = (sha256(retained) if retained.is_file() and not retained.is_symlink() else None) - record["artifact_kind"] = engineering.ADAPTER + record["artifact_kind"] = adapter.ADAPTER if generation_only: record["execution_mode"] = engineering.GENERATION_MODE atomic_write_json(trial_dir / "result.json", record) diff --git a/tests/check_installed.py b/tests/check_installed.py index d412f14ce..914d57417 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -9,11 +9,20 @@ import inferrsi from inferrsi.catalog import knowledge_records, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.harness.benchmarks import builtin_tasks, load_task from inferrsi.knowledge_operations import KnowledgeReader assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) +assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} +for benchmark_id in builtin_tasks(): + benchmark = load_task(benchmark_id) + benchmark.assert_unchanged() + assert (benchmark.root / "baseline.py").is_file() + assert (benchmark.root / "candidate.py").is_file() + assert (benchmark.root / "LICENSE").is_file() + records = knowledge_records() assert len(records) == 127, "Knowledge is missing from the installation" assert ( diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py new file mode 100644 index 000000000..97dc1f11b --- /dev/null +++ b/tests/test_engineering_benchmarks.py @@ -0,0 +1,445 @@ +"""Real payload checks and consumer regressions; no model/device calls.""" + +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.contract import load_builtin_task +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.harness.skill_evaluation import run_trial + + +@pytest.fixture +def numpy_runtime(): + return pytest.importorskip("numpy") + + +def read_result(execution): + return json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + + +def test_real_baseline_candidate_measurement_and_profile(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + checked = [] + measurement = task.manifest["measurement"] + expected_calls = len(task.manifest["workloads"]) + len(measurement["case_ids"]) * ( + measurement["warmups"] + measurement["samples"] + ) + for role in ("baseline", "candidate"): + candidate = task.root / (role + ".py") + grade = grade_artifact(task, candidate, tmp_path / role, timeout_seconds=20) + assert grade["verdict"] == "PASS" + assert len(grade["checks"]["assertions"]) == 4 * expected_calls + checked.append(json.loads((tmp_path / role / "result.json").read_text())) + candidate = task.root / "candidate.py" + measured = read_result( + run_action(task, candidate, "measure", tmp_path / "timed", timeout_seconds=20) + ) + assert measured["passed"] + assert measured["metrics"]["suite_latency_ms"] > 0 + assert [case["id"] for case in measured["cases"]] == measurement["case_ids"] + assert all(len(c["samples_ms"]) == 5 for c in measured["cases"]) + profile = read_result( + run_action(task, candidate, "profile", tmp_path / "profile", timeout_seconds=20) + ) + assert "metrics" not in profile + assert profile["diagnostic"]["performance_verdict"] is None + assert "compute" in (tmp_path / "profile/profile.txt").read_text() + for result in [*checked, profile, measured]: + assert result["workloads"] == measured["workloads"] + assert result["input_sequence"] == measured["input_sequence"] + assert result["executed_calls"] == measured["executed_calls"] + assert len(result["executed_calls"]) == expected_calls + hashes = [call["input_sha256"] for call in result["executed_calls"]] + assert len(hashes) == len(set(hashes)) + assert "zeros" not in measurement["case_ids"] + assert any(call["id"] == "zeros" for call in measured["executed_calls"]) + for case in measured["cases"]: + assert case["sample_input_sha256"] == [ + call["input_sha256"] + for call in measured["input_sequence"] + if call["id"] == case["id"] and call["phase"] == "measurement" + ] + + +@pytest.mark.parametrize("key", ["id(x)", "x.tobytes()"]) +def test_timed_calls_cannot_hit_precheck_or_warmup_memo(key, tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + candidate.write_text( + "import numpy as np\nfrom pathlib import Path\ncache = {}\n" + "def compute(x, *, eps):\n" + f" key = {key}\n" + " if key in cache:\n" + " Path(__file__).with_suffix('.hits').write_text('cache hit')\n" + " return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "completed" and result["passed"] + assert result["metrics"]["suite_latency_ms"] > 0 + # Correct memoization remains valid, but every timed call must be a miss. + assert not candidate.with_suffix(".hits").exists() + + +def test_shape_cached_output_replay_is_rejected_after_precheck(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + candidate.write_text( + "import numpy as np\ncache = {}\ndef compute(x, *, eps):\n" + " key = (x.shape, x.strides)\n" + " if key in cache: return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "failed" and not result["passed"] + assert "metrics" not in result + assert result["failed_call"]["case"] == "normal" + assert result["failed_call"]["phase"] == "warmup" + assert all(value for key, value in result["assertions"].items() if ".call-" not in key) + + +@pytest.mark.parametrize( + "body", + [ + "return np.zeros_like(x)", + "return np.full_like(x, np.nan)", + "return np.ones_like(x, dtype=np.float64)", + "return np.ones((1,), dtype=np.float32)", + "y = np.exp(x); return y / y.sum(axis=1, keepdims=True)", + "x[:] = 0; return np.full_like(x, 1 / x.shape[1])", + "y = np.exp(x - x.max(axis=0, keepdims=True)); return y / y.sum(axis=0, keepdims=True)", + ], +) +def test_bad_candidates_cannot_supply_performance(body, tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import numpy as np\ndef compute(x, *, eps):\n " + body + "\n") + task = load_task("softmax-cpu") + grade = grade_artifact(task, candidate, tmp_path / "grade", timeout_seconds=20) + assert grade["verdict"] == "FAIL" + execution = run_action(task, candidate, "measure", tmp_path / "measure", timeout_seconds=20) + assert execution["status"] == "failed" + assert "metrics" not in read_result(execution) + + +def test_checks_repeated_timed_calls_not_only_initial_call(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text( + "import numpy as np\nn = 0\ndef compute(x, *, eps):\n" + " global n\n n += 1\n" + " if n > 6: return np.zeros_like(x)\n" + " y = np.exp(x-x.max(axis=1,keepdims=True))\n" + " return y/y.sum(axis=1,keepdims=True)\n" + ) + execution = run_action( + load_task("softmax-cpu"), candidate, "measure", tmp_path / "run", timeout_seconds=20 + ) + assert execution["status"] == "failed" + result = read_result(execution) + assert not result["passed"] and "metrics" not in result + + +def test_frozen_package_and_candidate_drift(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + task = load_task(root) + candidate = root / "candidate.py" + command = task.command("check", candidate) + candidate.write_text(candidate.read_text() + "\n# changed\n") + with pytest.raises(ValueError, match="changed"): + task.command("check", candidate) + execution = execute_trial(command, root, tmp_path / "run", time.monotonic() + 20, None) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [ + ("schema_version", 2), + ("schema_version", True), + ("adapter", "unknown"), + ("adapter", "numpy-axis1-v1"), + ("environment", {"backend": "cuda", "runtime": "numpy", "dtype": "float32"}), + ("coverage", []), + ("workloads", []), + ], +) +def test_unsupported_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text(encoding="utf-8")) + document[field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_descriptor_does_not_mutate_task(): + task = load_task("softmax-cpu") + manifest = task.manifest + manifest["oracle"]["axis"] = 0 + assert task.manifest["oracle"]["axis"] == 1 + assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} + with pytest.raises(ValueError, match="Unknown built-in task package"): + load_builtin_task("rmsnorm-cpu") + + +@pytest.mark.parametrize( + "field,value", + [ + ("input_sequence", "repeated"), + ("case_ids", []), + ("case_ids", ["missing"]), + ("case_ids", ["normal", "normal"]), + ("case_ids", ["normal", "zeros"]), + ("samples", 1000), + ], +) +def test_invalid_input_sequence_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + document["measurement"][field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_duplicate_generated_input_bytes_refuse_measurement(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + # Same values under different layouts still prime a bytes-keyed memo. + document["workloads"][1]["seed"] = document["workloads"][0]["seed"] + (root / "task.json").write_text(json.dumps(document)) + task = load_task(root) + execution = run_action( + task, root / "candidate.py", "measure", tmp_path / "run", timeout_seconds=20 + ) + result = read_result(execution) + assert execution["status"] == "failed" + assert result["status"] == "error" and "metrics" not in result + assert result["error_type"] == "ValueError" + + +def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + task = load_task("softmax-cpu") + candidate = repo / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + spec = task.runspec( + candidate, + repo=repo, + output_dir=tmp_path / "runs", + repetitions=2, + max_minutes=1, + min_improvement_pct=0, + ) + run = run_experiment(spec, {"scope": "CPU software integration"}) + evaluation = json.loads((run / "evaluation.json").read_text()) + assert evaluation["source_integrity"]["verified"] + assert evaluation["baseline"]["excluded_trials"] == 0 + assert evaluation["candidate"]["excluded_trials"] == 0 + assert evaluation["statistical_significance"] == "not established" + events = [json.loads(line) for line in (run / "events.jsonl").read_text().splitlines()] + assert [e["side"] for e in events if e["phase"] == "measurement"] == [ + "baseline", + "candidate", + "candidate", + "baseline", + ] + identity = json.loads((run / "candidate/trial-001/result.json").read_text())["identity"] + assert identity == task.identity(candidate) + baseline = json.loads((run / "baseline/trial-001/result.json").read_text()) + for side in ("baseline", "candidate"): + for path in (run / side).glob("trial-*/result.json"): + result = json.loads(path.read_text()) + assert result["input_sequence"] == baseline["input_sequence"] + assert result["executed_calls"] == baseline["executed_calls"] + + +@pytest.mark.parametrize( + "case,expected_code,expected_state,expected_verdict", + [ + ("unmet-gain", 1, "completed", "FAIL"), + ("wrong-output", 2, "failed", "INCONCLUSIVE"), + ("timeout", 2, "timeout", "INCONCLUSIVE"), + ], +) +def test_compare_cli_reports_retained_workbench_outcome( + case, expected_code, expected_state, expected_verdict, tmp_path, numpy_runtime, capsys +): + from inferrsi.cli import main + + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + candidate = repo / "implementation.py" + candidate.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if case == "wrong-output" + else (load_task("softmax-cpu").root / "candidate.py").read_text(), + encoding="utf-8", + ) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", "-C", str(repo), "-c", "user.name=Fixture", "-c", + "user.email=fixture@example.invalid", "commit", "-qm", "fixture", + ], + check=True, + ) + code = main( + [ + "benchmarks", "compare", "softmax-cpu", "--candidate", str(candidate), + "--repo", str(repo), "--output-dir", str(tmp_path / "runs"), + "--repetitions", "1", "--max-minutes", "1e-10" if case == "timeout" else "1", + # A complete positive-latency comparison cannot reach a 100% reduction. + "--min-improvement-pct", "100", + ] + ) + response = json.loads(capsys.readouterr().out) + run_dir = Path(response["run_dir"]) + retained = json.loads((run_dir / "status.json").read_text()) + evaluation = json.loads((run_dir / "evaluation.json").read_text()) + assert response["status"] == retained + assert retained["state"] == expected_state + assert retained["verdict"] == evaluation["verdict"] == expected_verdict + assert code == expected_code + + +class RetainedControl: + """Local authored control for the real trial machinery; no model or token usage.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +@pytest.mark.parametrize("good", [True, False]) +def test_actual_skilleval_trial_uses_same_oracle(good, tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + candidate.write_text( + (package.root / "candidate.py").read_text() + if good + else "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + ) + task = skill_verifier.task_descriptor(package, candidate) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "task.json").write_text(json.dumps(task)) + client = RetainedControl(task["benchmark"]["identity"]["candidate_sha256"]) + record = run_trial( + client, + task, + "N", + 1, + workspace, + tmp_path / "trial", + tmp_path / "unused-schema.json", + Path(skill_verifier.__file__), + "Offline authored control", + time.monotonic() + 30, + ) + assert record["verdict"] == ("PASS" if good else "FAIL") + assert record["inputs_unchanged"] + assert record["observations"]["usage"] is None + raw = json.loads((tmp_path / "trial/verification/artifact/result.json").read_text()) + assert raw["identity"] == task["benchmark"]["identity"] + + +def test_missing_numpy_has_no_fallback(tmp_path): + # An import blocker exercises dependency failure in a real worker process. + blocker = tmp_path / "numpy.py" + blocker.write_text("raise ImportError('numpy deliberately unavailable')\n") + task = load_task("softmax-cpu") + result = tmp_path / "result.json" + env = {**os.environ, "PYTHONPATH": str(tmp_path), "INFERRSI_RESULT_PATH": str(result)} + process = subprocess.run( + task.command("measure", task.root / "candidate.py"), + env=env, + cwd=tmp_path, + capture_output=True, + ) + assert process.returncode == 1 + document = json.loads(result.read_text()) + assert document["status"] == "error" and "metrics" not in document + + +def test_timeout_is_incomplete_and_retains_execution(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import time\ndef compute(x, *, eps):\n time.sleep(10)\n") + result = grade_artifact( + load_task("softmax-cpu"), candidate, tmp_path / "timeout", timeout_seconds=0.5 + ) + assert result["verdict"] == "INCONCLUSIVE" + assert result["execution_status"] == "timeout" + assert result["checks"] is None + assert (tmp_path / "timeout/execution.json").is_file() + + +def test_runtime_and_grader_drift_rejected(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + for field in ("--runtime-sha256", "--grader-sha256", "--candidate-sha256"): + command = list(task.command("check", task.root / "candidate.py")) + command[command.index(field) + 1] = "0" * 64 + execution = execute_trial( + tuple(command), task.root, tmp_path / field[2:], time.monotonic() + 20, None + ) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +def test_skill_verifier_binds_answer_to_retained_artifact(tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = package.root / "candidate.py" + descriptor = skill_verifier.task_descriptor(package, candidate) + result = skill_verifier.verify(descriptor, {"candidate_sha256": "0" * 64}, tmp_path / "unused") + assert result == {"passed": False, "assertions": {"artifact_identity": False}} + assert not (tmp_path / "unused").exists() diff --git a/tests/test_npu_engineering_benchmark.py b/tests/test_npu_engineering_benchmark.py new file mode 100644 index 000000000..979eee2e3 --- /dev/null +++ b/tests/test_npu_engineering_benchmark.py @@ -0,0 +1,139 @@ +"""Device-free contract controls; actual NPU execution is explicitly opted in.""" + +import json +import os +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from inferrsi.harness.benchmarks import load_builtin_task, load_task +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.npu_task import check_output, device_kernels, reference +from inferrsi.harness.skill_eval_benchmark import task_descriptor +from tools.npu_benchmark_validate import validation_passed + + +def test_discovery_has_no_torch_dependency_and_formal_cpu_path_refuses_npu(): + task = load_builtin_task("rmsnorm-npu") + assert task.manifest["environment"]["backend"] == "ascend" + subprocess.run([sys.executable, "-c", "import sys; " + "from inferrsi.harness.benchmarks import load_builtin_task; " + "load_builtin_task('rmsnorm-npu'); " + "assert 'torch' not in sys.modules and 'torch_npu' not in sys.modules"], + check=True) + with pytest.raises(ValueError, match="CPU"): + task_descriptor("rmsnorm-npu") + + +@pytest.mark.parametrize("field,value", [ + ("environment", {"backend": "cpu", "runtime": "numpy", "dtype": "float32"}), + ("oracle.operation", "softmax"), ("measurement.boundary", "synchronous-compute-call"), + ("workloads.0.layout", "strided"), ("workloads.0.shape", [2, 3, 4]), +]) +def test_npu_contract_rejects_wrong_execution_semantics(field, value, tmp_path): + root = tmp_path / "task" + shutil.copytree(load_builtin_task("rmsnorm-npu").root, root) + manifest = json.loads((root / "task.json").read_text(encoding="utf-8")) + current = manifest + parts = field.split(".") + for key in parts[:-1]: + current = current[int(key)] if isinstance(current, list) else current[key] + current[parts[-1]] = value + (root / "task.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + load_task(root) + + +def test_unallocated_device_is_inconclusive_without_import_or_fallback(tmp_path, monkeypatch): + monkeypatch.delenv("ASCEND_RT_VISIBLE_DEVICES", raising=False) + task = load_builtin_task("rmsnorm-npu") + grade = grade_artifact(task, task.root / "candidate.py", tmp_path / "run", timeout_seconds=20) + assert grade["verdict"] == "INCONCLUSIVE" + result = json.loads((tmp_path / "run" / "result.json").read_text(encoding="utf-8")) + assert result["error_type"] == "RuntimeError" and "metrics" not in result + + +def test_scalar_affine_oracle_preserves_epsilon_and_weight(): + np = pytest.importorskip("numpy") + x = np.array([[0, 0], [1e-7, -1e-7]], dtype=np.float32) + weight = np.array([2, -3], dtype=np.float32) + y = reference(x, weight, 1e-5, np) + np.testing.assert_allclose(y[0], [0, 0]) + np.testing.assert_allclose(y[1], [2e-7 / (1e-5 + 1e-14)**0.5, + 3e-7 / (1e-5 + 1e-14)**0.5], rtol=1e-6) + + +def test_profile_requires_hardware_kernel_records(): + launch = {"pid": 1, "ph": "X", "name": "npu_rms_norm", "cat": "cpu_op"} + metadata = {"pid": 2, "name": "process_name", "args": {"name": "Ascend Hardware"}} + flow = {"pid": 2, "ph": "f", "cat": "async_npu"} + kernel = {"pid": 2, "ph": "X", "name": "RmsNorm", "dur": 5.0, + "args": {"Task Type": "KERNEL_AIVEC"}} + assert device_kernels([launch, metadata, flow]) == [] + assert device_kernels({"traceEvents": [launch, metadata, flow, kernel]}) == [kernel] + + +@pytest.mark.parametrize("status,code,passed,expected", [ + ("completed", 0, True, True), ("failed", 1, False, False), + ("timeout", None, True, False), ("completed", 0, False, False), +]) +def test_noop_qualification_requires_final_quality_pass(status, code, passed, expected): + assert validation_passed({"execution_status": status, "returncode": code, + "result": {"status": "completed", "passed": passed}}) is expected + + +def test_changed_input_shape_and_broadcast_output_cannot_pass(): + np = pytest.importorskip("numpy") + + class Tensor: + dtype = "float32" + device = "npu:0" + + def __init__(self, array): + self.array, self.shape = array, array.shape + + def cpu(self): + return self + + def detach(self): + return self + + def numpy(self): + return self.array + + def is_contiguous(self): + return True + + torch = SimpleNamespace(Tensor=Tensor, float32="float32", device=lambda value: value) + original = np.array([[1., 2., 3.]], dtype=np.float32) + weight = np.ones(3, dtype=np.float32) + # Identical bytes, wrong metadata, and values that allclose could broadcast. + squeezed = Tensor(original.reshape(3)) + checks = check_output(squeezed, squeezed, Tensor(weight), original, weight, original, + {"rtol": 1e-5, "atol": 1e-6}, torch, np) + assert checks["shape_dtype_device"] is False + assert checks["input_preserved"] is False + assert checks["values"] is False + + +@pytest.mark.skipif(os.environ.get("INFERRSI_REQUIRE_NPU_BENCHMARK") != "1", + reason="Requires explicitly scheduled Ascend NPU execution") +def test_real_npu_baseline_fused_wrong_output_and_profile(tmp_path): + task = load_builtin_task("rmsnorm-npu") + for name in ("baseline", "candidate"): + report = grade_artifact(task, task.root / f"{name}.py", tmp_path / name, + timeout_seconds=180) + assert report["verdict"] == "PASS" + wrong = tmp_path / "wrong.py" + wrong.write_text("import torch\ndef compute(x, weight, *, eps): return torch.zeros_like(x)\n") + result = run_action(task, wrong, "measure", tmp_path / "wrong", timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert not record["passed"] and "metrics" not in record + result = run_action(task, task.root / "candidate.py", "profile", tmp_path / "profile", + timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert record["passed"] and "metrics" not in record + assert any("RmsNorm" in kernel["name"] for kernel in record["diagnostic"]["kernels"]) diff --git a/tests/test_skill_eval_benchmark.py b/tests/test_skill_eval_benchmark.py new file mode 100644 index 000000000..04b1018d7 --- /dev/null +++ b/tests/test_skill_eval_benchmark.py @@ -0,0 +1,325 @@ +"""Shared-oracle and isolated artifact controls; never model-benefit evidence.""" + +import base64 +import json +import os +import shutil +from dataclasses import replace +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks import numpy_task +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task + +READY = {"ready": True, "numpy": np.__version__, "python": "3.10.0", "machine": "x86_64"} + + +def answer(name="softmax-cpu", source=None): + task = benchmark.task_descriptor(name) + return task, {"artifact": benchmark.ARTIFACT, "summary": "Authored software control", + "artifact_source": source or benchmark.starter(task).decode()} + + +def oracle_transport(task): + """Synthetic transport exercises parent grading only, never executes source.""" + package = load_builtin_task(task["benchmark"]["task_id"]) + prepared, _, _ = benchmark._prepare(package, np) + outputs = [{"output": {"shape": list(x.shape), "dtype": "float32", + "data": base64.b64encode(expected.tobytes()).decode()}, + "input_after": base64.b64encode(x.tobytes()).decode()} + for _, x, expected, _, _ in prepared] + return {"outputs": outputs} + + +def test_descriptor_freezes_builtin_task_and_shared_grader(): + name = "softmax-cpu" + task, _ = answer(name) + benchmark.validate_task(task) + package = load_task(name) + assert task["contract"] == package.manifest + assert task["benchmark"] == { + "task_id": name, "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + } + assert benchmark.starter(task) == (package.root / "candidate.py").read_bytes() + + +def shadow_softmax(directory): + """A reviewed-path task is valid but must never replace the formal rubric.""" + canonical = load_builtin_task("softmax-cpu") + shadow = directory / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + (shadow / "task.json").write_text(json.dumps(manifest), encoding="utf-8") + (shadow / "candidate.py").write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + return canonical, shadow + + +def test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + original, _ = answer("softmax-cpu") + monkeypatch.chdir(tmp_path) + # Trusted callers can still deliberately load an external reviewed task. + assert load_task("softmax-cpu").root == shadow + assert load_task(shadow).manifest["oracle"]["atol"] == 1 + assert load_builtin_task("softmax-cpu").root == canonical.root + task, output = answer("softmax-cpu") + assert task == original + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + assert benchmark.starter(task) == (canonical.root / "candidate.py").read_bytes() + assert benchmark.starter(task) != (shadow / "candidate.py").read_bytes() + # Synthetic returned arrays test parent grading, not sandbox source execution. + transport = oracle_transport(task) + for record in transport["outputs"]: + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + with mock.patch.object(benchmark, "_invoke", side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert not result["assertions"]["normal.values"] + assert "metrics" not in result + + +@pytest.mark.parametrize("change", ["adapter", "environment"]) +def test_formal_artifact_path_rejects_new_backend_packages(change, monkeypatch): + package = load_builtin_task("softmax-cpu") + manifest = package.manifest + if change == "adapter": + manifest["adapter"] = "torch-npu-rmsnorm-v1" + else: + manifest["environment"] = {"backend": "ascend", "runtime": "torch_npu", + "dtype": "float32"} + unsupported = replace(package, frozen_manifest=json.dumps(manifest)) + monkeypatch.setattr(benchmark, "load_builtin_task", lambda _: unsupported) + with pytest.raises(ValueError, match="NumPy CPU adapter"): + benchmark.task_descriptor("rmsnorm-npu") + + +@pytest.mark.parametrize("change", ["hash", "holdout", "path", "request", "extra", "contract"]) +def test_descriptor_drift_and_noncanonical_tasks_rejected(change): + task, _ = answer() + if change == "hash": + task["benchmark"]["task_sha256"] = "0" * 64 + elif change == "path": + task["benchmark"]["task_id"] = "../softmax-cpu" + elif change == "holdout": + task["split"] = "holdout" + elif change == "request": + task["request"] += " instead use a new oracle" + elif change == "contract": + task["contract"]["oracle"]["atol"] = 1 + else: + task["expected"] = True + with pytest.raises(ValueError): + benchmark.validate_task(task) + + +@pytest.mark.parametrize("change", [{"artifact": "parser.py"}, {"artifact_source": ""}, + {"artifact_source": "x" * 65537}, {"passed": True}]) +def test_invalid_artifact_does_not_start_source(change): + task, output = answer() + with mock.patch.object(benchmark, "_invoke") as invoke: + assert not benchmark.verify(task, {**output, **change})["passed"] + invoke.assert_not_called() + + +def test_no_sandbox_means_explicit_unavailable_never_host_execution(tmp_path): + marker = tmp_path / "must-not-exist" + task, output = answer(source=f"open({str(marker)!r}, 'w').write('bad')") + with mock.patch.object(benchmark, "_invoke", return_value=None): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.verify(task, output) + assert not marker.exists() + + +def test_missing_numpy_is_explicit_unavailable(): + task, output = answer() + with mock.patch.dict("sys.modules", {"numpy": None}): + with pytest.raises(benchmark.SandboxUnavailable, match="requires NumPy"): + benchmark.verify(task, output) + + +def test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call(): + name = "softmax-cpu" + task, output = answer(name) + transport = oracle_transport(task) + calls = [] + + def invoke(work, request=None, *, probe=False): + assert sorted(p.name for p in work.iterdir()) == ["implementation.py", "runner.py"] + assert (work / "implementation.py").read_bytes() == output["artifact_source"].encode() + if probe: + return READY + calls.append(request) + assert set(request) == {"eps", "inputs"} + assert all(set(x) == {"shape", "layout", "data"} for x in request["inputs"]) + return transport + + with mock.patch.object(benchmark, "_invoke", side_effect=invoke): + result = benchmark.verify(task, output) + package = load_task(name) + # Use only the repository-authored candidate for comparison, not submitted source. + trusted = numpy_task.evaluate(package, package.root / "candidate.py", "check", Path(".")) + assert result["passed"] and len(result["assertions"]) == 164 + assert result["assertions"] == trusted["assertions"] + for key in ("workloads", "input_sequence"): + assert result["benchmark"][key] == trusted[key] + assert result["benchmark"]["returned_calls"] == trusted["executed_calls"] + assert len(calls) == 1 and len(calls[0]["inputs"]) == 41 + assert result["benchmark"]["timing"] == "not measured" + + +@pytest.mark.parametrize("wrong", ["dtype", "values", "mutation", "truncated", "malformed"]) +def test_parent_rejects_wrong_or_malformed_returned_arrays(wrong): + task, output = answer() + transport = oracle_transport(task) + record = transport["outputs"][10] + if wrong == "dtype": + record["output"]["dtype"] = "float64" + elif wrong == "values": + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + elif wrong == "mutation": + record["input_after"] = base64.b64encode( + bytes(len(base64.b64decode(record["input_after"]))) + ).decode() + elif wrong == "truncated": + transport["outputs"].pop() + else: + record["output"]["data"] = "!bad-base64" + with mock.patch.object(benchmark, "_invoke", + side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert "metrics" not in result + + +def test_workspace_freeze_excludes_only_implementation(tmp_path): + (tmp_path / "task.json").write_text("{}") + (tmp_path / "implementation.py").write_text("original") + (tmp_path / "parser.py").write_text("frozen") + before = benchmark.input_hashes(tmp_path) + assert set(before) == {"task.json", "parser.py"} + (tmp_path / "implementation.py").write_text("candidate") + assert benchmark.input_hashes(tmp_path) == before + (tmp_path / "parser.py").write_text("changed") + assert benchmark.input_hashes(tmp_path) != before + + +def test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository(): + original = ["bwrap", "--unshare-all", "--clearenv", "--ro-bind", "/safe/work", "/work", + "--remount-ro", "/", "--", "/usr/bin/python3", "-I", "-S", "-B", + "/work/runner.py"] + runtime = mock.Mock(stdout=json.dumps({"version": np.__version__, + "root": "/opt/public-runtime/numpy"}).encode()) + with mock.patch.object(engineering, "sandbox_command", return_value=original), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime), \ + mock.patch.object(Path, "is_dir", return_value=True), \ + mock.patch.object(Path, "is_absolute", return_value=True), \ + mock.patch.object(Path, "resolve", return_value=Path("/opt/public-runtime/numpy.libs")): + command = benchmark.sandbox_command(Path("/safe/work")) + assert "--unshare-all" in command and "--clearenv" in command + assert command[-5:] == original[-5:] + assert command.count("--ro-bind") == 3 + assert "/runtime/numpy" in command and "/runtime/numpy.libs" in command + assert command.index("/runtime/numpy.libs") < command.index("--remount-ro") + assert "--share-net" not in command + assert "OPENBLAS_NUM_THREADS" in command + + +def test_runtime_version_mismatch_is_unavailable(): + runtime = mock.Mock(stdout=json.dumps({"version": "different", "root": "/runtime/numpy"})) + with mock.patch.object(engineering, "sandbox_command", return_value=["--"]), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.sandbox_command(Path("/safe/work")) + + +@pytest.mark.parametrize("ready", [False, True]) +def test_worker_startup_is_distinct_from_artifact_failure(ready): + process = mock.Mock(returncode=1) + + def start(*args, **kwargs): + assert kwargs["env"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} + if ready: + kwargs["stderr"].write(b"INFERRSI_BENCHMARK_READY\n") + return process + + with mock.patch.object(benchmark, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(benchmark.subprocess, "Popen", side_effect=start): + if ready: + assert benchmark._invoke(Path("/unused"), {}) is None + else: + with pytest.raises(benchmark.SandboxUnavailable): + benchmark._invoke(Path("/unused"), {}) + + +def run_real(task, output): + try: + return benchmark.verify(task, output) + except benchmark.SandboxUnavailable as error: + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail(str(error)) + pytest.skip(str(error)) + + +def test_real_sandbox_correct_and_wrong_authored_controls(): + task, output = answer() + result = run_real(task, output) + assert result["passed"] and len(result["assertions"]) == 164 + wrong = {**output, "artifact_source": + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n"} + rejected = run_real(task, wrong) + assert not rejected["passed"] and "metrics" not in rejected + + +def test_real_sandbox_rejects_zero_output_despite_cwd_shadow(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + monkeypatch.chdir(tmp_path) + task, wrong = answer("softmax-cpu", source=(shadow / "candidate.py").read_text()) + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + result = run_real(task, wrong) + assert not result["passed"] and not result["assertions"]["normal.values"] + assert "metrics" not in result + + +def test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay(): + source = """import numpy as np +cache = {} +def compute(x, *, eps): + key = (x.shape, x.strides) + if key not in cache: + y = np.exp(x - x.max(axis=1, keepdims=True)) + cache[key] = y / y.sum(axis=1, keepdims=True) + return cache[key] +""" + task, output = answer(source=source) + result = run_real(task, output) + assert not result["passed"] + assert result["assertions"]["normal.values"] + assert not result["assertions"]["normal.call-0.values"] + + +def test_real_sandbox_cannot_read_host_sentinel_or_oracle(tmp_path): + sentinel = tmp_path / "host-only-secret.txt" + sentinel.write_text("must not be visible to submitted code") + task, output = answer() + source = output["artifact_source"] + f""" +from pathlib import Path +assert not Path({str(sentinel)!r}).exists() +assert sorted(p.name for p in Path('/work').iterdir()) == ['implementation.py', 'runner.py'] +assert not Path('/proc/1/root{str(sentinel)}').exists() +""" + result = run_real(task, {**output, "artifact_source": source}) + assert result["passed"] diff --git a/tests/test_skill_eval_benchmark_compare.py b/tests/test_skill_eval_benchmark_compare.py new file mode 100644 index 000000000..1165286b2 --- /dev/null +++ b/tests/test_skill_eval_benchmark_compare.py @@ -0,0 +1,330 @@ +"""Formal two-arm plumbing with authored producers, not agent/skill efficacy. + +The producer is a real local subprocess. These controller tests replace the +artifact verifier; they do not establish that Linux/NumPy sandbox execution ran. +The separate benchmark verifier tests own real sandbox software controls. +""" + +import hashlib +import json +import os +import shutil +import subprocess +import sys + +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import configuration +from tools import skill_eval_compare as compare +from tools import skill_eval_local as local + +PRODUCER = """import json, pathlib, sys +work, final = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +source, mode = sys.argv[3:] +artifact = work / 'implementation.py' +artifact.write_text(source, encoding='utf-8', newline='') +answer = {'artifact': 'implementation.py', 'summary': 'Authored software control, no model'} +if mode == 'changed-input': (work / 'task.json').write_text('{}') +if mode == 'extra-file': (work / 'unexpected.txt').write_text('unexpected') +if mode == 'missing-artifact': artifact.unlink() +if mode == 'oversized': artifact.write_bytes(b'x' * 65537) +if mode == 'invalid-answer': answer['passed'] = True +final.write_text(json.dumps(answer), encoding='utf-8') +event = {'type': 'turn.completed'} +if mode != 'unknown-usage': event['usage'] = {'input_tokens': 3, 'output_tokens': 2} +print(json.dumps(event), flush=True) +""" + + +class AuthoredProducer: + """Write candidate bytes without importing or executing them on the host.""" + + timeout_seconds = 30 + + def __init__(self, modes=None): + self.modes = modes or {} + self.calls = [] + + def command(self, workspace, final, schema, prompt): + task = compare.read_json(workspace / "task.json") + role = "baseline" if "-baseline-" in final.parent.name else "candidate" + repeat = int(final.parent.name.rsplit("-", 1)[1]) + mode = self.modes.get( + (task["id"], role, repeat), self.modes.get((task["id"], role), "correct")) + source = ("import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if mode == "wrong-output" else benchmark.starter(task).decode("utf-8")) + self.calls.append({ + "task": task, "role": role, "repeat": repeat, "workspace": workspace, + "schema_sha256": sha256(schema), "source": source, + "inputs": {p.relative_to(workspace).as_posix(): p.read_bytes() + for p in workspace.rglob("*") if p.is_file()}, + }) + return (sys.executable, "-B", "-c", PRODUCER, str(workspace), str(final), source, mode) + + +def prepared(tmp_path, names=("softmax-cpu",), repeats=1): + compare.require_git_version(subprocess.check_output(("git", "--version"), text=True)) + config_path, repo = configuration(tmp_path, ids=names, repeats=repeats) + config = compare.read_json(config_path) + config["adapter"] = benchmark.ADAPTER + write_json(config_path, config) + tasks = [benchmark.task_descriptor(name) for name in names] + write_json(tmp_path / "cases.json", tasks) + output = tmp_path / "run" + digest = compare.prepare(config_path, repo, output) + return output, digest, config["snapshots"]["candidate"]["git_revision"] + + +def software_control_verifier(task, answer): + """A source-identity stub, deliberately not presented as numerical grading.""" + benchmark.validate_task(task) + passed = answer["artifact_source"] == benchmark.starter(task).decode("utf-8") + return { + "passed": passed, "assertions": {"authored_source_control": passed}, + "benchmark": {"scope": "mocked controller test only; no sandbox execution"}, + } + + +def test_real_authored_processes_use_frozen_task_and_retain_both_sources(tmp_path, monkeypatch): + # Two repetitions retain AB/BA isolation and both report outcomes with one task. + output, digest, _ = prepared(tmp_path, repeats=2) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "baseline", 1): "wrong-output"}) + result = compare.execute(output, digest, client=client) + assert result["adapter"] == benchmark.ADAPTER + assert result["counts"]["fix"] == result["counts"]["unchanged_correct"] == 1 + assert len(client.calls) == 4 + assert [call["repeat"] for call in client.calls] == [1, 1, 2, 2] + assert [call["role"] for call in client.calls] == [ + "baseline", "candidate", "candidate", "baseline"] + for left, right in (client.calls[:2], client.calls[2:]): + assert left["task"] == right["task"] + assert left["inputs"]["task.json"] == right["inputs"]["task.json"] + assert left["inputs"]["implementation.py"] == right["inputs"]["implementation.py"] + assert left["schema_sha256"] == right["schema_sha256"] + assert left["inputs"]["knowledge/selected/SKILL.md"] != ( + right["inputs"]["knowledge/selected/SKILL.md"]) + assert len({call["workspace"] for call in client.calls}) == 4 + assert all(not call["workspace"].exists() for call in client.calls) + for pair in result["pairs"]: + for role in compare.ROLES: + trial = pair[role] + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] + answer = json.loads(trial["output"]) + assert artifact.read_text(encoding="utf-8") == answer["artifact_source"] + assert trial["artifact_kind"] == benchmark.ADAPTER + assert trial["inputs_unchanged"] is True + assert trial["grader_trust"] == "reverified" + assert set(trial["checks"]) == {"passed", "assertions"} + assert "metrics" not in trial + before = (output / "report.json").read_bytes() + compare.report(output) + assert len(client.calls) == 4 + assert (output / "report.json").read_bytes() == before + assert str(tmp_path) not in before.decode("utf-8") + + +@pytest.mark.parametrize("kind", ["task", "schema", "starter"]) +def test_changed_frozen_input_prevents_any_producer_call(tmp_path, kind): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + name = {"task": "plan/tasks.json", "schema": "plan/schema.json"}.get(kind) + if name is None: + name = next(name for name in plan["files"] if name.endswith("/implementation.py")) + path = output / name + path.write_bytes(path.read_bytes() + b"\n") + client = AuthoredProducer() + with pytest.raises(ValueError, match="Frozen input changed"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +def test_task_grader_and_oracle_dependencies_are_frozen_before_producer(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + names = ( + "src/inferrsi/harness/skill_eval_benchmark.py", + "src/inferrsi/harness/benchmarks/numpy_task.py", + "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json", + ) + for name in names: + assert plan["controllers"][name] == sha256(compare.ROOT / name) + changed_path = compare.ROOT / name + + def modified_digest(path): + return "0" * 64 if path == changed_path else sha256(path) + + client = AuthoredProducer() + with monkeypatch.context() as scope: + scope.setattr(compare, "sha256", modified_digest) + with pytest.raises(ValueError, match="Controller changed after freeze"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +def test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow(tmp_path, monkeypatch): + canonical = load_builtin_task("softmax-cpu") + shadow = tmp_path / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + write_json(shadow / "task.json", manifest) + (shadow / "candidate.py").write_text("# substituted starter\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + output, _, _ = prepared(tmp_path, names=("softmax-cpu",)) + plan = compare.load_plan(output) + frozen = load_task(output / "plan/benchmark-tasks/softmax-cpu") + assert frozen.task_sha256 == canonical.task_sha256 + assert frozen.manifest["oracle"] == canonical.manifest["oracle"] + assert (output / "plan/starters/softmax-cpu/implementation.py").read_bytes() == ( + canonical.root / "candidate.py").read_bytes() + name = "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json" + assert plan["controllers"][name] == sha256(canonical.root / "task.json") + assert not any(name.startswith("softmax-cpu/") for name in plan["controllers"]) + # A later cwd change cannot make the frozen package validate against a new rubric. + manifest["oracle"]["atol"] = 2 + write_json(shadow / "task.json", manifest) + assert compare.load_plan(output) == plan + + +@pytest.mark.parametrize("mode,error", [ + ("changed-input", "inputs_changed"), + ("extra-file", "inputs_changed"), + ("missing-artifact", "missing_or_invalid_artifact"), + ("oversized", "missing_or_invalid_artifact"), + ("invalid-answer", "invalid_artifact_answer"), +]) +def test_invalid_handoff_is_not_a_clean_comparison(tmp_path, monkeypatch, mode, error): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "candidate"): mode}) + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + assert trial["verdict"] == "INCONCLUSIVE" + retained = compare.read_json(output / trial["evidence"]["result.json"]) + assert retained["evaluation_error"] == error + # Report prioritizes missing artifact integrity over the original capture + # error, which remains available in the unchanged result.json record. + expected_report_error = ("missing_retained_artifact" if mode in { + "missing-artifact", "oversized", "invalid-answer"} else error) + assert trial["evaluation_error"] == expected_report_error + assert result["counts"]["inconclusive"] == 1 + + +def test_no_unsafe_default_launch_or_silent_sandbox_fallback(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + with pytest.raises(ValueError, match="confined local client"): + compare.execute(output, digest) + assert not (output / "results/start.json").exists() + + def unavailable(*_): + raise benchmark.SandboxUnavailable("authored unsupported-host control") + + monkeypatch.setattr(benchmark, "verify", unavailable) + result = compare.execute(output, digest, client=AuthoredProducer()) + for role in compare.ROLES: + trial = result["pairs"][0][role] + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["artifact_sha256"] is not None + + +def test_retained_source_tamper_invalidates_report_without_rerunning_producer( + tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer() + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + artifact = output / trial["evidence"]["implementation.py"] + artifact.write_text("# substituted source\n", encoding="utf-8") + report = compare.read_report(output) + assert report["pairs"][0]["candidate"]["verdict"] == "INCONCLUSIVE" + assert report["pairs"][0]["candidate"]["evaluation_error"] == "retained_artifact_hash_mismatch" + assert len(client.calls) == 2 + + +def test_unknown_usage_stops_new_trials_without_inventing_cost(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("softmax-cpu", "baseline"): "unknown-usage"}) + result = compare.execute(output, digest, client=client) + assert len(client.calls) == 1 + assert result["status"]["stop_reason"] == "unknown_usage" + assert result["counts"]["missing_pair"] == 1 + assert result["conditions"]["baseline"]["cost"] is None + + +def test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + submission = tmp_path / "author-logs" + submission.mkdir() + write_json(submission / "execution.json", { + "model": compare.load_plan(output)["model"], "environment": "authored software control", + "access": "not a real agent run", "resources": "no model calls", "usage": None, + "elapsed_seconds": 1.0, "agent_exit_code": 0, "summary": "controller test only", + }) + for name in ("stdout.log", "stderr.log"): + (submission / name).write_text("authored control\n", encoding="utf-8") + for role in compare.ROLES: + workspace = tmp_path / role + local.stage(output, digest, "softmax-cpu", role, 1, workspace, head) + if role == "baseline": + (workspace / benchmark.ARTIFACT).write_bytes(b"# authored bad control\r\n") + result = local.capture(output, digest, "softmax-cpu", role, 1, workspace, head, submission) + assert result["counts"]["artifact_improved"] == 1 + pair = result["pairs"][0] + assert pair["comparability"] == "unverified_author_reported" + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert pair["candidate"]["author_reported_execution"] is True + assert pair["candidate"]["observations"]["usage"] is None + artifact = output / pair["baseline"]["evidence"]["implementation.py"] + assert artifact.read_bytes() == b"# authored bad control\r\n" + assert pair["baseline"]["artifact_sha256"] == hashlib.sha256(artifact.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("change", [ + {"adapter": "unregistered-benchmark"}, + {"execution_mode": "generation-only"}, + {"schema": "custom-schema.json"}, + {"grader": "custom-grader.py"}, +]) +def test_adapter_does_not_accept_unfrozen_alternate_paths(tmp_path, change): + config_path, _ = configuration(tmp_path, ids=("softmax-cpu",)) + config = compare.read_json(config_path) + config.update(adapter=benchmark.ADAPTER) + config.update(change) + with pytest.raises(ValueError): + compare.validate(config, [benchmark.task_descriptor("softmax-cpu")]) + + +def test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable(tmp_path): + """Actual authored source replay; never represents an LLM or skill benefit.""" + output, digest, _ = prepared(tmp_path) + client = AuthoredProducer({("softmax-cpu", "baseline"): "wrong-output"}) + result = compare.execute(output, digest, client=client) + pair = result["pairs"][0] + if any(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" + for role in compare.ROLES): + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail("Required benchmark sandbox unavailable; no host fallback") + pytest.skip("Benchmark sandbox unavailable; two-arm numerical execution unqualified") + assert len(client.calls) == 2 + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert result["counts"]["fix"] == 1 + assert len(pair["candidate"]["checks"]["assertions"]) == 164 + assert not all(pair["baseline"]["checks"]["assertions"].values()) + for role in compare.ROLES: + trial = pair[role] + assert trial["evaluation_error"] is None + assert trial["grader_trust"] == "reverified" + assert "suite_latency_ms" not in json.dumps(trial["checks"]) + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py new file mode 100644 index 000000000..2fb7af8a8 --- /dev/null +++ b/tools/engineering_benchmark_validate.py @@ -0,0 +1,192 @@ +"""Reproduce CPU task integration with retained, authored controls. + +No models, hardware allocators or downloaded programs are invoked. Output is a +local validation bundle, not campaign acceptance or a skill-benefit experiment. +""" + +import argparse +import json +import shutil +import sys +import time +from pathlib import Path + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.skill_evaluation import run_trial +from inferrsi.infra.artifacts import atomic_write_json + + +class AuthoredControl: + """A local fixture emits a retained artifact identity, never a claimed score.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +def replay(task, candidate, directory): + descriptor = skill_verifier.task_descriptor(task, candidate) + workspace = directory / "workspace" + workspace.mkdir(parents=True) + atomic_write_json(workspace / "task.json", descriptor) + return run_trial( + AuthoredControl(descriptor["benchmark"]["identity"]["candidate_sha256"]), + descriptor, + "N", + 1, + workspace, + directory / "trial", + directory / "unused-schema.json", + Path(skill_verifier.__file__), + "Reviewed offline control", + time.monotonic() + 40, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + args = parser.parse_args() + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + records = [] + for name in builtin_tasks(): + # This historical replay utility has a NumPy-only candidate protocol. + if load_task(name).manifest["adapter"] != "numpy-axis1-v2": + continue + directory = output / name + directory.mkdir() + original = load_task(name) + shutil.copytree( + original.root, directory / "task", ignore=shutil.ignore_patterns("__pycache__") + ) + task = load_task(directory / "task") + candidate = directory / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + # Freeze the plan before any payload execution. + spec = task.runspec( + candidate, + repo=args.repo, + output_dir=directory / "workbench", + repetitions=args.repetitions, + max_minutes=2, + min_improvement_pct=0, + ) + atomic_write_json(directory / "runspec.json", spec.to_dict()) + atomic_write_json(directory / "identity.json", task.identity(candidate)) + grade = grade_artifact(task, candidate, directory / "correctness", timeout_seconds=20) + profile = run_action(task, candidate, "profile", directory / "profile", timeout_seconds=20) + run = run_experiment( + spec, {"scope": "CPU development integration", "identity": task.identity(candidate)} + ) + evaluation = json.loads((run / "evaluation.json").read_text(encoding="utf-8")) + good = replay(task, candidate, directory / "skilleval-good") + bad_source = directory / "wrong.py" + bad_source.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + bad = replay(task, bad_source, directory / "skilleval-bad") + rejected = run_action( + task, bad_source, "measure", directory / "wrong-measure", timeout_seconds=20 + ) + measured = [ + json.loads(p.read_text(encoding="utf-8")) + for side in ("baseline", "candidate") + for p in sorted((run / side).glob("trial-*/result.json")) + ] + consumers = measured + [ + json.loads((directory / path).read_text(encoding="utf-8")) + for path in ( + "correctness/result.json", + "profile/result.json", + "skilleval-good/trial/verification/artifact/result.json", + "skilleval-bad/trial/verification/artifact/result.json", + ) + ] + frozen_identity = task.identity(candidate) + record = { + "task": name, + "adapter": task.manifest["adapter"], + "identity": task.identity(candidate), + "correctness": grade["verdict"], + "profile_status": profile["status"], + "skilleval_good": good["verdict"], + "skilleval_bad": bad["verdict"], + "bad_measurement_status": rejected["status"], + "measurement_trials": len(measured), + "measured_calls": sum(len(c["samples_ms"]) for r in measured for c in r["cases"]), + "source_integrity": evaluation["source_integrity"]["verified"], + "descriptive_workbench_verdict": evaluation["verdict"], + "workbench_run": run.relative_to(output).as_posix(), + "all_consumer_identities_match": all( + all( + r["identity"][key] == frozen_identity[key] + for key in ("task_id", "task_sha256", "grader_sha256", "runtime_sha256") + ) + for r in consumers + ), + "all_workloads_match": all( + r["workloads"] == consumers[0]["workloads"] for r in consumers + ), + "input_sequence": consumers[0]["input_sequence"], + "all_input_sequences_match": all( + r["input_sequence"] == consumers[0]["input_sequence"] for r in consumers + ), + "all_valid_executed_calls_match": all( + r["executed_calls"] == consumers[0]["executed_calls"] + for r in consumers + if r["passed"] + ), + "expected_measured_calls": ( + 2 + * args.repetitions + * len(task.manifest["measurement"]["case_ids"]) + * task.manifest["measurement"]["samples"] + ), + } + records.append(record) + print(json.dumps(record), flush=True) + succeeded = all( + r["correctness"] == r["skilleval_good"] == "PASS" + and r["skilleval_bad"] == "FAIL" + and r["bad_measurement_status"] == "failed" + and r["profile_status"] == "completed" + and r["source_integrity"] + and r["all_consumer_identities_match"] + and r["all_workloads_match"] + and r["all_input_sequences_match"] + and r["all_valid_executed_calls_match"] + and r["measured_calls"] == r["expected_measured_calls"] + and r["measurement_trials"] == 2 * args.repetitions + for r in records + ) + atomic_write_json( + output / "summary.json", + { + "scope": "local CPU integration with authored controls", + "checks_succeeded": succeeded, + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + "tasks": records, + }, + ) + return 0 if succeeded else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/npu_benchmark_validate.py b/tools/npu_benchmark_validate.py new file mode 100644 index 000000000..831de7d25 --- /dev/null +++ b/tools/npu_benchmark_validate.py @@ -0,0 +1,208 @@ +"""Qualify a trusted NPU task with controls, profiling and existing Workbench. + +Run inside a caller-owned NPU allocation. All policy values are explicit CLI +inputs, frozen before execution. This is an authored software-control exercise, +not a model-generated optimization campaign or formal NPU SkillEval sandbox. +""" + +import argparse +import json +import math +import shutil +import statistics +import subprocess +from pathlib import Path + +from inferrsi.harness.benchmarks import load_builtin_task +from inferrsi.harness.benchmarks.consumers import run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.infra.artifacts import atomic_write_json + + +def read(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def validation_passed(validation): + result = validation.get("result", {}) + return (validation.get("execution_status") == "completed" + and validation.get("returncode") == 0 + and result.get("status") == "completed" and result.get("passed") is True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + parser.add_argument("--min-improvement-pct", type=float, required=True) + parser.add_argument("--max-aa-difference-pct", type=float, required=True) + parser.add_argument("--max-minutes", type=float, required=True) + parser.add_argument("--timeout-seconds", type=float, required=True) + args = parser.parse_args() + if args.repetitions < 2 or any(not math.isfinite(v) or v <= 0 for v in ( + args.min_improvement_pct, args.max_aa_difference_pct, + args.max_minutes, args.timeout_seconds, + )): + parser.error("Use at least two pairs and positive finite policy values") + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + for directory in ("plan", "validator", "patch", "results"): + (output / directory).mkdir() + task = load_builtin_task("rmsnorm-npu") + baseline = task.root / "baseline.py" + candidate = task.root / "candidate.py" + controls = { + "zeros": "return torch.zeros_like(x)", + "missing-epsilon": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True)) * weight", + "missing-weight": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps)", + "wrong-axis": "return x * torch.rsqrt((x*x).mean(0,keepdim=True) + eps) * weight", + "input-shape": "x.squeeze_(0); " + "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight", + } + wrong_paths = {} + for name, body in controls.items(): + path = output / "patch" / f"{name}.py" + path.write_text(f"import torch\ndef compute(x, weight, *, eps):\n {body}\n") + wrong_paths[name] = path + # A correct precheck followed by stale output must also fail before scoring. + replay = output / "patch" / "replay.py" + replay.write_text( + "import torch\ncache = {}\n" + "def compute(x, weight, *, eps):\n" + " key = tuple(x.shape)\n" + " if key not in cache:\n" + " cache[key] = x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight\n" + " return cache[key]\n" + ) + wrong_paths["replay"] = replay + policy = {key: getattr(args, key) for key in ( + "repetitions", "min_improvement_pct", "max_aa_difference_pct", + "max_minutes", "timeout_seconds", + )} + implementation_paths = {"baseline": baseline, "reference": candidate, **wrong_paths} + frozen = { + "scope": "NPU software controls; no model/skill benefit or service claim", + "source_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=args.repo, text=True + ).strip(), + "policy": policy, + "implementations": {name: task.identity(path) + for name, path in implementation_paths.items()}, + } + atomic_write_json(output / "plan" / "frozen.json", frozen) + atomic_write_json(output / "validator" / "task.json", task.manifest) + for path in (baseline, candidate): + shutil.copyfile(path, output / "patch" / path.name) + actions = {} + + def action(name, path, mode): + execution = run_action(task, path, mode, output / "results" / name, + timeout_seconds=args.timeout_seconds) + result = read(execution["result_path"]) + record = {"implementation": next(key for key, value in implementation_paths.items() + if value == path), + "execution_status": execution["status"], + "returncode": execution["returncode"], "result": result} + actions[name] = record + print(json.dumps({"action": name, "status": execution["status"], + "passed": result.get("passed")}), flush=True) + return record + + for name, path in (("baseline", baseline), ("reference", candidate)): + action(name + "-check", path, "check") + action(name + "-profile", path, "profile") + for name, path in wrong_paths.items(): + action(name, path, "measure") + comparisons = {} + # Same implementation in both arms is the explicit no-op/A-A control. + for name, path in (("aa", baseline), ("reference", candidate)): + spec = task.runspec(path, repo=args.repo, output_dir=output / "results" / name, + repetitions=args.repetitions, max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct) + run = run_experiment(spec, {"benchmark": task.identity(path), + "backend": "ascend", "qualification": "pending"}) + evaluation = read(run / "evaluation.json") + validation_dir = run / "validation" + execution = (read(validation_dir / "execution.json") + if (validation_dir / "execution.json").is_file() else {}) + validated = (read(validation_dir / "result.json") + if (validation_dir / "result.json").is_file() else {}) + validation = {"execution_status": execution.get("status"), + "returncode": execution.get("returncode"), "result": validated} + trials = {side: [read(p) for p in sorted((run / side).glob("trial-*/result.json"))] + for side in ("baseline", "candidate")} + comparisons[name] = {"run": run.relative_to(output).as_posix(), + "evaluation": evaluation, "trials": trials, + "validation": validation} + print(json.dumps({"comparison": name, "verdict": evaluation["verdict"]}), flush=True) + + def correct(name): + record = actions[name] + return (record["execution_status"] == "completed" and record["returncode"] == 0 + and record["result"].get("status") == "completed" + and record["result"].get("passed") is True) + + checks = {"correctness": all(correct(name + "-check") for name in ("baseline", "reference")), + "device_profiles": all(correct(name + "-profile") + for name in ("baseline", "reference")), + "reject_wrong_outputs": all(actions[name]["execution_status"] == "failed" + and actions[name]["returncode"] == 1 + and actions[name]["result"].get("status") == "completed" + and actions[name]["result"].get("passed") is False + and "metrics" not in actions[name]["result"] for name in wrong_paths)} + all_results = [record["result"] for record in actions.values()] + all_results.extend(trial for comparison in comparisons.values() + for trials in comparison["trials"].values() for trial in trials) + all_results.extend(comparison["validation"]["result"] for comparison in comparisons.values()) + environments = [result.get("environment") for result in all_results] + checks["fixed_environment"] = bool(environments) and all( + env and env == environments[0] for env in environments + ) + identity_checks = [record["result"].get("identity") + == frozen["implementations"][record["implementation"]] + for record in actions.values()] + for name, comparison in comparisons.items(): + selected = "baseline" if name == "aa" else "reference" + for side, trials in comparison["trials"].items(): + expected = frozen["implementations"]["baseline" if side == "baseline" else selected] + identity_checks.extend(trial.get("identity") == expected for trial in trials) + identity_checks.append(comparison["validation"]["result"].get("identity") + == frozen["implementations"][selected]) + identity_checks.extend(task.identity(path) == frozen["implementations"][name] + for name, path in implementation_paths.items()) + checks["frozen_identities"] = all(identity_checks) + task.assert_unchanged() + shape_comparisons = {} + for name, comparison in comparisons.items(): + by_shape = {} + valid = validation_passed(comparison["validation"]) and all(len(trials) == args.repetitions + and all(trial.get("passed") and "cases" in trial for trial in trials) + for trials in comparison["trials"].values()) + for case in task.manifest["measurement"]["case_ids"] if valid else []: + medians = {side: statistics.median( + next(c["median_ms"] for c in trial["cases"] if c["id"] == case) + for trial in trials) for side, trials in comparison["trials"].items()} + medians["improvement_pct"] = (1 - medians["candidate"] / medians["baseline"]) * 100 + by_shape[case] = medians + shape_comparisons[name] = by_shape + checks[name + "_qualified"] = valid and bool(by_shape) and ( + all(abs(values["improvement_pct"]) <= args.max_aa_difference_pct + for values in by_shape.values()) and comparison["evaluation"]["verdict"] == "FAIL" + if name == "aa" else all(values["improvement_pct"] >= args.min_improvement_pct + for values in by_shape.values()) + and comparison["evaluation"]["verdict"] == "PASS" + ) + summary = {"scope": frozen["scope"], "frozen": frozen, "checks": checks, + "software_qualification": "PASS" if all(checks.values()) else "INCONCLUSIVE", + "campaign_acceptance": "not established", "agent_benefit": "not measured", + "shape_comparisons": shape_comparisons, + "actions": actions, "comparisons": comparisons} + atomic_write_json(output / "summary.json", summary) + print(json.dumps({"checks": checks, + "software_qualification": summary["software_qualification"]})) + return 0 if all(checks.values()) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index b5c0fb3e5..be75dfea8 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -12,8 +12,10 @@ from dataclasses import replace from pathlib import Path +from inferrsi.harness import skill_eval_benchmark as benchmark from inferrsi.harness import skill_eval_diagnostic as diagnostic from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks.contract import load_task from inferrsi.harness.skill_evaluation import ( CodexClient, export_bundle, @@ -38,6 +40,7 @@ ROOT = Path(__file__).resolve().parents[1] ROLES = ("baseline", "candidate") +ARTIFACT_ADAPTERS = {engineering.ADAPTER: engineering, benchmark.ADAPTER: benchmark} GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "tools/skill_eval_artifact_guidance.py", @@ -188,9 +191,9 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> "adapter", "execution_mode"}: raise ValueError("Unknown configuration field") adapter = config.get("adapter", "audit") - if adapter not in ("audit", diagnostic.ADAPTER, engineering.ADAPTER): + if adapter not in ("audit", diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): raise ValueError("Unknown experiment adapter") - if adapter in (diagnostic.ADAPTER, engineering.ADAPTER): + if adapter in (diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): if config["mode"] != "update": raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") if "grader" in config or "schema" in config: @@ -201,7 +204,7 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if generation_only and (not isinstance(config.get("client"), dict) or config["client"].get("kind") != FULL_CONTEXT_KIND): raise ValueError("Generation-only requires the bounded full-context provider") - if adapter == engineering.ADAPTER and "client" in config and not generation_only: + if adapter in ARTIFACT_ADAPTERS and "client" in config and not generation_only: raise ValueError("Engineering artifacts require local execution, not read/submit provider") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") @@ -236,8 +239,8 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if not tasks: raise ValueError("cases must be nonempty with unique IDs") for task in tasks: - if adapter == engineering.ADAPTER: - engineering.validate_task(task) + if adapter in ARTIFACT_ADAPTERS: + ARTIFACT_ADAPTERS[adapter].validate_task(task) elif adapter == diagnostic.ADAPTER: diagnostic.validate_task(task) else: @@ -326,7 +329,9 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: tasks = read_json(config_path.parent / config["cases"]) is_diagnostic = config.get("adapter") == diagnostic.ADAPTER is_engineering = config.get("adapter") == engineering.ADAPTER - grader = (Path(engineering.__file__) if is_engineering else + is_benchmark = config.get("adapter") == benchmark.ADAPTER + artifact_adapter = ARTIFACT_ADAPTERS.get(config.get("adapter")) + grader = (Path(artifact_adapter.__file__) if artifact_adapter else Path(diagnostic.__file__) if is_diagnostic else config_path.parent / config["grader"] if "grader" in config else ROOT / "tools/skill_eval_verifier.py") @@ -335,7 +340,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) generation_only = config.get("execution_mode") == engineering.GENERATION_MODE schema = (engineering.GENERATION_SCHEMA if generation_only else - engineering.SCHEMA if is_engineering else + artifact_adapter.SCHEMA if artifact_adapter else diagnostic.SCHEMA if is_diagnostic else read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: @@ -349,11 +354,19 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: if is_engineering: (output / "plan/starter").mkdir() (output / "plan/starter/parser.py").write_bytes(engineering.STARTER.encode("utf-8")) + if is_benchmark: + for task in tasks: + package = benchmark.task_package(task["benchmark"]["task_id"]) + shutil.copytree(package.root, output / "plan/benchmark-tasks" / task["id"], + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + starter.parent.mkdir(parents=True) + starter.write_bytes(benchmark.starter(task)) arms = {} with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): selected = config["snapshots"][role] - if is_diagnostic or is_engineering: + if is_diagnostic or artifact_adapter: closure = selected_snapshot( repo, selected["git_revision"], config["skill_root"], config["skill"], Path(temporary) / short, bundle_revision=selected.get("bundle_revision")) @@ -393,7 +406,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], "prompt": (engineering.GENERATION_PROMPT if generation_only else - engineering.PROMPT if is_engineering else + artifact_adapter.PROMPT if artifact_adapter else diagnostic.PROMPT if is_diagnostic else PROMPT), "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, @@ -403,14 +416,25 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: } if is_diagnostic: plan["adapter"] = diagnostic.ADAPTER - if is_engineering: - plan["adapter"] = engineering.ADAPTER - name = "src/inferrsi/harness/skill_eval_engineering.py" - plan["controllers"][name] = sha256(ROOT / name) + if artifact_adapter: + plan["adapter"] = artifact_adapter.ADAPTER local_source = "tools/skill_eval_local.py" plan["controllers"][local_source] = sha256(ROOT / local_source) plan["isolation"] += (" Artifact replay requires bubblewrap namespaces and fails closed. " - "Public tests, not blind evaluation; reconstructed CPU fixture.") + "Public CPU development checks, not blind evaluation.") + if is_benchmark: + dependencies = [ROOT / "src/inferrsi/harness/skill_eval_benchmark.py"] + dependencies += list((ROOT / "src/inferrsi/harness/benchmarks").glob("*.py")) + for task in tasks: + package = benchmark.task_package(task["benchmark"]["task_id"]) + dependencies += [p for p in package.root.rglob("*") + if p.is_file() and "__pycache__" not in p.parts + and p.suffix != ".pyc"] + for path in dependencies: + plan["controllers"][path.relative_to(ROOT).as_posix()] = sha256(path) + plan["isolation"] += ( + " Independent oracle stays in the parent; only inputs and implementation enter " + "the NumPy artifact sandbox. Correctness only; no performance or skill-benefit claim.") if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], config["limits"]["timeout_seconds"], @@ -500,6 +524,20 @@ def load_plan(output: Path) -> dict: for name, path in frozen_exam.items(): if plan["exam"].get(name) != plan["files"].get(path): raise ValueError(f"Frozen exam hash disagrees with the frozen file list: {name}") + if plan.get("adapter") == benchmark.ADAPTER: + if (plan["mode"] != "update" or "client" in plan or "execution_mode" in plan + or plan["exam"]["grader"] != sha256(Path(benchmark.__file__)) + or read_json(output / "plan/schema.json") != benchmark.SCHEMA + or plan["prompt"] != benchmark.PROMPT): + raise ValueError("Frozen benchmark adapter contract changed") + for task in tasks: + benchmark.validate_task(task) + package = load_task(output / "plan/benchmark-tasks" / task["id"]) + if package.task_sha256 != task["benchmark"]["task_sha256"]: + raise ValueError("Frozen benchmark package disagrees with task identity") + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + if starter.read_bytes() != benchmark.starter(task): + raise ValueError("Frozen benchmark starter changed") return plan @@ -579,7 +617,7 @@ def execute(output: Path, authorization: str, *, client=None, stop_on_unknown_usage = True if plan.get("execution_mode") == engineering.GENERATION_MODE and "client" not in plan: raise ValueError("Generation-only requires the frozen bounded provider") - if plan.get("adapter") == engineering.ADAPTER and client is None: + if plan.get("adapter") in ARTIFACT_ADAPTERS and client is None: raise ValueError("Engineering execution requires a verified confined local client; " "the default danger-full-access Codex client is not qualified") if client is None: @@ -613,9 +651,12 @@ def execute(output: Path, authorization: str, *, client=None, with tempfile.TemporaryDirectory(prefix="se-trial-") as temporary: workspace = Path(temporary) write_json(workspace / "task.json", task) - is_engineering = plan.get("adapter") == engineering.ADAPTER - if is_engineering: + artifact_adapter = ARTIFACT_ADAPTERS.get(plan.get("adapter")) + if artifact_adapter is engineering: shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + elif artifact_adapter is benchmark: + shutil.copyfile(output / "plan/starters" / task["id"] / benchmark.ARTIFACT, + workspace / benchmark.ARTIFACT) prompt = plan["prompt"] + "\nTask operation: " + task["request"] if bundle is not None: folder = ".agents/skills" if task["split"] == "discovery" else "knowledge" @@ -627,7 +668,7 @@ def execute(output: Path, authorization: str, *, client=None, directory, output / "plan/schema.json", output / "validator/verify.py", prompt, deadline, timeout_seconds=plan["limits"]["timeout_seconds"], - engineering_artifact=is_engineering, + artifact_adapter=artifact_adapter, generation_only=(plan.get("execution_mode") == engineering.GENERATION_MODE)) result["evidence_sha256"] = {name: sha256(directory / name) for name in EVIDENCE @@ -656,6 +697,9 @@ def execute(output: Path, authorization: str, *, client=None, def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_mode=None) -> dict: + artifact_adapter = ARTIFACT_ADAPTERS.get(expected_adapter) + artifact_name = (getattr(artifact_adapter, "ARTIFACT", "parser.py") + if artifact_adapter else "parser.py") directory = trial_directory(output, entry) path = directory / "result.json" if not path.is_file(): @@ -664,10 +708,10 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m "evaluation_error": "interrupted_attempt" if interrupted else None, "missing": not interrupted, "checks": None, "output": None, "evidence": {}, "elapsed_seconds": None, "observations": {"usage": None}} - if expected_adapter == engineering.ADAPTER: + if artifact_adapter: # A retained tree is untrusted input too: reject special/symlink evidence # before legacy report readers open it. Producers must already be stopped. - names = ("result.json", "receipt.json", "answer.json", "parser.py", + names = ("result.json", "receipt.json", "answer.json", artifact_name, "author-execution.json", *EVIDENCE) try: for name in names: @@ -702,6 +746,9 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m # Select report fields; raw commands and private workspace paths stay in local artifacts. selected = {key: record[key] for key in ("task_id", "condition", "repeat", "split", "verdict", "checks", "elapsed_seconds", "observations", "inputs_unchanged")} + if artifact_adapter is benchmark: + selected["artifact_kind"] = record.get("artifact_kind") + selected["artifact_sha256"] = record.get("artifact_sha256") # Legacy local imports used [] for unparsed logs, not an observed zero. unparsed_author_log = (record.get("execution_state") == "author_reported" and record["observations"].get("completed_turns") is None) @@ -713,9 +760,8 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m } record_error = record.get("evaluation_error") # Artifact integrity failures take precedence over a replayable host error. - error = None if expected_adapter == engineering.ADAPTER else record_error - if (expected_adapter == engineering.ADAPTER - and record.get("artifact_kind") != engineering.ADAPTER): + error = None if artifact_adapter else record_error + if (artifact_adapter and record.get("artifact_kind") != expected_adapter): error = error or "missing_retained_artifact_identity" if execution_mode == engineering.GENERATION_MODE: if record.get("execution_mode") != execution_mode: @@ -757,14 +803,14 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m error = error or "retained_evidence_hash_mismatch" elif name == "author-execution.json": evidence[name] = artifact.relative_to(output).as_posix() - if record.get("artifact_kind") == engineering.ADAPTER and record.get("artifact_sha256") is None: + if artifact_adapter and record.get("artifact_sha256") is None: error = error or "missing_retained_artifact" if record.get("artifact_sha256") is not None: try: artifact_answer = json.loads(data) if data is not None else None except ValueError: artifact_answer = None - artifact = directory / "parser.py" + artifact = directory / artifact_name if artifact.is_symlink() or not artifact.is_file(): error = error or "missing_retained_artifact" elif sha256(artifact) != record["artifact_sha256"]: @@ -773,7 +819,7 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m or artifact_answer.get("artifact_source") != artifact.read_bytes().decode("utf-8")): error = error or "retained_artifact_disagrees_with_answer" else: - evidence["parser.py"] = artifact.relative_to(output).as_posix() + evidence[artifact_name] = artifact.relative_to(output).as_posix() error = error or record_error if (error or record["execution_state"] not in ("completed", "author_reported") or not record["inputs_unchanged"]): @@ -868,14 +914,14 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "fix", "regression", "unchanged_correct", "unchanged_incorrect", "missing_pair", "inconclusive", *( ("artifact_improved", "artifact_regressed", "artifacts_both_pass", "artifacts_both_fail") - if plan.get("adapter") == engineering.ADAPTER else ()))}, + if plan.get("adapter") in ARTIFACT_ADAPTERS else ()))}, "discovery": discovery, "discovery_conditions": condition_summary(discovery), "status": status, "claim": "Pipeline results only; fixed-output offline clients do not establish " "skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.", } - if plan.get("adapter") == engineering.ADAPTER: - result["adapter"] = engineering.ADAPTER + if plan.get("adapter") in ARTIFACT_ADAPTERS: + result["adapter"] = plan["adapter"] if any(trial.get("author_reported_execution") for trial in trials): captured = sum(bool(trial.get("author_reported_execution")) for trial in trials) result["status"] = {"execution_state": "author_reported_unverified", @@ -887,6 +933,15 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "usage and costs are self-reported, not independently authenticated. Public tests " "may be overfit. Paired artifact differences do not prove skill causality, original " "engine behavior, hardware efficacy or whole-PR acceptance. Reviewer decides merge.") + if plan.get("adapter") == benchmark.ADAPTER: + result["claim"] = ( + "Shared CPU benchmark artifact correctness only. The trusted parent checks actual " + "outputs with the shared oracle; candidate source runs only in bubblewrap. " + "Public development inputs are not holdout. Authored controls and paired artifact " + "differences do not establish skill causality, reliable performance ranking, " + "hardware qualification or service gains. Author-reported provenance/usage remain " + "unverified; absent usage and cost remain unknown.") + result["benchmark_tasks"] = read_json(output / "plan/tasks.json") if plan.get("execution_mode") == engineering.GENERATION_MODE: result["execution_mode"] = engineering.GENERATION_MODE result["claim"] = ( @@ -917,6 +972,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: sha256(ROOT / "tools/skill_eval_verifier.py"): verify, sha256(Path(diagnostic.__file__)): diagnostic.verify, sha256(Path(engineering.__file__)): engineering.verify, + sha256(Path(benchmark.__file__)): benchmark.verify, }.get(plan["exam"]["grader"]) if trusted_verifier is None: for trial in trials: @@ -931,7 +987,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["evaluation_error"] = "unverified_custom_grader" return for trial in trials: - manual_artifact = (trusted_verifier is engineering.verify + manual_artifact = (trusted_verifier in (engineering.verify, benchmark.verify) and trial.get("author_reported_execution") and trial.get("evaluation_error") in ( None, "reported_agent_failure", "artifact_sandbox_unavailable", @@ -950,7 +1006,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: if trial["verdict"] not in ("PASS", "FAIL") and not replay_artifact: continue try: - checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + verified = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + checks = {key: verified[key] for key in ("passed", "assertions")} except engineering.SandboxUnavailable: trial["verdict"] = "INCONCLUSIVE" trial["grader_trust"] = "unverified" @@ -972,6 +1029,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: # without rewriting capture failure or upgrading pair comparability. trial["artifact_checks"] = checks trial["artifact_verdict"] = "PASS" if checks["passed"] else "FAIL" + if trusted_verifier is benchmark.verify and "benchmark" in verified: + trial["benchmark"] = verified["benchmark"] trial["grader_trust"] = "reverified" diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py index 44d85a222..0b26aa8ed 100644 --- a/tools/skill_eval_local.py +++ b/tools/skill_eval_local.py @@ -13,18 +13,27 @@ from pathlib import Path from inferrsi.harness import skill_eval_engineering as engineering -from inferrsi.harness.skill_evaluation import normalize_usage +from inferrsi.harness.skill_evaluation import normalize_usage, valid_checks from inferrsi.infra.artifacts import sha256, write_json +def _adapter(plan): + if plan.get("adapter") == engineering.ADAPTER: + return engineering + from inferrsi.harness import skill_eval_benchmark as benchmark + + if plan.get("adapter") == benchmark.ADAPTER: + return benchmark + raise ValueError("Local artifact handoff requires a supported engineering artifact adapter") + + def _selection(output, authorization, task_id, condition, repeat, current_head): from tools import skill_eval_compare as compare if authorization != sha256(output / "plan/freeze.json"): raise ValueError("Local handoff requires the exact frozen plan SHA-256") plan = compare.load_plan(output) - if plan.get("adapter") != engineering.ADAPTER: - raise ValueError("Local artifact handoff is only for the engineering adapter") + _adapter(plan) if current_head != plan["arms"]["candidate"]["git_revision"]: raise ValueError("Current PR head differs from the frozen candidate; revalidate first") selected = [entry for entry in plan["schedule"] if entry["task_id"] == task_id @@ -54,7 +63,7 @@ def _read_regular(path, bound=8 * 1024 * 1024): return data -def _hashes(workspace): +def _hashes(workspace, *, editable="parser.py"): result = {} for path in workspace.rglob("*"): if path.is_symlink(): @@ -62,7 +71,7 @@ def _hashes(workspace): if path.is_dir(): continue name = path.relative_to(workspace).as_posix() - if name != "parser.py": + if name != editable: result[name] = hashlib.sha256(_read_regular(path)).hexdigest() return result @@ -92,6 +101,8 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat workspace: Path, current_head: str): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") if directory.exists() or workspace.exists(): raise ValueError("Stage requires fresh trial and workspace; never overwrite an attempt") # Keep writable work separate from retained evidence/validator and the other arm. @@ -101,7 +112,9 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat # Frozen inputs are LF bytes on every host; never apply platform translation. (workspace / "task.json").write_bytes( (json.dumps(task, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8")) - shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + starter = (output / "plan/starter/parser.py" if adapter is engineering else + output / "plan/starters" / task["id"] / artifact_name) + shutil.copyfile(starter, workspace / artifact_name) bundle = plan["arms"][condition]["bundle"] prompt = _prompt(plan, task, condition) if bundle: @@ -109,11 +122,13 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat (workspace / "INSTRUCTIONS.txt").write_bytes(prompt.encode("utf-8")) directory.mkdir(parents=True) write_json(directory / "local-stage.json", { - "plan_sha256": authorization, "entry": entry, "input_hashes": _hashes(workspace), + "plan_sha256": authorization, "entry": entry, + "input_hashes": _hashes(workspace, editable=artifact_name), "provenance": "author-run; execution not observed by controller", }) return {"entry": entry, "plan_sha256": authorization, - "next": "Run approved local agent; only parser.py is writable. Keep logs separately. " + "next": f"Run approved local agent; only {artifact_name} is writable. " + "Keep logs separately. " "Capture does not authorize or pay for that run."} @@ -121,6 +136,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe workspace: Path, current_head: str, submission: Path): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") stage_record = json.loads(_read_regular(directory / "local-stage.json")) if stage_record["plan_sha256"] != authorization or stage_record["entry"] != entry: raise ValueError("Stage receipt does not match the frozen trial") @@ -153,21 +170,22 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe error = None try: expected = _expected_inputs(plan, task, condition) - unchanged = (_hashes(workspace) == expected + unchanged = (_hashes(workspace, editable=artifact_name) == expected and stage_record["input_hashes"] == expected) except ValueError: unchanged = False if not unchanged: error = "inputs_changed" - artifact = workspace / "parser.py" + artifact = workspace / artifact_name source = None artifact_bytes = None - if artifact.is_symlink() or not artifact.is_file() or artifact.stat().st_size > 65536: + if (artifact.is_symlink() or not artifact.is_file() + or artifact.stat().st_size > adapter.MAX_SOURCE_BYTES): error = error or "missing_or_invalid_artifact" else: try: - artifact_bytes = _read_regular(artifact, 65536) - if len(artifact_bytes) > 65536: + artifact_bytes = _read_regular(artifact, adapter.MAX_SOURCE_BYTES) + if len(artifact_bytes) > adapter.MAX_SOURCE_BYTES: raise UnicodeError("artifact grew beyond bound") source = artifact_bytes.decode("utf-8") except (UnicodeError, ValueError): @@ -180,22 +198,28 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe (directory / "agent").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "agent" / name).write_bytes(submitted[name]) - answer = {"artifact": "parser.py", "summary": metadata["summary"], + answer = {"artifact": artifact_name, "summary": metadata["summary"], "artifact_source": source} write_json(directory / "answer.json", answer) if source is not None: - (directory / "parser.py").write_bytes(artifact_bytes) - checks = None + (directory / artifact_name).write_bytes(artifact_bytes) + checks, raw_checks = None, None if error is None: try: - checks = engineering.verify(task, answer) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, answer) + except adapter.SandboxUnavailable: error = "artifact_sandbox_unavailable" + else: + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if valid_checks(proposed): + checks = proposed + else: + error = "invalid_verifier_result" (directory / "verification").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "verification" / name).write_text("", encoding="utf-8") - if checks is not None: - write_json(directory / "verification/result.json", checks) + if raw_checks is not None: + write_json(directory / "verification/result.json", raw_checks) artifact_verdict = None if checks is None else "PASS" if checks["passed"] else "FAIL" if metadata["agent_exit_code"] != 0: error = error or "reported_agent_failure" @@ -211,8 +235,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe "execution_state": "author_reported", "inputs_unchanged": unchanged, "evaluation_error": error, "input_hashes": stage_record["input_hashes"], "output_sha256": sha256(directory / "answer.json"), - "artifact_kind": engineering.ADAPTER, - "artifact_sha256": sha256(directory / "parser.py") if source is not None else None, + "artifact_kind": adapter.ADAPTER, + "artifact_sha256": sha256(directory / artifact_name) if source is not None else None, "provenance": "author-supplied; agent execution and costs not independently verified", "artifact_verdict": artifact_verdict, "evidence_sha256": {name: sha256(directory / name) for name in compare.EVIDENCE