From d0ce9d785e1bead9124eed1cf3f6e9f2e0cf05db Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Tue, 3 Nov 2026 22:52:10 +0800 Subject: [PATCH 01/20] feat: share engineering benchmark tasks across consumers Add frozen task packages, independent CPU oracles and profiling, with bindings to the existing Workbench and offline SkillEval verifier. Include research decisions, reproducible integration checks and explicit qualification limits. Agent-assisted-by: Codex (implementation and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/architecture.md | 6 + docs/engineering-benchmarks.md | 87 +++++ docs/operations.md | 5 + docs/plans/issue-252/01-research.md | 52 +++ docs/plans/issue-252/02-contract.md | 35 +++ docs/plans/issue-252/03-tasks.md | 36 +++ docs/plans/issue-252/04-integration.md | 29 ++ docs/plans/issue-252/README.md | 49 +++ docs/plans/issue-252/validation-summary.json | 53 ++++ docs/plans/issue-252/validation.md | 54 ++++ pyproject.toml | 5 +- src/inferrsi/arguments.py | 3 + src/inferrsi/cli.py | 11 + .../rmsnorm-cpu/LICENSE | 22 ++ .../rmsnorm-cpu/README.md | 26 ++ .../rmsnorm-cpu/baseline.py | 8 + .../rmsnorm-cpu/candidate.py | 11 + .../rmsnorm-cpu/task.json | 121 +++++++ .../softmax-cpu/LICENSE | 22 ++ .../softmax-cpu/README.md | 26 ++ .../softmax-cpu/baseline.py | 8 + .../softmax-cpu/candidate.py | 10 + .../softmax-cpu/task.json | 109 +++++++ src/inferrsi/harness/benchmarks/__init__.py | 5 + src/inferrsi/harness/benchmarks/cli.py | 82 +++++ src/inferrsi/harness/benchmarks/consumers.py | 69 ++++ src/inferrsi/harness/benchmarks/contract.py | 262 ++++++++++++++++ src/inferrsi/harness/benchmarks/numpy_task.py | 168 ++++++++++ .../harness/benchmarks/skill_verifier.py | 55 ++++ src/inferrsi/harness/benchmarks/worker.py | 60 ++++ tests/check_installed.py | 9 + tests/test_engineering_benchmarks.py | 296 ++++++++++++++++++ tools/engineering_benchmark_validate.py | 170 ++++++++++ 33 files changed, 1962 insertions(+), 2 deletions(-) create mode 100644 docs/engineering-benchmarks.md create mode 100644 docs/plans/issue-252/01-research.md create mode 100644 docs/plans/issue-252/02-contract.md create mode 100644 docs/plans/issue-252/03-tasks.md create mode 100644 docs/plans/issue-252/04-integration.md create mode 100644 docs/plans/issue-252/README.md create mode 100644 docs/plans/issue-252/validation-summary.json create mode 100644 docs/plans/issue-252/validation.md create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json create mode 100644 src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE create mode 100644 src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md create mode 100644 src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py create mode 100644 src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py create mode 100644 src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json create mode 100644 src/inferrsi/harness/benchmarks/__init__.py create mode 100644 src/inferrsi/harness/benchmarks/cli.py create mode 100644 src/inferrsi/harness/benchmarks/consumers.py create mode 100644 src/inferrsi/harness/benchmarks/contract.py create mode 100644 src/inferrsi/harness/benchmarks/numpy_task.py create mode 100644 src/inferrsi/harness/benchmarks/skill_verifier.py create mode 100644 src/inferrsi/harness/benchmarks/worker.py create mode 100644 tests/test_engineering_benchmarks.py create mode 100644 tools/engineering_benchmark_validate.py diff --git a/docs/architecture.md b/docs/architecture.md index 6589edcc9..1f47bfdaf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,5 +1,11 @@ # Modular architecture +The [shared engineering benchmark adapter](engineering-benchmarks.md) lives in +`harness/benchmarks`, with task data in `data/engineering_benchmarks`. It binds +task-owned oracles and workloads to the existing Workbench runner and offline +SkillEval verifier contract. It adds neither a search controller nor another +execution platform; its first qualified scope is the locally checked CPU subset. + The local experiment path now has concrete Harness, Model and Infra modules. The CLI, RunSpec, knowledge retrieval and existing command contracts remain stable. The diagrams below describe implemented code, with the historical compatibility diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md new file mode 100644 index 000000000..4a290d5b7 --- /dev/null +++ b/docs/engineering-benchmarks.md @@ -0,0 +1,87 @@ +# Shared engineering benchmarks + +The `benchmarks` command exposes versioned task packages for Workbench and +offline artifact evaluation. The initial `numpy-axis1-v1` adapter supplies +Softmax and RMSNorm CPU adaptations of attributed KernelBench tasks. Packages +own semantics, workload, oracle and metric. Consumers own comparison conditions; +search policy and agent execution remain separate. + +## Use the tasks + +Install with `pip install -e ".[dev,benchmarks]"`. Discovery and planning do not +import NumPy; execution requires it and never silently switches backend. + +```bash +inferrsi benchmarks list +inferrsi benchmarks show softmax-cpu +inferrsi benchmarks check softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/check --timeout-seconds 30 +inferrsi benchmarks profile softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/profile --timeout-seconds 30 +inferrsi benchmarks measure softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/measurement --timeout-seconds 30 +inferrsi benchmarks grade softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/grade --timeout-seconds 30 +inferrsi benchmarks plan softmax-cpu --candidate /path/to/implementation.py --repo /path/to/git-checkout --output-dir /path/to/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 +``` + +Use `compare` instead of `plan` to execute the emitted RunSpec using existing +Workbench. Single-action output directories must be new. These are example +budgets, not campaign acceptance gates. The supplied Git checkout identifies +the source context; task and candidate files are additionally bound by hashes. + +The candidate is a reviewed self-contained Python file with `compute(x, *, eps)`. +Use NumPy, math and local helpers. `load_task("softmax-cpu").root` locates its +baseline and example candidate. Copy `candidate.py` to `implementation.py` before +editing. The package and grader remain protected comparison inputs. This adapter +executes trusted source and is **not an untrusted-code sandbox**. Hash checks +detect drift, not malicious same-process code. Do not connect raw model output. + +## Measurement and evidence + +`TaskPackage` validates the schema/adapter/environment and freezes manifest, +package and grader identities. Commands bind candidate and runtime fingerprints. +The worker checks bindings before and after execution. Runtime captures Python, +NumPy version, OS and machine architecture; it does not prove an uncontended host +or a fully locked native dependency installation. + +Finite float32 inputs include strided layouts. RMSNorm reduces axis 1, including +for four-dimensional arrays. Scalar double precision with math.fsum supplies the +independent oracle. Numerical tolerances, shapes, seeds, warmups and sample counts +are task-specific and fixed across both consumers. + +`check` evaluates all public cases. `measure` checks first, times real synchronous +compute calls, and verifies every measured output outside the timer. Input +generation, oracle, validation, startup and warmups are excluded; allocations +inside compute are included. Every case retains its samples in milliseconds. +The objective sums case medians for this fixed suite; it is not a universal +score. Inspect per-case regressions before adopting a candidate. Repeated +warm-cache calls do not characterize cold-start serving. + +`profile` retains a separate CPU `profile.txt` with no performance metric/verdict. +It is not a device timeline. Hardware adapters need their own synchronization, +timing and qualification. Missing or failed execution never becomes qualification. + +`grade` produces artifact-correctness PASS/FAIL/INCONCLUSIVE and SkillEval's +existing `passed`/`assertions` structure. `skill_verifier.task_descriptor` binds a +controller-reviewed retained artifact; `skill_verifier.py` plugs into the existing +`skill_evaluation.run_trial`. The answer supplies only `candidate_sha256`. +The shared oracle independently grades actual implementation behavior. This +offline handoff does not modify production engineering sandbox or paid execution. +Live artifact production and skill/search benefit need separate controlled runs. + +Exceptions, missing dependencies, drift and timeouts leave incomplete evidence. +Wrong results emit no usable performance metric. Workbench statistics remain +descriptive; a local verdict is not service gain or campaign acceptance. + +## Reproduce the integration + +```bash +python tools/engineering_benchmark_validate.py --repo /path/to/checkout --output-dir /path/to/new-validation --repetitions 3 +``` + +The tool freezes copied tasks and candidate identities, executes both CPU tasks, +retains profiles and AB/BA samples, and runs the actual SkillEval trial path with +correct and deliberately wrong **authored controls**. No model is called. +`summary.json` separates integration checks from campaign acceptance. Local raw +bundles contain resolved paths; review/redact a copy before sharing. + +See [#252 research and subtask plans](plans/issue-252/README.md). Formal Issue +acceptance, independent review, accelerator qualification, communication and +memory-capacity coverage remain distinct from this local CPU subset. diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..181e08f4b 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -1,5 +1,10 @@ # Operation guide +Shared engineering tasks use `inferrsi benchmarks list|show|plan` for read-only +inspection and `check|measure|profile|grade|compare` for explicit local execution. +See the [engineering benchmark contract](engineering-benchmarks.md) for trusted +source requirements, task identities, CPU coverage and offline SkillEval reuse. + One concise map from an agent's intent to the supported entry point. This guide is navigation only: detailed contracts stay with their owning documents, and it is not a third primary matrix. The two authoritative matrices remain the diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md new file mode 100644 index 000000000..d90a10745 --- /dev/null +++ b/docs/plans/issue-252/01-research.md @@ -0,0 +1,52 @@ +# 子项 01：调研与方案决策 + +## 理解 + +研究目标是选出适合现有 InferRSI 的最小可用组合。不能把一个外部 benchmark 的评分、runner、agent 和 CUDA 假设整体搬进来。 + +## 计划 + +1. 重读 #252 正文与两条评论，区分已要求的任务契约和独立 policy RFC。 +2. 核对 KernelBench、AgentKernelArena、BackendBench、SkillsBench 的论文和源码边界。 +3. 用 AlphaEvolve、AIDE 检查任务评估和搜索策略的职责划分。 +4. 核对现有 RunSpec、Workbench 和 SkillEval，选择最薄的接入方式。 +5. 记录固定来源、许可证、适配变化以及未采用原因；让实现和反例验证检验决策。 + +## 一手来源与采用边界 + +2026-11-03 重新读取下列来源。论文解释方法，固定版本代码确定可复用接口。作者的测量结果不作为本地结果。 + +| 工作 | 证据与具体设计 | 本次决定 | +| --- | --- | --- | +| KernelBench | [论文 §3、§4.2](https://arxiv.org/html/2502.10517v1) 将正确性和速度分开；[Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py) 和 [RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) 明确 axis=1；[MIT 许可](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/LICENSE) 已读取 | 适配数学题意并保留许可。CPU/NumPy、小形状、边界分布是明确变更。原 Softmax 的 4096×393216 FP32 输入约 6 GiB，不直接搬用 | +| AgentKernelArena | [论文 §3.1](https://arxiv.org/html/2605.16819v1) 分离 workspace、baseline、agent、central evaluation；[task v2 contract](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) 把 semantics/candidate/baseline/reference/workload/evaluation 放在任务包 | 采用责任划分、baseline/oracle 分离、统一最终评估。没有复制 GPU 平台或评分器，没有声称满足其 GPU 资格标准 | +| BackendBench | [README](https://github.com/meta-pytorch/BackendBench/blob/2a8d7e19e2c13c789ff2dceb44883ea8ea04dab4/README.md) 明确用 OpInfo/FACTO 做边界检查，用真实模型 shapes 做性能 | 借鉴边界和布局验证。首批不引入整个 PyTorch backend override、FACTO 和 TorchBench，也不声称覆盖它们 | +| SkillsBench | [固定版本 README](https://github.com/benchflow-ai/skillsbench/blob/f860ecf02ef53412a030f8661c83d644b739ff01/README.md) 分离 task.md、environment、oracle、verifier；[论文](https://arxiv.org/abs/2602.12670) 是技能受控比较参考 | 保持题目与技能条件独立，沿用现有 SkillEval verifier，不接入另一套云执行平台 | +| AlphaEvolve | [§2.1、§2.4–2.6](https://arxiv.org/html/2506.13131v1) 分离 task/evaluate、evaluation cascade、程序库和演化控制 | 采用正确性先于测量。候选生成、选择和程序库不放进 #252 | +| AIDE | [§3.1–3.2](https://arxiv.org/html/2502.13138v1) 显式区分 evaluator h、search policy π、coding operator f、solution tree T | search policy 是独立模块。本次提供任务评估，不实现搜索树或停止规则 | +| InferRSI 当前源码 | RunSpec 要求 baseline/candidate；runner 已有 AB/BA、超时和快照；SkillEval run_trial 已有独立 verifier；生产 engineering verifier 有 Linux bubblewrap 边界 | 生成真实 RunSpec，诊断用 execute_trial；保留产物接入现有 verifier。生产 sandbox 和执行权限不变 | + +## 方案比较与结论 + +| 方案 | 优点 | 当前问题 | 决策 | +| --- | --- | --- | --- | +| 完整引入外部 benchmark runner | 复用成熟 GPU 生态 | 硬件未定，重复 Workbench，带入 agent/评分假设 | 不选 | +| 只写统一 schema | 成本低、无需设备 | 无法验证真实任务或消费者 | 不选 | +| 首版实现所有硬件与三大类别 | 覆盖广 | 无资格环境，容易只剩空协议 | 不选 | +| 小任务包 + adapter + 既有执行器 + 两个消费者 | 语义和证据共享，可立即跑通和检验反例 | 首批只证明 CPU 子集 | **采用** | + +1. **首批选 Softmax 和 RMSNorm**。都是推理相关归约算子。Softmax 暴露 exp 溢出，RMSNorm 四维输入暴露错误轴，使第二道题有不同验证价值。通信/显存不放空实现。 +2. **消费者接口通用，adapter 能力具体**。`command(check|measure|profile)` 和 `runspec` 是共享边界，`numpy-axis1-v1` 描述当前能力。未知 backend/adapter 被拒绝。未来适配器沿用消费者入口，但必须定义自身同步、计时、manifest 校验和环境资格。 +3. **独立 oracle**。baseline/candidate 用 NumPy 归约，oracle 用标量 double 与 math.fsum，减少共享实现错误。容差属于具体任务数值规范，非全仓库性能门槛；每次比较冻结。 +4. **检查真实计时路径**。计时后的实际输出再验证，能发现预检后变错的状态行为。公开输入、同进程评估不能抵御恶意作弊，入口限定已审核可信文件，不能直接接任意模型源码。 +5. **profiling 独立**。CPU cProfile 解释 CPU 调用，不进入速度统计、不输出 A/B verdict。设备 trace 待硬件 adapter。 +6. **SkillEval 采用保留产物 replay**。controller 固定审核后的候选及其哈希，已有 run_trial 调用独立 verifier。本次 authored control 验证真实通路，没有模型调用和技能收益结论。 +7. **共享身份与原始样本**。task、grader、candidate、runtime 分别有哈希，记录实际输入摘要及每次样本。漂移、缺依赖、超时保留不完整证据。没有新建 AB/BA 调度器或统计规则。 + +## 对评论中三个问题的处理 + +- 没有引入 `upper_bound < residual_target`，增量贡献仍由独立 policy 处理。 +- correctness/profile 有独立执行记录，不需要伪造 baseline；只有 measurement 构成 A/B。 +- 没有把某 cell 一条失败记录解释成搜索耗尽，也没有增加停止控制器。 + +本方案针对硬件未确定的当前约束；下一步设备和任务类别应由实际 campaign 决定。它不是维护者对 #252 的最终验收决定。 diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md new file mode 100644 index 000000000..63ebfe1ab --- /dev/null +++ b/docs/plans/issue-252/02-contract.md @@ -0,0 +1,35 @@ +# 子项 02：共享任务接口 + +## 理解 + +任务包负责题目事实。消费者负责执行组织或 agent 对照。任务接口不应依赖硬件最终选择，也不应承担搜索调度。 + +## 计划 + +- 用版本化 JSON manifest 记录任务、SearchSpace 关联、来源、环境、输入、可编辑范围、oracle、单位、计时边界与局限。 +- 用内容哈希冻结包、grader 和候选；任务改变后不得继续冒充原比较。 +- 暴露 correctness、measurement、diagnostic 三个操作。实际命令仍由现有执行器运行。 +- 将真正的 A/B 比较转成现有 RunSpec；不给诊断填假 baseline。 +- 为路径、格式、未支持环境、非有限指标和漂移编写有实际失败意义的检查。 + +## 验收 + +同一包身份被两个消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。真实运行和安装包验证见 validation.md。 + +## 已实施 + +代码在 `src/inferrsi/harness/benchmarks/contract.py`，CLI 入口为 `inferrsi benchmarks`。 + +| 对象 | 实际字段或操作 | 用途 | +| --- | --- | --- | +| TaskPackage | schema_version、adapter、source、environment、candidate、oracle、measurement、workloads、coverage | 消费者读取同一份题目事实 | +| 冻结身份 | task_sha256、grader_sha256、candidate_sha256、runtime_sha256 | 记录本次真正使用的包、实现和解释器/NumPy 版本 | +| 单次操作 | check、measure、profile | 都生成结构化结果，只有 measure 提供性能指标 | +| Workbench 绑定 | runspec(candidate, repo, output_dir, repetitions, max_minutes, min_improvement_pct) | 使用已有 AB/BA、快照、截止时间和报告 | +| 工程评估绑定 | grade_artifact；skill_verifier.task_descriptor | 用实际实现与独立 oracle 生成 SkillEval checks | + +选择 JSON 是为了保持核心包零第三方依赖并便于命令快照。任务包描述题目，不保存密钥或用户运行配置。执行预算和增益门槛仍由 CLI/Python 调用方提供。 + +首个 adapter 明确限定 CPU/NumPy/float32；未知后端直接拒绝。未来 backend 实现新 adapter，不改变上述消费者接口。当前没有承诺任意任务 manifest 都能执行，也没有提供空的通信/显存适配。 + +真实 payload 仍由已有 `execute_trial/execute_command` 运行，独立 worker 仅执行一道题。`profile` 不需要 RunSpec，`compare` 才使用 RunSpec。保留缺依赖、超时、候选/任务漂移等失败路径。哈希冻结不是恶意代码隔离。 diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md new file mode 100644 index 000000000..41be0cf40 --- /dev/null +++ b/docs/plans/issue-252/03-tasks.md @@ -0,0 +1,36 @@ +# 子项 03：首批真实任务 + +## 理解 + +首批任务应足够小，能在硬件未定时跑通共享契约，又包含会出错的真实计算。CPU 适配必须公开与上游 workload/backend 的差异。 + +## 计划 + +- 从成熟算子题目选择两个小任务，核对源码语义、许可和形状。 +- 公开 CPU 子集的 dtype、轴、形状、容差、布局和异常输入范围。 +- baseline 和候选可独立替换，oracle 使用不同实现路径。 +- 覆盖普通输入、数值边界、非连续布局、错误轴、NaN、输入修改等反例。 +- 实际执行正确性、计时和 CPU profiling，保存每个工作负载的样本。 + +## 验收 + +正确实现能通过，常见错误能被识别；测量执行相同候选函数并核对输出；不要求某个实现必须加速。GPU/NPU、低精度、通信和容量边界保持未验证。 + +## 已实施 + +任务位于 `src/inferrsi/data/engineering_benchmarks/softmax-cpu` 和 `rmsnorm-cpu`。每包包括 task.json、baseline.py、candidate.py、README.md、LICENSE。原始 KernelBench 来源固定到完整 commit，MIT 许可随包分发。 + +| 任务 | 算什么 | 为什么选它 | +| --- | --- | --- | +| Softmax | 沿 axis=1 的稳定指数归一化 | exp 溢出、错误轴、分母和形状都可能出错 | +| RMSNorm | x / sqrt(mean(x², axis=1) + eps)，无 affine 参数 | 四维且各轴长度不同，能识别把 feature 轴写成末轴的常见错误 | + +每题六个公开 workload，包括普通随机、非连续布局、全零、大数、小数和单元素归约轴。普通 Softmax 为 16×257；普通 RMSNorm 为 4×17×9×7。这是刻意缩小的 CPU 开发集，不是原题性能结果。 + +NumPy 基线与使用临时 buffer 的示例候选都由本次实现提供。参考计算使用标量 double、math.exp/math.sqrt/math.fsum，独立于 NumPy 归约。每题检查形状/dtype、有限输出、输入保持、数值误差四类断言。 + +计时使用 perf_counter_ns，仅包围同步 compute 调用。每个 case 预热 2 次、保留 5 次计时；单个 trial 指标为六个 case 中位数的和，所有原始样本仍保留。这些是任务版本的测量协议，非全仓库性能门槛。外部 trial 次数和预算另行冻结。 + +变异测试覆盖全零错误答案、NaN、dtype 错误、shape 错误、不稳定 exp、修改输入、错误轴，以及前六次正确之后变错。失败候选不输出可用性能指标；中途失败保留失败调用和已有样本。timeout/缺依赖与完整数值拒绝分别记录。 + +Profiling 实际运行同一候选，生成可读 CPU 调用表，完全独立于正式计时。它能帮助检查 Python 调用开销；它不能回答 GPU/NPU 空洞或设备 kernel 利用率。 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md new file mode 100644 index 000000000..d3ea11f37 --- /dev/null +++ b/docs/plans/issue-252/04-integration.md @@ -0,0 +1,29 @@ +# 子项 04：消费者接入与实际用途验证 + +## 理解 + +只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 + +## 计划 + +- Workbench 接入复用 run_experiment、AB/BA、超时、源快照、结果和报告。 +- 离线工程评估接入复用现有 SkillEval 的独立 checks/verdict 消费契约，明确本地 authored control 与真实 agent 输出的区别。 +- 用正确、错误、改变冻结文件三种情况验证两端一致性。 +- 用实际工作负载保留原始结果和 CPU profile，检查来自真实执行而非预制分数。 +- 验证非 editable wheel 安装，以及现有相关回归；最终完整测试结果如实记录。 + +## 验收 + +两个消费者共享 task/grader 身份，正确性结论一致；错误候选不能获得可用性能证据；缺失 profiling 不冒充已有诊断。付费模型、生产 SkillEval 执行授权和设备资格不由本次接入改变。 + +## 已实施 + +- `inferrsi benchmarks plan/compare` 生成并执行真正的既有 RunSpec；不改原 runner 的统计或阶段规则。 +- `grade_artifact` 将独立正确性结果转成既有 SkillEval checks 结构。 +- `skill_verifier.py` 接入现有 `skill_evaluation.run_trial`，任务描述锁定 controller 审核后的保留产物，答复只提交该产物哈希。 +- `tools/engineering_benchmark_validate.py` 冻结任务和候选，运行两个真实任务、CPU profile、AB/BA，再通过真实 SkillEval trial 处理正确/错误的 authored control。没有生成模型请求，也没有伪造 token 使用。 +- `tests/check_installed.py` 检查 wheel 中的任务数据、许可和源码；新增回归覆盖真实子进程与失败路径。 + +消费链为“同一 task.json 和 grader → Workbench 性能比较 / SkillEval 独立正确性验证”。后者是离线保留产物交接，不是在线 agent 写代码的新 sandbox。是否由技能帮助产出更好代码，需要未来真实 agent、匹配预算和对照条件的实验。 + +运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，最终结果见 [验证记录](validation.md)。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md new file mode 100644 index 000000000..e639b49e2 --- /dev/null +++ b/docs/plans/issue-252/README.md @@ -0,0 +1,49 @@ +# #252 共享工程基准交付计划 + +本地实施依据为用户在 2026-11-03 的明确委托。重新读取的 [Issue 正文](https://github.com/InferMatrixLab/InferRSI/issues/252) 与全部两条评论仍未记录维护者对首批子集的验收。本目录记录可供维护者评审的实现与证据，不代替项目接受或关闭 Issue。 + +基线为 `663f3b327998526a13aa31e72ad729025c18983c`。实现位于独立分支 `codex/issue252-engineering-benchmarks`，不改变其他工作区。 + +## 要解决的实际问题 + +Workbench 和工程评估需要对同一个候选实现回答两个问题：计算是否正确，以及指定工作负载下测得什么性能。任务语义、输入、参考答案和计时规则应只有一份。搜索策略决定下一步尝试什么，消费者不能各自复制或修改题目答案。 + +本次将这个模块命名为 **Engineering Benchmarks（工程基准）**。这是本次实现的命名，尚非维护者已接受的架构决策。 + +## 子项与执行顺序 + +| 子项 | 要交付什么 | 如何证明有用 | +| --- | --- | --- | +| [01 调研和决策](01-research.md) | 一手论文、固定版本源码对照，选择及不选择的理由 | 每项决策对应接口、任务或验证行为 | +| [02 共享接口](02-contract.md) | 任务包读取、身份冻结、命令生成、独立诊断 | 错误配置和任务漂移会被拒绝，能生成现有 RunSpec | +| [03 首批任务](03-tasks.md) | 少量真实算子任务，独立 oracle，基线和候选接口 | 正确实现通过，典型错误被抓到，实际执行并计时 | +| [04 消费者与验收](04-integration.md) | Workbench 和离线工程评估共享同一任务，复现入口和结果 | 真正调用现有执行器/评估器，保留原始样本、失败记录和诊断 | + +硬件尚未确定，按用户的“共享接口优先”约束先验证本机 CPU 子集。加速器适配不能用 CPU 结果冒充。通信、分布式拓扑、显存容量、GPU/NPU 计时不属于这个首批子集。 + +## 与评论中的 Search Policy 的关系 + +[RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306) 提出了候选卡、准入、成本层级、轮次、负结果和停止规则。[维护者回复](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922) 明确要求保持它与 #252 独立，并修正增量贡献、诊断记录、搜索耗尽三个问题。本次交付任务和 oracle，不顺带实现这套搜索控制器。 + +Profiling 保留为独立诊断操作，与 correctness 和配对 performance 分开。它不伪造 baseline，也不产生性能胜负。本次 CPU profiling 只能解释 CPU 调用；设备 trace 由未来环境适配器提供。 + +## 完成标准 + +- 每个子项有方案、理由、实际改动和验证记录。 +- 首批任务可从安装包读取，离开源码目录仍可使用。 +- 冻结任务、grader、候选和输入身份，保留失败与缺失状态。 +- 测量明确单位、边界、预热、重复与样本，不用文档或 CI 绿色代替真实任务运行。 +- 证明接口复用和错误识别；不宣称技能/agent/search 的收益、GPU/NPU 正确性或服务加速。 +- 正式 Issue 验收、维护者同意、独立 review 与本地实施完成分别报告。 + +最终结果见 [验证记录](validation.md)。 + +## 当前交付 + +公开子任务分别为 [#334 调研与决策](https://github.com/InferMatrixLab/InferRSI/issues/334)、[#335 共享接口](https://github.com/InferMatrixLab/InferRSI/issues/335)、[#336 首批任务](https://github.com/InferMatrixLab/InferRSI/issues/336)、[#337 消费者与验证](https://github.com/InferMatrixLab/InferRSI/issues/337)。四项均引用父任务 #252，保持打开供维护者评审与验收。当前提交账号无法写入原生父子关系，需维护者补充。 + +四个子项的本地实施已完成。已交付共享 TaskPackage、两个真实 CPU 算子任务、独立正确性与 profiling、Workbench/SkillEval 接入，以及可复现实测工具。使用入口见 [工程基准文档](../../engineering-benchmarks.md)。 + +建议先读本页理解范围，再读 01 看方案理由。要开始使用时，运行 `inferrsi benchmarks list` 和 `show softmax-cpu`，复制任务包中的示例候选到独立 implementation.py，按使用文档执行 check、profile、compare。修改任务协议应创建新比较，不能沿用旧身份。 + +这完成的是用户委托的首批共享接口和 CPU 子集；#252 的维护者验收、目标硬件选择与后续覆盖仍单独记录，不在本地冒充已接受。 diff --git a/docs/plans/issue-252/validation-summary.json b/docs/plans/issue-252/validation-summary.json new file mode 100644 index 000000000..5f2d868ec --- /dev/null +++ b/docs/plans/issue-252/validation-summary.json @@ -0,0 +1,53 @@ +{ + "scope": "local CPU integration with authored controls", + "checks_succeeded": true, + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + "tasks": [ + { + "task": "rmsnorm-cpu", + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", + "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", + "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "correctness": "PASS", + "profile_status": "completed", + "skilleval_good": "PASS", + "skilleval_bad": "FAIL", + "bad_measurement_status": "failed", + "measurement_trials": 6, + "measured_calls": 180, + "source_integrity": true, + "descriptive_workbench_verdict": "FAIL", + "workbench_run": "rmsnorm-cpu/workbench/20261103T141447Z-897e0a2641", + "all_consumer_identities_match": true, + "all_workloads_match": true + }, + { + "task": "softmax-cpu", + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "a0673d210792959bafa60851d2b9e33ded1ab5b7222076fb2858c27d125c6a2d", + "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", + "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "correctness": "PASS", + "profile_status": "completed", + "skilleval_good": "PASS", + "skilleval_bad": "FAIL", + "bad_measurement_status": "failed", + "measurement_trials": 6, + "measured_calls": 180, + "source_integrity": true, + "descriptive_workbench_verdict": "PASS", + "workbench_run": "softmax-cpu/workbench/20261103T141513Z-717a97f18d", + "all_consumer_identities_match": true, + "all_workloads_match": true + } + ] +} diff --git a/docs/plans/issue-252/validation.md b/docs/plans/issue-252/validation.md new file mode 100644 index 000000000..89f29d49e --- /dev/null +++ b/docs/plans/issue-252/validation.md @@ -0,0 +1,54 @@ +# #252 本地验证记录 + +验证日期 2026-11-03。实现基于 `663f3b327998526a13aa31e72ad729025c18983c`，具体新增内容以当前工作区和结果中的内容哈希为准。本记录说明共享任务子集的功能与集成，不是项目维护者的验收决定。 + +## 实际运行结果 + +最终原始目录名为 `inferrsi-252-validation-final`，保留在仓库外的本地 outputs 目录。可用 `tools/engineering_benchmark_validate.py` 重建一份新记录。该次执行使用 Windows、Python 3.11.18、NumPy 2.4.6。 + +[可公开的原始汇总](validation-summary.json) 保留任务/候选/grader/runtime 身份、执行数量、原始 verdict 和覆盖状态，不包含个人路径。完整原始日志和样本仍在本地，公开汇总不能代替独立重放或正式 campaign 验收。 + +| 验证内容 | Softmax CPU | RMSNorm CPU | +| --- | --- | --- | +| workload 数 | 6 | 6 | +| 每次独立正确性断言 | 24 | 24 | +| 正确候选在工程评估中 | 通过 | 通过 | +| 正确候选经过实际 SkillEval run_trial/verifier | 通过 | 通过 | +| 全零错误候选经过同一路径 | 拒绝 | 拒绝 | +| 错误候选请求测量 | 失败，无可用性能指标 | 失败，无可用性能指标 | +| Workbench 配对 trial | 3 baseline + 3 candidate | 3 baseline + 3 candidate | +| 保留的实际计时样本 | 180 | 180 | +| CPU profile | 已生成，42 次 compute 调用 | 已生成，42 次 compute 调用 | +| 所有消费者 task/grader/runtime 身份 | 一致 | 一致 | +| 所有消费者实际输入摘要、shape/dtype/strides | 一致 | 一致 | +| Workbench 源码完整性检查 | 通过 | 通过 | + +`summary.json` 的 `checks_succeeded` 为 true。每个任务目录包含冻结的任务包、候选源码、runspec.json、identity.json、correctness、profile、workbench、skilleval-good、skilleval-bad、wrong-measure。原始样本、命令、stderr、输入摘要和 profile 均保留，可追溯到实际执行。 + +工程评估与 SkillEval 的 PASS 仅表示相应公开用例的数值正确性。Workbench 的比较 verdict 原样保留，包括不满足增益门槛的结果。两次集成运行的性能胜负有变化，且宿主机同时有软件检查活动，不能据此宣布优化收益；本次目的为验证测量通路和失败处理，不是做隔离 CPU 性能实验。统计显著性仍为 not established。 + +## 软件检查 + +- 新增专项回归 `tests/test_engineering_benchmarks.py`：**26 passed**。覆盖真实任务、反例、预检之后变错、task/candidate/grader/runtime 漂移、缺 NumPy、超时、实际 Workbench AB/BA、真实 SkillEval verifier 和错误产物身份。 +- `ruff check src tests tools/engineering_benchmark_validate.py`：通过。 +- `git diff --check`：通过。 +- sdist 和 wheel 构建：通过。Windows 长路径导致的第一次构建失败保留；缩短构建临时目录后成功，没有删减包内容。 +- 全新非 editable wheel 环境，源码目录外运行 `python -I tests/check_installed.py`：通过，包括任务文件、许可证、127 条既有知识记录等安装检查。 +- 同一隔离安装中实际执行 Softmax 独立 oracle：通过。 +- 全量回归使用 Python 3.11、4 个 pytest worker 完成：**3479 passed、51 skipped、56 failed**，另有 281 个通过的 subtest。55 个失败用例产生 56 个失败报告，全部在未修改的原始基线复现。典型原因包括 Windows 的 CRLF 字节差异、符号链接权限、POSIX FIFO/process API、Linux sandbox 和路径假设；没有通过修改无关测试掩盖。 +- 基线对照在独立 detached worktree 执行，确认 import 指向原始基线 `src/inferrsi`；逐个匹配 JUnit 中的失败用例，未出现“本次失败但基线通过”的用例。全量结果仍为非绿色，不能称为所有检查通过。 + +最初仅有 Python 3.14 的环境缺少 tzdata，导致仓库中 6 个已有模块无法收集测试。后续使用仓库 CI 覆盖的 Python 3.11，并在本地测试环境安装 tzdata。未更改项目的时区业务逻辑。26 项专项验证的最终结果来自该 Python 3.11 环境。 + +全量串行尝试在发现原有失败后改用 4 个 worker 完整执行。最终日志为 `inferrsi-252-full.log/xml`；对照为 `inferrsi-252-baseline-all-failures.log/xml`；新增功能专项为 `inferrsi-252-focused.log`。早期失败、构建长路径问题和首次集成运行也保留，没有覆盖为成功。 + +## 四个子项的实际用途 + +1. 调研决定落实在代码边界中：保留 axis=1、独立 oracle、正确性先行、profiling 独立、已有 runner 复用，没有照搬外部搜索平台。 +2. 共享接口会拒绝漂移或未支持环境，避免两个消费者用不同题目版本得出不可比较的结果。 +3. 两个算子确实计算并计时；数值、布局和状态错误会被抓到，未用预制分数代替实现。 +4. Workbench 与 SkillEval 真正执行了同一任务包；逐项比较实际输入摘要验证复用，没有为第二个消费者复制答案。 + +## 本次未建立的结论 + +GPU/NPU 正确性和性能、通信/内存容量任务、真实模型/服务收益、技能或搜索策略收益均未验证。公开开发用例不是 holdout；可信代码执行不等于恶意 agent sandbox。本次公开提交用于评审；尚无独立 reviewer 的签署、维护者对首批子集的正式接受或关闭 #252 的授权，公开追踪入口见总方案。上述差别不影响已经完成的本地实现和功能验证，但不能被省略为“整个 tracking issue 已验收”。 diff --git a/pyproject.toml b/pyproject.toml index e44a7a806..166f438c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/engineering_benchmarks/**/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" @@ -35,8 +35,9 @@ prerelease = true [project.optional-dependencies] skills = ["PyYAML>=6.0"] -dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0"] +dev = ["pytest>=7.4", "ruff>=0.6", "PyYAML>=6.0", "numpy>=1.24"] image = ["numpy>=1.24", "Pillow>=10", "scikit-image>=0.21"] +benchmarks = ["numpy>=1.24"] [tool.pytest.ini_options] testpaths = ["tests"] diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 46b5e9336..6d7aece10 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -39,6 +39,9 @@ def parser() -> argparse.ArgumentParser: ) root.add_argument("--version", action="version", version=__version__) commands = root.add_subparsers(dest="command", required=True) + from .harness.benchmarks.cli import add_parser + + add_parser(commands) doctor = commands.add_parser( "doctor", help="Inspect a prepared local target without installing dependencies" ) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 349e3df34..63544082e 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -116,6 +116,17 @@ def main(argv: list[str] | None = None) -> int: argument_parser = parser() args = argument_parser.parse_args(argv) try: + if args.command == "benchmarks": + from .harness.benchmarks.cli import dispatch + + result = dispatch(args) + _print(result) + if isinstance(result, dict): + if "verdict" in result: + return 0 if result["verdict"] == "PASS" else 1 + if "status" in result: + return 0 if result["status"] == "completed" else 1 + return 0 if args.command == "skills": # Optional dependencies load only for the skills command; the core # CLI stays independent of them. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE new file mode 100644 index 000000000..eaae6b11e --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2023 Anne Ouyang, Simon Guo, Azalia Mirhoseini +Scaling Intelligence Lab, Stanford University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md new file mode 100644 index 000000000..983e5e607 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md @@ -0,0 +1,26 @@ +# rmsnorm-cpu + +CPU adaptation of [KernelBench 36_RMSNorm_.py](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py). +Source license is retained in LICENSE. No upstream scoring result transfers here. + +Implement `compute(x, *, eps)` in a self-contained `implementation.py`. Return a +float32 NumPy array of the same shape. Preserve input values. Compute x / sqrt(mean(x*x, axis=1, keepdims=True) + eps), with no affine scale. +Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; +the evaluator is not a security sandbox for model-produced arbitrary code. + +The task JSON owns all six public inputs, seeds, layouts, numerical tolerances +and timing policy. Baseline and example candidate are separate from the scalar +double-precision oracle. The oracle uses math.fsum and is not editable by a +candidate. The task package and grader are content-frozen per comparison. + +Each measured call actually runs compute and checks its output and input +preservation outside the timed region. Input generation, reference computation, +warmups, validation, process startup and profiling are excluded. Output allocation +inside compute is included. Samples use perf_counter_ns and report milliseconds. +Sum of per-case medians is only this task's fixed-suite objective, not a universal +score. Retain every per-case sample to inspect regressions. Repeated calls describe +a warm-cache CPU workload. No device synchronization is claimed. + +All tests are public. This small subset cannot demonstrate production inference +speedup, memory savings, or GPU/NPU support. A fresh task version and device +qualification are needed for changed shapes, dtype, backend or timing. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py new file mode 100644 index 000000000..7920c19b0 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py @@ -0,0 +1,8 @@ +"""NumPy CPU adaptation of the attributed KernelBench task; see LICENSE.""" + +import numpy as np + + +def compute(x, *, eps): + rms = np.sqrt(np.mean(x * x, axis=1, keepdims=True) + np.float32(eps)) + return x / rms diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py new file mode 100644 index 000000000..89f3c6cf1 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py @@ -0,0 +1,11 @@ +"""Reviewed example using temporary buffers; not an agent-generated result.""" + +import numpy as np + + +def compute(x, *, eps): + squared = np.square(x) + rms = np.mean(squared, axis=1, keepdims=True) + rms += np.float32(eps) + np.sqrt(rms, out=rms) + return np.divide(x, rms) diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json new file mode 100644 index 000000000..b1e56ffe3 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json @@ -0,0 +1,121 @@ +{ + "schema_version": 1, + "id": "rmsnorm-cpu", + "adapter": "numpy-axis1-v1", + "description": "Optimize rmsnorm on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", + "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "source": { + "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py", + "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", + "license": "MIT", + "adaptation": "Adapted mathematical task, not upstream scores. PyTorch/CUDA replaced by NumPy CPU; reduced shapes, public stress distributions and strided inputs added. Axis 1 retained. Baseline is a NumPy implementation; scalar math.fsum supplies independent reference." + }, + "environment": { + "backend": "cpu", + "runtime": "numpy", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "rmsnorm", + "axis": 1, + "eps": 0.00001, + "rtol": 0.00001, + "atol": 1e-7 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "synchronous-compute-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 2, + "samples": 5 + }, + "workloads": [ + { + "id": "normal", + "shape": [ + 4, + 17, + 9, + 7 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "strided", + "shape": [ + 4, + 17, + 9, + 7 + ], + "seed": 253, + "distribution": "normal", + "layout": "strided" + }, + { + "id": "zeros", + "shape": [ + 2, + 3, + 5, + 7 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 2, + 5, + 3, + 7 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 2, + 7, + 3, + 5 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + }, + { + "id": "singleton", + "shape": [ + 2, + 1, + 3, + 5 + ], + "seed": 257, + "distribution": "normal", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads, not held-out generalization", + "CPU float32 finite inputs only; no float16/bfloat16, NaN/Inf inputs or gradients", + "No CUDA/ROCm/Ascend execution or accelerator timing qualification", + "No service/E2E, communication, memory-capacity or skill/search benefit claim", + "Trusted self-contained reviewed source only; no hostile-agent isolation", + "Repeated warm-cache calls; allocations inside compute included, setup/oracle/profile excluded from timing" + ] +} diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE new file mode 100644 index 000000000..eaae6b11e --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2023 Anne Ouyang, Simon Guo, Azalia Mirhoseini +Scaling Intelligence Lab, Stanford University + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md new file mode 100644 index 000000000..7e6b0b251 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md @@ -0,0 +1,26 @@ +# softmax-cpu + +CPU adaptation of [KernelBench 23_Softmax.py](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py). +Source license is retained in LICENSE. No upstream scoring result transfers here. + +Implement `compute(x, *, eps)` in a self-contained `implementation.py`. Return a +float32 NumPy array of the same shape. Preserve input values. Compute stable softmax along axis 1. +Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; +the evaluator is not a security sandbox for model-produced arbitrary code. + +The task JSON owns all six public inputs, seeds, layouts, numerical tolerances +and timing policy. Baseline and example candidate are separate from the scalar +double-precision oracle. The oracle uses math.fsum and is not editable by a +candidate. The task package and grader are content-frozen per comparison. + +Each measured call actually runs compute and checks its output and input +preservation outside the timed region. Input generation, reference computation, +warmups, validation, process startup and profiling are excluded. Output allocation +inside compute is included. Samples use perf_counter_ns and report milliseconds. +Sum of per-case medians is only this task's fixed-suite objective, not a universal +score. Retain every per-case sample to inspect regressions. Repeated calls describe +a warm-cache CPU workload. No device synchronization is claimed. + +All tests are public. This small subset cannot demonstrate production inference +speedup, memory savings, or GPU/NPU support. A fresh task version and device +qualification are needed for changed shapes, dtype, backend or timing. diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py new file mode 100644 index 000000000..c6c630b0f --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/baseline.py @@ -0,0 +1,8 @@ +"""NumPy CPU adaptation of the attributed KernelBench task; see LICENSE.""" + +import numpy as np + + +def compute(x, *, eps): + values = np.exp(x - np.max(x, axis=1, keepdims=True)) + return values / np.sum(values, axis=1, keepdims=True) diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py new file mode 100644 index 000000000..f19af8c23 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/candidate.py @@ -0,0 +1,10 @@ +"""Reviewed example using temporary buffers; not an agent-generated result.""" + +import numpy as np + + +def compute(x, *, eps): + output = np.subtract(x, np.max(x, axis=1, keepdims=True)) + np.exp(output, out=output) + output /= np.sum(output, axis=1, keepdims=True) + return output diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json new file mode 100644 index 000000000..ea2818495 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json @@ -0,0 +1,109 @@ +{ + "schema_version": 1, + "id": "softmax-cpu", + "adapter": "numpy-axis1-v1", + "description": "Optimize softmax on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", + "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "source": { + "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py", + "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", + "license": "MIT", + "adaptation": "Adapted mathematical task, not upstream scores. PyTorch/CUDA replaced by NumPy CPU; reduced shapes, public stress distributions and strided inputs added. Axis 1 retained. Baseline is a NumPy implementation; scalar math.fsum supplies independent reference." + }, + "environment": { + "backend": "cpu", + "runtime": "numpy", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "softmax", + "axis": 1, + "eps": 0.00001, + "rtol": 0.00001, + "atol": 1e-7 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "synchronous-compute-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 2, + "samples": 5 + }, + "workloads": [ + { + "id": "normal", + "shape": [ + 16, + 257 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "strided", + "shape": [ + 16, + 257 + ], + "seed": 253, + "distribution": "normal", + "layout": "strided" + }, + { + "id": "zeros", + "shape": [ + 3, + 7 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 3, + 31 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 2, + 17 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + }, + { + "id": "singleton", + "shape": [ + 2, + 1 + ], + "seed": 257, + "distribution": "normal", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads, not held-out generalization", + "CPU float32 finite inputs only; no float16/bfloat16, NaN/Inf inputs or gradients", + "No CUDA/ROCm/Ascend execution or accelerator timing qualification", + "No service/E2E, communication, memory-capacity or skill/search benefit claim", + "Trusted self-contained reviewed source only; no hostile-agent isolation", + "Repeated warm-cache calls; allocations inside compute included, setup/oracle/profile excluded from timing" + ] +} diff --git a/src/inferrsi/harness/benchmarks/__init__.py b/src/inferrsi/harness/benchmarks/__init__.py new file mode 100644 index 000000000..e1113d79d --- /dev/null +++ b/src/inferrsi/harness/benchmarks/__init__.py @@ -0,0 +1,5 @@ +"""Shared engineering task contracts; execution belongs to the existing harness.""" + +from .contract import TaskPackage, builtin_tasks, load_task + +__all__ = ["TaskPackage", "builtin_tasks", "load_task"] diff --git a/src/inferrsi/harness/benchmarks/cli.py b/src/inferrsi/harness/benchmarks/cli.py new file mode 100644 index 000000000..4f7731284 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/cli.py @@ -0,0 +1,82 @@ +"""CLI-first engineering benchmark operations; all execution is explicit.""" + +import math +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..runner import run_experiment +from .consumers import grade_artifact, run_action +from .contract import builtin_tasks, load_task + + +def add_parser(commands): + root = commands.add_parser("benchmarks", help="Shared engineering tasks (trusted CPU sources)") + actions = root.add_subparsers(dest="benchmark_action", required=True) + actions.add_parser("list") + show = actions.add_parser("show") + show.add_argument("task") + for name in ("check", "measure", "profile", "grade", "plan", "compare"): + child = actions.add_parser(name) + child.add_argument("task", help="Built-in task ID or local task package directory") + child.add_argument( + "--candidate", + type=Path, + required=True, + help="Reviewed self-contained Python source; no security sandbox", + ) + child.add_argument("--output-dir", type=Path, required=True) + if name in {"plan", "compare"}: + child.add_argument("--repo", type=Path, required=True) + child.add_argument("--repetitions", type=int, required=True) + child.add_argument("--max-minutes", type=float, required=True) + child.add_argument("--min-improvement-pct", type=float, required=True) + else: + child.add_argument("--timeout-seconds", type=float, required=True) + + +def dispatch(args) -> dict | list: + action = args.benchmark_action + if action == "list": + return builtin_tasks() + task = load_task(args.task) + if action == "show": + return { + "manifest": task.manifest, + "task_sha256": task.task_sha256, + "grader_sha256": task.grader_sha256, + } + if action in {"plan", "compare"}: + spec = task.runspec( + args.candidate, + repo=args.repo, + output_dir=args.output_dir, + repetitions=args.repetitions, + max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct, + ) + if action == "plan": + return spec.to_dict() + identity = task.identity(args.candidate) + run_dir = run_experiment( + spec, + { + "benchmark": identity, + "backend": "cpu", + "qualification": "local CPU development workload only", + }, + ) + atomic_write_json(run_dir / "task.lock.json", task.manifest) + return { + "run_dir": str(run_dir), + "scope": "local-artifact-comparison", + "campaign_acceptance": "not established", + } + if not math.isfinite(args.timeout_seconds) or args.timeout_seconds <= 0: + raise ValueError("Timeout must be positive and finite") + if action == "grade": + return grade_artifact( + task, args.candidate, args.output_dir, timeout_seconds=args.timeout_seconds + ) + return run_action( + task, args.candidate, action, args.output_dir, timeout_seconds=args.timeout_seconds + ) diff --git a/src/inferrsi/harness/benchmarks/consumers.py b/src/inferrsi/harness/benchmarks/consumers.py new file mode 100644 index 000000000..8d2334c75 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/consumers.py @@ -0,0 +1,69 @@ +"""Thin consumers; no alternate runner, agent client or result truth.""" + +import json +import math +import time +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from ..scenarios.external import execute_trial +from ..skill_evaluation import valid_checks + + +def run_action(task, candidate: Path, action: str, output: Path, *, timeout_seconds: float) -> dict: + """Execute trusted local source. This API is not an untrusted-code sandbox.""" + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not math.isfinite(timeout_seconds) + or not 0 < timeout_seconds <= 7 * 24 * 3600 + ): + raise ValueError("Timeout must be positive, finite and at most seven days") + command = task.command(action, candidate) + return execute_trial( + command, + candidate.resolve().parent, + output.resolve(), + time.monotonic() + timeout_seconds, + task.manifest["measurement"]["key"] if action == "measure" else None, + ) + + +def grade_artifact(task, candidate: Path, output: Path, *, timeout_seconds: float) -> dict: + """Independent correctness for offline engineering evaluation. + + Checks have SkillEval's existing shape. An agent/skill comparison must still + establish its own task delivery, trace, budget and matched conditions. + """ + identity = task.identity(candidate) + execution = run_action(task, candidate, "check", output, timeout_seconds=timeout_seconds) + result, checks = None, None + try: + result = json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + task.assert_unchanged() + if ( + task.identity(candidate) == identity + and result.get("identity") == identity + and result.get("action") == "check" + and result.get("status") == "completed" + ): + proposed = {key: result.get(key) for key in ("passed", "assertions")} + if ( + valid_checks(proposed) + and execution["status"] in {"completed", "failed"} + and execution["returncode"] == (0 if proposed["passed"] else 1) + ): + checks = proposed + except (OSError, ValueError, TypeError): + pass + verdict = "INCONCLUSIVE" if checks is None else "PASS" if checks["passed"] else "FAIL" + record = { + "scope": "artifact-correctness-only", + "identity": identity, + "verdict": verdict, + "checks": checks, + "execution_status": execution["status"], + "agent_benefit": "not measured", + } + atomic_write_json(output / "artifact-evaluation.json", record) + return record diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py new file mode 100644 index 000000000..0767c88b2 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -0,0 +1,262 @@ +"""Versioned task truth and bindings to existing Workbench/SkillEval contracts.""" + +import hashlib +import json +import math +import platform +import re +import sys +from dataclasses import dataclass +from importlib.metadata import PackageNotFoundError, version +from importlib.resources import files +from pathlib import Path + +from ...runspec import RunSpec + + +def digest(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def tree_digest(root: Path) -> str: + entries = {} + for path in sorted(root.rglob("*")): + if "__pycache__" in path.parts or path.suffix == ".pyc": + continue + if path.is_symlink(): + raise ValueError("Task packages must not contain symlinks") + if path.is_file(): + entries[path.relative_to(root).as_posix()] = digest(path) + return hashlib.sha256(json.dumps(entries, sort_keys=True).encode()).hexdigest() + + +def grader_digest() -> str: + return tree_digest(Path(__file__).parent) + + +def runtime_digest() -> str: + try: + numpy_version = version("numpy") + except PackageNotFoundError: + numpy_version = "unavailable" + facts = [platform.python_version(), platform.system(), platform.machine(), numpy_version] + return hashlib.sha256(json.dumps(facts).encode()).hexdigest() + + +def builtin_tasks() -> list[str]: + root = files("inferrsi").joinpath("data/engineering_benchmarks") + return sorted(p.name for p in root.iterdir() if p.joinpath("task.json").is_file()) + + +def _text(value, field): + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{field} must be a nonempty string") + + +def _positive(value, field, *, integer=False): + if ( + isinstance(value, bool) + or not isinstance(value, int if integer else (int, float)) + or not math.isfinite(value) + or value <= 0 + ): + raise ValueError(f"{field} must be positive and finite") + + +@dataclass(frozen=True) +class TaskPackage: + root: Path + frozen_manifest: str + task_sha256: str + grader_sha256: str + + @property + def manifest(self) -> dict: + # Callers cannot mutate the task by changing a returned dictionary. + return json.loads(self.frozen_manifest) + + def assert_unchanged(self) -> None: + if tree_digest(self.root) != self.task_sha256 or grader_digest() != self.grader_sha256: + raise ValueError("Frozen task or grader changed") + + def identity(self, candidate: Path) -> dict: + self.assert_unchanged() + candidate = candidate.absolute() + if candidate.is_symlink() or not candidate.is_file(): + raise ValueError("Candidate must be a regular source file") + return { + "task_id": self.manifest["id"], + "task_sha256": self.task_sha256, + "grader_sha256": self.grader_sha256, + "candidate_sha256": digest(candidate), + "runtime_sha256": runtime_digest(), + } + + def command(self, action: str, candidate: Path) -> tuple[str, ...]: + if action not in {"check", "measure", "profile"}: + raise ValueError("Unknown task action") + identity = self.identity(candidate) + return ( + sys.executable, + "-B", + "-m", + "inferrsi.harness.benchmarks.worker", + action, + "--package", + str(self.root), + "--candidate", + str(candidate.resolve()), + "--task-sha256", + identity["task_sha256"], + "--grader-sha256", + identity["grader_sha256"], + "--candidate-sha256", + identity["candidate_sha256"], + "--runtime-sha256", + identity["runtime_sha256"], + ) + + def runspec( + self, + candidate: Path, + *, + repo: Path, + output_dir: Path, + repetitions: int, + max_minutes: float, + min_improvement_pct: float, + ) -> RunSpec: + """Bind one task to AB/BA; the caller owns repetition/budget/gain gates.""" + metric = self.manifest["measurement"] + spec = RunSpec( + repo=str(repo.resolve()), + repo_type="engineering_task", + backend=self.manifest["environment"]["backend"], + baseline_command=self.command("measure", self.root / "baseline.py"), + candidate_command=self.command("measure", candidate), + validation_command=self.command("check", candidate), + metric_key=metric["key"], + metric_unit=metric["unit"], + comparison_scope=self.manifest["id"] + ":" + metric["boundary"], + direction="minimize", + repetitions=repetitions, + max_minutes=max_minutes, + min_improvement_pct=min_improvement_pct, + workload_id=self.task_sha256, + output_dir=str(output_dir.resolve()), + mode="code", + ) + spec.validate() + return spec + + +def load_task(name: str | Path) -> TaskPackage: + root = Path(name) + if not root.is_dir(): + if str(name) not in builtin_tasks(): + raise ValueError(f"Unknown task package: {name}") + root = Path(str(files("inferrsi").joinpath("data/engineering_benchmarks", str(name)))) + if root.is_symlink(): + raise ValueError("Task root must not be a symlink") + root = root.resolve() + data = json.loads((root / "task.json").read_text(encoding="utf-8")) + required = { + "schema_version", + "id", + "description", + "searchspace", + "source", + "environment", + "candidate", + "oracle", + "measurement", + "workloads", + "coverage", + "adapter", + } + if ( + not isinstance(data, dict) + or set(data) != required + or type(data["schema_version"]) is not int + ): + raise ValueError("Invalid task fields") + if data["schema_version"] != 1 or data["adapter"] != "numpy-axis1-v1": + raise ValueError("Unsupported task schema or adapter") + if not isinstance(data["id"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", data["id"]): + raise ValueError("Invalid task id") + for field in ("description", "searchspace"): + _text(data[field], field) + source = data["source"] + if not isinstance(source, dict) or set(source) != {"url", "revision", "license", "adaptation"}: + raise ValueError("Source provenance is required") + for key, value in source.items(): + _text(value, "source." + key) + if not re.fullmatch(r"[0-9a-f]{40}", source["revision"]): + raise ValueError("Source revision must be a full commit") + env = data["environment"] + if env != {"backend": "cpu", "runtime": "numpy", "dtype": "float32"}: + raise ValueError("Unsupported environment; no implicit device fallback") + if data["candidate"] != {"entrypoint": "compute", "editable": ["implementation.py"]}: + raise ValueError("Unsupported candidate contract") + oracle = data["oracle"] + if ( + not isinstance(oracle, dict) + or set(oracle) != {"operation", "axis", "eps", "rtol", "atol"} + or oracle["operation"] not in {"softmax", "rmsnorm"} + or type(oracle["axis"]) is not int + or oracle["axis"] != 1 + ): + raise ValueError("Unsupported oracle") + for key in ("eps", "rtol", "atol"): + _positive(oracle[key], "oracle." + key) + metric = data["measurement"] + if ( + not isinstance(metric, dict) + or set(metric) != {"key", "unit", "boundary", "aggregation", "warmups", "samples"} + or metric["key"] != "metrics.suite_latency_ms" + or metric["unit"] != "ms" + or metric["boundary"] != "synchronous-compute-call" + or metric["aggregation"] != "sum-of-per-case-medians" + ): + raise ValueError("Unsupported measurement contract") + for key in ("warmups", "samples"): + _positive(metric[key], "measurement." + key, integer=True) + if not isinstance(data["workloads"], list) or not data["workloads"]: + raise ValueError("Workloads are required") + names = set() + for case in data["workloads"]: + if not isinstance(case, dict) or set(case) != { + "id", + "shape", + "seed", + "distribution", + "layout", + }: + raise ValueError("Invalid workload fields") + _text(case["id"], "workload id") + if case["id"] in names: + raise ValueError("Duplicate workload id") + names.add(case["id"]) + if ( + not isinstance(case["shape"], list) + or not 2 <= len(case["shape"]) <= 4 + or any(type(n) is not int or n < 1 for n in case["shape"]) + or math.prod(case["shape"]) > 1048576 + ): + raise ValueError("Invalid or oversized CPU workload") + if type(case["seed"]) is not int or case["seed"] < 0: + raise ValueError("Seed must be a nonnegative integer") + if case["distribution"] not in {"normal", "zeros", "large", "tiny"}: + raise ValueError("Unknown distribution") + if case["layout"] not in {"contiguous", "strided"}: + raise ValueError("Unknown layout") + if ( + not isinstance(data["coverage"], list) + or not data["coverage"] + or any(not isinstance(v, str) or not v.strip() for v in data["coverage"]) + ): + raise ValueError("Coverage limits are required") + for path in ("baseline.py", "candidate.py", "LICENSE", "README.md"): + if not (root / path).is_file(): + raise ValueError(f"Task package missing {path}") + return TaskPackage(root, json.dumps(data, sort_keys=True), tree_digest(root), grader_digest()) diff --git a/src/inferrsi/harness/benchmarks/numpy_task.py b/src/inferrsi/harness/benchmarks/numpy_task.py new file mode 100644 index 000000000..8340c2502 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/numpy_task.py @@ -0,0 +1,168 @@ +"""CPU task payload; never schedules processes or makes performance verdicts. + +Only reviewed, trusted implementations may run here. Hash checks detect drift, +not malicious code. The public input suite is not a hidden evaluation set. +""" + +import cProfile +import hashlib +import math +import platform +import pstats +import statistics +import time +from pathlib import Path + + +def inputs(case, np): + rng = np.random.default_rng(case["seed"]) + x = rng.standard_normal(case["shape"]).astype(np.float32) + if case["distribution"] == "zeros": + x.fill(0) + elif case["distribution"] == "large": + x = x * np.float32(100) + np.float32(1000) + elif case["distribution"] == "tiny": + x *= np.float32(1e-7) + if case["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + return x + + +def reference(x, oracle, np): + """Scalar double-precision oracle, independent of NumPy reduction kernels.""" + rows = np.moveaxis(x, 1, -1) + output = np.empty_like(rows) + for index in np.ndindex(rows.shape[:-1]): + values = [float(v) for v in rows[index]] + if oracle["operation"] == "softmax": + maximum = max(values) + exps = [math.exp(v - maximum) for v in values] + total = math.fsum(exps) + result = [v / total for v in exps] + else: + scale = math.sqrt(math.fsum(v * v for v in values) / len(values) + oracle["eps"]) + result = [v / scale for v in values] + output[index] = result + return np.moveaxis(output, -1, 1) + + +def check_output(output, x, original, expected, oracle, np): + return { + "shape_dtype": isinstance(output, np.ndarray) + and output.shape == x.shape + and output.dtype == x.dtype, + "finite": isinstance(output, np.ndarray) and bool(np.isfinite(output).all()), + "input_preserved": x.tobytes() == original.tobytes(), + "values": isinstance(output, np.ndarray) + and output.shape == expected.shape + and bool( + np.allclose(output, expected, rtol=oracle["rtol"], atol=oracle["atol"], equal_nan=False) + ), + } + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # Optional dependency is loaded only for execution, never task discovery. + import numpy as np + + manifest = task.manifest + source = candidate.read_bytes() + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(source, "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, *, eps)") + oracle = manifest["oracle"] + checks, cases, workload_records = {}, [], [] + prepared = [] + for case in manifest["workloads"]: + x = inputs(case, np) + original = x.copy() + expected = reference(x, oracle, np) + workload_records.append( + { + "id": case["id"], + "input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), + "shape": list(x.shape), + "dtype": str(x.dtype), + "strides": list(x.strides), + } + ) + output = compute(x, eps=oracle["eps"]) + assertions = check_output(output, x, original, expected, oracle, np) + checks.update({case["id"] + "." + k: v for k, v in assertions.items()}) + prepared.append((case, x, original, expected)) + result = { + "status": "completed", + "action": action, + "passed": all(checks.values()), + "assertions": checks, + "environment": { + "backend": "cpu", + "python": platform.python_version(), + "numpy": np.__version__, + "os": platform.system(), + "machine": platform.machine(), + }, + "coverage": manifest["coverage"], + "workloads": workload_records, + } + if not result["passed"] or action == "check": + return result + + # Correctness and profiling do not masquerade as a paired measurement. + profiler = cProfile.Profile() if action == "profile" else None + measurement = manifest["measurement"] + for case, x, original, expected in prepared: + samples = [] + for index in range(measurement["warmups"] + measurement["samples"]): + if profiler: + profiler.enable() + started = time.perf_counter_ns() + output = compute(x, eps=oracle["eps"]) + elapsed = (time.perf_counter_ns() - started) / 1e6 + if profiler: + profiler.disable() + assertions = check_output(output, x, original, expected, oracle, np) + if not all(assertions.values()): + checks.update({f"{case['id']}.call-{index}.{k}": v for k, v in assertions.items()}) + result.update( + passed=False, + cases=cases, + failed_call={ + "case": case["id"], + "call_index": index, + "phase": "warmup" if index < measurement["warmups"] else "measurement", + "elapsed_ms": elapsed, + "prior_samples_ms": samples, + "usable_measurement": False, + }, + ) + return result + if index >= measurement["warmups"]: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve a positive duration") + samples.append(elapsed) + cases.append( + {"id": case["id"], "samples_ms": samples, "median_ms": statistics.median(samples)} + ) + if profiler: + # Strip local installation paths from the textual profile before capture. + with (result_dir / "profile.txt").open("w", encoding="utf-8") as stream: + pstats.Stats(profiler, stream=stream).strip_dirs().sort_stats( + "cumulative" + ).print_stats() + result["diagnostic"] = { + "kind": "cpu-cprofile", + "artifact": "profile.txt", + "performance_verdict": None, + "limitation": "Instrumented CPU calls; no accelerator timeline", + } + else: + result["cases"] = cases + result["measurement"] = measurement + result["metrics"] = {"suite_latency_ms": sum(c["median_ms"] for c in cases)} + return result diff --git a/src/inferrsi/harness/benchmarks/skill_verifier.py b/src/inferrsi/harness/benchmarks/skill_verifier.py new file mode 100644 index 000000000..2ae3e3a74 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/skill_verifier.py @@ -0,0 +1,55 @@ +"""SkillEval verifier for a controller-selected, reviewed retained artifact. + +This is an offline handoff after artifact review, not live generated-code +execution. It deliberately leaves the production engineering sandbox unchanged. +""" + +import json +import os +import sys +from pathlib import Path + +from inferrsi.harness.benchmarks.consumers import grade_artifact +from inferrsi.harness.benchmarks.contract import load_task +from inferrsi.infra.artifacts import atomic_write_json + + +def task_descriptor(task, candidate: Path) -> dict: + return { + "id": task.manifest["id"], + "split": "development", + "request": "Submit the SHA-256 of the controller-reviewed retained implementation.", + "provenance": task.manifest["source"]["url"], + "benchmark": { + "package": str(task.root), + "candidate": str(candidate.resolve()), + "identity": task.identity(candidate), + }, + } + + +def verify(task: dict, answer: dict, output_dir: Path) -> dict: + binding = task["benchmark"] + package = load_task(binding["package"]) + candidate = Path(binding["candidate"]) + if package.identity(candidate) != binding["identity"]: + raise ValueError("Retained artifact or task identity changed") + if answer != {"candidate_sha256": binding["identity"]["candidate_sha256"]}: + return {"passed": False, "assertions": {"artifact_identity": False}} + record = grade_artifact(package, candidate, output_dir, timeout_seconds=25) + if record["checks"] is None: + raise ValueError("Artifact verification did not complete") + return record["checks"] + + +def main(): + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + task = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) + answer = json.loads(Path(sys.argv[2]).read_text(encoding="utf-8")) + result = verify(task, answer, destination.parent / "artifact") + atomic_write_json(destination, result) + return 0 if result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inferrsi/harness/benchmarks/worker.py b/src/inferrsi/harness/benchmarks/worker.py new file mode 100644 index 000000000..ffa05a4c2 --- /dev/null +++ b/src/inferrsi/harness/benchmarks/worker.py @@ -0,0 +1,60 @@ +"""Execute one frozen task action under an existing InferRSI executor.""" + +import argparse +import json +import os +from pathlib import Path + +from ...infra.artifacts import atomic_write_json +from .contract import load_task + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=("check", "measure", "profile")) + for field in ( + "package", + "candidate", + "task-sha256", + "grader-sha256", + "candidate-sha256", + "runtime-sha256", + ): + parser.add_argument("--" + field, required=True) + args = parser.parse_args() + destination = Path(os.environ["INFERRSI_RESULT_PATH"]) + result = {"status": "error", "action": args.action, "passed": False} + try: + task = load_task(args.package) + candidate = Path(args.candidate) + expected = { + "task_id": task.manifest["id"], + "task_sha256": args.task_sha256, + "grader_sha256": args.grader_sha256, + "candidate_sha256": args.candidate_sha256, + "runtime_sha256": args.runtime_sha256, + } + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed") + from .numpy_task import evaluate + + result = evaluate(task, candidate, args.action, destination.parent) + if task.identity(candidate) != expected: + raise ValueError("Frozen task, grader or candidate changed during execution") + result["identity"] = expected + except Exception as error: + # Errors are incomplete evidence, never numerical correctness failures. + # Do not capture arbitrary exception text that may expose private paths. + result = { + "status": "error", + "action": args.action, + "passed": False, + "error_type": type(error).__name__, + } + atomic_write_json(destination, result) + print(json.dumps({k: result[k] for k in ("status", "action", "passed")})) + return 0 if result["status"] == "completed" and result["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/check_installed.py b/tests/check_installed.py index d412f14ce..067fa09fb 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -9,11 +9,20 @@ import inferrsi from inferrsi.catalog import knowledge_records, repository_pack, search_knowledge from inferrsi.cli import main +from inferrsi.harness.benchmarks import builtin_tasks, load_task from inferrsi.knowledge_operations import KnowledgeReader assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) +assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu"} +for benchmark_id in builtin_tasks(): + benchmark = load_task(benchmark_id) + benchmark.assert_unchanged() + assert (benchmark.root / "baseline.py").is_file() + assert (benchmark.root / "candidate.py").is_file() + assert (benchmark.root / "LICENSE").is_file() + records = knowledge_records() assert len(records) == 127, "Knowledge is missing from the installation" assert ( diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py new file mode 100644 index 000000000..0e852f068 --- /dev/null +++ b/tests/test_engineering_benchmarks.py @@ -0,0 +1,296 @@ +"""Real payload checks and consumer regressions; no model/device calls.""" + +import json +import os +import shutil +import subprocess +import sys +import time +from pathlib import Path + +import pytest + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.scenarios.external import execute_trial +from inferrsi.harness.skill_evaluation import run_trial + + +@pytest.fixture +def numpy_runtime(): + return pytest.importorskip("numpy") + + +def read_result(execution): + return json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +def test_real_baseline_candidate_measurement_and_profile(name, tmp_path, numpy_runtime): + task = load_task(name) + for role in ("baseline", "candidate"): + candidate = task.root / (role + ".py") + grade = grade_artifact(task, candidate, tmp_path / role, timeout_seconds=20) + assert grade["verdict"] == "PASS" + assert len(grade["checks"]["assertions"]) == 24 + candidate = task.root / "candidate.py" + measured = read_result( + run_action(task, candidate, "measure", tmp_path / "timed", timeout_seconds=20) + ) + assert measured["passed"] + assert measured["metrics"]["suite_latency_ms"] > 0 + assert len(measured["cases"]) == len(task.manifest["workloads"]) + assert all(len(c["samples_ms"]) == 5 for c in measured["cases"]) + profile = read_result( + run_action(task, candidate, "profile", tmp_path / "profile", timeout_seconds=20) + ) + assert "metrics" not in profile + assert profile["diagnostic"]["performance_verdict"] is None + assert "compute" in (tmp_path / "profile/profile.txt").read_text() + + +@pytest.mark.parametrize( + "body", + [ + "return np.zeros_like(x)", + "return np.full_like(x, np.nan)", + "return np.ones_like(x, dtype=np.float64)", + "return np.ones((1,), dtype=np.float32)", + "y = np.exp(x); return y / y.sum(axis=1, keepdims=True)", + "x[:] = 0; return np.full_like(x, 1 / x.shape[1])", + "y = np.exp(x - x.max(axis=0, keepdims=True)); return y / y.sum(axis=0, keepdims=True)", + ], +) +def test_bad_candidates_cannot_supply_performance(body, tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import numpy as np\ndef compute(x, *, eps):\n " + body + "\n") + task = load_task("softmax-cpu") + grade = grade_artifact(task, candidate, tmp_path / "grade", timeout_seconds=20) + assert grade["verdict"] == "FAIL" + execution = run_action(task, candidate, "measure", tmp_path / "measure", timeout_seconds=20) + assert execution["status"] == "failed" + assert "metrics" not in read_result(execution) + + +def test_rmsnorm_last_axis_bug_detected(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text( + "import numpy as np\ndef compute(x, *, eps):\n" + " return x / np.sqrt(np.mean(x*x, axis=-1, keepdims=True) + eps)\n" + ) + assert ( + grade_artifact(load_task("rmsnorm-cpu"), candidate, tmp_path / "grade", timeout_seconds=20)[ + "verdict" + ] + == "FAIL" + ) + + +def test_checks_repeated_timed_calls_not_only_initial_call(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text( + "import numpy as np\nn = 0\ndef compute(x, *, eps):\n" + " global n\n n += 1\n" + " if n > 6: return np.zeros_like(x)\n" + " y = np.exp(x-x.max(axis=1,keepdims=True))\n" + " return y/y.sum(axis=1,keepdims=True)\n" + ) + execution = run_action( + load_task("softmax-cpu"), candidate, "measure", tmp_path / "run", timeout_seconds=20 + ) + assert execution["status"] == "failed" + result = read_result(execution) + assert not result["passed"] and "metrics" not in result + + +def test_frozen_package_and_candidate_drift(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + task = load_task(root) + candidate = root / "candidate.py" + command = task.command("check", candidate) + candidate.write_text(candidate.read_text() + "\n# changed\n") + with pytest.raises(ValueError, match="changed"): + task.command("check", candidate) + execution = execute_trial(command, root, tmp_path / "run", time.monotonic() + 20, None) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +@pytest.mark.parametrize( + "field,value", + [ + ("schema_version", 2), + ("schema_version", True), + ("adapter", "unknown"), + ("environment", {"backend": "cuda", "runtime": "numpy", "dtype": "float32"}), + ("coverage", []), + ("workloads", []), + ], +) +def test_unsupported_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text(encoding="utf-8")) + document[field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_descriptor_does_not_mutate_task(): + task = load_task("softmax-cpu") + manifest = task.manifest + manifest["oracle"]["axis"] = 0 + assert task.manifest["oracle"]["axis"] == 1 + assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu"} + + +def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + task = load_task("softmax-cpu") + candidate = repo / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", + "-C", + str(repo), + "-c", + "user.name=Fixture", + "-c", + "user.email=fixture@example.invalid", + "commit", + "-qm", + "fixture", + ], + check=True, + ) + spec = task.runspec( + candidate, + repo=repo, + output_dir=tmp_path / "runs", + repetitions=2, + max_minutes=1, + min_improvement_pct=0, + ) + run = run_experiment(spec, {"scope": "CPU software integration"}) + evaluation = json.loads((run / "evaluation.json").read_text()) + assert evaluation["source_integrity"]["verified"] + assert evaluation["baseline"]["excluded_trials"] == 0 + assert evaluation["candidate"]["excluded_trials"] == 0 + assert evaluation["statistical_significance"] == "not established" + events = [json.loads(line) for line in (run / "events.jsonl").read_text().splitlines()] + assert [e["side"] for e in events if e["phase"] == "measurement"] == [ + "baseline", + "candidate", + "candidate", + "baseline", + ] + identity = json.loads((run / "candidate/trial-001/result.json").read_text())["identity"] + assert identity == task.identity(candidate) + + +class RetainedControl: + """Local authored control for the real trial machinery; no model or token usage.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +@pytest.mark.parametrize("good", [True, False]) +def test_actual_skilleval_trial_uses_same_oracle(good, tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = tmp_path / "implementation.py" + candidate.write_text( + (package.root / "candidate.py").read_text() + if good + else "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + ) + task = skill_verifier.task_descriptor(package, candidate) + workspace = tmp_path / "workspace" + workspace.mkdir() + (workspace / "task.json").write_text(json.dumps(task)) + client = RetainedControl(task["benchmark"]["identity"]["candidate_sha256"]) + record = run_trial( + client, + task, + "N", + 1, + workspace, + tmp_path / "trial", + tmp_path / "unused-schema.json", + Path(skill_verifier.__file__), + "Offline authored control", + time.monotonic() + 30, + ) + assert record["verdict"] == ("PASS" if good else "FAIL") + assert record["inputs_unchanged"] + assert record["observations"]["usage"] is None + raw = json.loads((tmp_path / "trial/verification/artifact/result.json").read_text()) + assert raw["identity"] == task["benchmark"]["identity"] + + +def test_missing_numpy_has_no_fallback(tmp_path): + # An import blocker exercises dependency failure in a real worker process. + blocker = tmp_path / "numpy.py" + blocker.write_text("raise ImportError('numpy deliberately unavailable')\n") + task = load_task("softmax-cpu") + result = tmp_path / "result.json" + env = {**os.environ, "PYTHONPATH": str(tmp_path), "INFERRSI_RESULT_PATH": str(result)} + process = subprocess.run( + task.command("measure", task.root / "candidate.py"), + env=env, + cwd=tmp_path, + capture_output=True, + ) + assert process.returncode == 1 + document = json.loads(result.read_text()) + assert document["status"] == "error" and "metrics" not in document + + +def test_timeout_is_incomplete_and_retains_execution(tmp_path, numpy_runtime): + candidate = tmp_path / "implementation.py" + candidate.write_text("import time\ndef compute(x, *, eps):\n time.sleep(10)\n") + result = grade_artifact( + load_task("softmax-cpu"), candidate, tmp_path / "timeout", timeout_seconds=0.5 + ) + assert result["verdict"] == "INCONCLUSIVE" + assert result["execution_status"] == "timeout" + assert result["checks"] is None + assert (tmp_path / "timeout/execution.json").is_file() + + +def test_runtime_and_grader_drift_rejected(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") + for field in ("--runtime-sha256", "--grader-sha256", "--candidate-sha256"): + command = list(task.command("check", task.root / "candidate.py")) + command[command.index(field) + 1] = "0" * 64 + execution = execute_trial( + tuple(command), task.root, tmp_path / field[2:], time.monotonic() + 20, None + ) + assert execution["status"] == "failed" + assert read_result(execution)["status"] == "error" + + +def test_skill_verifier_binds_answer_to_retained_artifact(tmp_path, numpy_runtime): + package = load_task("softmax-cpu") + candidate = package.root / "candidate.py" + descriptor = skill_verifier.task_descriptor(package, candidate) + result = skill_verifier.verify(descriptor, {"candidate_sha256": "0" * 64}, tmp_path / "unused") + assert result == {"passed": False, "assertions": {"artifact_identity": False}} + assert not (tmp_path / "unused").exists() diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py new file mode 100644 index 000000000..57663d40c --- /dev/null +++ b/tools/engineering_benchmark_validate.py @@ -0,0 +1,170 @@ +"""Reproduce the #252 CPU task integration with retained, authored controls. + +No models, hardware allocators or downloaded programs are invoked. Output is a +local validation bundle, not campaign acceptance or a skill-benefit experiment. +""" + +import argparse +import json +import shutil +import sys +import time +from pathlib import Path + +from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.skill_evaluation import run_trial +from inferrsi.infra.artifacts import atomic_write_json + + +class AuthoredControl: + """A local fixture emits a retained artifact identity, never a claimed score.""" + + timeout_seconds = 20 + + def __init__(self, identity): + self.identity = identity + + def command(self, workspace, final, schema, prompt): + script = ( + "import json, pathlib, sys; " + "pathlib.Path(sys.argv[1]).write_text(json.dumps({'candidate_sha256':sys.argv[2]})); " + "print(json.dumps({'type':'turn.completed'}))" + ) + return sys.executable, "-B", "-c", script, str(final), self.identity + + +def replay(task, candidate, directory): + descriptor = skill_verifier.task_descriptor(task, candidate) + workspace = directory / "workspace" + workspace.mkdir(parents=True) + atomic_write_json(workspace / "task.json", descriptor) + return run_trial( + AuthoredControl(descriptor["benchmark"]["identity"]["candidate_sha256"]), + descriptor, + "N", + 1, + workspace, + directory / "trial", + directory / "unused-schema.json", + Path(skill_verifier.__file__), + "Reviewed offline control", + time.monotonic() + 40, + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + args = parser.parse_args() + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + records = [] + for name in builtin_tasks(): + directory = output / name + directory.mkdir() + original = load_task(name) + shutil.copytree( + original.root, directory / "task", ignore=shutil.ignore_patterns("__pycache__") + ) + task = load_task(directory / "task") + candidate = directory / "implementation.py" + shutil.copyfile(task.root / "candidate.py", candidate) + # Freeze the plan before any payload execution. + spec = task.runspec( + candidate, + repo=args.repo, + output_dir=directory / "workbench", + repetitions=args.repetitions, + max_minutes=2, + min_improvement_pct=0, + ) + atomic_write_json(directory / "runspec.json", spec.to_dict()) + atomic_write_json(directory / "identity.json", task.identity(candidate)) + grade = grade_artifact(task, candidate, directory / "correctness", timeout_seconds=20) + profile = run_action(task, candidate, "profile", directory / "profile", timeout_seconds=20) + run = run_experiment( + spec, {"scope": "CPU development integration", "identity": task.identity(candidate)} + ) + evaluation = json.loads((run / "evaluation.json").read_text(encoding="utf-8")) + good = replay(task, candidate, directory / "skilleval-good") + bad_source = directory / "wrong.py" + bad_source.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + bad = replay(task, bad_source, directory / "skilleval-bad") + rejected = run_action( + task, bad_source, "measure", directory / "wrong-measure", timeout_seconds=20 + ) + measured = [ + json.loads(p.read_text(encoding="utf-8")) + for side in ("baseline", "candidate") + for p in sorted((run / side).glob("trial-*/result.json")) + ] + consumers = measured + [ + json.loads((directory / path).read_text(encoding="utf-8")) + for path in ( + "correctness/result.json", + "profile/result.json", + "skilleval-good/trial/verification/artifact/result.json", + "skilleval-bad/trial/verification/artifact/result.json", + ) + ] + frozen_identity = task.identity(candidate) + record = { + "task": name, + "identity": task.identity(candidate), + "correctness": grade["verdict"], + "profile_status": profile["status"], + "skilleval_good": good["verdict"], + "skilleval_bad": bad["verdict"], + "bad_measurement_status": rejected["status"], + "measurement_trials": len(measured), + "measured_calls": sum(len(c["samples_ms"]) for r in measured for c in r["cases"]), + "source_integrity": evaluation["source_integrity"]["verified"], + "descriptive_workbench_verdict": evaluation["verdict"], + "workbench_run": run.relative_to(output).as_posix(), + "all_consumer_identities_match": all( + all( + r["identity"][key] == frozen_identity[key] + for key in ("task_id", "task_sha256", "grader_sha256", "runtime_sha256") + ) + for r in consumers + ), + "all_workloads_match": all( + r["workloads"] == consumers[0]["workloads"] for r in consumers + ), + } + records.append(record) + print(json.dumps(record), flush=True) + succeeded = all( + r["correctness"] == r["skilleval_good"] == "PASS" + and r["skilleval_bad"] == "FAIL" + and r["bad_measurement_status"] == "failed" + and r["profile_status"] == "completed" + and r["source_integrity"] + and r["all_consumer_identities_match"] + and r["all_workloads_match"] + and r["measurement_trials"] == 2 * args.repetitions + for r in records + ) + atomic_write_json( + output / "summary.json", + { + "scope": "local CPU integration with authored controls", + "checks_succeeded": succeeded, + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + "tasks": records, + }, + ) + return 0 if succeeded else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) From c06b72555d4643df133c59fe7cead5a91e7b86f5 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Tue, 3 Nov 2026 23:36:12 +0800 Subject: [PATCH 02/20] test: demonstrate benchmark utility and timing limits Agent-assisted-by: Codex (validation fixture and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- .gitattributes | 4 + docs/plans/issue-252/04-integration.md | 2 + docs/plans/issue-252/README.md | 2 + .../issue-252/usefulness-profiles/einsum.txt | 25 + .../usefulness-profiles/unchanged.txt | 19 + docs/plans/issue-252/usefulness-summary.json | 2134 +++++++++++++++++ docs/plans/issue-252/usefulness.md | 72 + docs/plans/issue-252/validation.md | 2 + .../engineering_benchmark_usefulness.py | 330 +++ 9 files changed, 2590 insertions(+) create mode 100644 docs/plans/issue-252/usefulness-profiles/einsum.txt create mode 100644 docs/plans/issue-252/usefulness-profiles/unchanged.txt create mode 100644 docs/plans/issue-252/usefulness-summary.json create mode 100644 docs/plans/issue-252/usefulness.md create mode 100644 tests/fixtures/engineering_benchmark_usefulness.py diff --git a/.gitattributes b/.gitattributes index 16433b97f..76dc5c839 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,5 +8,9 @@ /evidence/README.md text eol=lf /evidence/_template/** text eol=lf +# Preserve the recorded utility probe's JSON/profile bytes and their SHA-256s. +/docs/plans/issue-252/usefulness-summary.json -text whitespace=cr-at-eol +/docs/plans/issue-252/usefulness-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof + # Retain this upstream license's original CRLF Git blob bytes. /src/inferrsi/data/skill_library/frameworks/torch-npu/torch-npu/references/licenses/op-plugin-LICENSE.txt -text diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index d3ea11f37..8679e232d 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -27,3 +27,5 @@ 消费链为“同一 task.json 和 grader → Workbench 性能比较 / SkillEval 独立正确性验证”。后者是离线保留产物交接，不是在线 agent 写代码的新 sandbox。是否由技能帮助产出更好代码，需要未来真实 agent、匹配预算和对照条件的实验。 运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，最终结果见 [验证记录](validation.md)。 + +补充[实际工程用途对照](usefulness.md)把预先固定的 no-epsilon、einsum 和 unchanged 实现接入同一任务，证明普通冒烟漏检会被完整任务拒绝。它也暴露了计时稳定性限制：同一代码的比较出现大幅差异，可靠性能排序仍待后续协议验证，不能作为已完成的验收项。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index e639b49e2..d7c83af63 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -38,6 +38,8 @@ Profiling 保留为独立诊断操作，与 correctness 和配对 performance 最终结果见 [验证记录](validation.md)。 +[实际工程用途对照](usefulness.md)进一步验证了错误拦截和两个消费者的复用价值，并如实记录了原版对原版的计时噪声；可靠性能优选尚未建立。 + ## 当前交付 公开子任务分别为 [#334 调研与决策](https://github.com/InferMatrixLab/InferRSI/issues/334)、[#335 共享接口](https://github.com/InferMatrixLab/InferRSI/issues/335)、[#336 首批任务](https://github.com/InferMatrixLab/InferRSI/issues/336)、[#337 消费者与验证](https://github.com/InferMatrixLab/InferRSI/issues/337)。四项均引用父任务 #252，保持打开供维护者评审与验收。当前提交账号无法写入原生父子关系，需维护者补充。 diff --git a/docs/plans/issue-252/usefulness-profiles/einsum.txt b/docs/plans/issue-252/usefulness-profiles/einsum.txt new file mode 100644 index 000000000..ee7a4a8f4 --- /dev/null +++ b/docs/plans/issue-252/usefulness-profiles/einsum.txt @@ -0,0 +1,25 @@ + 1260 function calls in 0.001 seconds + + Ordered by: cumulative time + + ncalls tottime percall cumtime percall filename:lineno(function) + 42 0.000 0.000 0.001 0.000 implementation.py:2(compute) + 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:515(expand_dims) + 42 0.000 0.000 0.000 0.000 einsumfunc.py:1244(einsum) + 42 0.000 0.000 0.000 0.000 {built-in method numpy._core._multiarray_umath.c_einsum} + 42 0.000 0.000 0.000 0.000 numeric.py:1431(normalize_axis_tuple) + 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:600() + 84 0.000 0.000 0.000 0.000 numeric.py:1480() + 210 0.000 0.000 0.000 0.000 einsumfunc.py:1236(_einsum_dispatcher) + 126 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} + 42 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects} + 84 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} + 126 0.000 0.000 0.000 0.000 {built-in method builtins.len} + 126 0.000 0.000 0.000 0.000 {built-in method builtins.next} + 42 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} + 42 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} + 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:511(_expand_dims_dispatcher) + 42 0.000 0.000 0.000 0.000 {built-in method builtins.iter} + 42 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + diff --git a/docs/plans/issue-252/usefulness-profiles/unchanged.txt b/docs/plans/issue-252/usefulness-profiles/unchanged.txt new file mode 100644 index 000000000..9053ff523 --- /dev/null +++ b/docs/plans/issue-252/usefulness-profiles/unchanged.txt @@ -0,0 +1,19 @@ + 630 function calls in 0.002 seconds + + Ordered by: cumulative time + + ncalls tottime percall cumtime percall filename:lineno(function) + 42 0.000 0.000 0.002 0.000 implementation.py:6(compute) + 42 0.000 0.000 0.001 0.000 fromnumeric.py:3699(mean) + 42 0.001 0.000 0.001 0.000 _methods.py:115(_mean) + 42 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} + 42 0.000 0.000 0.000 0.000 _methods.py:73(_count_reduce_items) + 84 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} + 84 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} + 84 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass} + 42 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} + 42 0.000 0.000 0.000 0.000 fromnumeric.py:3694(_mean_dispatcher) + 42 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} + 42 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + diff --git a/docs/plans/issue-252/usefulness-summary.json b/docs/plans/issue-252/usefulness-summary.json new file mode 100644 index 000000000..c0ca64203 --- /dev/null +++ b/docs/plans/issue-252/usefulness-summary.json @@ -0,0 +1,2134 @@ +{ + "scope": "CPU engineering-use probe with trusted authored candidates", + "plan_sha256": "05fdde5a55f7420569fa85168c39b7873e6a4ed207d05940de5622ab9933b51e", + "script_sha256": "317285d0534f9935d06e1fc488ef35c56a76ab7cba2c2423e5a73ba09799ebc7", + "replay_helper_sha256": "6d7670a62203676de519e893ca5abfaa1d1de6fca94b2b9e136c30382f8959cd", + "smoke_config": { + "seed": 2026, + "shape": [ + 4, + 17, + 9, + 7 + ], + "scale": 4, + "dtype": "float32" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "checks": { + "valid_candidates_accepted": true, + "wrong_candidate_rejected": true, + "wrong_has_no_metric": true, + "profiles_complete": true, + "consumer_identities_match": true, + "actual_workloads_match": true, + "comparison_quality_passed": true, + "comparisons_complete": true + }, + "checks_succeeded": true, + "identities": { + "unchanged": { + "task_id": "rmsnorm-cpu", + "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", + "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", + "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "einsum": { + "task_id": "rmsnorm-cpu", + "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", + "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", + "candidate_sha256": "8f29ea405865f4493775ca809e37c0dd7f054dbb2b203e6290975b2e9e41b471", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "no-eps": { + "task_id": "rmsnorm-cpu", + "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", + "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", + "candidate_sha256": "9b78a97381b4e5775af022934ea504f6eb6269915adcfb5ffc580fc68693f046", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + } + }, + "comparison": { + "repetitions": 6, + "max_minutes": 2.0, + "min_improvement_pct": 0.0 + }, + "workloads": [ + { + "id": "normal", + "input_sha256": "a931597557d07c29727b4746588ff9f2db5f5a23c67b7148e22bbadf536f6b68", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ] + }, + { + "id": "strided", + "input_sha256": "5a1a5521d252443593cf93106a97ab37b39a3ef3d9acde5e50aa6aedc56f912a", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ] + }, + { + "id": "zeros", + "input_sha256": "16d0edc8b7ad7705b23a14058f366ff1c0dfa16a0ad14f741924c308754cf8d1", + "shape": [ + 2, + 3, + 5, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 140, + 28, + 4 + ] + }, + { + "id": "large", + "input_sha256": "4f3eac31387a99ca7259dd058b228f7e3c57d0d349131aa0b5ccb237ffb75660", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ] + }, + { + "id": "tiny", + "input_sha256": "558d3471bf22a7b3cb4ef169090e15f3dde3440de6f695c85d04718a5f300cb5", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ] + }, + { + "id": "singleton", + "input_sha256": "5159fde66d7906ff87e0aabde2557fa06e8294f024b0783ef97a2c23e408ba05", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ] + } + ], + "profiles": { + "unchanged": { + "status": "completed", + "diagnostic": { + "kind": "cpu-cprofile", + "artifact": "profile.txt", + "performance_verdict": null, + "limitation": "Instrumented CPU calls; no accelerator timeline" + }, + "has_metric": false, + "artifact_sha256": "7a89e01154b3bb0c0be3e08964d6e5c5c2f6b801889a0cc0c5fdfbebf41f4e7b" + }, + "einsum": { + "status": "completed", + "diagnostic": { + "kind": "cpu-cprofile", + "artifact": "profile.txt", + "performance_verdict": null, + "limitation": "Instrumented CPU calls; no accelerator timeline" + }, + "has_metric": false, + "artifact_sha256": "bdbf5421222ac36d3ef2efd118d8cbd23b15a6b5d2db4beaaedfce1331c0826d" + } + }, + "candidates": { + "unchanged": { + "correctness": "PASS", + "skilleval": "PASS", + "failed_assertions": [] + }, + "einsum": { + "correctness": "PASS", + "skilleval": "PASS", + "failed_assertions": [] + }, + "no-eps": { + "correctness": "FAIL", + "skilleval": "FAIL", + "failed_assertions": [ + "normal.values", + "strided.values", + "zeros.finite", + "zeros.values", + "tiny.values", + "singleton.values" + ] + } + }, + "ordinary_smoke": { + "input_sha256": "0c13945f4562c85d8fd2d9cf450a9e02847b5d34d9dc58f1b54f073c3502a51f", + "candidates": { + "unchanged": { + "passed": true, + "assertions": { + "shape_dtype": true, + "finite": true, + "input_preserved": true, + "values": true + } + }, + "einsum": { + "passed": true, + "assertions": { + "shape_dtype": true, + "finite": true, + "input_preserved": true, + "values": true + } + }, + "no-eps": { + "passed": true, + "assertions": { + "shape_dtype": true, + "finite": true, + "input_preserved": true, + "values": true + } + } + } + }, + "comparisons": { + "unchanged": { + "verdict": "FAIL", + "baseline": { + "values": [ + 0.063, + 0.0484, + 0.046200000000000005, + 0.0459, + 0.0482, + 0.0496 + ], + "median": 0.048299999999999996, + "min": 0.0459, + "max": 0.063, + "range_pct_of_median": 35.40372670807453, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.1169, + 0.11330000000000001, + 0.0622, + 0.0455, + 0.0458, + 0.0832 + ], + "median": 0.0727, + "min": 0.0455, + "max": 0.1169, + "range_pct_of_median": 98.21182943603853, + "excluded_trials": 0 + }, + "improvement_pct": -50.51759834368531, + "statistical_significance": "not established", + "final_quality_passed": true, + "source_integrity": true, + "trials": [ + { + "side": "baseline", + "trial": "trial-001", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0219, + 0.0176, + 0.0151, + 0.0148, + 0.0138 + ], + "median_ms": 0.0151 + }, + { + "id": "strided", + "samples_ms": [ + 0.018, + 0.018, + 0.0175, + 0.0189, + 0.0179 + ], + "median_ms": 0.018 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0089, + 0.0084, + 0.0085, + 0.0086, + 0.0088 + ], + "median_ms": 0.0086 + }, + { + "id": "large", + "samples_ms": [ + 0.009, + 0.0084, + 0.007, + 0.0074, + 0.0081 + ], + "median_ms": 0.0081 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0075, + 0.0073, + 0.0071, + 0.007, + 0.007 + ], + "median_ms": 0.0071 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0065, + 0.0067, + 0.006, + 0.006, + 0.0061 + ], + "median_ms": 0.0061 + } + ], + "metrics": { + "suite_latency_ms": 0.063 + } + }, + { + "side": "baseline", + "trial": "trial-002", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0162, + 0.0141, + 0.0126, + 0.0121, + 0.0105 + ], + "median_ms": 0.0126 + }, + { + "id": "strided", + "samples_ms": [ + 0.0122, + 0.0122, + 0.0124, + 0.0128, + 0.0124 + ], + "median_ms": 0.0124 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0059, + 0.0064, + 0.0061, + 0.0064, + 0.0059 + ], + "median_ms": 0.0061 + }, + { + "id": "large", + "samples_ms": [ + 0.0062, + 0.0061, + 0.0063, + 0.006, + 0.0067 + ], + "median_ms": 0.0062 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0059, + 0.006, + 0.006, + 0.0061 + ], + "median_ms": 0.006 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0049, + 0.0051, + 0.0051, + 0.0052, + 0.0049 + ], + "median_ms": 0.0051 + } + ], + "metrics": { + "suite_latency_ms": 0.0484 + } + }, + { + "side": "baseline", + "trial": "trial-003", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0139, + 0.012, + 0.0111, + 0.0108, + 0.01 + ], + "median_ms": 0.0111 + }, + { + "id": "strided", + "samples_ms": [ + 0.0131, + 0.0127, + 0.0126, + 0.0137, + 0.0123 + ], + "median_ms": 0.0127 + }, + { + "id": "zeros", + "samples_ms": [ + 0.006, + 0.0056, + 0.0057, + 0.0058, + 0.006 + ], + "median_ms": 0.0058 + }, + { + "id": "large", + "samples_ms": [ + 0.006, + 0.0061, + 0.0062, + 0.006, + 0.0064 + ], + "median_ms": 0.0061 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0055, + 0.0055, + 0.0058, + 0.0056 + ], + "median_ms": 0.0056 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0049, + 0.0049, + 0.0052, + 0.005, + 0.0049 + ], + "median_ms": 0.0049 + } + ], + "metrics": { + "suite_latency_ms": 0.046200000000000005 + } + }, + { + "side": "baseline", + "trial": "trial-004", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0137, + 0.012, + 0.0113, + 0.0107, + 0.0097 + ], + "median_ms": 0.0113 + }, + { + "id": "strided", + "samples_ms": [ + 0.0123, + 0.0119, + 0.0123, + 0.013, + 0.0123 + ], + "median_ms": 0.0123 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0059, + 0.0058, + 0.0058, + 0.0055, + 0.0057 + ], + "median_ms": 0.0058 + }, + { + "id": "large", + "samples_ms": [ + 0.006, + 0.006, + 0.0058, + 0.0059, + 0.0065 + ], + "median_ms": 0.006 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0055, + 0.0057, + 0.0056, + 0.0056, + 0.0057 + ], + "median_ms": 0.0056 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0049, + 0.0049, + 0.0049, + 0.0049, + 0.0048 + ], + "median_ms": 0.0049 + } + ], + "metrics": { + "suite_latency_ms": 0.0459 + } + }, + { + "side": "baseline", + "trial": "trial-005", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0147, + 0.0128, + 0.0115, + 0.0115, + 0.0106 + ], + "median_ms": 0.0115 + }, + { + "id": "strided", + "samples_ms": [ + 0.0138, + 0.0246, + 0.0161, + 0.0145, + 0.0132 + ], + "median_ms": 0.0145 + }, + { + "id": "zeros", + "samples_ms": [ + 0.006, + 0.0058, + 0.006, + 0.0058, + 0.0058 + ], + "median_ms": 0.0058 + }, + { + "id": "large", + "samples_ms": [ + 0.006, + 0.0062, + 0.0058, + 0.0058, + 0.0066 + ], + "median_ms": 0.006 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0059, + 0.0056, + 0.0056, + 0.0056, + 0.0055 + ], + "median_ms": 0.0056 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0049, + 0.0049, + 0.0047, + 0.0047, + 0.0048 + ], + "median_ms": 0.0048 + } + ], + "metrics": { + "suite_latency_ms": 0.0482 + } + }, + { + "side": "baseline", + "trial": "trial-006", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0157, + 0.0137, + 0.0123, + 0.0121, + 0.011 + ], + "median_ms": 0.0123 + }, + { + "id": "strided", + "samples_ms": [ + 0.0134, + 0.0133, + 0.0129, + 0.0137, + 0.0128 + ], + "median_ms": 0.0133 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0093, + 0.0075, + 0.0066, + 0.0063, + 0.0061 + ], + "median_ms": 0.0066 + }, + { + "id": "large", + "samples_ms": [ + 0.0064, + 0.0063, + 0.0061, + 0.006, + 0.0073 + ], + "median_ms": 0.0063 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0061, + 0.0061, + 0.006, + 0.0059, + 0.0058 + ], + "median_ms": 0.006 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0052, + 0.0052, + 0.0051, + 0.005, + 0.005 + ], + "median_ms": 0.0051 + } + ], + "metrics": { + "suite_latency_ms": 0.0496 + } + }, + { + "side": "candidate", + "trial": "trial-001", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0363, + 0.0314, + 0.028, + 0.0258, + 0.0246 + ], + "median_ms": 0.028 + }, + { + "id": "strided", + "samples_ms": [ + 0.0398, + 0.034, + 0.0318, + 0.0348, + 0.0331 + ], + "median_ms": 0.034 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0147, + 0.0147, + 0.0144, + 0.0138, + 0.0138 + ], + "median_ms": 0.0144 + }, + { + "id": "large", + "samples_ms": [ + 0.0147, + 0.0148, + 0.0144, + 0.0139, + 0.0163 + ], + "median_ms": 0.0147 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0146, + 0.0143, + 0.0141, + 0.0142, + 0.0139 + ], + "median_ms": 0.0142 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0115, + 0.0116, + 0.0117, + 0.0116, + 0.0117 + ], + "median_ms": 0.0116 + } + ], + "metrics": { + "suite_latency_ms": 0.1169 + } + }, + { + "side": "candidate", + "trial": "trial-002", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0321, + 0.0282, + 0.0256, + 0.0263, + 0.0239 + ], + "median_ms": 0.0263 + }, + { + "id": "strided", + "samples_ms": [ + 0.0313, + 0.0308, + 0.0307, + 0.0319, + 0.0316 + ], + "median_ms": 0.0313 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0154, + 0.0154, + 0.0147, + 0.0143, + 0.0151 + ], + "median_ms": 0.0151 + }, + { + "id": "large", + "samples_ms": [ + 0.0152, + 0.015, + 0.0146, + 0.0149, + 0.0159 + ], + "median_ms": 0.015 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0141, + 0.0138, + 0.014, + 0.0137, + 0.0135 + ], + "median_ms": 0.0138 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0118, + 0.0118, + 0.0125, + 0.0118, + 0.0229 + ], + "median_ms": 0.0118 + } + ], + "metrics": { + "suite_latency_ms": 0.11330000000000001 + } + }, + { + "side": "candidate", + "trial": "trial-003", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0207, + 0.0172, + 0.0153, + 0.0151, + 0.0136 + ], + "median_ms": 0.0153 + }, + { + "id": "strided", + "samples_ms": [ + 0.018, + 0.0173, + 0.0173, + 0.0184, + 0.0173 + ], + "median_ms": 0.0173 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0085, + 0.0081, + 0.0077, + 0.0073, + 0.0076 + ], + "median_ms": 0.0077 + }, + { + "id": "large", + "samples_ms": [ + 0.0077, + 0.0079, + 0.0079, + 0.0075, + 0.0083 + ], + "median_ms": 0.0079 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0076, + 0.0076, + 0.0073, + 0.0072, + 0.0076 + ], + "median_ms": 0.0076 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0066, + 0.0064, + 0.0064, + 0.0063, + 0.0063 + ], + "median_ms": 0.0064 + } + ], + "metrics": { + "suite_latency_ms": 0.0622 + } + }, + { + "side": "candidate", + "trial": "trial-004", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0135, + 0.0119, + 0.0113, + 0.0105, + 0.0103 + ], + "median_ms": 0.0113 + }, + { + "id": "strided", + "samples_ms": [ + 0.0126, + 0.014, + 0.012, + 0.0129, + 0.0125 + ], + "median_ms": 0.0126 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0057, + 0.0059, + 0.0056, + 0.0055, + 0.0056 + ], + "median_ms": 0.0056 + }, + { + "id": "large", + "samples_ms": [ + 0.0054, + 0.0056, + 0.0057, + 0.0056, + 0.0062 + ], + "median_ms": 0.0056 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0056, + 0.0058, + 0.0056, + 0.0054, + 0.0055 + ], + "median_ms": 0.0056 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0048, + 0.0048, + 0.0049, + 0.0049, + 0.0047 + ], + "median_ms": 0.0048 + } + ], + "metrics": { + "suite_latency_ms": 0.0455 + } + }, + { + "side": "candidate", + "trial": "trial-005", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0136, + 0.0116, + 0.0103, + 0.0103, + 0.0096 + ], + "median_ms": 0.0103 + }, + { + "id": "strided", + "samples_ms": [ + 0.0129, + 0.0128, + 0.0127, + 0.0134, + 0.0126 + ], + "median_ms": 0.0128 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0062, + 0.0058, + 0.0057, + 0.0058, + 0.0059 + ], + "median_ms": 0.0058 + }, + { + "id": "large", + "samples_ms": [ + 0.0062, + 0.0063, + 0.0059, + 0.0057, + 0.0068 + ], + "median_ms": 0.0062 + }, + { + "id": "tiny", + "samples_ms": [ + 0.006, + 0.0058, + 0.0059, + 0.0057, + 0.0057 + ], + "median_ms": 0.0058 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0048, + 0.0051, + 0.0049, + 0.005, + 0.0049 + ], + "median_ms": 0.0049 + } + ], + "metrics": { + "suite_latency_ms": 0.0458 + } + }, + { + "side": "candidate", + "trial": "trial-006", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0145, + 0.0124, + 0.0115, + 0.0107, + 0.0564 + ], + "median_ms": 0.0124 + }, + { + "id": "strided", + "samples_ms": [ + 0.0579, + 0.035, + 0.0278, + 0.0288, + 0.0259 + ], + "median_ms": 0.0288 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0113, + 0.0118, + 0.0109, + 0.0109, + 0.0108 + ], + "median_ms": 0.0109 + }, + { + "id": "large", + "samples_ms": [ + 0.0112, + 0.0113, + 0.0105, + 0.011, + 0.0136 + ], + "median_ms": 0.0112 + }, + { + "id": "tiny", + "samples_ms": [ + 0.011, + 0.0107, + 0.0108, + 0.011, + 0.0111 + ], + "median_ms": 0.011 + }, + { + "id": "singleton", + "samples_ms": [ + 0.009, + 0.0089, + 0.0089, + 0.0088, + 0.0089 + ], + "median_ms": 0.0089 + } + ], + "metrics": { + "suite_latency_ms": 0.0832 + } + } + ], + "measured_calls": 360 + }, + "einsum": { + "verdict": "FAIL", + "baseline": { + "values": [ + 0.046099999999999995, + 0.0773, + 0.0537, + 0.04529999999999999, + 0.0463, + 0.0522 + ], + "median": 0.04925, + "min": 0.04529999999999999, + "max": 0.0773, + "range_pct_of_median": 64.9746192893401, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.08750000000000001, + 0.052199999999999996, + 0.0439, + 0.0868, + 0.0891, + 0.0987 + ], + "median": 0.08715, + "min": 0.0439, + "max": 0.0987, + "range_pct_of_median": 62.88009179575443, + "excluded_trials": 0 + }, + "improvement_pct": -76.95431472081219, + "statistical_significance": "not established", + "final_quality_passed": true, + "source_integrity": true, + "trials": [ + { + "side": "baseline", + "trial": "trial-001", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0134, + 0.012, + 0.0104, + 0.0108, + 0.01 + ], + "median_ms": 0.0108 + }, + { + "id": "strided", + "samples_ms": [ + 0.0131, + 0.0126, + 0.0128, + 0.0128, + 0.0128 + ], + "median_ms": 0.0128 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0059, + 0.0061, + 0.0059, + 0.0063, + 0.0055 + ], + "median_ms": 0.0059 + }, + { + "id": "large", + "samples_ms": [ + 0.0056, + 0.006, + 0.0056, + 0.0058, + 0.0066 + ], + "median_ms": 0.0058 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0058, + 0.0057, + 0.0061, + 0.0057 + ], + "median_ms": 0.0058 + }, + { + "id": "singleton", + "samples_ms": [ + 0.005, + 0.005, + 0.0049, + 0.0049, + 0.0052 + ], + "median_ms": 0.005 + } + ], + "metrics": { + "suite_latency_ms": 0.046099999999999995 + } + }, + { + "side": "baseline", + "trial": "trial-002", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0259, + 0.021, + 0.0194, + 0.019, + 0.0167 + ], + "median_ms": 0.0194 + }, + { + "id": "strided", + "samples_ms": [ + 0.0207, + 0.021, + 0.0207, + 0.0225, + 0.0209 + ], + "median_ms": 0.0209 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0102, + 0.0096, + 0.0092, + 0.0096, + 0.0094 + ], + "median_ms": 0.0096 + }, + { + "id": "large", + "samples_ms": [ + 0.0099, + 0.0097, + 0.0096, + 0.0094, + 0.0111 + ], + "median_ms": 0.0097 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0098, + 0.0095, + 0.0094, + 0.0095, + 0.0094 + ], + "median_ms": 0.0095 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0082, + 0.0084, + 0.008, + 0.0082, + 0.0071 + ], + "median_ms": 0.0082 + } + ], + "metrics": { + "suite_latency_ms": 0.0773 + } + }, + { + "side": "baseline", + "trial": "trial-003", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0149, + 0.0127, + 0.0111, + 0.0112, + 0.0106 + ], + "median_ms": 0.0112 + }, + { + "id": "strided", + "samples_ms": [ + 0.0136, + 0.0133, + 0.0133, + 0.0138, + 0.0135 + ], + "median_ms": 0.0135 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0062, + 0.0065, + 0.0061, + 0.0062, + 0.0063 + ], + "median_ms": 0.0062 + }, + { + "id": "large", + "samples_ms": [ + 0.0063, + 0.0063, + 0.0061, + 0.0061, + 0.0075 + ], + "median_ms": 0.0063 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0064, + 0.0062, + 0.0317, + 0.0154, + 0.0112 + ], + "median_ms": 0.0112 + }, + { + "id": "singleton", + "samples_ms": [ + 0.006, + 0.0055, + 0.0053, + 0.0052, + 0.005 + ], + "median_ms": 0.0053 + } + ], + "metrics": { + "suite_latency_ms": 0.0537 + } + }, + { + "side": "baseline", + "trial": "trial-004", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0133, + 0.0121, + 0.0107, + 0.0107, + 0.0098 + ], + "median_ms": 0.0107 + }, + { + "id": "strided", + "samples_ms": [ + 0.0125, + 0.0127, + 0.0121, + 0.0131, + 0.0125 + ], + "median_ms": 0.0125 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0061, + 0.0059, + 0.0056, + 0.0059, + 0.0056 + ], + "median_ms": 0.0059 + }, + { + "id": "large", + "samples_ms": [ + 0.0056, + 0.0059, + 0.0056, + 0.0057, + 0.0064 + ], + "median_ms": 0.0057 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0057, + 0.0058, + 0.0057, + 0.0057, + 0.0057 + ], + "median_ms": 0.0057 + }, + { + "id": "singleton", + "samples_ms": [ + 0.005, + 0.0046, + 0.0048, + 0.0048, + 0.0046 + ], + "median_ms": 0.0048 + } + ], + "metrics": { + "suite_latency_ms": 0.04529999999999999 + } + }, + { + "side": "baseline", + "trial": "trial-005", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0138, + 0.0125, + 0.0109, + 0.0108, + 0.0102 + ], + "median_ms": 0.0109 + }, + { + "id": "strided", + "samples_ms": [ + 0.0128, + 0.0126, + 0.0125, + 0.0131, + 0.0124 + ], + "median_ms": 0.0126 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0061, + 0.006, + 0.0059, + 0.0058, + 0.0058 + ], + "median_ms": 0.0059 + }, + { + "id": "large", + "samples_ms": [ + 0.0062, + 0.0064, + 0.0059, + 0.0057, + 0.0066 + ], + "median_ms": 0.0062 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0062, + 0.0059, + 0.0058, + 0.0058 + ], + "median_ms": 0.0058 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0051, + 0.0052, + 0.0049, + 0.0049, + 0.0049 + ], + "median_ms": 0.0049 + } + ], + "metrics": { + "suite_latency_ms": 0.0463 + } + }, + { + "side": "baseline", + "trial": "trial-006", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0169, + 0.0141, + 0.0134, + 0.0125, + 0.0117 + ], + "median_ms": 0.0134 + }, + { + "id": "strided", + "samples_ms": [ + 0.0129, + 0.0132, + 0.0124, + 0.0137, + 0.0131 + ], + "median_ms": 0.0131 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0064, + 0.0063, + 0.0064, + 0.0061, + 0.0065 + ], + "median_ms": 0.0064 + }, + { + "id": "large", + "samples_ms": [ + 0.0067, + 0.0077, + 0.0065, + 0.0076, + 0.0073 + ], + "median_ms": 0.0073 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0065, + 0.0066, + 0.0065, + 0.0061, + 0.006 + ], + "median_ms": 0.0065 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0053, + 0.0057, + 0.0061, + 0.0055, + 0.0054 + ], + "median_ms": 0.0055 + } + ], + "metrics": { + "suite_latency_ms": 0.0522 + } + }, + { + "side": "candidate", + "trial": "trial-001", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0271, + 0.0242, + 0.0209, + 0.0216, + 0.0203 + ], + "median_ms": 0.0216 + }, + { + "id": "strided", + "samples_ms": [ + 0.0393, + 0.0244, + 0.0233, + 0.0265, + 0.0215 + ], + "median_ms": 0.0244 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0112, + 0.0113, + 0.0109, + 0.0104, + 0.0105 + ], + "median_ms": 0.0109 + }, + { + "id": "large", + "samples_ms": [ + 0.0105, + 0.0105, + 0.0109, + 0.0102, + 0.0126 + ], + "median_ms": 0.0105 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0109, + 0.0118, + 0.0115, + 0.0108, + 0.0109 + ], + "median_ms": 0.0109 + }, + { + "id": "singleton", + "samples_ms": [ + 0.01, + 0.0092, + 0.009, + 0.0088, + 0.0093 + ], + "median_ms": 0.0092 + } + ], + "metrics": { + "suite_latency_ms": 0.08750000000000001 + } + }, + { + "side": "candidate", + "trial": "trial-002", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0176, + 0.0163, + 0.0149, + 0.0146, + 0.0103 + ], + "median_ms": 0.0149 + }, + { + "id": "strided", + "samples_ms": [ + 0.0122, + 0.0126, + 0.0127, + 0.0133, + 0.0126 + ], + "median_ms": 0.0126 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0066, + 0.0063, + 0.0063, + 0.0064, + 0.0063 + ], + "median_ms": 0.0063 + }, + { + "id": "large", + "samples_ms": [ + 0.0062, + 0.0064, + 0.0066, + 0.0064, + 0.0075 + ], + "median_ms": 0.0064 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0065, + 0.0063, + 0.0063, + 0.0062, + 0.0063 + ], + "median_ms": 0.0063 + }, + { + "id": "singleton", + "samples_ms": [ + 0.006, + 0.0059, + 0.0056, + 0.0057, + 0.0055 + ], + "median_ms": 0.0057 + } + ], + "metrics": { + "suite_latency_ms": 0.052199999999999996 + } + }, + { + "side": "candidate", + "trial": "trial-003", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0133, + 0.0117, + 0.011, + 0.0101, + 0.009 + ], + "median_ms": 0.011 + }, + { + "id": "strided", + "samples_ms": [ + 0.011, + 0.0107, + 0.0101, + 0.0109, + 0.0109 + ], + "median_ms": 0.0109 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0059, + 0.0059, + 0.0059, + 0.0061, + 0.0054 + ], + "median_ms": 0.0059 + }, + { + "id": "large", + "samples_ms": [ + 0.0059, + 0.0056, + 0.0054, + 0.0052, + 0.0064 + ], + "median_ms": 0.0056 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0056, + 0.0054, + 0.0056, + 0.0071 + ], + "median_ms": 0.0056 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0049, + 0.0052, + 0.0052, + 0.0047, + 0.0048 + ], + "median_ms": 0.0049 + } + ], + "metrics": { + "suite_latency_ms": 0.0439 + } + }, + { + "side": "candidate", + "trial": "trial-004", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0269, + 0.0233, + 0.0202, + 0.0199, + 0.0195 + ], + "median_ms": 0.0202 + }, + { + "id": "strided", + "samples_ms": [ + 0.0236, + 0.0229, + 0.0219, + 0.0241, + 0.0231 + ], + "median_ms": 0.0231 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0135, + 0.0132, + 0.0311, + 0.0124, + 0.0121 + ], + "median_ms": 0.0132 + }, + { + "id": "large", + "samples_ms": [ + 0.0102, + 0.0101, + 0.0104, + 0.0102, + 0.0119 + ], + "median_ms": 0.0102 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0102, + 0.0107, + 0.0107, + 0.0121, + 0.0105 + ], + "median_ms": 0.0107 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0098, + 0.0094, + 0.0088, + 0.0096, + 0.0087 + ], + "median_ms": 0.0094 + } + ], + "metrics": { + "suite_latency_ms": 0.0868 + } + }, + { + "side": "candidate", + "trial": "trial-005", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0325, + 0.0266, + 0.0254, + 0.0223, + 0.019 + ], + "median_ms": 0.0254 + }, + { + "id": "strided", + "samples_ms": [ + 0.0217, + 0.0208, + 0.021, + 0.0225, + 0.0212 + ], + "median_ms": 0.0212 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0111, + 0.0106, + 0.0109, + 0.0106, + 0.0106 + ], + "median_ms": 0.0106 + }, + { + "id": "large", + "samples_ms": [ + 0.0107, + 0.0108, + 0.0107, + 0.0104, + 0.0134 + ], + "median_ms": 0.0107 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0111, + 0.0107, + 0.0112, + 0.011, + 0.0111 + ], + "median_ms": 0.0111 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0101, + 0.0101, + 0.0097, + 0.0097, + 0.0101 + ], + "median_ms": 0.0101 + } + ], + "metrics": { + "suite_latency_ms": 0.0891 + } + }, + { + "side": "candidate", + "trial": "trial-006", + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0315, + 0.024, + 0.0233, + 0.0205, + 0.0194 + ], + "median_ms": 0.0233 + }, + { + "id": "strided", + "samples_ms": [ + 0.0261, + 0.0272, + 0.0258, + 0.0245, + 0.0264 + ], + "median_ms": 0.0261 + }, + { + "id": "zeros", + "samples_ms": [ + 0.0135, + 0.0132, + 0.0147, + 0.0123, + 0.0126 + ], + "median_ms": 0.0132 + }, + { + "id": "large", + "samples_ms": [ + 0.0132, + 0.013, + 0.0122, + 0.0127, + 0.0149 + ], + "median_ms": 0.013 + }, + { + "id": "tiny", + "samples_ms": [ + 0.0129, + 0.0125, + 0.0122, + 0.0121, + 0.0144 + ], + "median_ms": 0.0125 + }, + { + "id": "singleton", + "samples_ms": [ + 0.0107, + 0.0109, + 0.0106, + 0.0106, + 0.0106 + ], + "median_ms": 0.0106 + } + ], + "metrics": { + "suite_latency_ms": 0.0987 + } + } + ], + "measured_calls": 360 + } + }, + "unchanged_raw_pass": false, + "interpretation": "Raw Workbench verdicts are descriptive; an unchanged PASS is a false-positive warning, not optimization. No speedup claim.", + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run" +} diff --git a/docs/plans/issue-252/usefulness.md b/docs/plans/issue-252/usefulness.md new file mode 100644 index 000000000..50d8d69d2 --- /dev/null +++ b/docs/plans/issue-252/usefulness.md @@ -0,0 +1,72 @@ +# #252：实际工程用途对照 + +本补证回答一个具体问题：共享任务能否帮助使用者识别一个看似合理的错误改动，并让正确候选经过同一套工程检查和实际性能比较？它不以 CI 绿色或某次计时 PASS 作为“有用”的定义。 + +## 预先固定的对照 + +选择既有 `rmsnorm-cpu` 任务，保持六个公开工作负载、axis=1、float32、epsilon、容差、oracle、预热与采样规则不变。所有代码均为审核过的本地实验实现；SkillEval 交接的是保留产物身份，没有调用生成模型。 + +| 实现 | 检查目的 | +| --- | --- | +| unchanged | 逐字复制原版，作为原版对原版的计时对照 | +| no-epsilon | 一个真实容易出现的错误：省掉 epsilon；先观察普通输入冒烟，再交给完整任务及两个消费者 | +| einsum | 用 axis-1 平方和归约替代显式 `x*x` 中间数组，保留 epsilon、类型、布局和输入语义；性能改善只是待检验假设 | + +普通输入冒烟预先定义为 seed 2026、shape `(4, 17, 9, 7)` 的 float32 正态分布乘以 4，使用任务的独立 oracle 和容差。它与完整任务的压力输入不同，结果会如实保留。即使错误实现未通过此冒烟，也不换一组输入重试。 + +先冻结计划、任务、实现、grader/runtime 身份和比较参数，再执行。对 unchanged 与 einsum 各做一次既有 Workbench 比较，按预定顺序运行，所有 trial 均保留。性能阈值只用于展示既有描述性裁决，不作为本补证的功能通过条件，不根据结果选择重跑或修改阈值。 + +## 怎样判断有实际用途 + +1. 正确候选在完整任务和实际 SkillEval verifier 中均通过；两个消费者使用同一任务、实际输入和 oracle。 +2. no-epsilon 在完整任务中被拒绝，错误测量没有可用性能指标；保留具体失败断言。 +3. 正确候选的实际比较保留每个工作负载的原始样本，不能只挑总分或快的 case。 +4. 原版对原版的结果同样公开；若也出现描述性 PASS，必须标明相同实现的计时波动，不能宣称优化收益。 +5. profiling 独立运行且没有性能胜负，保留调用路径及其分辨能力的限制。 + +## 诊断与决策边界 + +已有 CPU profile 只有 42 次记录的 compute 调用，总时间约毫秒量级，文本中的多个耗时显示为 `0.000`。这能说明实际调用了哪些函数，不能证明内存带宽或临时数组分配就是瓶颈。einsum 候选来自源码检查提出的假设，不能写成 profiler 已证实的优化方向。 + +小尺寸公开用例没有建立真实服务代表性。既有 Workbench 使用中位数和阈值，尚未实现统计显著性或多轮选择偏差控制；未改进实现也可能得到描述性 PASS。本补证保留这个事实，不以集成测试通过替代性能验收。 + +## 复现入口与结果 + +两阶段复现入口为 `python -m tests.fixtures.engineering_benchmark_usefulness`：`prepare` 只冻结输入，`run` 要求显式计划摘要并执行一次。使用全新输出目录，参数来自对应 Issue 的预登记记录。完整原始日志留在本地；公开汇总仅发布无私人路径的身份、结果、失败断言和逐 case 样本。 + +实际执行前的条件、身份与摘要见 [#337 预登记](https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6607885902)。本地原始目录名为 `inferrsi-252-usefulness-20261103-v2`；此前的目录只执行过 prepare，没有运行 payload。独立静态检查发现并修复了补证脚本的最终质量门槛缺口，然后重新冻结计划，实际 payload 只执行一次。 + +```bash +python -m tests.fixtures.engineering_benchmark_usefulness prepare \ + --repo --output-dir \ + --repetitions 6 --max-minutes 2 --min-improvement-pct 0 --timeout-seconds 25 +python -m tests.fixtures.engineering_benchmark_usefulness run \ + --output-dir --plan-sha256 +``` + +这些参数来自本次预登记，不是共享的性能验收默认值。新复现的路径或环境会改变计划摘要，应使用该次 prepare 的返回值。 + +## 实测结果与明确决策 + +2026-11-03 在 Windows / Python 3.11.18 / NumPy 2.4.6 上执行。期间没有并行运行本地测试，但宿主机没有专用资源隔离。运行时与任务仍为 PR 初始实现；本补证只新增 fixture 和说明。 + +| 检查 | 实际结果 | 说明 | +| --- | --- | --- | +| no-epsilon 普通输入冒烟 | 通过 | 单个普通输入确实漏掉了错误 | +| no-epsilon 完整任务 / SkillEval | 两边均 FAIL | 拒绝 `normal.values`、`strided.values`、`zeros.finite`、`zeros.values`、`tiny.values`、`singleton.values` 六个断言 | +| no-epsilon 测量 | 拒绝，无性能指标 | 错误实现没有凭借省略计算获得可用分数 | +| unchanged / einsum 正确性 | 两个实现均在两条路径通过 | 两个 Workbench 比较结束时的质量验证也通过 | +| 共享任务和实际输入 | 一致 | 包含正确性、profiling、计时和真实 SkillEval verifier | +| 原版对原版比较 | 12 个 trial，360 个原始样本；原始 verdict 为 FAIL | 报告的 improvement 为 -50.52%，同码对照说明当前测量条件不足以可靠分辨优化 | +| einsum 对原版比较 | 12 个 trial，360 个原始样本；原始 verdict 为 FAIL | 报告的 improvement 为 -76.95%；不能在上述噪声条件下把数值解释为稳定退化或加速 | +| 独立 CPU profiling | 两份均完成，无性能 verdict | 支持调用路径检查，未确认内存瓶颈 | + +**已经证明的用途：** 共享基准抓住了预先指定的普通冒烟漏检，同时让两个消费者用同一套题目和 oracle 识别正确与错误实现，并保留可审查的实验记录。八项功能检查全部通过。 + +**没有通过的用途：** 本机小尺寸协议尚不能支持可靠性能排序。即使正确性通过，也不能据此采用 einsum 候选；本次决定保留原版，不修改任务包默认实现。不重跑挑选好看的结果，也不把两个性能 FAIL 改成 PASS。 + +后续若要将这些任务用于性能搜索，需要单独预登记计时稳定性验证：先使原版对原版具备足够分辨能力，再比较候选；考虑足够长的采样、宿主机干扰、逐 case 噪声和跨轮次选择偏差。这是后续验收项，不是本次已经交付的统计控制器。 + +[公开原始汇总与全部 720 个计时样本](usefulness-summary.json) 为本次 summary 的原始副本；两份已去路径的原始 profile 为 [unchanged](usefulness-profiles/unchanged.txt) 和 [einsum](usefulness-profiles/einsum.txt)。它们保留原始数值与 SHA-256；完整命令、日志和源码快照仍在本地。此次探针代码摘要为 `317285d0534f9935d06e1fc488ef35c56a76ab7cba2c2423e5a73ba09799ebc7`，计划摘要见预登记与 JSON，不因后续文档更新而改写。 + +该检查不证明 agent/skill/search 的收益、GPU/NPU 资格或服务加速，也不实现全流程控制器。 diff --git a/docs/plans/issue-252/validation.md b/docs/plans/issue-252/validation.md index 89f29d49e..b823d68aa 100644 --- a/docs/plans/issue-252/validation.md +++ b/docs/plans/issue-252/validation.md @@ -44,6 +44,8 @@ ## 四个子项的实际用途 +后续新增了[实际工程用途对照](usefulness.md)：普通冒烟漏掉的 RMSNorm epsilon 错误被共享任务和 SkillEval 一致拒绝；原版对原版的大幅计时差异同时说明，本机小尺寸协议尚不能支持可靠性能排序。该次公开汇总保留全部 720 个原始样本，不改变本页早先集成运行的结果。 + 1. 调研决定落实在代码边界中：保留 axis=1、独立 oracle、正确性先行、profiling 独立、已有 runner 复用，没有照搬外部搜索平台。 2. 共享接口会拒绝漂移或未支持环境，避免两个消费者用不同题目版本得出不可比较的结果。 3. 两个算子确实计算并计时；数值、布局和状态错误会被抓到，未用预制分数代替实现。 diff --git a/tests/fixtures/engineering_benchmark_usefulness.py b/tests/fixtures/engineering_benchmark_usefulness.py new file mode 100644 index 000000000..f3e449468 --- /dev/null +++ b/tests/fixtures/engineering_benchmark_usefulness.py @@ -0,0 +1,330 @@ +"""Frozen CPU engineering-use probe; trusted authored candidates, no agent calls. + +Prepare only writes a plan. Run consumes that exact plan once and keeps raw evidence. +Functionality, not a performance winner, determines this fixture's exit status. +""" + +import argparse +import hashlib +import json +import shutil +from pathlib import Path + +from inferrsi.harness.benchmarks import load_task +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.contract import digest +from inferrsi.harness.benchmarks.numpy_task import check_output, reference +from inferrsi.harness.runner import run_experiment +from inferrsi.harness.skill_evaluation import valid_checks +from inferrsi.infra.artifacts import atomic_write_json +from tools import engineering_benchmark_validate as integration + +EINSUM = """import numpy as np +def compute(x, *, eps): + rms = np.einsum('ij...,ij...->i...', x, x) + rms /= np.float32(x.shape[1]) + rms += np.float32(eps) + np.sqrt(rms, out=rms) + return x / np.expand_dims(rms, axis=1) +""" +NO_EPS = """import numpy as np +def compute(x, *, eps): + return x / np.sqrt(np.mean(x * x, axis=1, keepdims=True)) +""" +ROLES = ("unchanged", "einsum", "no-eps") + + +def read(path): + return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} + + +def normalized(value): + return json.loads(json.dumps(value)) + + +def specs(task, root, plan): + return { + role: task.runspec( + root / (role + ".py"), + repo=Path(plan["repo"]), + output_dir=root / "results" / role / "workbench", + **plan["comparison"], + ) + for role in ROLES[:2] + } + + +def prepare(args): + root = args.output_dir.resolve() + root.mkdir(parents=True, exist_ok=False) + shutil.copytree( + load_task("rmsnorm-cpu").root, + root / "task", + ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), + ) + task = load_task(root / "task") + shutil.copyfile(task.root / "baseline.py", root / "unchanged.py") + (root / "einsum.py").write_text(EINSUM, encoding="utf-8") + (root / "no-eps.py").write_text(NO_EPS, encoding="utf-8") + plan = { + "schema_version": 1, + "repo": str(args.repo.resolve()), + "script_sha256": digest(Path(__file__)), + "replay_helper_sha256": digest(Path(integration.__file__)), + "comparison": { + key: getattr(args, key) for key in ("repetitions", "max_minutes", "min_improvement_pct") + }, + "action_timeout_seconds": args.timeout_seconds, + "replay_bounds_seconds": {"client": 20, "trial": 40}, + "identities": {role: task.identity(root / (role + ".py")) for role in ROLES}, + "smoke": {"seed": 2026, "shape": [4, 17, 9, 7], "scale": 4, "dtype": "float32"}, + "hypothesis": "Source suggests einsum may avoid the full-sized square temporary; " + "CPU cProfile does not establish a memory bottleneck.", + "sequence": [ + "profiles", + "correctness-and-replay", + "ordinary-smoke", + "reject-wrong-measurement", + "unchanged-comparison", + "einsum-comparison", + ], + "success": "Both valid candidates accepted; no-eps rejected without a metric; " + "complete profiles/comparisons and identical consumer tasks/workloads. " + "Performance winner and ordinary-smoke outcome are not success gates.", + } + if not 0 < args.timeout_seconds <= 7 * 24 * 3600: + raise ValueError("Action timeout must be positive and at most seven days") + plan["runspecs"] = {role: spec.to_dict() for role, spec in specs(task, root, plan).items()} + atomic_write_json(root / "plan.json", plan) + print(json.dumps({"plan_sha256": digest(root / "plan.json"), "payload_executed": False})) + return 0 + + +def verify(root, expected): + if digest(root / "plan.json") != expected: + raise ValueError("Frozen plan changed") + plan = read(root / "plan.json") + if ( + digest(Path(__file__)) != plan["script_sha256"] + or digest(Path(integration.__file__)) != plan["replay_helper_sha256"] + ): + raise ValueError("Probe or replay helper changed") + task = load_task(root / "task") + if any(task.identity(root / (role + ".py")) != plan["identities"][role] for role in ROLES): + raise ValueError("Task, candidate, grader or runtime changed") + current = specs(task, root, plan) + if normalized({role: spec.to_dict() for role, spec in current.items()}) != plan["runspecs"]: + raise ValueError("RunSpec changed") + return plan, task, current + + +def smoke(root, plan, task): + import numpy as np + + config = plan["smoke"] + x = ( + np.random.default_rng(config["seed"]).standard_normal(config["shape"]) * config["scale"] + ).astype(np.float32) + expected = reference(x, task.manifest["oracle"], np) + records = {} + for role in ROLES: + namespace = {} + exec(compile((root / (role + ".py")).read_bytes(), "implementation.py", "exec"), namespace) + actual_input = x.copy() + output = namespace["compute"](actual_input, eps=task.manifest["oracle"]["eps"]) + assertions = check_output(output, actual_input, x, expected, task.manifest["oracle"], np) + records[role] = {"passed": all(assertions.values()), "assertions": assertions} + return {"input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), "candidates": records} + + +def run(args): + root = args.output_dir.resolve() + plan, task, comparisons = verify(root, args.plan_sha256) + output = root / "results" + output.mkdir(exist_ok=False) # An interrupted attempt cannot silently resume or be replaced. + artifacts, profiles, candidates, measured = [], {}, {}, {} + + def checked(): + verify(root, args.plan_sha256) + + def retain(path, role): + document = read(path) + artifacts.append((role, document)) + return document + + for role in ROLES[:2]: + checked() + directory = output / role / "profile" + execution = run_action( + task, + root / (role + ".py"), + "profile", + directory, + timeout_seconds=plan["action_timeout_seconds"], + ) + result = retain(directory / "result.json", role) + profile = directory / "profile.txt" + profiles[role] = { + "status": execution["status"], + "diagnostic": result.get("diagnostic"), + "has_metric": "metrics" in result, + "artifact_sha256": digest(profile) if profile.is_file() else None, + } + for role in ROLES: + checked() + source, directory = root / (role + ".py"), output / role + grade = grade_artifact( + task, source, directory / "correctness", timeout_seconds=plan["action_timeout_seconds"] + ) + result = retain(directory / "correctness" / "result.json", role) + checked() + trial = integration.replay(task, source, directory / "skilleval") + retain(directory / "skilleval/trial/verification/artifact/result.json", role) + candidates[role] = { + "correctness": grade["verdict"], + "skilleval": trial["verdict"], + "failed_assertions": [ + key for key, value in result.get("assertions", {}).items() if not value + ], + } + checked() + smoke_result = smoke(root, plan, task) + atomic_write_json(output / "ordinary-smoke.json", smoke_result) + checked() + wrong = run_action( + task, + root / "no-eps.py", + "measure", + output / "no-eps/measure", + timeout_seconds=plan["action_timeout_seconds"], + ) + wrong_result = retain(output / "no-eps/measure/result.json", "no-eps") + for role in ROLES[:2]: + checked() + run_dir = run_experiment(comparisons[role], {"scope": "CPU engineering-use probe"}) + evaluation = read(run_dir / "evaluation.json") + trials = [] + for side in ("baseline", "candidate"): + for path in sorted((run_dir / side).glob("trial-*/result.json")): + result = retain(path, "unchanged" if side == "baseline" else role) + trials.append( + { + "side": side, + "trial": path.parent.name, + "cases": result.get("cases", []), + "metrics": result.get("metrics"), + } + ) + validation = retain(run_dir / "validation/result.json", role) + quality_checks = {key: validation.get(key) for key in ("passed", "assertions")} + measured[role] = { + key: evaluation.get(key) + for key in ( + "verdict", + "baseline", + "candidate", + "improvement_pct", + "statistical_significance", + ) + } + measured[role].update( + final_quality_passed=( + validation.get("status") == "completed" + and validation.get("action") == "check" + and validation.get("passed") is True + and valid_checks(quality_checks) + ), + source_integrity=evaluation.get("source_integrity", {}).get("verified"), + trials=trials, + measured_calls=sum( + len(case["samples_ms"]) for trial in trials for case in trial["cases"] + ), + ) + checked() + workloads = artifacts[0][1].get("workloads") + identities_match = all( + result.get("identity") == plan["identities"][role] for role, result in artifacts + ) + workloads_match = bool(workloads) and all( + result.get("workloads") == workloads for _, result in artifacts + ) + checks = { + "valid_candidates_accepted": all( + candidates[r]["correctness"] == candidates[r]["skilleval"] == "PASS" for r in ROLES[:2] + ), + "wrong_candidate_rejected": candidates["no-eps"]["correctness"] + == candidates["no-eps"]["skilleval"] + == "FAIL", + "wrong_has_no_metric": wrong["status"] == "failed" and "metrics" not in wrong_result, + "profiles_complete": all( + p["status"] == "completed" + and p["artifact_sha256"] + and p["diagnostic"] + and p["diagnostic"].get("performance_verdict") is None + and not p["has_metric"] + for p in profiles.values() + ), + "consumer_identities_match": identities_match, + "actual_workloads_match": workloads_match, + "comparison_quality_passed": all(m["final_quality_passed"] for m in measured.values()), + "comparisons_complete": all( + m["source_integrity"] + and len(m["trials"]) == 2 * plan["comparison"]["repetitions"] + and m["measured_calls"] + == 2 + * plan["comparison"]["repetitions"] + * len(task.manifest["workloads"]) + * task.manifest["measurement"]["samples"] + and all( + m[side] and m[side]["excluded_trials"] == 0 for side in ("baseline", "candidate") + ) + for m in measured.values() + ), + } + summary = { + "scope": "CPU engineering-use probe with trusted authored candidates", + "plan_sha256": args.plan_sha256, + "script_sha256": plan["script_sha256"], + "replay_helper_sha256": plan["replay_helper_sha256"], + "smoke_config": plan["smoke"], + "environment": artifacts[0][1].get("environment"), + "checks": checks, + "checks_succeeded": all(checks.values()), + "identities": plan["identities"], + "comparison": plan["comparison"], + "workloads": workloads, + "profiles": profiles, + "candidates": candidates, + "ordinary_smoke": smoke_result, + "comparisons": measured, + "unchanged_raw_pass": measured["unchanged"]["verdict"] == "PASS", + "interpretation": "Raw Workbench verdicts are descriptive; an unchanged PASS is " + "a false-positive warning, not optimization. No speedup claim.", + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + } + atomic_write_json(output / "summary.json", summary) + print(json.dumps({"checks_succeeded": summary["checks_succeeded"], "checks": checks})) + return 0 if summary["checks_succeeded"] else 1 + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + commands = parser.add_subparsers(dest="action", required=True) + setup = commands.add_parser("prepare") + setup.add_argument("--repo", type=Path, required=True) + setup.add_argument("--repetitions", type=int, required=True) + setup.add_argument("--max-minutes", type=float, required=True) + setup.add_argument("--min-improvement-pct", type=float, required=True) + setup.add_argument("--timeout-seconds", type=float, required=True) + execute = commands.add_parser("run") + execute.add_argument("--plan-sha256", required=True) + for command in (setup, execute): + command.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + return prepare(args) if args.action == "prepare" else run(args) + + +if __name__ == "__main__": + raise SystemExit(main()) From ce84506e7106c4fc48fafdfab55d2ea2f9dffd50 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 00:04:22 +0800 Subject: [PATCH 03/20] fix: bind benchmark timing to distinct paired inputs Document existing RSI consumers and remaining end-to-end acceptance. Agent-assisted-by: Codex (implementation, review and validation) Signed-off-by: dengyunyang <584797741@qq.com> --- .gitattributes | 1 + docs/engineering-benchmarks.md | 34 +- docs/plans/issue-252/03-tasks.md | 4 +- docs/plans/issue-252/04-integration.md | 2 + docs/plans/issue-252/README.md | 6 +- .../issue-252/replay-validation-summary.json | 1255 +++++++++++++++++ docs/plans/issue-252/replay-validation.md | 48 + docs/plans/issue-252/rsi-integration.md | 63 + docs/plans/issue-252/usefulness.md | 2 + docs/plans/issue-252/validation.md | 2 + .../rmsnorm-cpu/README.md | 13 +- .../rmsnorm-cpu/task.json | 22 +- .../softmax-cpu/README.md | 13 +- .../softmax-cpu/task.json | 22 +- src/inferrsi/harness/benchmarks/contract.py | 33 +- src/inferrsi/harness/benchmarks/numpy_task.py | 103 +- .../engineering_benchmark_usefulness.py | 13 +- tests/test_engineering_benchmarks.py | 125 +- tools/engineering_benchmark_validate.py | 19 + 19 files changed, 1712 insertions(+), 68 deletions(-) create mode 100644 docs/plans/issue-252/replay-validation-summary.json create mode 100644 docs/plans/issue-252/replay-validation.md create mode 100644 docs/plans/issue-252/rsi-integration.md diff --git a/.gitattributes b/.gitattributes index 76dc5c839..17ca18179 100644 --- a/.gitattributes +++ b/.gitattributes @@ -10,6 +10,7 @@ # Preserve the recorded utility probe's JSON/profile bytes and their SHA-256s. /docs/plans/issue-252/usefulness-summary.json -text whitespace=cr-at-eol +/docs/plans/issue-252/replay-validation-summary.json -text whitespace=cr-at-eol /docs/plans/issue-252/usefulness-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof # Retain this upstream license's original CRLF Git blob bytes. diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index 4a290d5b7..3a215b9f2 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -1,7 +1,7 @@ # Shared engineering benchmarks The `benchmarks` command exposes versioned task packages for Workbench and -offline artifact evaluation. The initial `numpy-axis1-v1` adapter supplies +offline artifact evaluation. The `numpy-axis1-v2` adapter supplies Softmax and RMSNorm CPU adaptations of attributed KernelBench tasks. Packages own semantics, workload, oracle and metric. Consumers own comparison conditions; search policy and agent execution remain separate. @@ -46,13 +46,26 @@ for four-dimensional arrays. Scalar double precision with math.fsum supplies the independent oracle. Numerical tolerances, shapes, seeds, warmups and sample counts are task-specific and fixed across both consumers. -`check` evaluates all public cases. `measure` checks first, times real synchronous -compute calls, and verifies every measured output outside the timer. Input -generation, oracle, validation, startup and warmups are excluded; allocations -inside compute are included. Every case retains its samples in milliseconds. -The objective sums case medians for this fixed suite; it is not a universal -score. Inspect per-case regressions before adopting a candidate. Repeated -warm-cache calls do not characterize cold-start serving. +`check` evaluates all six base cases plus the same varying-input sequence used +by measurement and profiling. `measure` checks first, times synchronous compute +calls, and verifies every invocation outside the timer. Inputs and references +are prepared in advance; all input objects stay alive and every timed input has +distinct bytes from precheck/warmup/other sampled inputs. This prevents an earlier +call from priming an identity/content memo for a later timed call. It does not +detect hostile code or public-suite overfitting. + +The task's `measurement.case_ids` declares five varying workloads. The invariant +zero-input case remains a correctness case and contributes no timing score. Every +result retains the planned `input_sequence` and actual `executed_calls`; each +sample records its input hash. Invalid candidates may stop before the full +sequence, so planned inputs must not be described as completed execution. + +Input generation, oracle, validation, startup and warmups are excluded; allocations +inside compute are included. The objective sums the five per-case medians for this +fixed suite, not a universal score. Inspect per-case regressions before adopting a +candidate. This protocol does not establish cold-start serving or statistical +resolution. The earlier `numpy-axis1-v1` repeated-input evidence remains historical; +its task/grader identities and six-case scores cannot be pooled with v2. `profile` retains a separate CPU `profile.txt` with no performance metric/verdict. It is not a device timeline. Hardware adapters need their own synchronization, @@ -66,6 +79,11 @@ The shared oracle independently grades actual implementation behavior. This offline handoff does not modify production engineering sandbox or paid execution. Live artifact production and skill/search benefit need separate controlled runs. +This low-level verifier integration does not register these tasks with the existing +`skill_eval_compare` adapter or the offline harness-improvement controller. See +[the RSI integration audit](plans/issue-252/rsi-integration.md) for callers, +decisions and the remaining end-to-end acceptance work. + Exceptions, missing dependencies, drift and timeouts leave incomplete evidence. Wrong results emit no usable performance metric. Workbench statistics remain descriptive; a local verdict is not service gain or campaign acceptance. diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index 41be0cf40..d0dc121c6 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -29,8 +29,10 @@ NumPy 基线与使用临时 buffer 的示例候选都由本次实现提供。参考计算使用标量 double、math.exp/math.sqrt/math.fsum，独立于 NumPy 归约。每题检查形状/dtype、有限输出、输入保持、数值误差四类断言。 -计时使用 perf_counter_ns，仅包围同步 compute 调用。每个 case 预热 2 次、保留 5 次计时；单个 trial 指标为六个 case 中位数的和，所有原始样本仍保留。这些是任务版本的测量协议，非全仓库性能门槛。外部 trial 次数和预算另行冻结。 +当前 v2 计时使用 perf_counter_ns，仅包围同步 compute 调用。任务明确五个计时 case，各预热 2 次、保留 5 次计时；全零 case 保留为正确性检查，不进入性能分数。单个 trial 指标为五个 case 中位数的和，所有原始样本与对应输入哈希均保留。所有调用的变化输入和 oracle 在执行前生成并保持存活；预检、预热和计时的输入字节不重复，双方使用相同的固定序列。这些是任务版本的测量协议，非全仓库性能门槛。外部 trial 次数和预算另行冻结。 变异测试覆盖全零错误答案、NaN、dtype 错误、shape 错误、不稳定 exp、修改输入、错误轴，以及前六次正确之后变错。失败候选不输出可用性能指标；中途失败保留失败调用和已有样本。timeout/缺依赖与完整数值拒绝分别记录。 +新增缓存重放回归：按 shape/layout 返回旧输出会在变化输入上被拒绝；按 id/完整内容 memoize 的正确候选可以正常计算，但验证其在本次序列没有缓存命中，不能从预检借用答案。公开序列仍不是隐藏测试或恶意代码隔离。 + Profiling 实际运行同一候选，生成可读 CPU 调用表，完全独立于正式计时。它能帮助检查 Python 调用开销；它不能回答 GPU/NPU 空洞或设备 kernel 利用率。 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index 8679e232d..6243cd907 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -4,6 +4,8 @@ 只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 +[RSI 接入审计](rsi-integration.md)进一步区分：本次已调用的 `run_trial/verifier` 是低层接口，不等价于正式 `skill_eval_compare` 两臂入口或 `skill_eval_improve` 已支持这些工程任务。系统接入必须说明结果如何影响真实候选的筛选与接受。 + ## 计划 - Workbench 接入复用 run_experiment、AB/BA、超时、源快照、结果和报告。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index d7c83af63..cf1ef45aa 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -6,6 +6,8 @@ ## 要解决的实际问题 +先读 [在现有 RSI 中的调用与验收](rsi-integration.md)：#252 同时为引擎候选比较和优化者评估提供共享任务。当前已验证的是组件及底层消费者通路，尚未完成正式 SkillEval 两臂入口、既有改进 controller 或 agent 收益的端到端验收。 + Workbench 和工程评估需要对同一个候选实现回答两个问题：计算是否正确，以及指定工作负载下测得什么性能。任务语义、输入、参考答案和计时规则应只有一份。搜索策略决定下一步尝试什么，消费者不能各自复制或修改题目答案。 本次将这个模块命名为 **Engineering Benchmarks（工程基准）**。这是本次实现的命名，尚非维护者已接受的架构决策。 @@ -44,8 +46,8 @@ Profiling 保留为独立诊断操作，与 correctness 和配对 performance 公开子任务分别为 [#334 调研与决策](https://github.com/InferMatrixLab/InferRSI/issues/334)、[#335 共享接口](https://github.com/InferMatrixLab/InferRSI/issues/335)、[#336 首批任务](https://github.com/InferMatrixLab/InferRSI/issues/336)、[#337 消费者与验证](https://github.com/InferMatrixLab/InferRSI/issues/337)。四项均引用父任务 #252，保持打开供维护者评审与验收。当前提交账号无法写入原生父子关系，需维护者补充。 -四个子项的本地实施已完成。已交付共享 TaskPackage、两个真实 CPU 算子任务、独立正确性与 profiling、Workbench/SkillEval 接入，以及可复现实测工具。使用入口见 [工程基准文档](../../engineering-benchmarks.md)。 +四个子项已有组件实现：共享 TaskPackage、两个真实 CPU 算子任务、独立正确性与 profiling、Workbench 和 SkillEval 低层 verifier 接入，以及可复现实测工具。测量语义 review、可靠性能排序和系统接入验收分别追踪，不能把局部实现表述为整个 RSI 已经闭环。使用入口见 [工程基准文档](../../engineering-benchmarks.md)。 建议先读本页理解范围，再读 01 看方案理由。要开始使用时，运行 `inferrsi benchmarks list` 和 `show softmax-cpu`，复制任务包中的示例候选到独立 implementation.py，按使用文档执行 check、profile、compare。修改任务协议应创建新比较，不能沿用旧身份。 -这完成的是用户委托的首批共享接口和 CPU 子集；#252 的维护者验收、目标硬件选择与后续覆盖仍单独记录，不在本地冒充已接受。 +这份交付覆盖共享接口和 CPU 子集的组件实现；正式 SkillEval 两臂接入、优化者收益和可靠性能排序尚未建立。#252 的维护者验收、目标硬件选择与后续覆盖仍单独记录，不在本地冒充已接受。 diff --git a/docs/plans/issue-252/replay-validation-summary.json b/docs/plans/issue-252/replay-validation-summary.json new file mode 100644 index 000000000..94df1658e --- /dev/null +++ b/docs/plans/issue-252/replay-validation-summary.json @@ -0,0 +1,1255 @@ +{ + "scope": "local CPU integration with authored controls", + "checks_succeeded": true, + "campaign_acceptance": "INCONCLUSIVE", + "agent_or_skill_benefit": "not measured", + "accelerator_qualification": "not run", + "tasks": [ + { + "task": "rmsnorm-cpu", + "adapter": "numpy-axis1-v2", + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "correctness": "PASS", + "profile_status": "completed", + "skilleval_good": "PASS", + "skilleval_bad": "FAIL", + "bad_measurement_status": "failed", + "measurement_trials": 6, + "measured_calls": 150, + "source_integrity": true, + "descriptive_workbench_verdict": "PASS", + "workbench_run": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc", + "all_consumer_identities_match": true, + "all_workloads_match": true, + "input_sequence": [ + { + "id": "normal", + "input_sha256": "bfb0c12b819449a751bd50f84ad4e79a6ce0a84f430ce4f62b2e81f3a73463f4", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "normal", + "input_sha256": "cf2888897320784dc66658e689680430c618691b7a2de9a134f9558ba7a0126c", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "normal", + "input_sha256": "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 4284, + 252, + 28, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "0cf53d712ace6cdd013fc50c2fa4ef1e9a84d36cedec99fd1f9d3e3e271475fd", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "strided", + "input_sha256": "597416a7eadfc8f71919803a2b09509b05793a65bb778718e224071f88b0e583", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "strided", + "input_sha256": "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b", + "shape": [ + 4, + 17, + 9, + 7 + ], + "dtype": "float32", + "strides": [ + 8568, + 504, + 56, + 8 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "5c28285af6ccbe887e6354fb953aab049c81b31b0503b51ab8d9c61196472126", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "large", + "input_sha256": "a89e3333ecc48304b7a6bcb33c03c1ab4f29570abfe066b529603a0bd0f0a082", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "large", + "input_sha256": "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba", + "shape": [ + 2, + 5, + 3, + 7 + ], + "dtype": "float32", + "strides": [ + 420, + 84, + 28, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "6a3b8e2783bd417a366bbe5cc38a65c0effe57848d11d0058289e8281ff530f5", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "tiny", + "input_sha256": "527e17491f351e73097806f3e5046060dec073b42881fb8ac0e8e770abf43689", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "tiny", + "input_sha256": "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f", + "shape": [ + 2, + 7, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 420, + 60, + 20, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "68835074c397f1aa5e920e28416092ffbd7d28bd392528e2da6986e75edbdf1a", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "singleton", + "input_sha256": "2a2a58087a23241da691f034142ccc3319ce9992f950004fc4f974fe00234577", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "singleton", + "input_sha256": "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6", + "shape": [ + 2, + 1, + 3, + 5 + ], + "dtype": "float32", + "strides": [ + 60, + 60, + 20, + 4 + ], + "call_index": 6, + "phase": "measurement" + } + ], + "all_input_sequences_match": true, + "all_valid_executed_calls_match": true, + "expected_measured_calls": 150 + }, + { + "task": "softmax-cpu", + "adapter": "numpy-axis1-v2", + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "correctness": "PASS", + "profile_status": "completed", + "skilleval_good": "PASS", + "skilleval_bad": "FAIL", + "bad_measurement_status": "failed", + "measurement_trials": 6, + "measured_calls": 150, + "source_integrity": true, + "descriptive_workbench_verdict": "PASS", + "workbench_run": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9", + "all_consumer_identities_match": true, + "all_workloads_match": true, + "input_sequence": [ + { + "id": "normal", + "input_sha256": "c25b8959c8e773366dfa4dd26d41366e74d72258ced92ba8ecc6be7555112100", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "normal", + "input_sha256": "adcf65bddfc618eafd449d562205d1b2ab8dbdd24bd2fb3868b3f26d3c6bcce1", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "normal", + "input_sha256": "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "normal", + "input_sha256": "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 1028, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "768e1830be4762059c5b81b967bf098d187efc162a3b3cd5b12ea0dd16b878d9", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "strided", + "input_sha256": "ace3f9750c6d76101c01a30ed1237c64fb4f3937daddec762d10f1b17f6345d9", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "strided", + "input_sha256": "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "strided", + "input_sha256": "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37", + "shape": [ + 16, + 257 + ], + "dtype": "float32", + "strides": [ + 2056, + 8 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "4e82ca66db9606b5566293f2edc5eea1e9df6649e56f2112f6f2e9e938d8438c", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "large", + "input_sha256": "af087bae1e27a00565a87dfa7113ebc69b26c01903362fc1ab9549c577ded188", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "large", + "input_sha256": "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "large", + "input_sha256": "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf", + "shape": [ + 3, + 31 + ], + "dtype": "float32", + "strides": [ + 124, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "bf4febd2050386a0849e2ce1be8d75657f3346c903bcf1b0a3d2d743bbea80ab", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "tiny", + "input_sha256": "5d962ceb305b5ddbc5776378338a91ab3dabc64f5245a31aa4aecdd17642769e", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "tiny", + "input_sha256": "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "tiny", + "input_sha256": "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4", + "shape": [ + 2, + 17 + ], + "dtype": "float32", + "strides": [ + 68, + 4 + ], + "call_index": 6, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "e1cfcaed0914dbc73942f399349276d1f2788d07ff18cd095d89183a6adf53af", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 0, + "phase": "warmup" + }, + { + "id": "singleton", + "input_sha256": "4cf5dbc59fe7b8cdf8b14fdefce446edd0a50982d37dbb5c0681872338ee500c", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 1, + "phase": "warmup" + }, + { + "id": "singleton", + "input_sha256": "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 2, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 3, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 4, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 5, + "phase": "measurement" + }, + { + "id": "singleton", + "input_sha256": "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200", + "shape": [ + 2, + 1 + ], + "dtype": "float32", + "strides": [ + 4, + 4 + ], + "call_index": 6, + "phase": "measurement" + } + ], + "all_input_sequences_match": true, + "all_valid_executed_calls_match": true, + "expected_measured_calls": 150 + } + ] +} diff --git a/docs/plans/issue-252/replay-validation.md b/docs/plans/issue-252/replay-validation.md new file mode 100644 index 000000000..982a52867 --- /dev/null +++ b/docs/plans/issue-252/replay-validation.md @@ -0,0 +1,48 @@ +# v2：按 RSI 测量边界修复输出重放 + +当前 RSI 的候选比较要求测量实际实现的计算，不能让最终输出缓存冒充优化。[PR review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4456416093) 指出 v1 预检、预热和计时复用同一输入，正确的 memoizing 实现可能只在预检计算一次。这是任务测量语义的缺陷，与有没有 CPU/GPU、是否有独立 oracle 无关。 + +## 修复与版本边界 + +- 适配器更新为 `numpy-axis1-v2`；每个任务显式声明 `seeded-distinct-per-call-v1` 输入协议和五个计时 case。全零输入仍检查正确性，但不计入性能分数。 +- 执行前生成全部输入、逐输入独立 oracle 和预定顺序。输入对象一直保留，字节不与预检、预热和其他采样重复；baseline 与 candidate 使用同一序列。 +- `check`、`measure`、`profile` 检查同一序列。结果区分计划的 `input_sequence` 与实际 `executed_calls`，每个计时样本绑定输入摘要；失败只留下实际执行的前缀。 +- 对按 id/完整内容缓存的正确实现，不禁止代码写缓存，而是验证本序列没有命中旧结果。按 shape/layout 直接重放旧答案的实现会被变化输入拒绝，且不能输出性能指标。 +- 输入和 oracle 生成在计时之外；计时仍只包围 compute。该机制不提供恶意代码隔离或隐藏测试，公开输入仍可能被过拟合。 + +任务、grader 身份和指标的五 case 聚合均已变化。v1 的 [集成记录](validation.md) 和 [用途对照](usefulness.md) 完整保留，不改写、不拼入 v2 的分数。正确性与性能排序仍分别判断。 + +## 实际验证 + +2026-11-03（UTC），Windows / Python 3.11.18 / NumPy 2.4.6。针对 v2 的专项回归 **40 项通过**，包括真实 baseline/candidate、缓存命中、旧输出重放、逐次输入配对、边界协议、漂移、超时、Workbench 和 SkillEval verifier。另行运行的 CLI catalog 和 SkillEval 相关回归也通过；Ruff 和 diff 检查通过。 + +开发时首轮回归暴露了 `SeedSequence` 的 NumPy 命名空间错误，修正为 `np.random.SeedSequence` 后完整重跑通过；未把首轮失败归因于环境或既有基线。日志保留在仓库外的本地 outputs 中。 + +随后单独执行现有复现工具： + +```bash +python tools/engineering_benchmark_validate.py \ + --repo --output-dir --repetitions 3 +``` + +这是固定示例候选的协议/消费者验证，不是新的性能优化搜索。工具在执行前冻结任务、候选及 RunSpec；实际原始目录名为 `inferrsi-252-replay-v2-validation`。 + +| 检查 | Softmax CPU | RMSNorm CPU | +| --- | --- | --- | +| 正确性范围 | 6 个基础 case + 35 次变化输入调用 | 相同 | +| 正确实现独立断言 | 164 项通过 | 164 项通过 | +| 正确/全零错误产物在 SkillEval | PASS / FAIL | PASS / FAIL | +| 错误测量 | 拒绝，无可用指标 | 拒绝，无可用指标 | +| 实际 Workbench trial / 计时样本 | 6 / 150 | 6 / 150 | +| 独立 profile | 完成 | 完成 | +| 所有消费者 task/grader/runtime 与基础输入 | 一致 | 一致 | +| 计划输入序列与正确实现的实际调用序列 | 一致 | 一致 | +| Workbench 源码完整性 | 通过 | 通过 | + +[原始可公开汇总](replay-validation-summary.json) 保留每次变化输入的摘要及身份、数量和原始 verdict，未包含私人路径。完整原始计时、日志和 profile 仍在本地。两次 Workbench 描述性 verdict 都为 PASS，未将其解释为可靠性能提升；本次没有完成 v2 同码噪声资格验证，也没有建立统计显著性。 + +## 对 RSI 的实际含义 + +现在普通 id/content 缓存不能因预检复用而得到重放时间，错误的旧输出也不能成为优化证据。这提高了 Workbench 和离线 verifier 共同消费的任务语义可靠性。 + +这仍不是“RSI 已经自动变好”的证据。可靠性能排序、正式 SkillEval 两臂产物交接、真实技能/搜索收益及目标硬件验收仍未完成；调用关系及下一步验收沿 [RSI 接入审计](rsi-integration.md) 组织，不另建执行器或搜索控制器。 diff --git a/docs/plans/issue-252/rsi-integration.md b/docs/plans/issue-252/rsi-integration.md new file mode 100644 index 000000000..b1fad6e81 --- /dev/null +++ b/docs/plans/issue-252/rsi-integration.md @@ -0,0 +1,63 @@ +# #252 在现有 InferRSI 中怎样发挥作用 + +验收起点是“谁调用任务包，拿结果作什么决定”。两个算子能运行、两个底层消费者复用 oracle，只完成组件级验证。它们尚不能证明 InferRSI 的优化者因此变得更有效。 + +## 两条现有路径，共用一份任务定义 + +| 路径 | 变化的对象 | #252 的职责 | 结果影响的决定 | +| --- | --- | --- | --- | +| 引擎优化：主 Agent → Workbench | 引擎的代码或配置 | 定义适用的任务、输入、oracle、诊断与测量边界 | 该实现是否正确；是否有足够证据保留它；下一轮需要什么诊断 | +| 优化者评估：SkillEval | skill、知识或 harness 的候选版本 | 用相同任务和独立 oracle 检查两边实际产出的实现 | 新指导是否帮助完成任务；质量和资源成本如何变化 | + +两条路径中改变的对象不能混淆。引擎代码变快，不等于某个 skill 有效；skill 版本比较也不能通过替换任务或判分规则制造进步。整体 agent/harness 收益仍由 #248 的受控实验回答，#249 负责并行候选搜索的相关范围。 + +已有 [harness-improvement controller](../../harness-improvement.md) 可以消费其支持的 SkillEval grades，进行 development 选择、重跑和一次 holdout 接受/拒绝。本次 benchmark 不能假定已经接入它。不要为接入 #252 再造一套总控制器或执行平台。 + +## 对照真实入口审计当前状态 + +| 入口 | 现在已经接通什么 | 尚未接通什么 | +| --- | --- | --- | +| `benchmarks plan/compare` → `TaskPackage.runspec` → `run_experiment` | 同一任务生成既有 RunSpec，由 Workbench 执行配对比较并保留证据 | 自动从用户目标/profile 选出适用任务；服务与目标硬件验收 | +| `benchmarks check/profile/measure` | 分开的数值校验、CPU 诊断、测量操作 | 强制宿主 Agent 按整个优化流程执行 | +| `skill_verifier.task_descriptor` → `skill_evaluation.run_trial` | 对 controller 已审核的保留实现运行真实 verifier；答案提交该实现的 SHA | Agent 根据不同技能实际产出两份实现的比较 | +| `tools.skill_eval_compare` | 已有冻结两臂、模型/预算、产物与报告机制 | 新 benchmark descriptor 尚未成为正式支持的 adapter；现有 engineering adapter 限定 parser 任务及隔离执行 | +| `tools.skill_eval_improve` | 已有面向支持的离线 audit 任务的选择、重跑和 holdout 流程 | 共享工程基准尚未接入该控制器的任务/grader 契约 | + +依据为 [CLI 适配](../../../src/inferrsi/harness/benchmarks/cli.py)、[保留产物 verifier](../../../src/inferrsi/harness/benchmarks/skill_verifier.py)、[SkillEval 比较入口](../../../tools/skill_eval_compare.py)、[现有 engineering adapter](../../../src/inferrsi/harness/skill_eval_engineering.py) 和 [改进入口](../../../tools/skill_eval_improve.py)。主 Agent 的调查与代码修改职责见[现有插件入口](../../../plugins/inferrsi/skills/inferrsi/SKILL.md)。 + +## 一次引擎优化中应该怎样使用 + +1. **从目标和基线 profile 开始。** 主 Agent 明确模型、硬件、工作负载和质量要求，再确认该任务是否覆盖所需语义、dtype、布局和环境。当前 CPU RMSNorm 任务不能因为名称相同就代表真实 GPU/NPU 服务。 +2. **冻结任务与实验条件。** 在候选尝试前固定任务、oracle、输入序列、版本、比较条件和预算。候选只改允许的实现；不同时修改裁判或基线。 +3. **利用知识形成可检验假设。** Agent 读取适用技能、论文和历史失败记录，说明该变更如何作用于已观测的瓶颈。CPU cProfile 无法确认的内存瓶颈继续保持未知。 +4. **先挡住错误，再测量。** 数值不正确的候选不能获得可用于选择的性能分数。计时必须测到实际计算；预检不能给后续同输入缓存重放预热。测量规则的修改产生新的任务/grader 身份，不重写旧证据。 +5. **区分原始报告与优化决定。** 保留 Workbench 原始 PASS/FAIL/INCONCLUSIVE；主 Agent 同时判断证据是否足以支持保留/放弃。若同码对照无法分辨，决策是“测量不足，先修协议”，不能把某次 FAIL 归纳成整个优化机制无效。 +6. **记录对下一轮有用的事实。** 保留候选、环境、任务、正确性失败、原始样本及未解决的诊断。经验整理与晋升走现有流程；不从一次算子结果自动推广成服务收益或通用技能。 + +## 按 RSI 需求补证的顺序 + +### A. 先使任务适合被优化者消费 + +要求 oracle 独立、候选修改边界清楚、实际计算不被预检缓存替代、输入在消费者之间配对一致、测量失败不产生可用性能证据。这是 #252/#337 的基础交付，不是另一个 search-policy 项目。 + +[缓存重放 review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4456416093) 暴露了这一层的具体缺口，现已通过 [v2 变化输入协议及回归](replay-validation.md)修复。此前的普通冒烟漏检对照证明了正确性检查用途；同码噪声与可靠性能排序仍须单独验证。 + +### B. 证明结果确实进入优化决定 + +最小演练应沿已有入口保留“profile → 假设 → 产物 → check → compare → 决策”链，并至少包含：错误产物被拒绝、正确但证据不足的产物不被当作改进、负结果留下适用条件。既有 [RMSNorm 对照](usefulness.md) 提供了部分输入与结果；当前没有程序化的全流程强制或真实 agent 对照，不能据此宣布完整闭环验收。 + +### C. 再接到已有 SkillEval 两臂入口 + +沿 `skill_eval_compare` 扩展任务适配与产物交接，保留其冻结、模型/预算一致性、独立 verifier、隔离和日志机制。两边分别接收旧/新 skill（或无/有 skill），各自交付实现，再交给同一任务包判分。 + +当前可信 CPU worker 不能直接接未经审核的模型代码，也不能代替现有 engineering adapter 的隔离。没有支持的产物执行环境时，应保留未验证状态；换一个 scripted client 只证明通路，不证明技能帮助。 + +### D. 最后评估优化者本身是否改善 + +实际 agent 实验要固定任务集、模型、总预算和对照，记录所有尝试、质量、可分辨的性能与资源使用。若用于选择可复用技能，还要有适当的独立验证任务；同一道公开题换一个 seed 不自动等于独立 holdout。先明确新 adapter 支持的范围，再评估能否被既有 harness-improvement controller 消费。 + +这一层沿 #88/#248 及已有 controller 的职责推进，不把整个搜索控制器、自动知识晋升或服务验收塞进 #252。 + +## 当前结论 + +共享任务与底层消费者复用已有实际证据，已补正影响测量语义的输入重放缺陷；端到端 SkillEval 接入、优化者收益和可靠性能排序尚未完成。四个子项的组件实现不能替代这些系统验收。PR 继续作为 Draft 评审，按现有 RSI 接口补齐接入，不把独立题目数量或另建编排系统作为进展指标。 diff --git a/docs/plans/issue-252/usefulness.md b/docs/plans/issue-252/usefulness.md index 50d8d69d2..81e687d7a 100644 --- a/docs/plans/issue-252/usefulness.md +++ b/docs/plans/issue-252/usefulness.md @@ -1,5 +1,7 @@ # #252：实际工程用途对照 +本页是 **v1 历史对照**，对应源码 [c06b7255](https://github.com/Bounty-hunter/InferRSI/tree/c06b72555d4643df133c59fe7cead5a91e7b86f5)。下文 720 个样本、脚本摘要和计划均属于该版本。精确复现应在此版本运行；最新 fixture 使用 v2 五个计时 case，会产生不同任务/脚本身份和样本数。后续修复见 [v2 验证](replay-validation.md)，不修改本页历史结果。 + 本补证回答一个具体问题：共享任务能否帮助使用者识别一个看似合理的错误改动，并让正确候选经过同一套工程检查和实际性能比较？它不以 CI 绿色或某次计时 PASS 作为“有用”的定义。 ## 预先固定的对照 diff --git a/docs/plans/issue-252/validation.md b/docs/plans/issue-252/validation.md index b823d68aa..7071461c9 100644 --- a/docs/plans/issue-252/validation.md +++ b/docs/plans/issue-252/validation.md @@ -1,5 +1,7 @@ # #252 本地验证记录 +本页保留 **v1 历史运行**（六个计时 case、重复输入）的原始记录，不能代表后续 v2 变化输入协议。当前 RSI 接入边界见 [接入审计](rsi-integration.md)，协议修复与新证据见 [v2 验证](replay-validation.md)。旧数值、身份与 verdict 不重写或合并到新结果。 + 验证日期 2026-11-03。实现基于 `663f3b327998526a13aa31e72ad729025c18983c`，具体新增内容以当前工作区和结果中的内容哈希为准。本记录说明共享任务子集的功能与集成，不是项目维护者的验收决定。 ## 实际运行结果 diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md index 983e5e607..2f914e28d 100644 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md @@ -8,8 +8,9 @@ float32 NumPy array of the same shape. Preserve input values. Compute x / sqrt(m Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; the evaluator is not a security sandbox for model-produced arbitrary code. -The task JSON owns all six public inputs, seeds, layouts, numerical tolerances -and timing policy. Baseline and example candidate are separate from the scalar +The task JSON owns six base correctness workloads, seeds, layouts, numerical +tolerances and the v2 timing policy. Five varying workloads have a frozen +per-call input sequence; invariant zero inputs are correctness-only. Baseline and example candidate are separate from the scalar double-precision oracle. The oracle uses math.fsum and is not editable by a candidate. The task package and grader are content-frozen per comparison. @@ -18,8 +19,12 @@ preservation outside the timed region. Input generation, reference computation, warmups, validation, process startup and profiling are excluded. Output allocation inside compute is included. Samples use perf_counter_ns and report milliseconds. Sum of per-case medians is only this task's fixed-suite objective, not a universal -score. Retain every per-case sample to inspect regressions. Repeated calls describe -a warm-cache CPU workload. No device synchronization is claimed. +score. Retain every per-case sample and input hash to inspect regressions. Inputs +are generated before timing and kept alive; each precheck, warmup and sampled +input has distinct bytes. Baseline and candidate consume the same sequence. +This prevents ordinary identity/content memo hits, not hostile-code execution. +Small CPU timings still need repeatability qualification; no cold-start serving +or device synchronization is claimed. All tests are public. This small subset cannot demonstrate production inference speedup, memory savings, or GPU/NPU support. A fresh task version and device diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json index b1e56ffe3..a17888f01 100644 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json @@ -1,9 +1,9 @@ { "schema_version": 1, "id": "rmsnorm-cpu", - "adapter": "numpy-axis1-v1", + "adapter": "numpy-axis1-v2", "description": "Optimize rmsnorm on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", - "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "searchspace": "docs/searchspace-matrix.md \u2014 Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", "source": { "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py", "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", @@ -24,9 +24,9 @@ "oracle": { "operation": "rmsnorm", "axis": 1, - "eps": 0.00001, - "rtol": 0.00001, - "atol": 1e-7 + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-07 }, "measurement": { "key": "metrics.suite_latency_ms", @@ -34,7 +34,15 @@ "boundary": "synchronous-compute-call", "aggregation": "sum-of-per-case-medians", "warmups": 2, - "samples": 5 + "samples": 5, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ] }, "workloads": [ { @@ -116,6 +124,6 @@ "No CUDA/ROCm/Ascend execution or accelerator timing qualification", "No service/E2E, communication, memory-capacity or skill/search benefit claim", "Trusted self-contained reviewed source only; no hostile-agent isolation", - "Repeated warm-cache calls; allocations inside compute included, setup/oracle/profile excluded from timing" + "Distinct frozen per-call inputs; zeros correctness-only; allocations inside compute included, setup/oracle/profile excluded from timing" ] } diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md index 7e6b0b251..91cef765a 100644 --- a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md @@ -8,8 +8,9 @@ float32 NumPy array of the same shape. Preserve input values. Compute stable sof Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; the evaluator is not a security sandbox for model-produced arbitrary code. -The task JSON owns all six public inputs, seeds, layouts, numerical tolerances -and timing policy. Baseline and example candidate are separate from the scalar +The task JSON owns six base correctness workloads, seeds, layouts, numerical +tolerances and the v2 timing policy. Five varying workloads have a frozen +per-call input sequence; invariant zero inputs are correctness-only. Baseline and example candidate are separate from the scalar double-precision oracle. The oracle uses math.fsum and is not editable by a candidate. The task package and grader are content-frozen per comparison. @@ -18,8 +19,12 @@ preservation outside the timed region. Input generation, reference computation, warmups, validation, process startup and profiling are excluded. Output allocation inside compute is included. Samples use perf_counter_ns and report milliseconds. Sum of per-case medians is only this task's fixed-suite objective, not a universal -score. Retain every per-case sample to inspect regressions. Repeated calls describe -a warm-cache CPU workload. No device synchronization is claimed. +score. Retain every per-case sample and input hash to inspect regressions. Inputs +are generated before timing and kept alive; each precheck, warmup and sampled +input has distinct bytes. Baseline and candidate consume the same sequence. +This prevents ordinary identity/content memo hits, not hostile-code execution. +Small CPU timings still need repeatability qualification; no cold-start serving +or device synchronization is claimed. All tests are public. This small subset cannot demonstrate production inference speedup, memory savings, or GPU/NPU support. A fresh task version and device diff --git a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json index ea2818495..e25252d59 100644 --- a/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json +++ b/src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json @@ -1,9 +1,9 @@ { "schema_version": 1, "id": "softmax-cpu", - "adapter": "numpy-axis1-v1", + "adapter": "numpy-axis1-v2", "description": "Optimize softmax on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", - "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", + "searchspace": "docs/searchspace-matrix.md \u2014 Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", "source": { "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py", "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", @@ -24,9 +24,9 @@ "oracle": { "operation": "softmax", "axis": 1, - "eps": 0.00001, - "rtol": 0.00001, - "atol": 1e-7 + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-07 }, "measurement": { "key": "metrics.suite_latency_ms", @@ -34,7 +34,15 @@ "boundary": "synchronous-compute-call", "aggregation": "sum-of-per-case-medians", "warmups": 2, - "samples": 5 + "samples": 5, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ] }, "workloads": [ { @@ -104,6 +112,6 @@ "No CUDA/ROCm/Ascend execution or accelerator timing qualification", "No service/E2E, communication, memory-capacity or skill/search benefit claim", "Trusted self-contained reviewed source only; no hostile-agent isolation", - "Repeated warm-cache calls; allocations inside compute included, setup/oracle/profile excluded from timing" + "Distinct frozen per-call inputs; zeros correctness-only; allocations inside compute included, setup/oracle/profile excluded from timing" ] } diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py index 0767c88b2..e4dc5b9fd 100644 --- a/src/inferrsi/harness/benchmarks/contract.py +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -180,7 +180,7 @@ def load_task(name: str | Path) -> TaskPackage: or type(data["schema_version"]) is not int ): raise ValueError("Invalid task fields") - if data["schema_version"] != 1 or data["adapter"] != "numpy-axis1-v1": + if data["schema_version"] != 1 or data["adapter"] != "numpy-axis1-v2": raise ValueError("Unsupported task schema or adapter") if not isinstance(data["id"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", data["id"]): raise ValueError("Invalid task id") @@ -212,15 +212,33 @@ def load_task(name: str | Path) -> TaskPackage: metric = data["measurement"] if ( not isinstance(metric, dict) - or set(metric) != {"key", "unit", "boundary", "aggregation", "warmups", "samples"} + or set(metric) + != { + "key", + "unit", + "boundary", + "aggregation", + "warmups", + "samples", + "input_sequence", + "case_ids", + } or metric["key"] != "metrics.suite_latency_ms" or metric["unit"] != "ms" or metric["boundary"] != "synchronous-compute-call" or metric["aggregation"] != "sum-of-per-case-medians" + or metric["input_sequence"] != "seeded-distinct-per-call-v1" ): raise ValueError("Unsupported measurement contract") for key in ("warmups", "samples"): _positive(metric[key], "measurement." + key, integer=True) + if ( + not isinstance(metric["case_ids"], list) + or not metric["case_ids"] + or any(not isinstance(name, str) for name in metric["case_ids"]) + or len(set(metric["case_ids"])) != len(metric["case_ids"]) + ): + raise ValueError("Distinct measured case IDs are required") if not isinstance(data["workloads"], list) or not data["workloads"]: raise ValueError("Workloads are required") names = set() @@ -250,6 +268,17 @@ def load_task(name: str | Path) -> TaskPackage: raise ValueError("Unknown distribution") if case["layout"] not in {"contiguous", "strided"}: raise ValueError("Unknown layout") + measured_cases = [case for case in data["workloads"] if case["id"] in metric["case_ids"]] + if [case["id"] for case in measured_cases] != metric["case_ids"]: + raise ValueError("Measured case IDs must follow workload order") + if any(case["distribution"] == "zeros" for case in measured_cases): + raise ValueError("Invariant zero workloads are correctness-only") + prepared_elements = sum(math.prod(case["shape"]) for case in data["workloads"]) + prepared_elements += (metric["warmups"] + metric["samples"]) * sum( + math.prod(case["shape"]) for case in measured_cases + ) + if prepared_elements > 1048576: + raise ValueError("Prepared input sequence exceeds the CPU workload bound") if ( not isinstance(data["coverage"], list) or not data["coverage"] diff --git a/src/inferrsi/harness/benchmarks/numpy_task.py b/src/inferrsi/harness/benchmarks/numpy_task.py index 8340c2502..883c7f559 100644 --- a/src/inferrsi/harness/benchmarks/numpy_task.py +++ b/src/inferrsi/harness/benchmarks/numpy_task.py @@ -14,8 +14,9 @@ from pathlib import Path -def inputs(case, np): - rng = np.random.default_rng(case["seed"]) +def inputs(case, np, *, sequence=0): + seed = case["seed"] if sequence == 0 else np.random.SeedSequence([case["seed"], sequence]) + rng = np.random.default_rng(seed) x = rng.standard_normal(case["shape"]).astype(np.float32) if case["distribution"] == "zeros": x.fill(0) @@ -64,6 +65,16 @@ def check_output(output, x, original, expected, oracle, np): } +def input_record(case, x): + return { + "id": case["id"], + "input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), + "shape": list(x.shape), + "dtype": str(x.dtype), + "strides": list(x.strides), + } + + def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: # Optional dependency is loaded only for execution, never task discovery. import numpy as np @@ -76,25 +87,40 @@ def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: if not callable(compute): raise ValueError("Candidate must define compute(x, *, eps)") oracle = manifest["oracle"] - checks, cases, workload_records = {}, [], [] - prepared = [] + measurement = manifest["measurement"] + checks, cases, workload_records, sequence_records, executed = {}, [], [], [], [] + prepared, sequences = [], [] + # Prepare all inputs/oracles before candidate calls. Keep every input alive: + # a previous call must not prime a later call's object/content memo entry. for case in manifest["workloads"]: x = inputs(case, np) original = x.copy() expected = reference(x, oracle, np) - workload_records.append( - { - "id": case["id"], - "input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), - "shape": list(x.shape), - "dtype": str(x.dtype), - "strides": list(x.strides), - } - ) + workload_records.append(input_record(case, x)) + prepared.append((case, x, original, expected)) + seen = {record["input_sha256"] for record in workload_records} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + calls = [] + for index in range(measurement["warmups"] + measurement["samples"]): + x = inputs(case, np, sequence=index + 1) + record = input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier input bytes") + seen.add(record["input_sha256"]) + record.update( + call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement", + ) + sequence_records.append(record) + calls.append((x, x.copy(), reference(x, oracle, np), record)) + sequences.append((case, calls)) + for (case, x, original, expected), record in zip(prepared, workload_records, strict=True): + executed.append({**record, "phase": "precheck"}) output = compute(x, eps=oracle["eps"]) assertions = check_output(output, x, original, expected, oracle, np) checks.update({case["id"] + "." + k: v for k, v in assertions.items()}) - prepared.append((case, x, original, expected)) result = { "status": "completed", "action": action, @@ -109,26 +135,32 @@ def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: }, "coverage": manifest["coverage"], "workloads": workload_records, + "input_sequence": sequence_records, + "executed_calls": executed, } - if not result["passed"] or action == "check": + if not result["passed"]: return result # Correctness and profiling do not masquerade as a paired measurement. profiler = cProfile.Profile() if action == "profile" else None - measurement = manifest["measurement"] - for case, x, original, expected in prepared: + for case, calls in sequences: samples = [] - for index in range(measurement["warmups"] + measurement["samples"]): - if profiler: - profiler.enable() - started = time.perf_counter_ns() - output = compute(x, eps=oracle["eps"]) - elapsed = (time.perf_counter_ns() - started) / 1e6 - if profiler: - profiler.disable() + for index, (x, original, expected, record) in enumerate(calls): + executed.append(record) + elapsed = None + if action == "check": + output = compute(x, eps=oracle["eps"]) + else: + if profiler: + profiler.enable() + started = time.perf_counter_ns() + output = compute(x, eps=oracle["eps"]) + elapsed = (time.perf_counter_ns() - started) / 1e6 + if profiler: + profiler.disable() assertions = check_output(output, x, original, expected, oracle, np) + checks.update({f"{case['id']}.call-{index}.{k}": v for k, v in assertions.items()}) if not all(assertions.values()): - checks.update({f"{case['id']}.call-{index}.{k}": v for k, v in assertions.items()}) result.update( passed=False, cases=cases, @@ -142,13 +174,24 @@ def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: }, ) return result - if index >= measurement["warmups"]: + if action != "check" and index >= measurement["warmups"]: if not math.isfinite(elapsed) or elapsed <= 0: raise ValueError("Timer did not resolve a positive duration") samples.append(elapsed) - cases.append( - {"id": case["id"], "samples_ms": samples, "median_ms": statistics.median(samples)} - ) + if action != "check": + cases.append( + { + "id": case["id"], + "samples_ms": samples, + "median_ms": statistics.median(samples), + "sample_input_sha256": [ + record["input_sha256"] + for _, _, _, record in calls[measurement["warmups"] :] + ], + } + ) + if action == "check": + return result if profiler: # Strip local installation paths from the textual profile before capture. with (result_dir / "profile.txt").open("w", encoding="utf-8") as stream: diff --git a/tests/fixtures/engineering_benchmark_usefulness.py b/tests/fixtures/engineering_benchmark_usefulness.py index f3e449468..c19e5d6da 100644 --- a/tests/fixtures/engineering_benchmark_usefulness.py +++ b/tests/fixtures/engineering_benchmark_usefulness.py @@ -248,6 +248,8 @@ def retain(path, role): workloads_match = bool(workloads) and all( result.get("workloads") == workloads for _, result in artifacts ) + sequence = artifacts[0][1].get("input_sequence") + executed = artifacts[0][1].get("executed_calls") checks = { "valid_candidates_accepted": all( candidates[r]["correctness"] == candidates[r]["skilleval"] == "PASS" for r in ROLES[:2] @@ -266,6 +268,14 @@ def retain(path, role): ), "consumer_identities_match": identities_match, "actual_workloads_match": workloads_match, + "planned_input_sequences_match": bool(sequence) + and all(result.get("input_sequence") == sequence for _, result in artifacts), + "valid_executed_inputs_match": bool(executed) + and all( + result.get("executed_calls") == executed + for _, result in artifacts + if result.get("passed") + ), "comparison_quality_passed": all(m["final_quality_passed"] for m in measured.values()), "comparisons_complete": all( m["source_integrity"] @@ -273,7 +283,7 @@ def retain(path, role): and m["measured_calls"] == 2 * plan["comparison"]["repetitions"] - * len(task.manifest["workloads"]) + * len(task.manifest["measurement"]["case_ids"]) * task.manifest["measurement"]["samples"] and all( m[side] and m[side]["excluded_trials"] == 0 for side in ("baseline", "candidate") @@ -293,6 +303,7 @@ def retain(path, role): "identities": plan["identities"], "comparison": plan["comparison"], "workloads": workloads, + "input_sequence": sequence, "profiles": profiles, "candidates": candidates, "ordinary_smoke": smoke_result, diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py index 0e852f068..2c63bb651 100644 --- a/tests/test_engineering_benchmarks.py +++ b/tests/test_engineering_benchmarks.py @@ -29,18 +29,24 @@ def read_result(execution): @pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) def test_real_baseline_candidate_measurement_and_profile(name, tmp_path, numpy_runtime): task = load_task(name) + checked = [] + measurement = task.manifest["measurement"] + expected_calls = len(task.manifest["workloads"]) + len(measurement["case_ids"]) * ( + measurement["warmups"] + measurement["samples"] + ) for role in ("baseline", "candidate"): candidate = task.root / (role + ".py") grade = grade_artifact(task, candidate, tmp_path / role, timeout_seconds=20) assert grade["verdict"] == "PASS" - assert len(grade["checks"]["assertions"]) == 24 + assert len(grade["checks"]["assertions"]) == 4 * expected_calls + checked.append(json.loads((tmp_path / role / "result.json").read_text())) candidate = task.root / "candidate.py" measured = read_result( run_action(task, candidate, "measure", tmp_path / "timed", timeout_seconds=20) ) assert measured["passed"] assert measured["metrics"]["suite_latency_ms"] > 0 - assert len(measured["cases"]) == len(task.manifest["workloads"]) + assert [case["id"] for case in measured["cases"]] == measurement["case_ids"] assert all(len(c["samples_ms"]) == 5 for c in measured["cases"]) profile = read_result( run_action(task, candidate, "profile", tmp_path / "profile", timeout_seconds=20) @@ -48,6 +54,76 @@ def test_real_baseline_candidate_measurement_and_profile(name, tmp_path, numpy_r assert "metrics" not in profile assert profile["diagnostic"]["performance_verdict"] is None assert "compute" in (tmp_path / "profile/profile.txt").read_text() + for result in [*checked, profile, measured]: + assert result["workloads"] == measured["workloads"] + assert result["input_sequence"] == measured["input_sequence"] + assert result["executed_calls"] == measured["executed_calls"] + assert len(result["executed_calls"]) == expected_calls + hashes = [call["input_sha256"] for call in result["executed_calls"]] + assert len(hashes) == len(set(hashes)) + assert "zeros" not in measurement["case_ids"] + assert any(call["id"] == "zeros" for call in measured["executed_calls"]) + for case in measured["cases"]: + assert case["sample_input_sha256"] == [ + call["input_sha256"] + for call in measured["input_sequence"] + if call["id"] == case["id"] and call["phase"] == "measurement" + ] + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +@pytest.mark.parametrize("key", ["id(x)", "x.tobytes()"]) +def test_timed_calls_cannot_hit_precheck_or_warmup_memo(name, key, tmp_path, numpy_runtime): + task = load_task(name) + candidate = tmp_path / "implementation.py" + operation = ( + "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + if name == "softmax-cpu" + else "y = x / np.sqrt(np.mean(x*x,axis=1,keepdims=True) + np.float32(eps))" + ) + candidate.write_text( + "import numpy as np\nfrom pathlib import Path\ncache = {}\n" + "def compute(x, *, eps):\n" + f" key = {key}\n" + " if key in cache:\n" + " Path(__file__).with_suffix('.hits').write_text('cache hit')\n" + " return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "completed" and result["passed"] + assert result["metrics"]["suite_latency_ms"] > 0 + # Correct memoization remains valid, but every timed call must be a miss. + assert not candidate.with_suffix(".hits").exists() + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +def test_shape_cached_output_replay_is_rejected_after_precheck(name, tmp_path, numpy_runtime): + task = load_task(name) + candidate = tmp_path / "implementation.py" + operation = ( + "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" + if name == "softmax-cpu" + else "y = x / np.sqrt(np.mean(x*x,axis=1,keepdims=True) + np.float32(eps))" + ) + candidate.write_text( + "import numpy as np\ncache = {}\ndef compute(x, *, eps):\n" + " key = (x.shape, x.strides)\n" + " if key in cache: return cache[key]\n" + f" {operation}\n" + " cache[key] = y\n" + " return y\n" + ) + execution = run_action(task, candidate, "measure", tmp_path / "run", timeout_seconds=20) + result = read_result(execution) + assert execution["status"] == "failed" and not result["passed"] + assert "metrics" not in result + assert result["failed_call"]["case"] == "normal" + assert result["failed_call"]["phase"] == "warmup" + assert all(value for key, value in result["assertions"].items() if ".call-" not in key) @pytest.mark.parametrize( @@ -124,6 +200,7 @@ def test_frozen_package_and_candidate_drift(tmp_path, numpy_runtime): ("schema_version", 2), ("schema_version", True), ("adapter", "unknown"), + ("adapter", "numpy-axis1-v1"), ("environment", {"backend": "cuda", "runtime": "numpy", "dtype": "float32"}), ("coverage", []), ("workloads", []), @@ -147,6 +224,44 @@ def test_descriptor_does_not_mutate_task(): assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu"} +@pytest.mark.parametrize( + "field,value", + [ + ("input_sequence", "repeated"), + ("case_ids", []), + ("case_ids", ["missing"]), + ("case_ids", ["normal", "normal"]), + ("case_ids", ["normal", "zeros"]), + ("samples", 1000), + ], +) +def test_invalid_input_sequence_contract_rejected(field, value, tmp_path): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + document["measurement"][field] = value + (root / "task.json").write_text(json.dumps(document)) + with pytest.raises(ValueError): + load_task(root) + + +def test_duplicate_generated_input_bytes_refuse_measurement(tmp_path, numpy_runtime): + root = tmp_path / "package" + shutil.copytree(load_task("softmax-cpu").root, root) + document = json.loads((root / "task.json").read_text()) + # Same values under different layouts still prime a bytes-keyed memo. + document["workloads"][1]["seed"] = document["workloads"][0]["seed"] + (root / "task.json").write_text(json.dumps(document)) + task = load_task(root) + execution = run_action( + task, root / "candidate.py", "measure", tmp_path / "run", timeout_seconds=20 + ) + result = read_result(execution) + assert execution["status"] == "failed" + assert result["status"] == "error" and "metrics" not in result + assert result["error_type"] == "ValueError" + + def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): repo = tmp_path / "repo" repo.mkdir() @@ -193,6 +308,12 @@ def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): ] identity = json.loads((run / "candidate/trial-001/result.json").read_text())["identity"] assert identity == task.identity(candidate) + baseline = json.loads((run / "baseline/trial-001/result.json").read_text()) + for side in ("baseline", "candidate"): + for path in (run / side).glob("trial-*/result.json"): + result = json.loads(path.read_text()) + assert result["input_sequence"] == baseline["input_sequence"] + assert result["executed_calls"] == baseline["executed_calls"] class RetainedControl: diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py index 57663d40c..b43c966eb 100644 --- a/tools/engineering_benchmark_validate.py +++ b/tools/engineering_benchmark_validate.py @@ -117,6 +117,7 @@ def main(): frozen_identity = task.identity(candidate) record = { "task": name, + "adapter": task.manifest["adapter"], "identity": task.identity(candidate), "correctness": grade["verdict"], "profile_status": profile["status"], @@ -138,6 +139,21 @@ def main(): "all_workloads_match": all( r["workloads"] == consumers[0]["workloads"] for r in consumers ), + "input_sequence": consumers[0]["input_sequence"], + "all_input_sequences_match": all( + r["input_sequence"] == consumers[0]["input_sequence"] for r in consumers + ), + "all_valid_executed_calls_match": all( + r["executed_calls"] == consumers[0]["executed_calls"] + for r in consumers + if r["passed"] + ), + "expected_measured_calls": ( + 2 + * args.repetitions + * len(task.manifest["measurement"]["case_ids"]) + * task.manifest["measurement"]["samples"] + ), } records.append(record) print(json.dumps(record), flush=True) @@ -149,6 +165,9 @@ def main(): and r["source_integrity"] and r["all_consumer_identities_match"] and r["all_workloads_match"] + and r["all_input_sequences_match"] + and r["all_valid_executed_calls_match"] + and r["measured_calls"] == r["expected_measured_calls"] and r["measurement_trials"] == 2 * args.repetitions for r in records ) From ac03ae23ce0bb5864a55c2c97d5f3d8d3b550d39 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 00:14:32 +0800 Subject: [PATCH 04/20] docs: clarify benchmark deliverables and their role in RSI Agent-assisted-by: Codex (documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/01-research.md | 16 +++++- docs/plans/issue-252/02-contract.md | 14 +++++- docs/plans/issue-252/03-tasks.md | 14 +++++- docs/plans/issue-252/04-integration.md | 14 +++++- docs/plans/issue-252/README.md | 70 ++++++++++++++------------ 5 files changed, 91 insertions(+), 37 deletions(-) diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md index d90a10745..74a58bfd9 100644 --- a/docs/plans/issue-252/01-research.md +++ b/docs/plans/issue-252/01-research.md @@ -1,4 +1,16 @@ -# 子项 01：调研与方案决策 +# 子项 01：选定第一批任务和复用方案 + +**这项工作的交付是选型结论：先做 Softmax/RMSNorm CPU 两道任务，复用成熟项目的任务定义和评测分工，并说明为何适合当前 InferRSI。** + +| 要说清楚的问题 | 本子项的答案 | +| --- | --- | +| 具体做什么 | 阅读论文与固定版本源码，比较任务、正确性参考、测量和执行边界，确定首批范围 | +| 交付什么 | 下文的一手来源表、方案比较、采用/不采用理由和任务选择 | +| 谁使用 | 子项 02 的接口开发者、子项 03 的任务开发者，以及范围评审者 | +| 实际作用 | 让实现围绕明确的两个优化问题展开，复用现有 Workbench/SkillEval，而不是从头搭另一个平台 | +| 当前状态 | 调研和方案已完成；维护者对首批范围的接受仍待记录 | + +它是一次设计交付，运行时不会执行“调研模块”。具体成果落实在后面三个子项。 ## 理解 @@ -36,7 +48,7 @@ | 小任务包 + adapter + 既有执行器 + 两个消费者 | 语义和证据共享，可立即跑通和检验反例 | 首批只证明 CPU 子集 | **采用** | 1. **首批选 Softmax 和 RMSNorm**。都是推理相关归约算子。Softmax 暴露 exp 溢出，RMSNorm 四维输入暴露错误轴，使第二道题有不同验证价值。通信/显存不放空实现。 -2. **消费者接口通用，adapter 能力具体**。`command(check|measure|profile)` 和 `runspec` 是共享边界，`numpy-axis1-v1` 描述当前能力。未知 backend/adapter 被拒绝。未来适配器沿用消费者入口，但必须定义自身同步、计时、manifest 校验和环境资格。 +2. **消费者接口通用，adapter 能力具体**。`command(check|measure|profile)` 和 `runspec` 是共享边界，当前 `numpy-axis1-v2` 以变化输入修复最初 v1 的重放问题，见 [协议验证](replay-validation.md)。未知 backend/adapter 被拒绝。未来适配器沿用消费者入口，但必须定义自身同步、计时、manifest 校验和环境资格。 3. **独立 oracle**。baseline/candidate 用 NumPy 归约，oracle 用标量 double 与 math.fsum，减少共享实现错误。容差属于具体任务数值规范，非全仓库性能门槛；每次比较冻结。 4. **检查真实计时路径**。计时后的实际输出再验证，能发现预检后变错的状态行为。公开输入、同进程评估不能抵御恶意作弊，入口限定已审核可信文件，不能直接接任意模型源码。 5. **profiling 独立**。CPU cProfile 解释 CPU 调用，不进入速度统计、不输出 A/B verdict。设备 trace 待硬件 adapter。 diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md index 63ebfe1ab..5e8d569df 100644 --- a/docs/plans/issue-252/02-contract.md +++ b/docs/plans/issue-252/02-contract.md @@ -1,4 +1,16 @@ -# 子项 02：共享任务接口 +# 子项 02：规定任务怎样提交、运行和判定 + +**这项工作的交付是一套共同规则：同一道优化任务交给 Workbench 或评估工具，都使用同一份输入、答案、计时边界和版本。** + +| 要说清楚的问题 | 本子项的答案 | +| --- | --- | +| 具体做什么 | 定义任务包字段、候选函数入口、检查/诊断/测量操作和结果身份 | +| 交付什么 | `TaskPackage`、`check/profile/measure` 操作及生成既有 RunSpec 的接口 | +| 谁使用 | Workbench 适配代码、离线 verifier 和后续任务作者 | +| 实际作用 | 一次比较中的题目和裁判固定；源码或规则改变时会被识别，两个调用者不用分别维护答案 | +| 当前状态 | 已实现并验证内容漂移、错误配置和未支持环境的拒绝；只承诺已实现的 CPU adapter | + +例如同一个 RMSNorm 候选在两条路径中都按 axis=1、相同 epsilon 和输入序列检查。接口不决定候选怎么生成。 ## 理解 diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index d0dc121c6..107f526eb 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -1,4 +1,16 @@ -# 子项 03：首批真实任务 +# 子项 03：把第一批任务做成可执行的包 + +**这项工作的交付是两道真正能运行的优化任务：Softmax 和 RMSNorm，每道题都带输入、性能基线、独立参考答案与测量规则。** + +| 要说清楚的问题 | 本子项的答案 | +| --- | --- | +| 具体做什么 | 将上游数学任务适配为 CPU float32 任务，补充布局、数值和逐次变化输入 | +| 交付什么 | 两个任务目录中的 `task.json`、基线、示例候选、独立 oracle 适配及来源许可 | +| 谁使用 | 负责修改实现的 Agent/开发者，以及执行检查的 Workbench/verifier | +| 实际作用 | Agent 有明确的代码修改目标，评测能抓住错误轴、遗漏 epsilon、溢出、输入修改与旧输出重放 | +| 当前状态 | 已实现并通过实际计算验证；当前计时还不能保证可靠性能排序 | + +例如省略 RMSNorm 的 epsilon 看似减少了一步计算，但完整任务会发现错误并拒绝给出可用性能指标。 ## 理解 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index 6243cd907..97d1410ba 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -1,4 +1,16 @@ -# 子项 04：消费者接入与实际用途验证 +# 子项 04：接入现有 RSI，并验证结果能被使用 + +**这项工作的交付是调用链和实际证据：让现有 Workbench 执行任务，让现有 verifier 按同一题目检查实现，并明确这些结果支持哪些决定。** + +| 要说清楚的问题 | 本子项的答案 | +| --- | --- | +| 具体做什么 | 把任务操作绑定到既有执行、比较、验证和报告入口，实际运行正确与错误候选 | +| 交付什么 | Workbench RunSpec 绑定、SkillEval 低层保留产物 verifier、复现工具和结果记录 | +| 谁使用 | 主 Agent 的引擎优化流程、评测开发者、PR 评审者 | +| 实际作用 | 工作台能够执行比较并保留证据；错误实现被共同裁判拒绝；结果可供 RSI 决定保留、放弃或继续诊断 | +| 当前状态 | Workbench 与底层 verifier 已接通；正式 SkillEval 两臂产物交接、真实技能收益与可靠性能排序未完成 | + +例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 ## 理解 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index cf1ef45aa..6478462bf 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -1,53 +1,59 @@ -# #252 共享工程基准交付计划 +# #252：给 InferRSI 提供共享优化任务和独立评测 -本地实施依据为用户在 2026-11-03 的明确委托。重新读取的 [Issue 正文](https://github.com/InferMatrixLab/InferRSI/issues/252) 与全部两条评论仍未记录维护者对首批子集的验收。本目录记录可供维护者评审的实现与证据，不代替项目接受或关闭 Issue。 +**我们要交付一套可复用的优化任务和独立评测工具：Agent 交付实现，工具检查算得对不对、在固定条件下测得多快，InferRSI 再据此决定下一步。** -基线为 `663f3b327998526a13aa31e72ad729025c18983c`。实现位于独立分支 `codex/issue252-engineering-benchmarks`，不改变其他工作区。 +本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。它的输入是选定的任务包和候选实现，输出是正确性结果、独立 profile、原始计时及对应版本。名称是本次交付提议，正式范围仍待维护者接受。 -## 要解决的实际问题 +## 实际交付给使用者什么 -先读 [在现有 RSI 中的调用与验收](rsi-integration.md)：#252 同时为引擎候选比较和优化者评估提供共享任务。当前已验证的是组件及底层消费者通路，尚未完成正式 SkillEval 两臂入口、既有改进 controller 或 agent 收益的端到端验收。 +以当前 `rmsnorm-cpu` 为例： -Workbench 和工程评估需要对同一个候选实现回答两个问题：计算是否正确，以及指定工作负载下测得什么性能。任务语义、输入、参考答案和计时规则应只有一份。搜索策略决定下一步尝试什么，消费者不能各自复制或修改题目答案。 +- **任务是什么：** 在规定的 CPU float32 输入上计算沿 axis=1 的 RMSNorm，保留 epsilon 和输入内容。 +- **Agent 可以改什么：** 独立 `implementation.py` 中的 `compute(x, *, eps)` 实现。 +- **Agent 交付什么：** 一份候选源码；输入、oracle、容差和计时规则来自受保护的任务包。 +- **谁检查：** `check` 使用独立参考答案验证；`profile` 生成单独诊断；`compare` 交给现有 Workbench 比较原版与候选版。 +- **结果如何发挥作用：** 错误实现不能获得可用性能分数；正确实现的计时和完整性证据供 RSI 判断是否保留。若计时无法分辨，继续记录“证据不足”，不能宣布优化成功。 -本次将这个模块命名为 **Engineering Benchmarks（工程基准）**。这是本次实现的命名，尚非维护者已接受的架构决策。 +一次真实对照中，省略 epsilon 的实现通过了普通输入冒烟，却被完整任务和 SkillEval 底层 verifier 一致拒绝。这证明了错误识别用途。该对照同时暴露计时噪声；可靠性能排序仍未建立，见[用途验证](usefulness.md)。之后还修复了重复输入导致缓存重放可被计时的问题，见 [v2 验证](replay-validation.md)。 -## 子项与执行顺序 +## 四个子任务分别做什么 -| 子项 | 要交付什么 | 如何证明有用 | -| --- | --- | --- | -| [01 调研和决策](01-research.md) | 一手论文、固定版本源码对照，选择及不选择的理由 | 每项决策对应接口、任务或验证行为 | -| [02 共享接口](02-contract.md) | 任务包读取、身份冻结、命令生成、独立诊断 | 错误配置和任务漂移会被拒绝，能生成现有 RunSpec | -| [03 首批任务](03-tasks.md) | 少量真实算子任务，独立 oracle，基线和候选接口 | 正确实现通过，典型错误被抓到，实际执行并计时 | -| [04 消费者与验收](04-integration.md) | Workbench 和离线工程评估共享同一任务，复现入口和结果 | 真正调用现有执行器/评估器，保留原始样本、失败记录和诊断 | +**这四项是交付步骤，不是四个运行时模块。** 第一项决定做什么，第二项规定规则，第三项交付可执行任务，第四项接入已有系统。 + +| 子任务 | 要交付的具体东西 | 谁使用、实际发挥什么作用 | 当前状态 | +| --- | --- | --- | --- | +| **① 选定第一批任务和复用方案** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [详细说明](01-research.md) | 论文/开源对照、任务选择、复用和适配决定 | 开发者据此确定先做哪几题、复用哪些设计，以及当前硬件范围 | 调研与方案已完成，范围待维护者接受 | +| **② 规定任务怎样提交、运行和判定** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [详细说明](02-contract.md) | 统一任务格式、候选入口、check/profile/measure 接口和版本记录 | Workbench 与评估工具读取同一题目和裁判；题目或源码变更后不能继续当作原比较 | 已实现共享接口和内容冻结 | +| **③ 把第一批任务做成可执行的包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [详细说明](03-tasks.md) | Softmax、RMSNorm CPU 任务；每题带输入、基线、独立参考答案和测量规则 | Agent 有明确修改对象，工具有真实可运行的正确性、诊断和测量任务 | 已实现并实测；可靠性能排序未建立 | +| **④ 接入现有 RSI，并验证结果能被使用** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [详细说明](04-integration.md) | Workbench 绑定、离线 verifier、复现入口和证据记录 | 现有实验流程执行任务、拒绝错误实现并保留证据；后续用于评估不同技能产出的实现 | Workbench 和底层 verifier 已接通；正式 SkillEval 两臂接入未完成 | -硬件尚未确定，按用户的“共享接口优先”约束先验证本机 CPU 子集。加速器适配不能用 CPU 结果冒充。通信、分布式拓扑、显存容量、GPU/NPU 计时不属于这个首批子集。 +## 放在我们 RSI 的哪里 -## 与评论中的 Search Policy 的关系 +同一套任务服务两种不同的比较： + +| RSI 要回答的问题 | 现有调用者 | #252 的作用 | +| --- | --- | --- | +| 这个引擎实现是否正确、有可验证的改进？ | 主 Agent + Workbench | 提供共同工作负载、独立正确性检查和测量规则 | +| 新 skill/知识是否帮助 Agent 产出更好的实现？ | SkillEval 两臂比较；后续衔接已有改进流程 | 两边产出的实现使用同一套任务和裁判，不随技能条件更换答案 | -[RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306) 提出了候选卡、准入、成本层级、轮次、负结果和停止规则。[维护者回复](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922) 明确要求保持它与 #252 独立，并修正增量贡献、诊断记录、搜索耗尽三个问题。本次交付任务和 oracle，不顺带实现这套搜索控制器。 +第一条路径已有实际执行。第二条目前只验证了已审核产物经过底层 verifier 的通路；尚未比较旧/新技能如何影响真实 Agent 的交付。完整调用关系和分层验收见 [RSI 接入审计](rsi-integration.md)。 -Profiling 保留为独立诊断操作，与 correctness 和配对 performance 分开。它不伪造 baseline，也不产生性能胜负。本次 CPU profiling 只能解释 CPU 调用；设备 trace 由未来环境适配器提供。 +主 Agent 和 Search Policy 决定下一步尝试什么；任务包决定要算什么、怎样检查和测量。Profiling 是独立诊断，不冒充速度比较。它们在现有 RSI 中分工合作，#252 不负责另建搜索控制器。 -## 完成标准 +## 参考 AMD 项目时具体学什么 -- 每个子项有方案、理由、实际改动和验证记录。 -- 首批任务可从安装包读取，离开源码目录仍可使用。 -- 冻结任务、grader、候选和输入身份，保留失败与缺失状态。 -- 测量明确单位、边界、预热、重复与样本，不用文档或 CI 绿色代替真实任务运行。 -- 证明接口复用和错误识别；不宣称技能/agent/search 的收益、GPU/NPU 正确性或服务加速。 -- 正式 Issue 验收、维护者同意、独立 review 与本地实施完成分别报告。 +之前调研的 AMD 项目是 [AgentKernelArena](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/README.md)。它明确描述：给 Agent GPU 优化任务，在工作区产出实现，由独立 evaluator 检查、计时并产生结果；外部系统负责训练或策略更新。 -最终结果见 [验证记录](validation.md)。 +本次借鉴它的[任务契约](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md)：明确计算语义、可编辑实现、性能基线、正确性参考、工作负载和评测操作。我们把这层能力接到已有 InferRSI；当前交付范围是共享任务及其接口，不等同于 Arena 完整的 GPU/Agent 执行平台。 -[实际工程用途对照](usefulness.md)进一步验证了错误拦截和两个消费者的复用价值，并如实记录了原版对原版的计时噪声；可靠性能优选尚未建立。 +## 怎么开始使用和检查结果 -## 当前交付 +运行 `inferrsi benchmarks list`、`show rmsnorm-cpu`，将示例候选复制成独立 `implementation.py`，再按[使用文档](../../engineering-benchmarks.md)执行 `check`、`profile` 和 `compare`。共享接口优先，硬件尚未确定，所以目前只支持明确的 CPU 子集。 -公开子任务分别为 [#334 调研与决策](https://github.com/InferMatrixLab/InferRSI/issues/334)、[#335 共享接口](https://github.com/InferMatrixLab/InferRSI/issues/335)、[#336 首批任务](https://github.com/InferMatrixLab/InferRSI/issues/336)、[#337 消费者与验证](https://github.com/InferMatrixLab/InferRSI/issues/337)。四项均引用父任务 #252，保持打开供维护者评审与验收。当前提交账号无法写入原生父子关系，需维护者补充。 +当前证据见 [v2 协议验证](replay-validation.md)；历史证据见 [v1 集成记录](validation.md) 和 [v1 用途对照](usefulness.md)。本地测试、组件实现、正式系统接入与维护者验收分别记录。GPU/NPU、通信/容量任务、真实 agent/skill/search 收益和可靠性能排序均不能由当前 CPU 结果代替。 -四个子项已有组件实现：共享 TaskPackage、两个真实 CPU 算子任务、独立正确性与 profiling、Workbench 和 SkillEval 低层 verifier 接入，以及可复现实测工具。测量语义 review、可靠性能排序和系统接入验收分别追踪，不能把局部实现表述为整个 RSI 已经闭环。使用入口见 [工程基准文档](../../engineering-benchmarks.md)。 +## 跟踪与范围 -建议先读本页理解范围，再读 01 看方案理由。要开始使用时，运行 `inferrsi benchmarks list` 和 `show softmax-cpu`，复制任务包中的示例候选到独立 implementation.py，按使用文档执行 check、profile、compare。修改任务协议应创建新比较，不能沿用旧身份。 +实现及文档在 [Draft PR #338](https://github.com/InferMatrixLab/InferRSI/pull/338)，四个子任务保持打开。原生父子关系写入受账号权限限制，目前通过相互引用和父任务索引追踪。 -这份交付覆盖共享接口和 CPU 子集的组件实现；正式 SkillEval 两臂接入、优化者收益和可靠性能排序尚未建立。#252 的维护者验收、目标硬件选择与后续覆盖仍单独记录，不在本地冒充已接受。 +初始基线为 `663f3b327998526a13aa31e72ad729025c18983c`，工作分支为 `codex/issue252-engineering-benchmarks`。本地实施与提交依据为用户的明确委托；这不等价于维护者已接受整个 #252。父 Issue 的 [Search Policy RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306) 按[维护者范围意见](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922)保持独立。 From 9454672f9b0fa00dd2e443bff4ce62569ab0fafd Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 07:32:09 +0800 Subject: [PATCH 05/20] fix: report benchmark comparison outcomes and document delivery Agent-assisted-by: Codex (review, implementation and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- .gitattributes | 2 + docs/engineering-benchmarks.md | 9 + docs/plans/issue-252/02-contract.md | 2 +- docs/plans/issue-252/04-integration.md | 2 +- docs/plans/issue-252/README.md | 8 +- docs/plans/issue-252/replay-measurements.json | 1742 +++++++++++++++++ .../issue-252/replay-profiles/rmsnorm-cpu.txt | 19 + .../issue-252/replay-profiles/softmax-cpu.txt | 19 + docs/plans/issue-252/replay-validation.md | 6 +- docs/plans/issue-252/review.md | 76 + docs/plans/issue-252/validation.md | 4 +- docs/plans/issue-252/walkthrough.md | 192 ++ src/inferrsi/cli.py | 3 + src/inferrsi/harness/benchmarks/cli.py | 2 + tests/test_engineering_benchmarks.py | 50 + 15 files changed, 2128 insertions(+), 8 deletions(-) create mode 100644 docs/plans/issue-252/replay-measurements.json create mode 100644 docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt create mode 100644 docs/plans/issue-252/replay-profiles/softmax-cpu.txt create mode 100644 docs/plans/issue-252/review.md create mode 100644 docs/plans/issue-252/walkthrough.md diff --git a/.gitattributes b/.gitattributes index 17ca18179..1dd2a9bab 100644 --- a/.gitattributes +++ b/.gitattributes @@ -12,6 +12,8 @@ /docs/plans/issue-252/usefulness-summary.json -text whitespace=cr-at-eol /docs/plans/issue-252/replay-validation-summary.json -text whitespace=cr-at-eol /docs/plans/issue-252/usefulness-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof +/docs/plans/issue-252/replay-measurements.json -text +/docs/plans/issue-252/replay-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof # Retain this upstream license's original CRLF Git blob bytes. /src/inferrsi/data/skill_library/frameworks/torch-npu/torch-npu/references/licenses/op-plugin-LICENSE.txt -text diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index 3a215b9f2..12f763909 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -26,6 +26,12 @@ Workbench. Single-action output directories must be new. These are example budgets, not campaign acceptance gates. The supplied Git checkout identifies the source context; task and candidate files are additionally bound by hashes. +`compare` returns the retained `status.json` alongside `run_dir`. Its exit code +matches Workbench: PASS is 0, FAIL is 1, and INCONCLUSIVE is 2. A completed +comparison missing its gain gate is FAIL; an incorrect candidate that prevents +measurement leaves the comparison INCONCLUSIVE, even though artifact correctness +grading can independently report FAIL. Inspect the retained state and errors. + The candidate is a reviewed self-contained Python file with `compute(x, *, eps)`. Use NumPy, math and local helpers. `load_task("softmax-cpu").root` locates its baseline and example candidate. Copy `candidate.py` to `implementation.py` before @@ -103,3 +109,6 @@ bundles contain resolved paths; review/redact a copy before sharing. See [#252 research and subtask plans](plans/issue-252/README.md). Formal Issue acceptance, independent review, accelerator qualification, communication and memory-capacity coverage remain distinct from this local CPU subset. + +For a Chinese walkthrough of each component and an RMSNorm example, start with +[the reading guide](plans/issue-252/walkthrough.md). diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md index 5e8d569df..f6ccead72 100644 --- a/docs/plans/issue-252/02-contract.md +++ b/docs/plans/issue-252/02-contract.md @@ -26,7 +26,7 @@ ## 验收 -同一包身份被两个消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。真实运行和安装包验证见 validation.md。 +同一包身份被两个消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。当前变化输入协议及实际消费者验证见 [v2 验证](replay-validation.md)；早期安装包检查和 v1 运行保留在[历史验证记录](validation.md)，不作为当前协议的执行结果。 ## 已实施 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index 97d1410ba..da6a2a974 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -40,6 +40,6 @@ 消费链为“同一 task.json 和 grader → Workbench 性能比较 / SkillEval 独立正确性验证”。后者是离线保留产物交接，不是在线 agent 写代码的新 sandbox。是否由技能帮助产出更好代码，需要未来真实 agent、匹配预算和对照条件的实验。 -运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，最终结果见 [验证记录](validation.md)。 +运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，当前协议的实际结果见 [v2 验证](replay-validation.md)。早期安装包检查和 v1 集成结果另存于[历史验证记录](validation.md)。 补充[实际工程用途对照](usefulness.md)把预先固定的 no-epsilon、einsum 和 unchanged 实现接入同一任务，证明普通冒烟漏检会被完整任务拒绝。它也暴露了计时稳定性限制：同一代码的比较出现大幅差异，可靠性能排序仍待后续协议验证，不能作为已完成的验收项。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 6478462bf..2c077bb6d 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -1,9 +1,11 @@ # #252：给 InferRSI 提供共享优化任务和独立评测 -**我们要交付一套可复用的优化任务和独立评测工具：Agent 交付实现，工具检查算得对不对、在固定条件下测得多快，InferRSI 再据此决定下一步。** +**我们要交付一套可复用的优化任务和独立评测工具：Agent 交付实现，工具检查算得对不对、在固定条件下测得多快，主 Agent 再依据这些证据决定下一步。** 本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。它的输入是选定的任务包和候选实现，输出是正确性结果、独立 profile、原始计时及对应版本。名称是本次交付提议，正式范围仍待维护者接受。 +**第一次阅读请从[完整导读](walkthrough.md)开始**：沿 RMSNorm 例子解释四个子任务、每个流程框、profiling、实际命令和结果。想核对本轮发现及修复的问题，请读[审查记录](review.md)。 + ## 实际交付给使用者什么 以当前 `rmsnorm-cpu` 为例： @@ -12,7 +14,7 @@ - **Agent 可以改什么：** 独立 `implementation.py` 中的 `compute(x, *, eps)` 实现。 - **Agent 交付什么：** 一份候选源码；输入、oracle、容差和计时规则来自受保护的任务包。 - **谁检查：** `check` 使用独立参考答案验证；`profile` 生成单独诊断；`compare` 交给现有 Workbench 比较原版与候选版。 -- **结果如何发挥作用：** 错误实现不能获得可用性能分数；正确实现的计时和完整性证据供 RSI 判断是否保留。若计时无法分辨，继续记录“证据不足”，不能宣布优化成功。 +- **结果如何发挥作用：** 错误实现不能获得可用性能分数；主 Agent 根据正确实现的计时和完整性证据判断是否保留。若计时无法分辨，继续记录“证据不足”，不能宣布优化成功。 一次真实对照中，省略 epsilon 的实现通过了普通输入冒烟，却被完整任务和 SkillEval 底层 verifier 一致拒绝。这证明了错误识别用途。该对照同时暴露计时噪声；可靠性能排序仍未建立，见[用途验证](usefulness.md)。之后还修复了重复输入导致缓存重放可被计时的问题，见 [v2 验证](replay-validation.md)。 @@ -38,7 +40,7 @@ 第一条路径已有实际执行。第二条目前只验证了已审核产物经过底层 verifier 的通路；尚未比较旧/新技能如何影响真实 Agent 的交付。完整调用关系和分层验收见 [RSI 接入审计](rsi-integration.md)。 -主 Agent 和 Search Policy 决定下一步尝试什么；任务包决定要算什么、怎样检查和测量。Profiling 是独立诊断，不冒充速度比较。它们在现有 RSI 中分工合作，#252 不负责另建搜索控制器。 +当前由主 Agent 决定下一步尝试什么；任务包决定要算什么、怎样检查和测量。Optimization Search Policy 的提案讨论如何选择下一次尝试，本次没有实现该策略。Profiling 是独立诊断，不冒充速度比较。#252 提供可调用的评测操作，不负责强制主 Agent 遵守整个流程或另建搜索控制器。 ## 参考 AMD 项目时具体学什么 diff --git a/docs/plans/issue-252/replay-measurements.json b/docs/plans/issue-252/replay-measurements.json new file mode 100644 index 000000000..f8ef51647 --- /dev/null +++ b/docs/plans/issue-252/replay-measurements.json @@ -0,0 +1,1742 @@ +{ + "scope": "Read-only export of the original 2026-11-03 UTC v2 integration; no new run", + "protocol": "numpy-axis1-v2", + "source_summary": "replay-validation-summary.json", + "source_summary_sha256": "a9e9d7d17f3de30344229332056071ab6450b283fbe3272352f48b68855f4842", + "provenance": "Measurements predate ce84506e; task/grader bytes match the frozen identities published there. No evidence is rebound to a later source revision.", + "transformation": "Retain all 12 trials and 300 samples unchanged; project result fields to omit repeated assertions/input records and local paths. Evaluations and stripped-path profiles are retained unchanged. Original full-result hashes identify local source files; those full files are not published.", + "campaign_acceptance": "INCONCLUSIVE", + "timing_qualification": "not established", + "agent_or_skill_benefit": "not measured", + "measured_calls": 300, + "tasks": [ + { + "task": "rmsnorm-cpu", + "trials": [ + { + "side": "baseline", + "trial": "trial-001", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-001/result.json", + "source_sha256": "7d7f50721b3fdcf42621799a05d416707b4eb8ba2616958ffe40a7cacf438296", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.045700000000000005 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0145, + 0.0126, + 0.012, + 0.0119, + 0.0117 + ], + "median_ms": 0.012, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0155, + 0.016, + 0.0158, + 0.016, + 0.0154 + ], + "median_ms": 0.0158, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0065, + 0.0068, + 0.0064, + 0.0065, + 0.006 + ], + "median_ms": 0.0065, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0062, + 0.0061, + 0.0061, + 0.0064, + 0.0071 + ], + "median_ms": 0.0062, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0052, + 0.0054, + 0.0054, + 0.0049, + 0.0051 + ], + "median_ms": 0.0052, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "baseline", + "trial": "trial-002", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-002/result.json", + "source_sha256": "c167e1d49bb7a2381287fba0ce1d716ca55c3469a90c1f53b95d86049464d67b", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.0453 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0134, + 0.0127, + 0.0115, + 0.0112, + 0.0104 + ], + "median_ms": 0.0115, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0156, + 0.0148, + 0.0149, + 0.0152, + 0.015 + ], + "median_ms": 0.015, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0059, + 0.0156, + 0.0093, + 0.0074, + 0.0064 + ], + "median_ms": 0.0074, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0066, + 0.0061, + 0.0057, + 0.0063, + 0.0073 + ], + "median_ms": 0.0063, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0055, + 0.0053, + 0.005, + 0.0051, + 0.005 + ], + "median_ms": 0.0051, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "baseline", + "trial": "trial-003", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-003/result.json", + "source_sha256": "fd967164167654667c47ad3934ffa4dd54993f01cf5c0f0681d87d2653f8c951", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.04800000000000001 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.016, + 0.0145, + 0.0134, + 0.0123, + 0.0123 + ], + "median_ms": 0.0134, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0162, + 0.0167, + 0.0159, + 0.017, + 0.0157 + ], + "median_ms": 0.0162, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0066, + 0.0075, + 0.0063, + 0.0066, + 0.0063 + ], + "median_ms": 0.0066, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0065, + 0.0064, + 0.0062, + 0.0062, + 0.0067 + ], + "median_ms": 0.0064, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0055, + 0.0054, + 0.0053, + 0.0054, + 0.0052 + ], + "median_ms": 0.0054, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-001", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-001/result.json", + "source_sha256": "5a886688a35a8f033a09da2390ac5375212efe9c7c965c26e8509fb11bd19daa", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.1256 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0146, + 0.0129, + 0.0122, + 0.0343, + 0.0158 + ], + "median_ms": 0.0146, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0149, + 0.0147, + 0.1118, + 0.061, + 0.0512 + ], + "median_ms": 0.0512, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0208, + 0.0191, + 0.0219, + 0.0218, + 0.0213 + ], + "median_ms": 0.0213, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0211, + 0.0205, + 0.0197, + 0.021, + 0.0253 + ], + "median_ms": 0.021, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0184, + 0.0175, + 0.0164, + 0.0175, + 0.0163 + ], + "median_ms": 0.0175, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-002", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-002/result.json", + "source_sha256": "80db25ef61f6ee371214574d4003f0dc561a9c73f8884dfb969036a55f6ad908", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.0436 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0148, + 0.014, + 0.0121, + 0.0124, + 0.0116 + ], + "median_ms": 0.0124, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0144, + 0.0136, + 0.0137, + 0.015, + 0.0133 + ], + "median_ms": 0.0137, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0074, + 0.0068, + 0.0071, + 0.0057, + 0.0056 + ], + "median_ms": 0.0068, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0068, + 0.0059, + 0.0056, + 0.0057, + 0.0068 + ], + "median_ms": 0.0059, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.005, + 0.0053, + 0.0048, + 0.0048, + 0.0048 + ], + "median_ms": 0.0048, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-003", + "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-003/result.json", + "source_sha256": "26671faef9a4cb403d1c731747143218a5da07f83f69eb34c3cb492bdc3ebc41", + "status": "completed", + "passed": true, + "identity": { + "task_id": "rmsnorm-cpu", + "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.0439 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0143, + 0.0128, + 0.0127, + 0.0126, + 0.0119 + ], + "median_ms": 0.0127, + "sample_input_sha256": [ + "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", + "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", + "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", + "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", + "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0138, + 0.0135, + 0.0138, + 0.0142, + 0.0135 + ], + "median_ms": 0.0138, + "sample_input_sha256": [ + "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", + "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", + "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", + "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", + "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0065, + 0.0063, + 0.0061, + 0.006, + 0.0062 + ], + "median_ms": 0.0062, + "sample_input_sha256": [ + "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", + "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", + "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", + "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", + "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0064, + 0.0058, + 0.0056, + 0.0058, + 0.0066 + ], + "median_ms": 0.0058, + "sample_input_sha256": [ + "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", + "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", + "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", + "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", + "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0055, + 0.0054, + 0.0054, + 0.0051, + 0.0051 + ], + "median_ms": 0.0054, + "sample_input_sha256": [ + "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", + "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", + "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", + "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", + "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + } + ], + "evaluation": { + "verdict": "PASS", + "reasons": [], + "metric_key": "metrics.suite_latency_ms", + "metric_unit": "ms", + "comparison_scope": "rmsnorm-cpu:synchronous-compute-call", + "direction": "minimize", + "min_improvement_pct": 0, + "baseline": { + "values": [ + 0.045700000000000005, + 0.0453, + 0.04800000000000001 + ], + "median": 0.045700000000000005, + "min": 0.0453, + "max": 0.04800000000000001, + "range_pct_of_median": 5.908096280087545, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.1256, + 0.0436, + 0.0439 + ], + "median": 0.0439, + "min": 0.0436, + "max": 0.1256, + "range_pct_of_median": 186.78815489749428, + "excluded_trials": 0 + }, + "improvement_pct": 3.938730853391691, + "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", + "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", + "source_integrity": { + "verified": true, + "reason": "Git source snapshots unchanged", + "command_checks": 7 + }, + "statistical_significance": "not established" + }, + "source_evaluation_sha256": "a488026ab37d256990f631aa1e44e0b9300784de3d9f063c8ca9c385854c0f41", + "profile": { + "path": "replay-profiles/rmsnorm-cpu.txt", + "sha256": "7e566463a8200a0a777902642c4686bcc88aae6e836871b5804b5996f9426064", + "copied_without_byte_changes": true, + "compute_calls": 35 + } + }, + { + "task": "softmax-cpu", + "trials": [ + { + "side": "baseline", + "trial": "trial-001", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-001/result.json", + "source_sha256": "e9138fab5651d7c7f54d4dd55a058d7df47e599f63e2c140479c8cf3a9f3b963", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.044000000000000004 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0145, + 0.0141, + 0.0129, + 0.0139, + 0.0139 + ], + "median_ms": 0.0139, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0148, + 0.0151, + 0.0153, + 0.0148, + 0.0148 + ], + "median_ms": 0.0148, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0058, + 0.006, + 0.0059, + 0.0056, + 0.0059 + ], + "median_ms": 0.0059, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0051, + 0.0052, + 0.0052, + 0.0051, + 0.0053 + ], + "median_ms": 0.0052, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0043, + 0.0042, + 0.0041, + 0.0042, + 0.0043 + ], + "median_ms": 0.0042, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "baseline", + "trial": "trial-002", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-002/result.json", + "source_sha256": "128420def0c712683758bed15189b6cf97ef2ca26c2a79bc95cdfbe3bbf50170", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.045500000000000006 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0146, + 0.014, + 0.0133, + 0.0152, + 0.0137 + ], + "median_ms": 0.014, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0162, + 0.0167, + 0.0165, + 0.0167, + 0.0166 + ], + "median_ms": 0.0166, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0056, + 0.0061, + 0.0056, + 0.0056, + 0.0058 + ], + "median_ms": 0.0056, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0058, + 0.0056, + 0.0048, + 0.0052, + 0.0052 + ], + "median_ms": 0.0052, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0047, + 0.004, + 0.0043, + 0.0041, + 0.0041 + ], + "median_ms": 0.0041, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "baseline", + "trial": "trial-003", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-003/result.json", + "source_sha256": "cb51e799d2270c94c1a552fc00dcaf832fcf7d8afca3f1c53584e27d4bd45129", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.1093 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0332, + 0.0326, + 0.0302, + 0.0324, + 0.0305 + ], + "median_ms": 0.0324, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0341, + 0.0344, + 0.0343, + 0.0335, + 0.0343 + ], + "median_ms": 0.0343, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0157, + 0.0163, + 0.0154, + 0.0314, + 0.0191 + ], + "median_ms": 0.0163, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.014, + 0.015, + 0.0144, + 0.0152, + 0.0156 + ], + "median_ms": 0.015, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0127, + 0.0123, + 0.0113, + 0.0113, + 0.011 + ], + "median_ms": 0.0113, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-001", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-001/result.json", + "source_sha256": "3dd12e1fd31a06c3bcf07a42317b2caf8db41afb4406a3dc5ff210a15c724b35", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.0426 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0136, + 0.0133, + 0.0125, + 0.0119, + 0.0119 + ], + "median_ms": 0.0125, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.015, + 0.0147, + 0.0146, + 0.0152, + 0.0144 + ], + "median_ms": 0.0147, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0058, + 0.0059, + 0.0058, + 0.0056, + 0.0058 + ], + "median_ms": 0.0058, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0053, + 0.005, + 0.005, + 0.0055, + 0.0054 + ], + "median_ms": 0.0053, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0041, + 0.0039, + 0.0108, + 0.005, + 0.0043 + ], + "median_ms": 0.0043, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-002", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-002/result.json", + "source_sha256": "232c9351a68fa3776912588492578172618bb3a396d2bf1c66159f393a7c1f7a", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.0422 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0137, + 0.013, + 0.0128, + 0.0136, + 0.0125 + ], + "median_ms": 0.013, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0148, + 0.014, + 0.0146, + 0.0153, + 0.0143 + ], + "median_ms": 0.0146, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0055, + 0.0057, + 0.0055, + 0.0055, + 0.0056 + ], + "median_ms": 0.0055, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0051, + 0.0054, + 0.0051, + 0.0051, + 0.0053 + ], + "median_ms": 0.0051, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0041, + 0.0039, + 0.0041, + 0.004, + 0.004 + ], + "median_ms": 0.004, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + }, + { + "side": "candidate", + "trial": "trial-003", + "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-003/result.json", + "source_sha256": "5c937be24fb41fa0960c7beceac63b83eef26f4bb8d85706875a96b2a2c7994b", + "status": "completed", + "passed": true, + "identity": { + "task_id": "softmax-cpu", + "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", + "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", + "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", + "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" + }, + "environment": { + "backend": "cpu", + "python": "3.11.18", + "numpy": "2.4.6", + "os": "Windows", + "machine": "AMD64" + }, + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "synchronous-compute-call", + "case_ids": [ + "normal", + "strided", + "large", + "tiny", + "singleton" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 5, + "unit": "ms", + "warmups": 2 + }, + "metrics": { + "suite_latency_ms": 0.044800000000000006 + }, + "cases": [ + { + "id": "normal", + "samples_ms": [ + 0.0143, + 0.0172, + 0.0136, + 0.0144, + 0.014 + ], + "median_ms": 0.0143, + "sample_input_sha256": [ + "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", + "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", + "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", + "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", + "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" + ] + }, + { + "id": "strided", + "samples_ms": [ + 0.0156, + 0.0153, + 0.0151, + 0.0154, + 0.0147 + ], + "median_ms": 0.0153, + "sample_input_sha256": [ + "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", + "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", + "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", + "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", + "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" + ] + }, + { + "id": "large", + "samples_ms": [ + 0.0059, + 0.0062, + 0.0058, + 0.006, + 0.006 + ], + "median_ms": 0.006, + "sample_input_sha256": [ + "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", + "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", + "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", + "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", + "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" + ] + }, + { + "id": "tiny", + "samples_ms": [ + 0.0052, + 0.0053, + 0.0052, + 0.005, + 0.0056 + ], + "median_ms": 0.0052, + "sample_input_sha256": [ + "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", + "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", + "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", + "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", + "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" + ] + }, + { + "id": "singleton", + "samples_ms": [ + 0.0042, + 0.004, + 0.0039, + 0.004, + 0.004 + ], + "median_ms": 0.004, + "sample_input_sha256": [ + "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", + "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", + "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", + "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", + "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" + ] + } + ], + "assertions_passed": 164, + "planned_and_executed_input_sequence_matches_summary": true + } + ], + "evaluation": { + "verdict": "PASS", + "reasons": [], + "metric_key": "metrics.suite_latency_ms", + "metric_unit": "ms", + "comparison_scope": "softmax-cpu:synchronous-compute-call", + "direction": "minimize", + "min_improvement_pct": 0, + "baseline": { + "values": [ + 0.044000000000000004, + 0.045500000000000006, + 0.1093 + ], + "median": 0.045500000000000006, + "min": 0.044000000000000004, + "max": 0.1093, + "range_pct_of_median": 143.5164835164835, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.0426, + 0.0422, + 0.044800000000000006 + ], + "median": 0.0426, + "min": 0.0422, + "max": 0.044800000000000006, + "range_pct_of_median": 6.1032863849765375, + "excluded_trials": 0 + }, + "improvement_pct": 6.373626373626387, + "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", + "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", + "source_integrity": { + "verified": true, + "reason": "Git source snapshots unchanged", + "command_checks": 7 + }, + "statistical_significance": "not established" + }, + "source_evaluation_sha256": "2015ce893c327c7c5c65800d1f7b1e81c630e94c4e8dca0b98b513c573e462c7", + "profile": { + "path": "replay-profiles/softmax-cpu.txt", + "sha256": "a0d40fe469c8bb794dd2a7c9a918fc4e0d98c0f0f1fb378e96ea21e6547e37d1", + "copied_without_byte_changes": true, + "compute_calls": 35 + } + } + ] +} diff --git a/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt b/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt new file mode 100644 index 000000000..37dd11e88 --- /dev/null +++ b/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt @@ -0,0 +1,19 @@ + 525 function calls in 0.000 seconds + + Ordered by: cumulative time + + ncalls tottime percall cumtime percall filename:lineno(function) + 35 0.000 0.000 0.000 0.000 implementation.py:6(compute) + 35 0.000 0.000 0.000 0.000 fromnumeric.py:3699(mean) + 35 0.000 0.000 0.000 0.000 _methods.py:115(_mean) + 35 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} + 35 0.000 0.000 0.000 0.000 _methods.py:73(_count_reduce_items) + 70 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} + 70 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass} + 70 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} + 35 0.000 0.000 0.000 0.000 fromnumeric.py:3694(_mean_dispatcher) + 35 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} + 35 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} + 35 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + diff --git a/docs/plans/issue-252/replay-profiles/softmax-cpu.txt b/docs/plans/issue-252/replay-profiles/softmax-cpu.txt new file mode 100644 index 000000000..5ce9b4d3d --- /dev/null +++ b/docs/plans/issue-252/replay-profiles/softmax-cpu.txt @@ -0,0 +1,19 @@ + 595 function calls in 0.000 seconds + + Ordered by: cumulative time + + ncalls tottime percall cumtime percall filename:lineno(function) + 35 0.000 0.000 0.000 0.000 implementation.py:6(compute) + 70 0.000 0.000 0.000 0.000 fromnumeric.py:66(_wrapreduction) + 35 0.000 0.000 0.000 0.000 fromnumeric.py:3011(max) + 70 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} + 35 0.000 0.000 0.000 0.000 fromnumeric.py:2304(sum) + 70 0.000 0.000 0.000 0.000 fromnumeric.py:67() + 70 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} + 70 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects} + 35 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} + 35 0.000 0.000 0.000 0.000 fromnumeric.py:2299(_sum_dispatcher) + 35 0.000 0.000 0.000 0.000 fromnumeric.py:3006(_max_dispatcher) + 35 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} + + diff --git a/docs/plans/issue-252/replay-validation.md b/docs/plans/issue-252/replay-validation.md index 982a52867..47be485ed 100644 --- a/docs/plans/issue-252/replay-validation.md +++ b/docs/plans/issue-252/replay-validation.md @@ -14,6 +14,8 @@ ## 实际验证 +**版本边界：本页记录的是 2026-11-03 UTC 的既有 v2 运行，grader 为 `0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407`，对应 ce84506e 发布的测量实现，ac03ae23 未改变这些代码。** 后续 [CLI 状态修复](review.md)改变了 grader 目录中的 `cli.py`，因此新提交有新 grader 身份；计算、oracle 和计时文件不变，但本页 300 个样本不能称为修复后新 head 的运行证据。新 head 的专项回归单独记录。 + 2026-11-03（UTC），Windows / Python 3.11.18 / NumPy 2.4.6。针对 v2 的专项回归 **40 项通过**，包括真实 baseline/candidate、缓存命中、旧输出重放、逐次输入配对、边界协议、漂移、超时、Workbench 和 SkillEval verifier。另行运行的 CLI catalog 和 SkillEval 相关回归也通过；Ruff 和 diff 检查通过。 开发时首轮回归暴露了 `SeedSequence` 的 NumPy 命名空间错误，修正为 `np.random.SeedSequence` 后完整重跑通过；未把首轮失败归因于环境或既有基线。日志保留在仓库外的本地 outputs 中。 @@ -39,7 +41,9 @@ python tools/engineering_benchmark_validate.py \ | 计划输入序列与正确实现的实际调用序列 | 一致 | 一致 | | Workbench 源码完整性 | 通过 | 通过 | -[原始可公开汇总](replay-validation-summary.json) 保留每次变化输入的摘要及身份、数量和原始 verdict，未包含私人路径。完整原始计时、日志和 profile 仍在本地。两次 Workbench 描述性 verdict 都为 PASS，未将其解释为可靠性能提升；本次没有完成 v2 同码噪声资格验证，也没有建立统计显著性。 +[原始可公开汇总](replay-validation-summary.json) 保留每次变化输入的摘要及身份、数量和原始 verdict，未包含私人路径。后续 review 从同一次本地记录只读导出了[全部 300 个计时样本及原始聚合](replay-measurements.json)、[RMSNorm profile](replay-profiles/rmsnorm-cpu.txt) 和 [Softmax profile](replay-profiles/softmax-cpu.txt)。原汇总保持逐字节不变；补充文件记录来源摘要、每个 trial 的原始结果哈希、样本输入哈希和完整 evaluation，未重跑、删慢样本或更改 verdict。未公开完整执行日志和含本地路径的运行文件；原始文件哈希只是来源标识，不代表读者已获得那些完整文件。 + +导出时逐项核对 12 个 trial 的 300 个样本、每 case 中位数、每 trial 中位数之和、Workbench 两侧聚合，以及计划/实际输入配对。两次 Workbench 描述性 verdict 都为 PASS，未将其解释为可靠性能提升；本次没有完成 v2 同码噪声资格验证，也没有建立统计显著性。原始 RMSNorm candidate trial 值为 0.1256、0.0436、0.0439 ms，Softmax baseline 为 0.044000000000000004、0.045500000000000006、0.1093 ms，完整保留其中的离散值。profile 的文本精度下时间多显示 0.000，只支持调用结构检查，不支持瓶颈定位结论。 ## 对 RSI 的实际含义 diff --git a/docs/plans/issue-252/review.md b/docs/plans/issue-252/review.md new file mode 100644 index 000000000..52872e1cd --- /dev/null +++ b/docs/plans/issue-252/review.md @@ -0,0 +1,76 @@ +# #252 本轮审查记录与阅读入口 + +本轮继续审查 PR #338，并准备供首次接触 InferRSI 的读者使用的[完整导读](walkthrough.md)。结论是：发现并修复一个 CLI 失败状态传播缺陷，补齐公开可检查的既有 v2 样本/profile，修正文档的历史版本指向。PR 仍保持 Draft；这些工作不代表维护者已经接受范围或批准合并。 + +## 审查对象和覆盖 + +审查起点为 `ac03ae23ce0bb5864a55c2c97d5f3d8d3b550d39`，对照原始基线 `663f3b327998526a13aa31e72ad729025c18983c`，共 42 个变更文件。审查分工与本轮修复复核均在同一工作树进行；新导读、样本导出和修复属于本轮增量。 + +| 范围 | 具体检查 | 结果 | +| --- | --- | --- | +| 任务契约、worker、NumPy 评测、两套任务包 | manifest 拒绝路径、输入上界、身份冻结、独立 oracle、axis=1/4D/跨步布局、变化输入、逐调用检查、profile/计时边界 | 未发现额外需修复缺陷；原 40 项专项测试独立重跑通过 | +| consumers、离线 verifier、CLI、接线和打包 | 真实 executor/RunSpec/run_trial 接入、结果与状态、包内资源、验证脚本 | 发现 R1；其余未发现新的实质问题 | +| 文档与证据 | 四项交付含义、RSI 调用关系、历史版本、声称完成的范围、样本可检查性 | 发现 R2/R3 并修正；保留正式集成与验收缺口 | +| 本轮 R1 修复 | 独立复核 diff、读取真实状态文件、退出码与既有 run 命令一致性 | 修复复核未发现新问题；新增实际执行回归通过 | + +“未发现”描述本轮检查结果，不是对所有输入、设备或恶意源码的保证。代码入口仍限定已审核可信实现。 + +## 已处理的问题 + +### R1 · P2：compare 在失败时仍返回退出码 0 + +**触发：** 通过 `inferrsi benchmarks compare` 运行返回全零的错误 Softmax 候选。Workbench 实际记录 `state=failed`、`verdict=INCONCLUSIVE`；旧 CLI 仅返回运行目录和范围信息，最终进程退出码却为 0。 + +**影响：** 脚本或 Agent 如果以退出码判断成功，会把失败比较误当成功。这与已有 `inferrsi run` 的退出约定不一致。 + +**修复：** benchmark CLI 读取并原样返回保留的 `status.json`；顶层 CLI 按实际 verdict 返回 PASS=0、FAIL=1、INCONCLUSIVE=2。不修改 Workbench 的判定规则，不把未完成比较改写为数值 FAIL。 + +**验证：** 三个新回归均执行真实 CLI/Workbench，并检查响应与磁盘状态及 evaluation 一致： + +| 情况 | 状态与 verdict | 退出码 | +| --- | --- | --- | +| 正确候选完成比较，但无法达到 100% 延迟下降的测试门槛 | completed / FAIL | 1 | +| 全零错误候选导致测量失败 | failed / INCONCLUSIVE | 2 | +| 极短预算在执行前耗尽 | timeout / INCONCLUSIVE | 2 | + +测试中的 100% 和极短预算是为确定性触发分支设置的测试条件，不是性能验收标准。`grade` 对错误产物本身仍可报告正确性 FAIL；不完整的 A/B 比较则是 INCONCLUSIVE。 + +### R2：v2 可公开记录只有数量和身份，没有原始计时 + +**影响：** 读者无法从仓库字节复算 300 个样本的聚合或查看 profile。 + +**处理：** 从原始本地 v2 运行只读导出 [replay-measurements.json](replay-measurements.json) 和两份 [RMSNorm](replay-profiles/rmsnorm-cpu.txt) / [Softmax](replay-profiles/softmax-cpu.txt) profile。保留全部 12 个 trial、300 个样本、逐样本输入哈希、原始 evaluation 与来源哈希；profile 逐字节复制，既有 summary 不改写。不重复运行，不筛除离散值，不声称公开了未附带的完整日志。 + +**核验：** 样本数、每 case 中位数、trial 指标、Workbench 两侧聚合、输入序列与旧汇总逐项一致。补充材料提高可检查性，不会使原本未建立的测量资格自动成立。 + +### R3：当前接入文档仍指向历史 v1 验证 + +**影响：** 读者可能以为当前工具可以生成旧协议的 360 个样本，或将 v1 和 v2 结果合用。 + +**处理：** 子项②/④指向 v2 记录，历史安装证据单独标明；v1 页面固定到原始实现，说明重放需使用当时的工具、题目和 grader。总览明确主 Agent 作决策，Search Policy 是独立提案。新导读将四个交付步骤与运行时模块分开解释。 + +## 本轮验证及证据边界 + +| 检查 | 结果与边界 | +| --- | --- | +| 修复前核心专项重跑 | ac03ae23 上 40 项通过 | +| 修复后专项回归 | `tests/test_engineering_benchmarks.py` 43 项通过，含 3 个实际 Workbench CLI 回归 | +| 顶层 CLI 回归 | `tests/test_cli_catalog.py` 19 项通过 | +| 静态与文档检查 | Ruff、diff 空白检查、本地 Markdown 目标检查通过；证据导出核验通过 | +| 已有 hosted CI | ac03ae23 的 Linux Python 3.10/3.11 full-check 和 installed-package 通过；[运行记录](https://github.com/InferMatrixLab/InferRSI/actions/runs/41672237031)；不将该 CI 结果冒充本轮新提交检查 | +| 历史 Windows 全量 | 3479 passed、51 skipped、56 failure reports；失败在未修改原始基线复现，原运行仍不绿，本轮未重复整个 Windows 全量 | +| 新性能实验 | 未执行。本轮修复不修改输入/oracle/计算/计时行为；真实 CLI 状态分支通过上述回归验证 | + +**grader 版本变化必须保留：** `grader_digest()` 对整个 benchmark 目录取哈希，因此修改 `cli.py` 也会改变 grader。旧 v2 300 个样本始终属于 `0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407`；本轮修复后的测试身份为 `34928e6956f54aab565a4d48ea68f74ee72aa91578f8ce63fd013934c71b4d55`。不能把旧样本重新绑定成新提交的运行记录。 + +外部 reviewer 在 [r4459426716](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4459426716) 确认了 ac03ae23 的缓存重放修复；该确认明确只覆盖这一问题，不能描述为完整 PR approval。本轮独立分工 review 也不替代维护者的项目验收。 + +## 仍需分别解决的事项 + +- 首批 CPU 任务及接口范围的维护者 accept/revise/reject 决定；原生父子 Issue 关系和标签由有权限的维护者协调。 +- 同码噪声与可分辨性能排序；现有 Workbench 的描述性 PASS/FAIL 不足以支持稳定优化收益。 +- `tools.skill_eval_compare` 的正式任务适配、产物交接与受支持隔离；当前只是底层保留产物 verifier。 +- 真实旧/新 skill 的受控 Agent 实验及资源成本；固定作者产物不能证明技能收益。 +- 硬件确定后的同步、计时、真实负载与设备/服务验收。 + +下一步工作沿已有 RSI 入口推进，详见 [接入审计](rsi-integration.md)。四个子 Issue 保持打开；本轮不作完成、合并或可靠性能提升的声明。 diff --git a/docs/plans/issue-252/validation.md b/docs/plans/issue-252/validation.md index 7071461c9..9ad9424f6 100644 --- a/docs/plans/issue-252/validation.md +++ b/docs/plans/issue-252/validation.md @@ -2,11 +2,11 @@ 本页保留 **v1 历史运行**（六个计时 case、重复输入）的原始记录，不能代表后续 v2 变化输入协议。当前 RSI 接入边界见 [接入审计](rsi-integration.md)，协议修复与新证据见 [v2 验证](replay-validation.md)。旧数值、身份与 verdict 不重写或合并到新结果。 -验证日期 2026-11-03。实现基于 `663f3b327998526a13aa31e72ad729025c18983c`，具体新增内容以当前工作区和结果中的内容哈希为准。本记录说明共享任务子集的功能与集成，不是项目维护者的验收决定。 +验证日期 2026-11-03。实现基于 `663f3b327998526a13aa31e72ad729025c18983c`，对应初始实现 [d0ce9d7](https://github.com/Bounty-hunter/InferRSI/tree/d0ce9d785e1bead9124eed1cf3f6e9f2e0cf05db)；实际运行文件的身份以结果中的内容哈希为准。本记录说明共享任务子集的功能与集成，不是项目维护者的验收决定。 ## 实际运行结果 -最终原始目录名为 `inferrsi-252-validation-final`，保留在仓库外的本地 outputs 目录。可用 `tools/engineering_benchmark_validate.py` 重建一份新记录。该次执行使用 Windows、Python 3.11.18、NumPy 2.4.6。 +最终原始目录名为 `inferrsi-252-validation-final`，保留在仓库外的本地 outputs 目录。重放这套 v1 协议须使用上述原始版本的 `tools/engineering_benchmark_validate.py`、任务包和 grader，并核对记录中的文件身份；新执行产生自己的记录，不会重建原始计时值。当前分支的工具执行 v2 协议，不能用于复现本页的样本数量或身份。该次执行使用 Windows、Python 3.11.18、NumPy 2.4.6。 [可公开的原始汇总](validation-summary.json) 保留任务/候选/grader/runtime 身份、执行数量、原始 verdict 和覆盖状态，不包含个人路径。完整原始日志和样本仍在本地，公开汇总不能代替独立重放或正式 campaign 验收。 diff --git a/docs/plans/issue-252/walkthrough.md b/docs/plans/issue-252/walkthrough.md new file mode 100644 index 000000000..2caab7233 --- /dev/null +++ b/docs/plans/issue-252/walkthrough.md @@ -0,0 +1,192 @@ +# 从零读懂 #252：做了什么，怎样在 InferRSI 中使用 + +这份导读先讲用途，再沿一个 RMSNorm 例子走到代码与结果。总览见 [README](README.md)，本轮审查结论见 [review](review.md)。 + +**我们给 InferRSI 增加的是 Engineering Benchmarks（共享工程评测任务）：一份题目、一份候选实现，交给固定的独立评测，得到可追溯的正确性、诊断和计时结果。** 主 Agent 根据这些证据决定修改、保留或放弃实现。 + +## 1. 先理解它解决哪一个问题 + +假设你对 Agent 说：“用 InferRSI 优化 RMSNorm。”只有这个名字还不够：沿哪个维度归约？输入是什么精度和布局？可以改哪些文件？怎样判断算错？究竟计时哪一段？ + +没有共享任务时，调用者仍然可以使用已有 Workbench，但需要自己提供这些命令和规则。不同实验若自行定义输入与答案，结果很难直接比较。 + +这次把规则放进一个任务包。当前 `rmsnorm-cpu` 明确要求：CPU、NumPy、float32、沿 axis=1 归约，保留 epsilon，不修改输入。Agent 只交付 `implementation.py` 中的 `compute(x, *, eps)`。工具检查这个文件的实际行为，并记录是哪一版题目、裁判、源码和运行环境产生了结果。 + +它在 RSI 中的位置是**任务与评测层**。Workbench 继续负责运行实验；主 Agent 继续负责调查、读论文、分析 profile、写代码和决定下一轮。评估 skill 是否有效时，可以复用这层任务，但还需要正式的两臂产物交接。 + +## 2. 四个子任务，具体交付了什么 + +四项分别可以简称为“选题复用、统一规则、交付题包、接入验证”。它们是交付步骤；运行一次优化时不会依次执行这四项。 + +| 子任务 | 实际做的事情 | 交付物 | 对使用者的作用 | 当前边界 | +| --- | --- | --- | --- | --- | +| ① **选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) | 读论文和固定版本源码，对照现有 RSI，决定首批范围 | [调研与选型](01-research.md) | 确定先做 Softmax/RMSNorm，复用任务分工与数学题意，避免重复已有执行器 | CPU 子集是提议，范围待维护者接受 | +| ② **统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) | 定义题目格式、候选入口、执行动作和内容身份 | [契约说明](02-contract.md)、`TaskPackage`、worker、CLI | 不同消费者使用同一题目和裁判；比较中源码或规则漂移会被发现 | 当前只接受 `numpy-axis1-v2`，新设备需要自己的 adapter | +| ③ **交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) | 将两道题变成可运行包，补边界输入与独立答案 | [任务说明](03-tasks.md)、两套 manifest/基线/候选/许可 | Agent 有具体修改对象；错误轴、漏 epsilon、输出重放等错误可被独立检查 | 只覆盖声明的 CPU 输入；不能外推 GPU、通信、容量或服务 | +| ④ **接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) | 生成真实 RunSpec，接现有 Workbench 与底层 SkillEval verifier，运行对照 | [集成说明](04-integration.md)、复现工具、验证记录 | 实验结果进入已有结果目录，错误产物被拒绝，正确实现留下计时证据 | Workbench 和底层 verifier 已接通；正式 SkillEval 两臂、可靠性能排序和技能收益未完成 | + +因此，“四项都有代码或文档交付”和“整个 #252 已验收”是两个状态。尤其第④项只完成了表中明确列出的接入层级。 + +## 3. RSI 的两条循环，分别在优化什么 + +| 循环 | 本轮改变什么 | 比较什么 | 谁用结果作决定 | +| --- | --- | --- | --- | +| **优化引擎实现** | `implementation.py` 或真实引擎中的实现 | 原版和候选版是否正确、是否有可分辨的性能差异 | 主 Agent / 实验负责人 | +| **优化优化者** | Agent 使用的 skill、知识或 harness | 两种条件下，Agent 产出的实现质量、速度、成功率和资源成本 | SkillEval 实验负责人及其支持的改进流程 | + +第一条路径已有实际执行。第二条目前只验证了“已审核文件能通过底层 verifier 判分”。我们用固定脚本提交文件哈希来验证通路；没有让两组 Agent 分别使用旧/新 skill 生成代码，所以没有得到 skill 有效性的结论。 + +即便一份事先写好的候选在可靠测量中更快，也不能说明“某个 skill 帮助 Agent 写出了它”。需要先固定任务、模型和预算，再比较 Agent 在两种技能条件下真实交付的文件。 + +### 优化一次实现时，应该怎样运作 + +下面展示主 Agent 应采用的工作顺序。箭头表示数据和决定的传递；目前没有后台控制器自动强制走完整张图。 + +```mermaid +flowchart TD + U[用户目标：优化某个实现] --> A[主 Agent：确定场景与验收条件] + A --> P[主 Agent 调用 profile：取得诊断] + P --> H[主 Agent：结合知识提出假设] + H --> C[主 Agent：修改 implementation.py] + T[冻结任务包：语义、输入、oracle、测量规则] --> K[check：检查实际输出] + T -->|诊断输入与检查规则| P + C -->|候选源码| K + K -->|错误断言| H + K -->|通过后发起比较| W[Workbench：执行原版与候选版] + T --> W + W -->|原始样本、校验结果、版本记录| D[主 Agent：判断证据是否足够] + D -->|需要新诊断或下一轮尝试| P + D -->|保留、拒绝或证据不足| R[保存实现与证据，进入已有后续流程] +``` + +| 图里的框 | 谁调用、输入是什么 | 框里实际做什么 | 交出什么，下一步怎么用 | +| --- | --- | --- | --- | +| 确定场景 | 用户与主 Agent；目标模型、硬件、负载、质量要求 | 判断题目是否覆盖真实需求，冻结实验范围和预算 | 可检查的实验计划；当前 CPU 题不自动覆盖同名 GPU 算子 | +| profile | 主 Agent；基线或候选源码与任务 | 独立运行 CPU cProfile，查看调用结构 | `profile.txt`；用来形成或检查假设，不参与速度分数 | +| 提出假设 | 主 Agent；profile、论文、知识和既往失败 | 解释准备改哪一处、预期为什么有用 | 一项可检验的改动，例如尝试减少中间数组 | +| 修改实现 | 主 Agent；受约束的候选文件 | 修改 `compute`，保持题目语义 | 待审核候选源码；题目、oracle 和基线仍是固定比较输入 | +| check | CLI 或消费者；题目与候选源码 | 用独立参考答案检查基础和变化输入 | `passed`、具体断言、输入身份；错误实现需要修正 | +| Workbench | `compare` 生成 RunSpec 后调用既有 runner | 按 AB/BA 顺序运行原版/候选；各 worker 内先检查，再计时并验证每次输出 | `result.json`、原始样本、`evaluation.json`、`status.json`、源码快照 | +| 判断与保存 | 主 Agent / 负责人；上述完整记录 | 分开判断正确性、测量可信度和任务是否代表真实目标 | 保留、拒绝或证据不足，并保留理由供下一轮使用 | + +AB/BA 是调换执行顺序：一轮先原版再候选，下一轮先候选再原版，减少固定先后顺序的影响。它本身不保证消除噪声。 + +### 谁确保 Agent 按流程走 + +当前有两层约束： + +- **代码强制的局部规则：** manifest/adapter 校验；执行前后身份校验；候选数值检查；检查失败不输出可用性能指标；超时与执行错误保留状态。 +- **由主 Agent 遵循的工作流程：** 先研究与 profiling、冻结假设和验收门槛、根据证据决定下一轮。现有插件入口和工作流文档提供指导，没有 hooks/daemon 强制每次都先做研究或 profile。 + +直接调用 `compare` 不会先自动运行 `profile`。通用 Workbench runner 是先执行 AB/BA，再运行可选 quality command；本次任务 worker 在每个测量进程内部执行正确性检查。不能据此声称通用 runner 已经实现了完整的全局阶段控制。 + +评论中的 **Optimization search policy** 负责“下一步试什么、分多少预算、何时停止”。它消费测量与失败信息。#252 提供任务和评测结果，独立 policy RFC 讨论搜索决策；当前实现没有新的 search controller。真实入口和剩余接入见 [RSI 接入审计](rsi-integration.md)。 + +## 4. 以 RMSNorm 为例，评测器到底在检查什么 + +计算语义是 `y = x / sqrt(mean(x*x, axis=1, keepdims=True) + eps)`。这里 axis=1 很关键：对四维 `(N,C,H,W)` 输入，沿 C 维归约，不能换成常见的“最后一维归约”实现。 + +三个文件/角色各有职责： + +| 角色 | 当前实现 | 用途 | +| --- | --- | --- | +| **baseline** | 任务包中的 NumPy 实现 | 性能比较中的原版 | +| **candidate** | 用户/Agent 交付的 `compute` | 待优化和受检查的实现 | +| **oracle** | 评测器中的标量 double、`math.fsum` 参考算法 | 判断答案是否正确；不拿它的慢速执行时间作为性能基线 | + +“独立 oracle”指算法实现独立，减少基线和候选共享同一错误的风险。当前 worker 执行的是已审核可信源码；它没有恶意代码隔离，不能直接接收任意模型输出。 + +六类基础输入覆盖普通值、跨步布局、全零、大数值、极小值和单元素归约。其中 `large` 指数值大，`tiny` 指数值小；不是张量规模大小。极小值可揭示遗漏 epsilon；跨步输入可揭示对内存连续性的错误假设。 + +### 一次调用如何得到结果 + +以当前两道内置题的默认规则为例： + +1. 预先生成全部输入、独立答案和输入摘要。这些准备不计入计算时间。 +2. 检查 6 个基础输入。每次检查 shape/dtype、数值有限、输入未变、答案在容差内。 +3. 对 5 个有变化的 workload，各准备 2 次预热 + 5 次采样，共 35 次调用。每次输入字节不同，所有输入对象保留存活；预检、预热或早先采样不会给后面的同输入缓存命中创造条件。 +4. 每次实际调用后，在计时范围外验证输出。正确性错误立即使这次测量不可用。 +5. `check` 共执行 41 次并产生 164 个断言；`measure` 留下 25 个计时样本；`profile` 单独记录 35 次变化输入调用，不生成性能指标。全零 case 只参与正确性检查。 + +预定输入写入 `input_sequence`，实际调用写入 `executed_calls`。失败可能只完成前缀；有一份完整计划不代表完整执行。每个计时样本还绑定自身的 `sample_input_sha256`。 + +计时范围是同步 `compute` 调用，函数内部申请数组的成本包含在内；进程启动、输入生成、oracle、预热和输出检查不包含在分数内。每个 workload 先取 5 个样本的中位数，再相加得到 `suite_latency_ms`。这是固定小任务集的指标，不能解释成某模型的请求延迟。 + +## 5. 自己从哪里开始看和运行 + +先读任务，再复制候选。安装/运行命令的详细参数见[使用文档](../../engineering-benchmarks.md)。下列演示在仓库根目录执行；输出目录必须未存在，重复试验应换新名字。 + +```bash +python -m pip install -e ".[dev,benchmarks]" +inferrsi benchmarks list +inferrsi benchmarks show rmsnorm-cpu +python -c "from pathlib import Path; import shutil; from inferrsi.harness.benchmarks import load_task; p=Path('runs/benchmark-demo'); p.mkdir(parents=True); shutil.copyfile(load_task('rmsnorm-cpu').root / 'candidate.py', p / 'implementation.py')" +inferrsi benchmarks check rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --output-dir runs/benchmark-demo/check --timeout-seconds 30 +inferrsi benchmarks profile rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --output-dir runs/benchmark-demo/profile --timeout-seconds 30 +inferrsi benchmarks plan rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --repo . --output-dir runs/benchmark-demo/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 +inferrsi benchmarks compare rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --repo . --output-dir runs/benchmark-demo/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 +``` + +这是学习接口的演示，数字是演示预算，`min-improvement-pct 0` 不是正式验收门槛。`plan` 只打印 RunSpec；`compare` 重新生成并执行 RunSpec。正式实验必须在搜索前保存并冻结计划，不应把两条命令之间重新生成的内容误当成自动锁定的历史计划。 + +先打开 `runs/benchmark-demo/check/result.json` 看失败断言或通过范围，再打开 `runs/benchmark-demo/profile/profile.txt`。`compare` 返回实际 run 目录和保留状态：PASS 退出 0，FAIL 退出 1，INCONCLUSIVE 退出 2。错误实现若使测量未完成，比较通常为 INCONCLUSIVE；其独立正确性 grade 可为 FAIL。在 run 目录中查看： + +| 文件 | 读它是为了什么 | +| --- | --- | +| `run.lock.json`、`task.lock.json` | 确认比较命令、任务、预算与规则 | +| `baseline/trial-*/result.json`、`candidate/trial-*/result.json` | 查看每题样本、每次输入身份和正确性检查 | +| `evaluation.json` | 查看原始聚合和描述性 verdict，包括统计显著性未建立的声明 | +| `status.json`、各 trial 的 `execution.json` | 确认是否完成，区分数值拒绝、执行失败和预算问题 | +| `source_checks.json` | 确认比较期间源码是否变化 | + +学习后再编辑候选并用新的输出目录验证。示例使用仓库已忽略的 `runs/` 目录。修改受 Git 跟踪的文件会进入 Workbench 源码快照；每次正式实验应保存独立计划和结果。 + +## 6. 证据说明它有什么实际用处 + +本节的 v2 归档来自 2026-11-03 UTC，使用旧 grader `0b75b5b…`，对应 ce84506e 发布、ac03ae23 保留的测量实现。本轮 CLI 修复改变了 grader 身份；输入与计时协议仍为 v2，但这 300 个样本不是修复后新 head 的运行证据。完整身份边界见 [审查记录](review.md)。 + +| 实际做过的检查 | 观察到了什么 | 能支持的结论 | 不能推出什么 | +| --- | --- | --- | --- | +| 历史 v1：RMSNorm 漏 epsilon 的合理错误实现 | 普通输入冒烟通过，完整任务和底层 SkillEval verifier 都拒绝；无性能指标 | 任务边界用例能补普通冒烟的漏洞 | 没有证明所有 RMSNorm 错误都能捕获 | +| 既有 v2 记录：id/content memo 和按 shape 重放旧输出 | 正确 memo 没有旧输入缓存命中；错误重放被拒绝 | 计时不再因预检复用输入而只测旧答案回放 | 没有隐藏题集或防恶意作弊保证 | +| 既有 v2 记录：两题经过真实 Workbench 和底层 verifier | 规则、输入配对一致；保留 12 个 trial、300 个计时样本及两份 profile | 两个消费者实际使用了同一套任务规则 | 没有证明正式 SkillEval 两臂已接通 | +| 历史 v1：同码与 einsum 对照 | 同码也出现很大的时间差，两次原始性能 verdict 均 FAIL | 必须保留负结果并先解决测量可信度 | 不能认定 einsum 稳定变慢，也不能宣布优化成功 | +| 既有 v2 记录：profile | 保留 CPU 调用路径；文本时间精度下多项显示 0.000 | 诊断操作确实独立运行，可检查调用结构 | 不能据此认定内存带宽瓶颈或函数耗时占比 | + +既有 v2 记录的 [300 个原始样本及聚合](replay-measurements.json)、[输入与身份摘要](replay-validation-summary.json)、[RMSNorm profile](replay-profiles/rmsnorm-cpu.txt) 和 [Softmax profile](replay-profiles/softmax-cpu.txt) 已可直接检查。它们来自同一次既有运行，此次整理没有重跑或筛除慢样本。 + +v2 Workbench 的两次原始 verdict 是 PASS，但仍是描述性结果。某组 trial 的范围相对中位数超过 100%，同码噪声资格还未完成，所以我们的决定仍是：**没有证据宣布可靠性能提升。** 历史 v1 的 360/720 个样本不可与 v2 的 300 个样本混合评分。 + +源码 CI 与实验验收也要分开：已记录的 hosted Linux Python 3.10/3.11 全量检查通过；历史 Windows 全量结果仍是 3479 passed、51 skipped、56 failure reports，所有失败在原始基线复现，但该次全量仍不绿。具体 revision 与本轮回归见 [review](review.md)。 + +## 7. 为什么选择这套设计，和 AMD 的项目是什么关系 + +AMD [AgentKernelArena 的固定版本任务契约](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) 把任务语义、Agent 可修改的实现、性能基线、正确性参考、工作负载和评测操作分清楚。这种分工让人容易理解“给 Agent 什么、要求交什么、谁来验收”。本次借鉴的就是这层职责。 + +| 来源/方案 | 在本次具体落在哪里 | 选择理由 | +| --- | --- | --- | +| AgentKernelArena | 任务包明确输入、候选、baseline、reference、评测 | 给用户和 Agent 明确的工作对象 | +| KernelBench | 两道归约算子的数学题意、axis=1 与来源许可 | 题目真实明确，CPU 适配可在硬件未定时检验共享边界 | +| BackendBench | 边界与布局检查思路 | 提高普通输入之外的错误识别能力 | +| SkillsBench | 题目/答案与技能条件分离 | 让两臂比较使用共同裁判 | +| AlphaEvolve、AIDE | evaluator 与候选搜索策略分开 | #252 保持任务评测职责，搜索策略走独立设计 | +| InferRSI 已有 Workbench/SkillEval | RunSpec、AB/BA、执行状态、verifier | 复用现有运行机制，让任务能在本项目发挥作用 | + +完整论文与固定源码引用、未采用方案见 [01-research](01-research.md)。当前约束下选择“小任务包 + 具体 adapter + 现有执行器”有可验证的交付；它不表示 CPU 小题已足以代表真实推理系统，也不表示完整引入另一套 GPU 平台更适合我们。 + +## 8. 看代码的顺序，以及剩下的工作 + +先看一题的 manifest 和候选，再看执行，再看接入。无需先逐行读历史 JSON。 + +| 顺序 | 入口 | 重点 | +| --- | --- | --- | +| 1 | [RMSNorm task.json](../../../src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json)、[candidate.py](../../../src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py) | 题目规定了什么，Agent 改哪里 | +| 2 | [numpy_task.py](../../../src/inferrsi/harness/benchmarks/numpy_task.py) | 输入、独立 oracle、check、profile 与计时的真实边界 | +| 3 | [contract.py](../../../src/inferrsi/harness/benchmarks/contract.py)、[worker.py](../../../src/inferrsi/harness/benchmarks/worker.py) | manifest 合法性、内容身份、执行前后检查、RunSpec | +| 4 | [consumers.py](../../../src/inferrsi/harness/benchmarks/consumers.py)、[CLI](../../../src/inferrsi/harness/benchmarks/cli.py) | 如何复用执行器，如何呈现状态和结果 | +| 5 | [skill_verifier.py](../../../src/inferrsi/harness/benchmarks/skill_verifier.py) | 已审核产物如何交给现有底层 verifier | +| 6 | [测试](../../../tests/test_engineering_benchmarks.py)、[复现入口](../../../tools/engineering_benchmark_validate.py) | 正确/错误对照和实际消费者验证 | + +下一步优先级是：维护者确认首批范围；建立能分辨差异的 CPU 测量条件与同码对照；沿既有 `skill_eval_compare` 补正式产物交接和受支持的隔离环境；再用受控 Agent 实验检验技能收益。目标硬件确定后，新增设备 adapter、真实负载和对应层级验收。 + +本次 PR 保持 Draft，四个子 Issue 保持打开。我们已经交付了共享任务、真实评测入口及组件证据；正式两臂系统接入、可靠性能排序、真实优化者收益和设备/服务收益各自仍需完成。 diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 63544082e..87c369c90 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -121,6 +121,9 @@ def main(argv: list[str] | None = None) -> int: result = dispatch(args) _print(result) + if args.benchmark_action == "compare": + verdict = result["status"].get("verdict") + return 0 if verdict == "PASS" else (1 if verdict == "FAIL" else 2) if isinstance(result, dict): if "verdict" in result: return 0 if result["verdict"] == "PASS" else 1 diff --git a/src/inferrsi/harness/benchmarks/cli.py b/src/inferrsi/harness/benchmarks/cli.py index 4f7731284..ac523e730 100644 --- a/src/inferrsi/harness/benchmarks/cli.py +++ b/src/inferrsi/harness/benchmarks/cli.py @@ -1,5 +1,6 @@ """CLI-first engineering benchmark operations; all execution is explicit.""" +import json import math from pathlib import Path @@ -68,6 +69,7 @@ def dispatch(args) -> dict | list: atomic_write_json(run_dir / "task.lock.json", task.manifest) return { "run_dir": str(run_dir), + "status": json.loads((run_dir / "status.json").read_text(encoding="utf-8")), "scope": "local-artifact-comparison", "campaign_acceptance": "not established", } diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py index 2c63bb651..ea5f3c842 100644 --- a/tests/test_engineering_benchmarks.py +++ b/tests/test_engineering_benchmarks.py @@ -316,6 +316,56 @@ def test_real_workbench_reuses_frozen_task(tmp_path, numpy_runtime): assert result["executed_calls"] == baseline["executed_calls"] +@pytest.mark.parametrize( + "case,expected_code,expected_state,expected_verdict", + [ + ("unmet-gain", 1, "completed", "FAIL"), + ("wrong-output", 2, "failed", "INCONCLUSIVE"), + ("timeout", 2, "timeout", "INCONCLUSIVE"), + ], +) +def test_compare_cli_reports_retained_workbench_outcome( + case, expected_code, expected_state, expected_verdict, tmp_path, numpy_runtime, capsys +): + from inferrsi.cli import main + + repo = tmp_path / "repo" + repo.mkdir() + subprocess.run(["git", "init", "-q", str(repo)], check=True) + candidate = repo / "implementation.py" + candidate.write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if case == "wrong-output" + else (load_task("softmax-cpu").root / "candidate.py").read_text(), + encoding="utf-8", + ) + subprocess.run(["git", "-C", str(repo), "add", "."], check=True) + subprocess.run( + [ + "git", "-C", str(repo), "-c", "user.name=Fixture", "-c", + "user.email=fixture@example.invalid", "commit", "-qm", "fixture", + ], + check=True, + ) + code = main( + [ + "benchmarks", "compare", "softmax-cpu", "--candidate", str(candidate), + "--repo", str(repo), "--output-dir", str(tmp_path / "runs"), + "--repetitions", "1", "--max-minutes", "1e-10" if case == "timeout" else "1", + # A complete positive-latency comparison cannot reach a 100% reduction. + "--min-improvement-pct", "100", + ] + ) + response = json.loads(capsys.readouterr().out) + run_dir = Path(response["run_dir"]) + retained = json.loads((run_dir / "status.json").read_text()) + evaluation = json.loads((run_dir / "evaluation.json").read_text()) + assert response["status"] == retained + assert retained["state"] == expected_state + assert retained["verdict"] == evaluation["verdict"] == expected_verdict + assert code == expected_code + + class RetainedControl: """Local authored control for the real trial machinery; no model or token usage.""" From ddc9126d5ab50c914a605998c998825dd77198d9 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 08:13:53 +0800 Subject: [PATCH 06/20] feat: connect shared benchmark artifacts to SkillEval comparisons Agent-assisted-by: Codex (implementation, validation and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- .github/workflows/ci-ready.yml | 30 ++ docs/engineering-benchmarks.md | 16 +- docs/plans/issue-252/04-integration.md | 9 +- docs/plans/issue-252/README.md | 4 +- docs/plans/issue-252/formal-integration.md | 154 +++++++++ docs/plans/issue-252/review.md | 2 + docs/plans/issue-252/rsi-integration.md | 10 +- docs/plans/issue-252/walkthrough.md | 15 +- docs/skill-eval-comparison.md | 9 + src/inferrsi/harness/skill_eval_benchmark.py | 324 ++++++++++++++++++ .../harness/skill_eval_engineering.py | 4 +- src/inferrsi/harness/skill_evaluation.py | 45 ++- tests/test_skill_eval_benchmark.py | 257 ++++++++++++++ tests/test_skill_eval_benchmark_compare.py | 299 ++++++++++++++++ tools/skill_eval_compare.py | 120 +++++-- tools/skill_eval_local.py | 68 ++-- 16 files changed, 1271 insertions(+), 95 deletions(-) create mode 100644 docs/plans/issue-252/formal-integration.md create mode 100644 src/inferrsi/harness/skill_eval_benchmark.py create mode 100644 tests/test_skill_eval_benchmark.py create mode 100644 tests/test_skill_eval_benchmark_compare.py diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 4f84c3dac..000cd9feb 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -68,6 +68,36 @@ jobs: PR_BODY: ${{ github.event.pull_request.body }} run: python3 tools/check_pr_references.py + benchmark-artifact-controls: + # Software correctness only: no model, secrets, device or performance gates. + runs-on: ubuntu-22.04 + timeout-minutes: 10 + permissions: + contents: read + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + with: + persist-credentials: false + - name: Install isolated artifact test dependencies + run: | + sudo apt-get update + sudo apt-get install --yes --no-install-recommends bubblewrap + sudo /usr/bin/python3 -m pip install -e ".[dev]" + /usr/bin/python3 -I -c "import numpy; print(numpy.__version__)" + - name: Real NumPy sandbox and authored two-arm software controls + env: + INFERRSI_REQUIRE_BENCHMARK_SANDBOX: "1" + run: | + /usr/bin/python3 -m pytest tests/test_skill_eval_benchmark.py tests/test_skill_eval_benchmark_compare.py --junitxml=benchmark-controls.xml + - name: Retain software-control results including failures + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: benchmark-artifact-controls-${{ github.run_id }}-${{ github.run_attempt }} + path: benchmark-controls.xml + if-no-files-found: error + retention-days: 14 + kv-task-qualification: # Bounded software-control qualification; not a branch-protection or efficacy gate. if: github.event.pull_request.draft == false diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index 12f763909..32b833288 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -85,10 +85,18 @@ The shared oracle independently grades actual implementation behavior. This offline handoff does not modify production engineering sandbox or paid execution. Live artifact production and skill/search benefit need separate controlled runs. -This low-level verifier integration does not register these tasks with the existing -`skill_eval_compare` adapter or the offline harness-improvement controller. See -[the RSI integration audit](plans/issue-252/rsi-integration.md) for callers, -decisions and the remaining end-to-end acceptance work. +For formal paired artifact evaluation, `tools.skill_eval_compare` now supports +`engineering-benchmark-v1`: freeze the common public task and two skill bundles, +retain each arm's actual `implementation.py`, and independently check its output +outside the artifact's Linux bubblewrap sandbox. Local stage/capture uses the +same controller. This adapter evaluates correctness only; it does not accept +child-reported timing, launch an unqualified default agent, or fall back to host +execution when the sandbox/NumPy runtime is unavailable. It is separate from the +trusted-source worker above. See [setup and handoff](plans/issue-252/formal-integration.md). + +The offline harness-improvement controller and provider generation mode do not +yet support these tasks. See [the RSI integration audit](plans/issue-252/rsi-integration.md) +for decisions and remaining acceptance work. Exceptions, missing dependencies, drift and timeouts leave incomplete evidence. Wrong results emit no usable performance metric. Workbench statistics remain diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index da6a2a974..fd8abd751 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -5,10 +5,10 @@ | 要说清楚的问题 | 本子项的答案 | | --- | --- | | 具体做什么 | 把任务操作绑定到既有执行、比较、验证和报告入口，实际运行正确与错误候选 | -| 交付什么 | Workbench RunSpec 绑定、SkillEval 低层保留产物 verifier、复现工具和结果记录 | +| 交付什么 | Workbench RunSpec 绑定、SkillEval 低层 verifier、正式两臂产物 adapter、复现工具和结果记录 | | 谁使用 | 主 Agent 的引擎优化流程、评测开发者、PR 评审者 | | 实际作用 | 工作台能够执行比较并保留证据；错误实现被共同裁判拒绝；结果可供 RSI 决定保留、放弃或继续诊断 | -| 当前状态 | Workbench 与底层 verifier 已接通；正式 SkillEval 两臂产物交接、真实技能收益与可靠性能排序未完成 | +| 当前状态 | Workbench、底层 verifier、正式 SkillEval 正确性产物交接已实现；真实技能收益与可靠性能排序未完成 | 例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 @@ -16,7 +16,7 @@ 只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 -[RSI 接入审计](rsi-integration.md)进一步区分：本次已调用的 `run_trial/verifier` 是低层接口，不等价于正式 `skill_eval_compare` 两臂入口或 `skill_eval_improve` 已支持这些工程任务。系统接入必须说明结果如何影响真实候选的筛选与接受。 +[RSI 接入审计](rsi-integration.md)区分早期的 `run_trial/verifier` 低层接线与后续[正式两臂接入](formal-integration.md)：新 `engineering-benchmark-v1` 复用 `skill_eval_compare` 的冻结、产物留存与配对报告，使用共享 oracle 在隔离进程外检查候选实际输出。`skill_eval_improve` 仍不支持这些公开 development 任务；系统收益仍需要真实受控 Agent 实验。 ## 计划 @@ -35,10 +35,11 @@ - `inferrsi benchmarks plan/compare` 生成并执行真正的既有 RunSpec；不改原 runner 的统计或阶段规则。 - `grade_artifact` 将独立正确性结果转成既有 SkillEval checks 结构。 - `skill_verifier.py` 接入现有 `skill_evaluation.run_trial`，任务描述锁定 controller 审核后的保留产物，答复只提交该产物哈希。 +- 后续 `engineering-benchmark-v1` 正式 adapter 接入 `skill_eval_compare` 和 `skill_eval_local`，冻结题包/两臂 skill、留存真实 `implementation.py`、隔离运行候选、在父进程复用 oracle，并在报告中重验；方法与平台记录见[正式接入说明](formal-integration.md)。 - `tools/engineering_benchmark_validate.py` 冻结任务和候选，运行两个真实任务、CPU profile、AB/BA，再通过真实 SkillEval trial 处理正确/错误的 authored control。没有生成模型请求，也没有伪造 token 使用。 - `tests/check_installed.py` 检查 wheel 中的任务数据、许可和源码；新增回归覆盖真实子进程与失败路径。 -消费链为“同一 task.json 和 grader → Workbench 性能比较 / SkillEval 独立正确性验证”。后者是离线保留产物交接，不是在线 agent 写代码的新 sandbox。是否由技能帮助产出更好代码，需要未来真实 agent、匹配预算和对照条件的实验。 +消费链为“同一题目与 oracle → Workbench 性能比较 / SkillEval 独立正确性验证”。Workbench 和早期离线 verifier 执行已审核可信源码；正式两臂 adapter 接收真实产物，复用既有 bubblewrap 边界，仅让候选计算在子进程运行，参考答案留在父进程。它隔离产物 replay，不自动认证或限制生产者 Agent 的全部权限。是否由技能帮助产出更好代码，需要真实 Agent、匹配预算和对照条件的实验。 运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，当前协议的实际结果见 [v2 验证](replay-validation.md)。早期安装包检查和 v1 集成结果另存于[历史验证记录](validation.md)。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 2c077bb6d..13d665a1d 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -27,7 +27,7 @@ | **① 选定第一批任务和复用方案** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [详细说明](01-research.md) | 论文/开源对照、任务选择、复用和适配决定 | 开发者据此确定先做哪几题、复用哪些设计，以及当前硬件范围 | 调研与方案已完成，范围待维护者接受 | | **② 规定任务怎样提交、运行和判定** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [详细说明](02-contract.md) | 统一任务格式、候选入口、check/profile/measure 接口和版本记录 | Workbench 与评估工具读取同一题目和裁判；题目或源码变更后不能继续当作原比较 | 已实现共享接口和内容冻结 | | **③ 把第一批任务做成可执行的包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [详细说明](03-tasks.md) | Softmax、RMSNorm CPU 任务；每题带输入、基线、独立参考答案和测量规则 | Agent 有明确修改对象，工具有真实可运行的正确性、诊断和测量任务 | 已实现并实测；可靠性能排序未建立 | -| **④ 接入现有 RSI，并验证结果能被使用** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [详细说明](04-integration.md) | Workbench 绑定、离线 verifier、复现入口和证据记录 | 现有实验流程执行任务、拒绝错误实现并保留证据；后续用于评估不同技能产出的实现 | Workbench 和底层 verifier 已接通；正式 SkillEval 两臂接入未完成 | +| **④ 接入现有 RSI，并验证结果能被使用** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [详细说明](04-integration.md) | Workbench 绑定、正式两臂产物 adapter、复现入口和证据记录 | 现有实验流程执行任务；两臂分别交付实现，由同一裁判检查并保留证据 | Workbench、底层 verifier、正式 SkillEval 正确性产物入口已实现；真实技能收益和性能排序仍未完成 | ## 放在我们 RSI 的哪里 @@ -38,7 +38,7 @@ | 这个引擎实现是否正确、有可验证的改进？ | 主 Agent + Workbench | 提供共同工作负载、独立正确性检查和测量规则 | | 新 skill/知识是否帮助 Agent 产出更好的实现？ | SkillEval 两臂比较；后续衔接已有改进流程 | 两边产出的实现使用同一套任务和裁判，不随技能条件更换答案 | -第一条路径已有实际执行。第二条目前只验证了已审核产物经过底层 verifier 的通路；尚未比较旧/新技能如何影响真实 Agent 的交付。完整调用关系和分层验收见 [RSI 接入审计](rsi-integration.md)。 +第一条路径已有实际执行。第二条现已增加 `skill_eval_compare` 的正式产物交接：两臂分别交付 `implementation.py`，候选在隔离进程运行，父进程以共享 oracle 检查，报告重验留存源码。接入方法及平台验证边界见[正式接入说明](formal-integration.md)；尚未比较旧/新技能对真实 Agent 交付的影响。完整调用关系和分层验收见 [RSI 接入审计](rsi-integration.md)。 当前由主 Agent 决定下一步尝试什么；任务包决定要算什么、怎样检查和测量。Optimization Search Policy 的提案讨论如何选择下一次尝试，本次没有实现该策略。Profiling 是独立诊断，不冒充速度比较。#252 提供可调用的评测操作，不负责强制主 Agent 遵守整个流程或另建搜索控制器。 diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md new file mode 100644 index 000000000..f212f8328 --- /dev/null +++ b/docs/plans/issue-252/formal-integration.md @@ -0,0 +1,154 @@ +# 将共享任务接入正式 SkillEval 两臂比较 + +这一步把 `rmsnorm-cpu` / `softmax-cpu` 接入现有 `tools.skill_eval_compare`：两边使用各自的旧/新 skill，交付真实 `implementation.py`，由共同的独立 verifier 检查，再进入现有配对报告。调用者不再需要手工拼接底层 `run_trial`。 + +**本次新增接入只判定产物正确性。** 它复用共享题目、变化输入与标量 oracle，并增加隔离执行的产物交接。性能仍由明确的 Workbench 实验回答；不能把 sandbox 启动时间或子进程声称的 latency 当作优化分数。 + +## 接入前后，分别能做什么 + +| 入口 | 原有能力 | 本次新增能力 | +| --- | --- | --- | +| `benchmarks compare` | 对已审核可信实现执行 Workbench 比较 | 仍承担已审核源码的正确性与测量，不改它的计时协议 | +| `benchmarks.skill_verifier` | 单个 controller 指定的已审核文件经过底层 verifier | 历史离线接口保留，不把它直接开放给任意模型源码 | +| `tools.skill_eval_compare` | audit、diagnostic，以及固定 parser 工程任务 | 新增 `engineering-benchmark-v1`，可冻结共享任务、两臂 skill、模型/预算、schema、starter 和 grader，并接收两边实际文件 | +| `tools.skill_eval_local` | parser 的 stage/capture 本地交接 | 共享 benchmark 也能导出每臂 workspace、留存实现和作者日志，再独立检查 | +| `tools.skill_eval_improve` | 离线 audit 的选择、重跑和 holdout 控制 | 本次仍不支持共享 benchmark；公开 development 题不能冒充独立 holdout | + +两臂改变的是提供给 Agent 的 skill。题目、正确性标准、候选入口和输入序列保持相同。固定作者脚本可以检查接线行为，但不是 skill 收益实验。 + +starter 本身是正确实现，因此原样保留也能通过正确性检查。这层入口能检查优化过程有没有破坏语义、拒绝错误产物，并保留两边真实实现；它不能区分“正确但未优化”和“正确且更快”。性能选择必须另有经过资格验证的测量，不能把正确性 PASS 当作优化成功。 + +本次设计沿用[既有调研](01-research.md)中的任务、候选、独立评测分工，并结合现有 RSI 接口作了以下选择： + +| 接入方式 | 当前决定与理由 | +| --- | --- | +| 将模型生成源码直接交给可信 CPU worker | 不采用；该 worker 的前提是已审核可信源码，不能充当正式生成产物的隔离边界 | +| 新建一套两臂控制器 | 不采用；现有 compare 已有 skill 快照、预算、日志、配对和留存复核 | +| 把整个 grader 与候选一同放进子进程并直接接受 verdict | 不采用；应由受信父进程持有答案并计算断言，候选不能提交自己的通过结论 | +| 沿现有 artifact 交接扩展 adapter，隔离执行候选，父进程检查数组 | 采用；共享数值规则并复用既有控制器，同时明确保护宿主与判定正确性的职责 | + +先交付正确性接入，是因为当前计时仍未完成可靠性能排序资格。引入隔离子进程不会自动解决测量噪声；性能比较需要独立定义和验证计时边界。 + +## 一次比较怎样运行 + +```mermaid +flowchart TD + T[同一共享题目及独立 oracle] --> F[prepare 冻结计划与全部输入] + S[旧 skill / 新 skill 的 Git 快照] --> F + F --> B[baseline workspace：旧 skill] + F --> C[candidate workspace：新 skill] + B --> BI[Agent 或已授权本地执行者交付 implementation.py] + C --> CI[Agent 或已授权本地执行者交付 implementation.py] + BI --> V[独立产物 verifier] + CI --> V + V --> R[配对报告：正确性差异、错误、版本、预算与证据] +``` + +| 环节 | 实际输入和操作 | 保留什么 | +| --- | --- | --- | +| prepare | canonical task descriptor、旧/新 skill revision、模型与共同预算；冻结题包、schema、starter、grader/controller | `plan/freeze.json`、`plan/tasks.json`、各臂 bundle、每题 starter 和题包副本 | +| stage / 调用客户端 | 每次独立 workspace，只给同一任务、starter 和该臂 skill；候选只能改 `implementation.py` | 冻结输入哈希与执行日志；默认不启动不受限 Agent | +| capture | 在 producer 停止后有界读取真实文件，拒绝危险路径/特殊节点，保留源码字节与 SHA-256 | `implementation.py`、原始答案、扩展后的留存答案、作者日志或 controller trace | +| verifier | 受信父进程生成输入与 oracle；Linux bubblewrap 子进程只执行候选并返回数组；父进程独立比较 | 164 个正确性断言及输入/任务身份；没有性能分数 | +| report | 检查留存文件与 receipt，使用已冻结、受信的 verifier 复核实际产物 | 两臂结果、错误原因、版本、资源记录和相对证据链接；不再次调用 Agent | + +当前内置题每次完整检查为 6 个基础输入 + 35 个变化输入，runner 按该顺序调用 41 次并由父进程检查 164 个断言。该顺序复用 `numpy-axis1-v2` 的规则。输入对象在单个子进程中保留，runner 只加载一次 `compute`，避免每次起进程而掩盖普通状态错误。 + +## 产物执行的边界 + +正式 adapter 为 `inferrsi.harness.skill_eval_benchmark`，候选文件固定为 `implementation.py`，入口仍是 `compute(x, *, eps)`。本地 producer 的答案仅为： + +```json +{"artifact": "implementation.py", "summary": "描述实际改动"} +``` + +controller 从文件读取源码，再添加留存用的 `artifact_source`；Agent 不能用一句“测试通过”替代实际产物。 + +受信父进程持有题包、输入生成器、独立 oracle 和断言。候选只在既有 Linux bubblewrap namespace/read-only mount 机制内运行，挂载所需系统 Python、匹配版本的 NumPy、runner 和实现；不挂载仓库、另一臂、答案或 grader。输出规模、进程资源和时间有界，父进程只解释数组数据，不执行候选提供的校验代码。 + +留存的 `returned_calls` 表示父进程收到并检查的格式合法回复，不是独立设备 trace，也不能证明任意候选都按约定执行了 `compute`。OS 隔离保护宿主资源；公开题集及同一子进程里的 Python runner 不提供隐藏测试、恶意协议伪造防护或独立的输入修改证明。结果的适用范围必须保留这些限制。 + +这仍不是整台机器上所有 Agent 活动的隔离保证。**产物 replay 的隔离与产物生产者的权限是两个边界。** 注入客户端需由调用者负责实际执行权限与预算；手工提交的作者日志无法认证模型身份、访问权限或全部尝试。没有合格本地客户端时，正式入口拒绝启动默认 danger-full-access Agent。 + +Windows、缺少 bubblewrap、不能创建 namespace、系统 NumPy 不满足运行条件时，都保留 `artifact_sandbox_unavailable` / INCONCLUSIVE，不回退到宿主执行。源码仍可留存，之后在支持的 reviewer 环境中复核；原始 unavailable 记录不能改写成当时已通过。 + +## 如何从现有入口开始 + +以下是手工两臂交接示例。占位符须替换为实际已冻结的 skill 路径、Git commit、模型设置和预算；它不是默认实验门槛。无需编造一份自定义题目，直接生成当前 canonical descriptor： + +```bash +python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('rmsnorm-cpu')])" +``` + +配置示意： + +```json +{ + "adapter": "engineering-benchmark-v1", + "mode": "update", + "skill": "", + "skill_root": "", + "cases": "cases.json", + "snapshots": { + "baseline": {"git_revision": ""}, + "candidate": {"git_revision": ""} + }, + "model": {"name": "", "effort": ""}, + "limits": { + "repeats": 1, + "max_task_calls": 2, + "timeout_seconds": 60, + "campaign_seconds": 180, + "token_stop_new_trials": 10000 + } +} +``` + +冻结、导出 baseline workspace： + +```bash +python -m tools.skill_eval_compare prepare --config config.json --repo --output +python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id rmsnorm-cpu --condition baseline --repeat 1 --current-head +``` + +Agent 在该 workspace 内读取任务和分配的 skill，只修改 `implementation.py`。记录实际 stdout/stderr 和作者 `execution.json`，字段与边界沿用[本地交接文档](../../local-skill-evaluation.md)。完成后 capture： + +```bash +python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id rmsnorm-cpu --condition baseline --repeat 1 --current-head +``` + +对 candidate 使用新的 workspace 和独立日志，把 `--condition` 改为 `candidate`。两臂完成后： + +```bash +python -m tools.skill_eval_compare report --output +``` + +`--authorize` 绑定 prepare 输出的计划哈希；配置、任务或代码变动后必须重新 prepare。它不是重新授权远端模型费用的替代机制。本次没有新增 provider、generation-only 或 `skill_eval_improve` 支持。 + +## 怎样判断结果有实际用途 + +| 观察 | 正确解释 | +| --- | --- | +| 正确实现 PASS，错误实现 FAIL | 共用裁判检查了实际源码行为 | +| baseline FAIL、candidate PASS | 产物正确性出现差异；固定脚本对照仅证明流程作用 | +| 手工两臂报告 artifact_improved | 作者提交的产物检查改善；身份、环境和成本仍是作者报告，不能直接证明 skill 因果收益 | +| sandbox unavailable、预算不完整、输入漂移 | INCONCLUSIVE 或拒绝执行；不能计作干净胜负 | +| 两边均 PASS | 在当前公开输入上都正确；并不表示候选更快 | + +接入验证分两层：跨平台 controller 测试使用真实作者脚本进程写候选，模拟 verifier 响应，验证冻结、交接、报告、预算和错误传播；Linux 软件对照执行真实 sandbox 与独立 oracle，验证正确/错误实现确实产生不同结论。前者不能代替后者。 + +复现测试入口： + +```bash +python -m pytest tests/test_skill_eval_benchmark_compare.py tests/test_skill_eval_benchmark.py +``` + +严格 Linux 软件对照设置 `INFERRSI_REQUIRE_BENCHMARK_SANDBOX=1`：缺少隔离环境时测试失败，不能用 skip 形成绿色资格结论。实际执行结果由本轮验证记录/CI 链接另行记录；本文的命令和测试设计本身不代表执行已经通过。 + +## 本轮交付结束后，仍然分开验收的内容 + +共享任务已经有正式两臂产物入口的设计与实现位置；是否在某个具体平台完成真实 sandbox 软件对照，以对应执行记录为准。性能排序、真实旧/新 skill 的 Agent 实验、生产者隔离资格、成本效果、改进控制器及目标硬件仍需各自证据。 + +Profiling 仍由主 Agent 根据实际瓶颈调用：对已审核实现可使用 `benchmarks profile`，真实设备需对应 profiler。当前 formal adapter 不会自动做 profile、选择下一次候选或晋升 skill。它提供“同题、两臂、真实产物、独立检查”的接入层，供这些现有流程使用。 + +在当前 RSI 中继续推进一次实现优化时，主 Agent 从报告的 `implementation.py` 证据链接取得候选，审核其源码与适用范围后，再用现有 Workbench 做 profile 和配对测量。formal adapter 的通过结果不会自动把未经审核的代码送入可信 worker，也不会自动批准该实现。 diff --git a/docs/plans/issue-252/review.md b/docs/plans/issue-252/review.md index 52872e1cd..a3c8d501b 100644 --- a/docs/plans/issue-252/review.md +++ b/docs/plans/issue-252/review.md @@ -1,5 +1,7 @@ # #252 本轮审查记录与阅读入口 +本页固定记录 `9454672f` 这一轮审查。之后用户要求继续正式接入，新增实现与验证见[正式 SkillEval 接入](formal-integration.md)；下文“未接入”是当时状态，不能代替当前接入说明。 + 本轮继续审查 PR #338，并准备供首次接触 InferRSI 的读者使用的[完整导读](walkthrough.md)。结论是：发现并修复一个 CLI 失败状态传播缺陷，补齐公开可检查的既有 v2 样本/profile，修正文档的历史版本指向。PR 仍保持 Draft；这些工作不代表维护者已经接受范围或批准合并。 ## 审查对象和覆盖 diff --git a/docs/plans/issue-252/rsi-integration.md b/docs/plans/issue-252/rsi-integration.md index b1fad6e81..c4150e0f6 100644 --- a/docs/plans/issue-252/rsi-integration.md +++ b/docs/plans/issue-252/rsi-integration.md @@ -20,7 +20,7 @@ | `benchmarks plan/compare` → `TaskPackage.runspec` → `run_experiment` | 同一任务生成既有 RunSpec，由 Workbench 执行配对比较并保留证据 | 自动从用户目标/profile 选出适用任务；服务与目标硬件验收 | | `benchmarks check/profile/measure` | 分开的数值校验、CPU 诊断、测量操作 | 强制宿主 Agent 按整个优化流程执行 | | `skill_verifier.task_descriptor` → `skill_evaluation.run_trial` | 对 controller 已审核的保留实现运行真实 verifier；答案提交该实现的 SHA | Agent 根据不同技能实际产出两份实现的比较 | -| `tools.skill_eval_compare` | 已有冻结两臂、模型/预算、产物与报告机制 | 新 benchmark descriptor 尚未成为正式支持的 adapter；现有 engineering adapter 限定 parser 任务及隔离执行 | +| `tools.skill_eval_compare` | 新增 `engineering-benchmark-v1`：共享题目/两臂 skill 冻结、实际文件留存、隔离计算与父进程 oracle、配对报告重验；支持现有 local stage/capture | 当前仅正确性；真实受控 Agent/skill 收益、性能排序、provider generation 模式未接入 | | `tools.skill_eval_improve` | 已有面向支持的离线 audit 任务的选择、重跑和 holdout 流程 | 共享工程基准尚未接入该控制器的任务/grader 契约 | 依据为 [CLI 适配](../../../src/inferrsi/harness/benchmarks/cli.py)、[保留产物 verifier](../../../src/inferrsi/harness/benchmarks/skill_verifier.py)、[SkillEval 比较入口](../../../tools/skill_eval_compare.py)、[现有 engineering adapter](../../../src/inferrsi/harness/skill_eval_engineering.py) 和 [改进入口](../../../tools/skill_eval_improve.py)。主 Agent 的调查与代码修改职责见[现有插件入口](../../../plugins/inferrsi/skills/inferrsi/SKILL.md)。 @@ -46,11 +46,11 @@ 最小演练应沿已有入口保留“profile → 假设 → 产物 → check → compare → 决策”链，并至少包含：错误产物被拒绝、正确但证据不足的产物不被当作改进、负结果留下适用条件。既有 [RMSNorm 对照](usefulness.md) 提供了部分输入与结果；当前没有程序化的全流程强制或真实 agent 对照，不能据此宣布完整闭环验收。 -### C. 再接到已有 SkillEval 两臂入口 +### C. 接到已有 SkillEval 两臂入口 -沿 `skill_eval_compare` 扩展任务适配与产物交接，保留其冻结、模型/预算一致性、独立 verifier、隔离和日志机制。两边分别接收旧/新 skill（或无/有 skill），各自交付实现，再交给同一任务包判分。 +已沿 `skill_eval_compare` 扩展 `engineering-benchmark-v1`，支持 update 模式的旧/新 skill 两臂，保留冻结、模型/预算、日志与报告。两边分别交付 `implementation.py`，控制器留存实际字节；候选在既有 bubblewrap 边界内计算，父进程使用共享输入与 oracle 独立检查。`skill_eval_local` 也支持 stage/capture；完整使用方法和验证分层见[正式接入说明](formal-integration.md)。 -当前可信 CPU worker 不能直接接未经审核的模型代码，也不能代替现有 engineering adapter 的隔离。没有支持的产物执行环境时，应保留未验证状态；换一个 scripted client 只证明通路，不证明技能帮助。 +原有可信 CPU worker 仍不能直接接未经审核的模型代码；新 formal adapter 为产物 replay 使用隔离路径，没有宿主回退。当前只输出正确性，不使用候选自报 latency；真实 Agent 执行权限、技能收益和性能排序另行验证。固定作者脚本只证明通路和裁判行为，不能证明技能帮助。 ### D. 最后评估优化者本身是否改善 @@ -60,4 +60,4 @@ ## 当前结论 -共享任务与底层消费者复用已有实际证据，已补正影响测量语义的输入重放缺陷；端到端 SkillEval 接入、优化者收益和可靠性能排序尚未完成。四个子项的组件实现不能替代这些系统验收。PR 继续作为 Draft 评审，按现有 RSI 接口补齐接入，不把独立题目数量或另建编排系统作为进展指标。 +共享任务与底层消费者复用已有实际证据，输入重放缺陷已修复；后续正式两臂 adapter 也已实现产物交接与独立正确性检查。真实 Agent/skill 收益、可靠性能排序、provider generation 和改进控制器接入仍未完成。具体平台软件对照结果见正式接入记录，不能用接口实现代替系统收益验收。PR 继续作为 Draft 评审，不另建编排系统。 diff --git a/docs/plans/issue-252/walkthrough.md b/docs/plans/issue-252/walkthrough.md index 2caab7233..cdc57f306 100644 --- a/docs/plans/issue-252/walkthrough.md +++ b/docs/plans/issue-252/walkthrough.md @@ -12,7 +12,7 @@ 这次把规则放进一个任务包。当前 `rmsnorm-cpu` 明确要求：CPU、NumPy、float32、沿 axis=1 归约，保留 epsilon，不修改输入。Agent 只交付 `implementation.py` 中的 `compute(x, *, eps)`。工具检查这个文件的实际行为，并记录是哪一版题目、裁判、源码和运行环境产生了结果。 -它在 RSI 中的位置是**任务与评测层**。Workbench 继续负责运行实验；主 Agent 继续负责调查、读论文、分析 profile、写代码和决定下一轮。评估 skill 是否有效时，可以复用这层任务，但还需要正式的两臂产物交接。 +它在 RSI 中的位置是**任务与评测层**。Workbench 继续负责运行实验；主 Agent 继续负责调查、读论文、分析 profile、写代码和决定下一轮。评估 skill 是否有效时，新增的[正式两臂产物 adapter](formal-integration.md)复用这层任务；实际技能收益仍需要受控 Agent 实验。 ## 2. 四个子任务，具体交付了什么 @@ -23,7 +23,7 @@ | ① **选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) | 读论文和固定版本源码，对照现有 RSI，决定首批范围 | [调研与选型](01-research.md) | 确定先做 Softmax/RMSNorm，复用任务分工与数学题意，避免重复已有执行器 | CPU 子集是提议，范围待维护者接受 | | ② **统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) | 定义题目格式、候选入口、执行动作和内容身份 | [契约说明](02-contract.md)、`TaskPackage`、worker、CLI | 不同消费者使用同一题目和裁判；比较中源码或规则漂移会被发现 | 当前只接受 `numpy-axis1-v2`，新设备需要自己的 adapter | | ③ **交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) | 将两道题变成可运行包，补边界输入与独立答案 | [任务说明](03-tasks.md)、两套 manifest/基线/候选/许可 | Agent 有具体修改对象；错误轴、漏 epsilon、输出重放等错误可被独立检查 | 只覆盖声明的 CPU 输入；不能外推 GPU、通信、容量或服务 | -| ④ **接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) | 生成真实 RunSpec，接现有 Workbench 与底层 SkillEval verifier，运行对照 | [集成说明](04-integration.md)、复现工具、验证记录 | 实验结果进入已有结果目录，错误产物被拒绝，正确实现留下计时证据 | Workbench 和底层 verifier 已接通；正式 SkillEval 两臂、可靠性能排序和技能收益未完成 | +| ④ **接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) | 生成真实 RunSpec，接现有 Workbench 与底层 SkillEval verifier，运行对照 | [集成说明](04-integration.md)、复现工具、验证记录 | 实验结果进入已有结果目录，错误产物被拒绝，正确实现留下计时证据 | Workbench、底层 verifier 和正式两臂正确性产物入口已实现；可靠性能排序和技能收益未完成 | 因此，“四项都有代码或文档交付”和“整个 #252 已验收”是两个状态。尤其第④项只完成了表中明确列出的接入层级。 @@ -34,7 +34,7 @@ | **优化引擎实现** | `implementation.py` 或真实引擎中的实现 | 原版和候选版是否正确、是否有可分辨的性能差异 | 主 Agent / 实验负责人 | | **优化优化者** | Agent 使用的 skill、知识或 harness | 两种条件下，Agent 产出的实现质量、速度、成功率和资源成本 | SkillEval 实验负责人及其支持的改进流程 | -第一条路径已有实际执行。第二条目前只验证了“已审核文件能通过底层 verifier 判分”。我们用固定脚本提交文件哈希来验证通路；没有让两组 Agent 分别使用旧/新 skill 生成代码，所以没有得到 skill 有效性的结论。 +第一条路径已有实际执行。第二条早期只有底层 verifier，后续已补上正式 `skill_eval_compare` 与 local stage/capture：两臂交付真实文件，隔离执行候选，父进程独立检查并形成配对报告。验证使用作者脚本和正确/错误软件对照；没有让两组真实 Agent 分别使用旧/新 skill 生成代码，所以没有 skill 有效性的结论。具体平台验证见[正式接入说明](formal-integration.md)。 即便一份事先写好的候选在可靠测量中更快，也不能说明“某个 skill 帮助 Agent 写出了它”。需要先固定任务、模型和预算，再比较 Agent 在两种技能条件下真实交付的文件。 @@ -94,7 +94,7 @@ AB/BA 是调换执行顺序：一轮先原版再候选，下一轮先候选再 | **candidate** | 用户/Agent 交付的 `compute` | 待优化和受检查的实现 | | **oracle** | 评测器中的标量 double、`math.fsum` 参考算法 | 判断答案是否正确；不拿它的慢速执行时间作为性能基线 | -“独立 oracle”指算法实现独立，减少基线和候选共享同一错误的风险。当前 worker 执行的是已审核可信源码；它没有恶意代码隔离，不能直接接收任意模型输出。 +“独立 oracle”指算法实现独立，减少基线和候选共享同一错误的风险。Workbench 的 worker 执行已审核可信源码，没有恶意代码隔离。正式 SkillEval 的新 adapter 使用另一条产物隔离路径，在父进程保留 oracle；两条执行边界详见[正式接入说明](formal-integration.md)。 六类基础输入覆盖普通值、跨步布局、全零、大数值、极小值和单元素归约。其中 `large` 指数值大，`tiny` 指数值小；不是张量规模大小。极小值可揭示遗漏 epsilon；跨步输入可揭示对内存连续性的错误假设。 @@ -185,8 +185,9 @@ AMD [AgentKernelArena 的固定版本任务契约](https://github.com/AMD-AGI/Ag | 3 | [contract.py](../../../src/inferrsi/harness/benchmarks/contract.py)、[worker.py](../../../src/inferrsi/harness/benchmarks/worker.py) | manifest 合法性、内容身份、执行前后检查、RunSpec | | 4 | [consumers.py](../../../src/inferrsi/harness/benchmarks/consumers.py)、[CLI](../../../src/inferrsi/harness/benchmarks/cli.py) | 如何复用执行器，如何呈现状态和结果 | | 5 | [skill_verifier.py](../../../src/inferrsi/harness/benchmarks/skill_verifier.py) | 已审核产物如何交给现有底层 verifier | -| 6 | [测试](../../../tests/test_engineering_benchmarks.py)、[复现入口](../../../tools/engineering_benchmark_validate.py) | 正确/错误对照和实际消费者验证 | +| 6 | [正式 adapter](../../../src/inferrsi/harness/skill_eval_benchmark.py)、[正式接入说明](formal-integration.md) | 两臂产物交接、隔离计算与父进程检查 | +| 7 | [测试](../../../tests/test_engineering_benchmarks.py)、[复现入口](../../../tools/engineering_benchmark_validate.py) | 正确/错误对照和实际消费者验证 | -下一步优先级是：维护者确认首批范围；建立能分辨差异的 CPU 测量条件与同码对照；沿既有 `skill_eval_compare` 补正式产物交接和受支持的隔离环境；再用受控 Agent 实验检验技能收益。目标硬件确定后，新增设备 adapter、真实负载和对应层级验收。 +下一步优先级是：维护者确认首批范围；建立能分辨差异的 CPU 测量条件与同码对照；在已接入的正式两臂入口上选定受支持的生产者与隔离运行环境，再用受控 Agent 实验检验技能收益。目标硬件确定后，新增设备 adapter、真实负载和对应层级验收。 -本次 PR 保持 Draft，四个子 Issue 保持打开。我们已经交付了共享任务、真实评测入口及组件证据；正式两臂系统接入、可靠性能排序、真实优化者收益和设备/服务收益各自仍需完成。 +本次 PR 保持 Draft，四个子 Issue 保持打开。我们已经交付了共享任务、真实评测入口及组件证据；正式两臂正确性产物入口已实现，实际平台控制结果见接入记录；可靠性能排序、真实优化者收益、改进控制器和设备/服务收益仍需完成。 diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index 41b0aa4c9..9b07d9166 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -10,6 +10,15 @@ authorization and retained-result publication around these interfaces. The [offline improvement MVP](harness-improvement.md) reuses the same interfaces for bounded INV/DOM probes, successor rerun and a one-shot held-out gate. +Shared Softmax/RMSNorm CPU artifacts use the `engineering-benchmark-v1` adapter +in **update mode**, with the same prepare/execute/report and local stage/capture +interfaces. The trusted parent grades output from a Linux bubblewrap NumPy +worker; unsupported sandbox/runtime leaves INCONCLUSIVE without host fallback. +This is correctness-only artifact evaluation, not timing or skill-efficacy +qualification. See the [task contract and complete handoff example](plans/issue-252/formal-integration.md). +The provider generation mode and offline improvement controller do not support +this adapter yet. + ```mermaid flowchart LR A["New: small JSON config"] --> B["New: fixed Git snapshots"] diff --git a/src/inferrsi/harness/skill_eval_benchmark.py b/src/inferrsi/harness/skill_eval_benchmark.py new file mode 100644 index 000000000..97012b9b7 --- /dev/null +++ b/src/inferrsi/harness/skill_eval_benchmark.py @@ -0,0 +1,324 @@ +"""Shared CPU tasks for SkillEval's captured-artifact path. + +The trusted parent owns inputs and the scalar oracle. Submitted source runs only +inside the existing Linux bubblewrap boundary, without the repository or grader. +This adapter grades correctness, never child-reported timing or performance. +""" + +import base64 +import binascii +import hashlib +import json +import os +import signal +import subprocess +import tempfile +from pathlib import Path + +from . import skill_eval_engineering as engineering +from .benchmarks import numpy_task +from .benchmarks.contract import builtin_tasks, load_task + +ADAPTER = "engineering-benchmark-v1" +ARTIFACT = "implementation.py" +MAX_SOURCE_BYTES = engineering.MAX_SOURCE_BYTES +MAX_OUTPUT_BYTES = 32 * 1024 * 1024 +SandboxUnavailable = engineering.SandboxUnavailable +read_regular = engineering.read_regular +SCHEMA = { + "type": "object", "additionalProperties": False, + "required": ["artifact", "summary"], + "properties": { + "artifact": {"type": "string", "enum": [ARTIFACT]}, + "summary": {"type": "string"}, + }, +} +PROMPT = ( + "Read task.json and the assigned knowledge. Change only implementation.py, " + "implementing compute(x, *, eps) under the public shared CPU task contract. " + "Use NumPy, math and self-contained helpers; preserve input contents. " + "Do not change frozen inputs or create other files. No network, installs, " + "other agents, commits or publication. Return artifact='implementation.py' " + "and a concise summary, not a verdict or claimed speedup. The controller " + "retains the actual file and independently checks its behavior in isolation." +) + + +def task_descriptor(name: str) -> dict: + if name not in builtin_tasks(): + raise ValueError("Only built-in shared CPU benchmark tasks are supported") + package = load_task(name) + manifest = package.manifest + return { + "id": name, + "split": "development", + "request": ( + manifest["description"] + " Implement compute(x, *, eps) in implementation.py. " + "Inputs and output are NumPy float32 arrays; preserve shape and input bytes. " + "Reduce axis 1; RMSNorm workloads include four-dimensional inputs. Use eps for " + "RMSNorm. Public finite inputs include strided layouts, zeros, large and tiny " + "values and singleton reductions. The shared task's base and distinct-per-call " + "input sequence is checked. This SkillEval adapter grades correctness only; " + "no latency, hardware, holdout generalization or skill-benefit claim follows." + ), + "provenance": manifest["source"]["url"], + "contract": manifest, + "benchmark": { + "task_id": name, + "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + }, + } + + +def validate_task(task): + if not isinstance(task, dict) or not isinstance(task.get("benchmark"), dict): + raise ValueError("Shared benchmark task requires a canonical descriptor") + name = task["benchmark"].get("task_id") + if not isinstance(name, str) or task != task_descriptor(name): + raise ValueError("Shared benchmark descriptor or frozen task identity changed") + + +def starter(task) -> bytes: + validate_task(task) + return (load_task(task["benchmark"]["task_id"]).root / "candidate.py").read_bytes() + + +def input_hashes(workspace: Path) -> dict: + return engineering.input_hashes(workspace, editable=ARTIFACT) + + +def _numpy(): + try: + import numpy as np + except (ImportError, OSError): + raise SandboxUnavailable("Shared benchmark verification requires NumPy") from None + return np + + +# Expected outputs, task packages, and the grader are never mounted in the child. +# All arrays remain alive and compute is loaded once for the complete sequence. +RUNNER = '''import base64, json, platform, resource, sys +resource.setrlimit(resource.RLIMIT_CPU, (10, 10)) +resource.setrlimit(resource.RLIMIT_AS, (1073741824, 1073741824)) +resource.setrlimit(resource.RLIMIT_FSIZE, (33554432, 33554432)) +resource.setrlimit(resource.RLIMIT_NOFILE, (64, 64)) +resource.setrlimit(resource.RLIMIT_NPROC, (0, 0)) +resource.setrlimit(resource.RLIMIT_CORE, (0, 0)) +sys.path.insert(0, "/runtime") +import numpy as np +if sys.argv[1:] == ["probe"]: + print(json.dumps({"ready": True, "numpy": np.__version__, + "python": platform.python_version(), "machine": platform.machine()})) +else: + print("INFERRSI_BENCHMARK_READY", file=sys.stderr, flush=True) + request = json.loads(sys.stdin.read(16777217)) + arrays = [] + for record in request["inputs"]: + x = np.frombuffer(base64.b64decode(record["data"], validate=True), + dtype=np.float32).reshape(record["shape"]).copy() + if record["layout"] == "strided": + storage = np.empty((*x.shape[:-1], x.shape[-1] * 2), dtype=np.float32) + view = storage[..., ::2] + view[...] = x + x = view + arrays.append(x) + namespace = {"__name__": "benchmark_candidate", "__file__": "/work/implementation.py"} + with open("/work/implementation.py", encoding="utf-8") as source: + exec(compile(source.read(), "implementation.py", "exec"), namespace) + compute = namespace["compute"] + outputs = [] + for x in arrays: + y = compute(x, eps=request["eps"]) + output = None + if isinstance(y, np.ndarray) and y.shape == x.shape and y.dtype == np.float32: + output = {"shape": list(y.shape), "dtype": str(y.dtype), + "data": base64.b64encode(y.tobytes()).decode("ascii")} + outputs.append({"output": output, + "input_after": base64.b64encode(x.tobytes()).decode("ascii")}) + print(json.dumps({"outputs": outputs}, allow_nan=False)) +''' + + +def sandbox_command(work: Path, *, probe=False): + # Reuse the established system-Python namespaces and read-only mount policy. + command = engineering.sandbox_command(work, probe=probe) + np = _numpy() + + query = ( + "import json,numpy,pathlib; " + "print(json.dumps({'version':numpy.__version__," + "'root':str(pathlib.Path(numpy.__file__).resolve().parent)}))" + ) + try: + runtime = subprocess.run( + ["/usr/bin/python3", "-I", "-c", query], check=True, + stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, + timeout=5, env={"PATH": "/usr/bin:/bin", "LANG": "C"}, + ) + info = json.loads(runtime.stdout) + root = Path(info["root"]) + if (info["version"] != np.__version__ or root.name != "numpy" + or not root.is_absolute() or not root.is_dir()): + raise ValueError("Unsupported NumPy runtime") + except (OSError, subprocess.SubprocessError, ValueError, KeyError, TypeError): + raise SandboxUnavailable("Matching system Python NumPy runtime is required") from None + mounts = ["--ro-bind", str(root), "/runtime/numpy"] + libraries = root.parent / "numpy.libs" + if libraries.is_dir(): + mounts += ["--ro-bind", str(libraries.resolve()), "/runtime/numpy.libs"] + mounts += ["--setenv", "OPENBLAS_NUM_THREADS", "1", "--setenv", "OMP_NUM_THREADS", "1"] + # Create the narrow mount destinations before the final read-only root remount. + position = command.index("--remount-ro") + return command[:position] + mounts + command[position:] + + +def _invoke(work, request=None, *, probe=False): + with tempfile.TemporaryFile() as stdout, tempfile.TemporaryFile() as stderr: + try: + process = subprocess.Popen( + sandbox_command(work, probe=probe), stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + env={"PATH": "/usr/bin:/bin", "LANG": "C"}, start_new_session=True, + ) + except OSError: + raise SandboxUnavailable("Cannot start benchmark artifact sandbox") from None + timed_out = False + try: + process.communicate(json.dumps(request).encode(), timeout=15) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.communicate() + timed_out = True + if not probe: + stderr.seek(0) + if not stderr.read(128).startswith(b"INFERRSI_BENCHMARK_READY\n"): + raise SandboxUnavailable("Benchmark worker did not establish its trusted boundary") + if timed_out: + return None + stdout.seek(0) + raw = stdout.read(MAX_OUTPUT_BYTES + 1) + if process.returncode or len(raw) > MAX_OUTPUT_BYTES: + return None + try: + return json.loads(raw) + except (ValueError, UnicodeError, RecursionError): + return None + + +def _prepare(package, np): + manifest = package.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + prepared, base, sequence = [], [], [] + for case in manifest["workloads"]: + x = numpy_task.inputs(case, np) + record = numpy_task.input_record(case, x) + base.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), + {**record, "phase": "precheck"}, case["id"])) + seen = {record["input_sha256"] for record in base} + for case in manifest["workloads"]: + if case["id"] not in measurement["case_ids"]: + continue + for index in range(measurement["warmups"] + measurement["samples"]): + x = numpy_task.inputs(case, np, sequence=index + 1) + record = numpy_task.input_record(case, x) + if record["input_sha256"] in seen: + raise ValueError("Input sequence repeats earlier bytes") + seen.add(record["input_sha256"]) + record.update(call_index=index, + phase="warmup" if index < measurement["warmups"] else "measurement") + sequence.append(record) + prepared.append((case, x, numpy_task.reference(x, oracle, np), record, + f"{case['id']}.call-{index}")) + return prepared, base, sequence + + +def _array(value, original, np): + if (not isinstance(value, dict) or set(value) != {"shape", "dtype", "data"} + or value["shape"] != list(original.shape) or value["dtype"] != "float32" + or not isinstance(value["data"], str)): + return None + try: + raw = base64.b64decode(value["data"], validate=True) + except (ValueError, binascii.Error): + return None + if len(raw) != original.nbytes: + return None + return np.frombuffer(raw, dtype=np.float32).reshape(original.shape) + + +def verify(task, output): + validate_task(task) + valid = (isinstance(output, dict) + and set(output) == {"artifact", "summary", "artifact_source"} + and output["artifact"] == ARTIFACT + and isinstance(output["summary"], str) + and isinstance(output["artifact_source"], str)) + if not valid: + return {"passed": False, "assertions": {"artifact_schema": False}} + try: + source = output["artifact_source"].encode("utf-8") + except UnicodeError: + source = b"" + if not 0 < len(source) <= MAX_SOURCE_BYTES: + return {"passed": False, "assertions": {"artifact_size": False}} + np = _numpy() + + package = load_task(task["benchmark"]["task_id"]) + prepared, base, sequence = _prepare(package, np) + request = { + "eps": package.manifest["oracle"]["eps"], + "inputs": [{"shape": list(x.shape), "layout": case["layout"], + "data": base64.b64encode(x.tobytes()).decode("ascii")} + for case, x, _, _, _ in prepared], + } + with tempfile.TemporaryDirectory(prefix="skill-benchmark-") as temporary: + work = Path(temporary) + (work / "runner.py").write_text(RUNNER, encoding="utf-8") + (work / ARTIFACT).write_bytes(source) + runtime = _invoke(work, probe=True) + if (not isinstance(runtime, dict) + or set(runtime) != {"ready", "numpy", "python", "machine"} + or runtime["ready"] is not True or runtime["numpy"] != np.__version__ + or any(not isinstance(runtime[k], str) or not runtime[k] + for k in ("python", "machine"))): + raise SandboxUnavailable("Benchmark sandbox preflight failed; no host fallback") + actual = _invoke(work, request) + package.assert_unchanged() + receipt = {"identity": task["benchmark"], "workloads": base, "input_sequence": sequence, + "artifact_sha256": hashlib.sha256(source).hexdigest(), + "runtime": {k: runtime[k] for k in ("numpy", "python", "machine")}, + "returned_calls": [], "scope": "isolated-artifact-correctness-only", + "timing": "not measured", "public_inputs": True, + "coverage": ( + "returned_calls indexes well-formed response rows, not independently " + "observed invocations; a crash may leave no rows. Isolation protects " + "the host and keeps the oracle outside, but does not prevent runtime " + "protocol spoofing by submitted source or public-suite overfitting." + )} + if (not isinstance(actual, dict) or set(actual) != {"outputs"} + or not isinstance(actual["outputs"], list) + or len(actual["outputs"]) != len(prepared)): + return {"passed": False, "assertions": {"artifact_response": False}, "benchmark": receipt} + checks = {} + for (case, original, expected, record, prefix), result in zip( + prepared, actual["outputs"], strict=True + ): + if not isinstance(result, dict) or set(result) != {"output", "input_after"}: + checks[prefix + ".response"] = False + continue + received = _array(result["output"], original, np) + after = _array({"shape": list(original.shape), "dtype": "float32", + "data": result["input_after"]}, original, np) + assertions = numpy_task.check_output( + received, original if after is None else after, original, expected, + package.manifest["oracle"], np, + ) + assertions["input_preserved"] = after is not None and assertions["input_preserved"] + checks.update({prefix + "." + key: value for key, value in assertions.items()}) + receipt["returned_calls"].append(record) + return {"passed": all(checks.values()), "assertions": checks, "benchmark": receipt} diff --git a/src/inferrsi/harness/skill_eval_engineering.py b/src/inferrsi/harness/skill_eval_engineering.py index 3fdb79904..218075ab6 100644 --- a/src/inferrsi/harness/skill_eval_engineering.py +++ b/src/inferrsi/harness/skill_eval_engineering.py @@ -171,7 +171,7 @@ def read_regular(path: Path, limit: int) -> bytes: return data -def input_hashes(workspace: Path) -> dict: +def input_hashes(workspace: Path, *, editable: str = "parser.py") -> dict: """Frozen engineering inputs only; special nodes are never opened.""" if any(path.is_symlink() for path in (workspace.absolute(), *workspace.absolute().parents)): raise ValueError("Workspace paths must not contain symlinks") @@ -181,7 +181,7 @@ def input_hashes(workspace: Path) -> dict: mode = path.lstat().st_mode if stat.S_ISLNK(mode) or not (stat.S_ISREG(mode) or stat.S_ISDIR(mode)): raise ValueError("Workspace contains a symlink or special node") - if stat.S_ISREG(mode) and name != "parser.py": + if stat.S_ISREG(mode) and name != editable: hashes[name] = hashlib.sha256(read_regular(path, 16 * 1024 * 1024)).hexdigest() return hashes diff --git a/src/inferrsi/harness/skill_evaluation.py b/src/inferrsi/harness/skill_evaluation.py index 27ce1ed3e..7ca4da9f3 100644 --- a/src/inferrsi/harness/skill_evaluation.py +++ b/src/inferrsi/harness/skill_evaluation.py @@ -190,12 +190,16 @@ def run_trial( timeout_seconds: float | None = None, engineering_artifact: bool = False, generation_only: bool = False, + artifact_adapter=None, ) -> dict: - if generation_only and not engineering_artifact: + adapter = artifact_adapter if artifact_adapter is not None else engineering + is_artifact = engineering_artifact or artifact_adapter is not None + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") + if generation_only and (not is_artifact or adapter is not engineering): raise ValueError("Generation-only requires the engineering artifact adapter") trial_dir.mkdir(parents=True) final = trial_dir / "answer.json" - before = engineering.input_hashes(workspace) if engineering_artifact else { + before = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -238,7 +242,7 @@ def run_trial( try: answer_limit = 1048576 if generation_only else 65536 answer_bytes = (engineering.read_regular(final, answer_limit) - if engineering_artifact + if is_artifact else final.read_bytes()) output = json.loads(answer_bytes.decode("utf-8")) except (ValueError, UnicodeError): @@ -248,25 +252,25 @@ def run_trial( output = engineering.materialize_generated_artifact(workspace, output) except (ValueError, OSError): evaluation_error = "invalid_generated_artifact" - if evaluation_error is None and engineering_artifact: - artifact = workspace / "parser.py" + if evaluation_error is None and is_artifact: + artifact = workspace / artifact_name if not isinstance(output, dict) or set(output) != {"artifact", "summary"}: evaluation_error = "invalid_artifact_answer" else: try: - artifact_bytes = engineering.read_regular(artifact, engineering.MAX_SOURCE_BYTES) + artifact_bytes = engineering.read_regular(artifact, adapter.MAX_SOURCE_BYTES) source = artifact_bytes.decode("utf-8") if any(path.is_symlink() for path in (trial_dir, *trial_dir.absolute().parents)): raise ValueError("Retained directory contains a symlink") if any((trial_dir / name).exists() or (trial_dir / name).is_symlink() - for name in ("agent-answer.json", "parser.py", "verification")): + for name in ("agent-answer.json", artifact_name, "verification")): raise ValueError("Agent created reserved evidence paths") except (ValueError, UnicodeError): evaluation_error = "missing_or_invalid_artifact" else: # A single bounded snapshot supplies both retained bytes and grading. (trial_dir / "agent-answer.json").write_bytes(answer_bytes) - (trial_dir / "parser.py").write_bytes(artifact_bytes) + (trial_dir / artifact_name).write_bytes(artifact_bytes) output = {**output, "artifact_source": source} atomic_write_json(final, output) verification_dir = trial_dir / "verification" @@ -279,14 +283,19 @@ def run_trial( evaluation_error = "artifact_verification_deferred" else: try: - checks = engineering.verify(task, output) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, output) + except adapter.SandboxUnavailable: evaluation_error = "artifact_sandbox_unavailable" else: - atomic_write_json(verification_dir / "result.json", checks) - verification = {"status": "completed" if checks["passed"] else "failed", - "returncode": 0 if checks["passed"] else 1} - if evaluation_error is None and not engineering_artifact: + atomic_write_json(verification_dir / "result.json", raw_checks) + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if not valid_checks(proposed): + evaluation_error = "invalid_verifier_result" + else: + checks = proposed + verification = {"status": "completed" if checks["passed"] else "failed", + "returncode": 0 if checks["passed"] else 1} + if evaluation_error is None and not is_artifact: verifier_environment = getattr(client, "verifier_environment", None) options = {"environment": verifier_environment()} if verifier_environment else {} verification = execute_command( @@ -315,7 +324,7 @@ def run_trial( if verification["returncode"] != (0 if checks["passed"] else 1): evaluation_error = "verifier_exit" try: - after = engineering.input_hashes(workspace) if engineering_artifact else { + after = adapter.input_hashes(workspace) if is_artifact else { p.relative_to(workspace).as_posix(): sha256(p) for p in workspace.rglob("*") if p.is_file() } @@ -343,11 +352,11 @@ def run_trial( "input_hashes": before, "output_sha256": sha256(final) if final.is_file() and not final.is_symlink() else None, } - if engineering_artifact: - retained = trial_dir / "parser.py" + if is_artifact: + retained = trial_dir / artifact_name record["artifact_sha256"] = (sha256(retained) if retained.is_file() and not retained.is_symlink() else None) - record["artifact_kind"] = engineering.ADAPTER + record["artifact_kind"] = adapter.ADAPTER if generation_only: record["execution_mode"] = engineering.GENERATION_MODE atomic_write_json(trial_dir / "result.json", record) diff --git a/tests/test_skill_eval_benchmark.py b/tests/test_skill_eval_benchmark.py new file mode 100644 index 000000000..eb30b9b84 --- /dev/null +++ b/tests/test_skill_eval_benchmark.py @@ -0,0 +1,257 @@ +"""Shared-oracle and isolated artifact controls; never model-benefit evidence.""" + +import base64 +import json +import os +from pathlib import Path +from unittest import mock + +import numpy as np +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks import numpy_task +from inferrsi.harness.benchmarks.contract import load_task + +READY = {"ready": True, "numpy": np.__version__, "python": "3.10.0", "machine": "x86_64"} + + +def answer(name="rmsnorm-cpu", source=None): + task = benchmark.task_descriptor(name) + return task, {"artifact": benchmark.ARTIFACT, "summary": "Authored software control", + "artifact_source": source or benchmark.starter(task).decode()} + + +def oracle_transport(task): + """Synthetic transport exercises parent grading only, never executes source.""" + package = load_task(task["benchmark"]["task_id"]) + prepared, _, _ = benchmark._prepare(package, np) + outputs = [{"output": {"shape": list(x.shape), "dtype": "float32", + "data": base64.b64encode(expected.tobytes()).decode()}, + "input_after": base64.b64encode(x.tobytes()).decode()} + for _, x, expected, _, _ in prepared] + return {"outputs": outputs} + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +def test_descriptor_freezes_builtin_task_and_shared_grader(name): + task, _ = answer(name) + benchmark.validate_task(task) + package = load_task(name) + assert task["contract"] == package.manifest + assert task["benchmark"] == { + "task_id": name, "task_sha256": package.task_sha256, + "grader_sha256": package.grader_sha256, + } + assert benchmark.starter(task) == (package.root / "candidate.py").read_bytes() + + +@pytest.mark.parametrize("change", ["hash", "holdout", "path", "request", "extra", "contract"]) +def test_descriptor_drift_and_noncanonical_tasks_rejected(change): + task, _ = answer() + if change == "hash": + task["benchmark"]["task_sha256"] = "0" * 64 + elif change == "path": + task["benchmark"]["task_id"] = "../rmsnorm-cpu" + elif change == "holdout": + task["split"] = "holdout" + elif change == "request": + task["request"] += " instead use a new oracle" + elif change == "contract": + task["contract"]["oracle"]["atol"] = 1 + else: + task["expected"] = True + with pytest.raises(ValueError): + benchmark.validate_task(task) + + +@pytest.mark.parametrize("change", [{"artifact": "parser.py"}, {"artifact_source": ""}, + {"artifact_source": "x" * 65537}, {"passed": True}]) +def test_invalid_artifact_does_not_start_source(change): + task, output = answer() + with mock.patch.object(benchmark, "_invoke") as invoke: + assert not benchmark.verify(task, {**output, **change})["passed"] + invoke.assert_not_called() + + +def test_no_sandbox_means_explicit_unavailable_never_host_execution(tmp_path): + marker = tmp_path / "must-not-exist" + task, output = answer(source=f"open({str(marker)!r}, 'w').write('bad')") + with mock.patch.object(benchmark, "_invoke", return_value=None): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.verify(task, output) + assert not marker.exists() + + +def test_missing_numpy_is_explicit_unavailable(): + task, output = answer() + with mock.patch.dict("sys.modules", {"numpy": None}): + with pytest.raises(benchmark.SandboxUnavailable, match="requires NumPy"): + benchmark.verify(task, output) + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +def test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call(name): + task, output = answer(name) + transport = oracle_transport(task) + calls = [] + + def invoke(work, request=None, *, probe=False): + assert sorted(p.name for p in work.iterdir()) == ["implementation.py", "runner.py"] + assert (work / "implementation.py").read_bytes() == output["artifact_source"].encode() + if probe: + return READY + calls.append(request) + assert set(request) == {"eps", "inputs"} + assert all(set(x) == {"shape", "layout", "data"} for x in request["inputs"]) + return transport + + with mock.patch.object(benchmark, "_invoke", side_effect=invoke): + result = benchmark.verify(task, output) + package = load_task(name) + # Use only the repository-authored candidate for comparison, not submitted source. + trusted = numpy_task.evaluate(package, package.root / "candidate.py", "check", Path(".")) + assert result["passed"] and len(result["assertions"]) == 164 + assert result["assertions"] == trusted["assertions"] + for key in ("workloads", "input_sequence"): + assert result["benchmark"][key] == trusted[key] + assert result["benchmark"]["returned_calls"] == trusted["executed_calls"] + assert len(calls) == 1 and len(calls[0]["inputs"]) == 41 + assert result["benchmark"]["timing"] == "not measured" + + +@pytest.mark.parametrize("wrong", ["dtype", "values", "mutation", "truncated", "malformed"]) +def test_parent_rejects_wrong_or_malformed_returned_arrays(wrong): + task, output = answer() + transport = oracle_transport(task) + record = transport["outputs"][10] + if wrong == "dtype": + record["output"]["dtype"] = "float64" + elif wrong == "values": + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + elif wrong == "mutation": + record["input_after"] = base64.b64encode( + bytes(len(base64.b64decode(record["input_after"]))) + ).decode() + elif wrong == "truncated": + transport["outputs"].pop() + else: + record["output"]["data"] = "!bad-base64" + with mock.patch.object(benchmark, "_invoke", + side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert "metrics" not in result + + +def test_workspace_freeze_excludes_only_implementation(tmp_path): + (tmp_path / "task.json").write_text("{}") + (tmp_path / "implementation.py").write_text("original") + (tmp_path / "parser.py").write_text("frozen") + before = benchmark.input_hashes(tmp_path) + assert set(before) == {"task.json", "parser.py"} + (tmp_path / "implementation.py").write_text("candidate") + assert benchmark.input_hashes(tmp_path) == before + (tmp_path / "parser.py").write_text("changed") + assert benchmark.input_hashes(tmp_path) != before + + +def test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository(): + original = ["bwrap", "--unshare-all", "--clearenv", "--ro-bind", "/safe/work", "/work", + "--remount-ro", "/", "--", "/usr/bin/python3", "-I", "-S", "-B", + "/work/runner.py"] + runtime = mock.Mock(stdout=json.dumps({"version": np.__version__, + "root": "/opt/public-runtime/numpy"}).encode()) + with mock.patch.object(engineering, "sandbox_command", return_value=original), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime), \ + mock.patch.object(Path, "is_dir", return_value=True), \ + mock.patch.object(Path, "is_absolute", return_value=True), \ + mock.patch.object(Path, "resolve", return_value=Path("/opt/public-runtime/numpy.libs")): + command = benchmark.sandbox_command(Path("/safe/work")) + assert "--unshare-all" in command and "--clearenv" in command + assert command[-5:] == original[-5:] + assert command.count("--ro-bind") == 3 + assert "/runtime/numpy" in command and "/runtime/numpy.libs" in command + assert command.index("/runtime/numpy.libs") < command.index("--remount-ro") + assert "--share-net" not in command + assert "OPENBLAS_NUM_THREADS" in command + + +def test_runtime_version_mismatch_is_unavailable(): + runtime = mock.Mock(stdout=json.dumps({"version": "different", "root": "/runtime/numpy"})) + with mock.patch.object(engineering, "sandbox_command", return_value=["--"]), \ + mock.patch.object(benchmark.subprocess, "run", return_value=runtime): + with pytest.raises(benchmark.SandboxUnavailable): + benchmark.sandbox_command(Path("/safe/work")) + + +@pytest.mark.parametrize("ready", [False, True]) +def test_worker_startup_is_distinct_from_artifact_failure(ready): + process = mock.Mock(returncode=1) + + def start(*args, **kwargs): + assert kwargs["env"] == {"PATH": "/usr/bin:/bin", "LANG": "C"} + if ready: + kwargs["stderr"].write(b"INFERRSI_BENCHMARK_READY\n") + return process + + with mock.patch.object(benchmark, "sandbox_command", return_value=["sandbox"]), \ + mock.patch.object(benchmark.subprocess, "Popen", side_effect=start): + if ready: + assert benchmark._invoke(Path("/unused"), {}) is None + else: + with pytest.raises(benchmark.SandboxUnavailable): + benchmark._invoke(Path("/unused"), {}) + + +def run_real(task, output): + try: + return benchmark.verify(task, output) + except benchmark.SandboxUnavailable as error: + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail(str(error)) + pytest.skip(str(error)) + + +@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) +def test_real_sandbox_correct_and_wrong_authored_controls(name): + task, output = answer(name) + result = run_real(task, output) + assert result["passed"] and len(result["assertions"]) == 164 + wrong = {**output, "artifact_source": + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n"} + rejected = run_real(task, wrong) + assert not rejected["passed"] and "metrics" not in rejected + + +def test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay(): + source = """import numpy as np +cache = {} +def compute(x, *, eps): + key = (x.shape, x.strides) + if key not in cache: + cache[key] = x / np.sqrt(np.mean(x*x, axis=1, keepdims=True) + eps) + return cache[key] +""" + task, output = answer(source=source) + result = run_real(task, output) + assert not result["passed"] + assert result["assertions"]["normal.values"] + assert not result["assertions"]["normal.call-0.values"] + + +def test_real_sandbox_cannot_read_host_sentinel_or_oracle(tmp_path): + sentinel = tmp_path / "host-only-secret.txt" + sentinel.write_text("must not be visible to submitted code") + task, output = answer() + source = output["artifact_source"] + f""" +from pathlib import Path +assert not Path({str(sentinel)!r}).exists() +assert sorted(p.name for p in Path('/work').iterdir()) == ['implementation.py', 'runner.py'] +assert not Path('/proc/1/root{str(sentinel)}').exists() +""" + result = run_real(task, {**output, "artifact_source": source}) + assert result["passed"] diff --git a/tests/test_skill_eval_benchmark_compare.py b/tests/test_skill_eval_benchmark_compare.py new file mode 100644 index 000000000..219a4190c --- /dev/null +++ b/tests/test_skill_eval_benchmark_compare.py @@ -0,0 +1,299 @@ +"""Formal two-arm plumbing with authored producers, not agent/skill efficacy. + +The producer is a real local subprocess. These controller tests replace the +artifact verifier; they do not establish that Linux/NumPy sandbox execution ran. +The separate benchmark verifier tests own real sandbox software controls. +""" + +import hashlib +import json +import os +import subprocess +import sys + +import pytest + +from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.infra.artifacts import sha256, write_json +from tests.fixtures.skill_eval_compare_support import configuration +from tools import skill_eval_compare as compare +from tools import skill_eval_local as local + +PRODUCER = """import json, pathlib, sys +work, final = pathlib.Path(sys.argv[1]), pathlib.Path(sys.argv[2]) +source, mode = sys.argv[3:] +artifact = work / 'implementation.py' +artifact.write_text(source, encoding='utf-8', newline='') +answer = {'artifact': 'implementation.py', 'summary': 'Authored software control, no model'} +if mode == 'changed-input': (work / 'task.json').write_text('{}') +if mode == 'extra-file': (work / 'unexpected.txt').write_text('unexpected') +if mode == 'missing-artifact': artifact.unlink() +if mode == 'oversized': artifact.write_bytes(b'x' * 65537) +if mode == 'invalid-answer': answer['passed'] = True +final.write_text(json.dumps(answer), encoding='utf-8') +event = {'type': 'turn.completed'} +if mode != 'unknown-usage': event['usage'] = {'input_tokens': 3, 'output_tokens': 2} +print(json.dumps(event), flush=True) +""" + + +class AuthoredProducer: + """Write candidate bytes without importing or executing them on the host.""" + + timeout_seconds = 30 + + def __init__(self, modes=None): + self.modes = modes or {} + self.calls = [] + + def command(self, workspace, final, schema, prompt): + task = compare.read_json(workspace / "task.json") + role = "baseline" if "-baseline-" in final.parent.name else "candidate" + mode = self.modes.get((task["id"], role), "correct") + source = ("import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" + if mode == "wrong-output" else benchmark.starter(task).decode("utf-8")) + self.calls.append({ + "task": task, "role": role, "workspace": workspace, + "schema_sha256": sha256(schema), "source": source, + "inputs": {p.relative_to(workspace).as_posix(): p.read_bytes() + for p in workspace.rglob("*") if p.is_file()}, + }) + return (sys.executable, "-B", "-c", PRODUCER, str(workspace), str(final), source, mode) + + +def prepared(tmp_path, names=("rmsnorm-cpu",), repeats=1): + compare.require_git_version(subprocess.check_output(("git", "--version"), text=True)) + config_path, repo = configuration(tmp_path, ids=names, repeats=repeats) + config = compare.read_json(config_path) + config["adapter"] = benchmark.ADAPTER + write_json(config_path, config) + tasks = [benchmark.task_descriptor(name) for name in names] + write_json(tmp_path / "cases.json", tasks) + output = tmp_path / "run" + digest = compare.prepare(config_path, repo, output) + return output, digest, config["snapshots"]["candidate"]["git_revision"] + + +def software_control_verifier(task, answer): + """A source-identity stub, deliberately not presented as numerical grading.""" + benchmark.validate_task(task) + passed = answer["artifact_source"] == benchmark.starter(task).decode("utf-8") + return { + "passed": passed, "assertions": {"authored_source_control": passed}, + "benchmark": {"scope": "mocked controller test only; no sandbox execution"}, + } + + +def test_real_authored_processes_use_frozen_task_and_retain_both_sources(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path, names=("rmsnorm-cpu", "softmax-cpu")) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "wrong-output"}) + result = compare.execute(output, digest, client=client) + assert result["adapter"] == benchmark.ADAPTER + assert result["counts"]["fix"] == result["counts"]["unchanged_correct"] == 1 + assert len(client.calls) == 4 + assert [call["role"] for call in client.calls] == [ + "baseline", "candidate", "candidate", "baseline"] + for left, right in (client.calls[:2], client.calls[2:]): + assert left["task"] == right["task"] + assert left["inputs"]["task.json"] == right["inputs"]["task.json"] + assert left["inputs"]["implementation.py"] == right["inputs"]["implementation.py"] + assert left["schema_sha256"] == right["schema_sha256"] + assert left["inputs"]["knowledge/selected/SKILL.md"] != ( + right["inputs"]["knowledge/selected/SKILL.md"]) + assert len({call["workspace"] for call in client.calls}) == 4 + assert all(not call["workspace"].exists() for call in client.calls) + for pair in result["pairs"]: + for role in compare.ROLES: + trial = pair[role] + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] + answer = json.loads(trial["output"]) + assert artifact.read_text(encoding="utf-8") == answer["artifact_source"] + assert trial["artifact_kind"] == benchmark.ADAPTER + assert trial["inputs_unchanged"] is True + assert trial["grader_trust"] == "reverified" + assert set(trial["checks"]) == {"passed", "assertions"} + assert "metrics" not in trial + before = (output / "report.json").read_bytes() + compare.report(output) + assert len(client.calls) == 4 + assert (output / "report.json").read_bytes() == before + assert str(tmp_path) not in before.decode("utf-8") + + +@pytest.mark.parametrize("kind", ["task", "schema", "starter"]) +def test_changed_frozen_input_prevents_any_producer_call(tmp_path, kind): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + name = {"task": "plan/tasks.json", "schema": "plan/schema.json"}.get(kind) + if name is None: + name = next(name for name in plan["files"] if name.endswith("/implementation.py")) + path = output / name + path.write_bytes(path.read_bytes() + b"\n") + client = AuthoredProducer() + with pytest.raises(ValueError, match="Frozen input changed"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +def test_task_grader_and_oracle_dependencies_are_frozen_before_producer(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + plan = compare.load_plan(output) + names = ( + "src/inferrsi/harness/skill_eval_benchmark.py", + "src/inferrsi/harness/benchmarks/numpy_task.py", + "src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json", + ) + for name in names: + assert plan["controllers"][name] == sha256(compare.ROOT / name) + changed_path = compare.ROOT / name + + def modified_digest(path): + return "0" * 64 if path == changed_path else sha256(path) + + client = AuthoredProducer() + with monkeypatch.context() as scope: + scope.setattr(compare, "sha256", modified_digest) + with pytest.raises(ValueError, match="Controller changed after freeze"): + compare.execute(output, digest, client=client) + assert client.calls == [] + + +@pytest.mark.parametrize("mode,error", [ + ("changed-input", "inputs_changed"), + ("extra-file", "inputs_changed"), + ("missing-artifact", "missing_or_invalid_artifact"), + ("oversized", "missing_or_invalid_artifact"), + ("invalid-answer", "invalid_artifact_answer"), +]) +def test_invalid_handoff_is_not_a_clean_comparison(tmp_path, monkeypatch, mode, error): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("rmsnorm-cpu", "candidate"): mode}) + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + assert trial["verdict"] == "INCONCLUSIVE" + retained = compare.read_json(output / trial["evidence"]["result.json"]) + assert retained["evaluation_error"] == error + # Report prioritizes missing artifact integrity over the original capture + # error, which remains available in the unchanged result.json record. + expected_report_error = ("missing_retained_artifact" if mode in { + "missing-artifact", "oversized", "invalid-answer"} else error) + assert trial["evaluation_error"] == expected_report_error + assert result["counts"]["inconclusive"] == 1 + + +def test_no_unsafe_default_launch_or_silent_sandbox_fallback(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + with pytest.raises(ValueError, match="confined local client"): + compare.execute(output, digest) + assert not (output / "results/start.json").exists() + + def unavailable(*_): + raise benchmark.SandboxUnavailable("authored unsupported-host control") + + monkeypatch.setattr(benchmark, "verify", unavailable) + result = compare.execute(output, digest, client=AuthoredProducer()) + for role in compare.ROLES: + trial = result["pairs"][0][role] + assert trial["verdict"] == "INCONCLUSIVE" + assert trial["evaluation_error"] == "artifact_sandbox_unavailable" + assert trial["artifact_sha256"] is not None + + +def test_retained_source_tamper_invalidates_report_without_rerunning_producer( + tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer() + result = compare.execute(output, digest, client=client) + trial = result["pairs"][0]["candidate"] + artifact = output / trial["evidence"]["implementation.py"] + artifact.write_text("# substituted source\n", encoding="utf-8") + report = compare.read_report(output) + assert report["pairs"][0]["candidate"]["verdict"] == "INCONCLUSIVE" + assert report["pairs"][0]["candidate"]["evaluation_error"] == "retained_artifact_hash_mismatch" + assert len(client.calls) == 2 + + +def test_unknown_usage_stops_new_trials_without_inventing_cost(tmp_path, monkeypatch): + output, digest, _ = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "unknown-usage"}) + result = compare.execute(output, digest, client=client) + assert len(client.calls) == 1 + assert result["status"]["stop_reason"] == "unknown_usage" + assert result["counts"]["missing_pair"] == 1 + assert result["conditions"]["baseline"]["cost"] is None + + +def test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks(tmp_path, monkeypatch): + output, digest, head = prepared(tmp_path) + monkeypatch.setattr(benchmark, "verify", software_control_verifier) + submission = tmp_path / "author-logs" + submission.mkdir() + write_json(submission / "execution.json", { + "model": compare.load_plan(output)["model"], "environment": "authored software control", + "access": "not a real agent run", "resources": "no model calls", "usage": None, + "elapsed_seconds": 1.0, "agent_exit_code": 0, "summary": "controller test only", + }) + for name in ("stdout.log", "stderr.log"): + (submission / name).write_text("authored control\n", encoding="utf-8") + for role in compare.ROLES: + workspace = tmp_path / role + local.stage(output, digest, "rmsnorm-cpu", role, 1, workspace, head) + if role == "baseline": + (workspace / benchmark.ARTIFACT).write_bytes(b"# authored bad control\r\n") + result = local.capture(output, digest, "rmsnorm-cpu", role, 1, workspace, head, submission) + assert result["counts"]["artifact_improved"] == 1 + pair = result["pairs"][0] + assert pair["comparability"] == "unverified_author_reported" + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert pair["candidate"]["author_reported_execution"] is True + assert pair["candidate"]["observations"]["usage"] is None + artifact = output / pair["baseline"]["evidence"]["implementation.py"] + assert artifact.read_bytes() == b"# authored bad control\r\n" + assert pair["baseline"]["artifact_sha256"] == hashlib.sha256(artifact.read_bytes()).hexdigest() + + +@pytest.mark.parametrize("change", [ + {"adapter": "unregistered-benchmark"}, + {"execution_mode": "generation-only"}, + {"schema": "custom-schema.json"}, + {"grader": "custom-grader.py"}, +]) +def test_adapter_does_not_accept_unfrozen_alternate_paths(tmp_path, change): + config_path, _ = configuration(tmp_path, ids=("rmsnorm-cpu",)) + config = compare.read_json(config_path) + config.update(adapter=benchmark.ADAPTER) + config.update(change) + with pytest.raises(ValueError): + compare.validate(config, [benchmark.task_descriptor("rmsnorm-cpu")]) + + +def test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable(tmp_path): + """Actual authored source replay; never represents an LLM or skill benefit.""" + output, digest, _ = prepared(tmp_path) + client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "wrong-output"}) + result = compare.execute(output, digest, client=client) + pair = result["pairs"][0] + if any(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" + for role in compare.ROLES): + if os.environ.get("INFERRSI_REQUIRE_BENCHMARK_SANDBOX") == "1": + pytest.fail("Required benchmark sandbox unavailable; no host fallback") + pytest.skip("Benchmark sandbox unavailable; two-arm numerical execution unqualified") + assert len(client.calls) == 2 + assert pair["baseline"]["verdict"] == "FAIL" + assert pair["candidate"]["verdict"] == "PASS" + assert result["counts"]["fix"] == 1 + assert len(pair["candidate"]["checks"]["assertions"]) == 164 + assert not all(pair["baseline"]["checks"]["assertions"].values()) + for role in compare.ROLES: + trial = pair[role] + assert trial["evaluation_error"] is None + assert trial["grader_trust"] == "reverified" + assert "suite_latency_ms" not in json.dumps(trial["checks"]) + artifact = output / trial["evidence"]["implementation.py"] + assert sha256(artifact) == trial["artifact_sha256"] diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index b5c0fb3e5..7f8c27a11 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -12,8 +12,10 @@ from dataclasses import replace from pathlib import Path +from inferrsi.harness import skill_eval_benchmark as benchmark from inferrsi.harness import skill_eval_diagnostic as diagnostic from inferrsi.harness import skill_eval_engineering as engineering +from inferrsi.harness.benchmarks.contract import load_task from inferrsi.harness.skill_evaluation import ( CodexClient, export_bundle, @@ -38,6 +40,7 @@ ROOT = Path(__file__).resolve().parents[1] ROLES = ("baseline", "candidate") +ARTIFACT_ADAPTERS = {engineering.ADAPTER: engineering, benchmark.ADAPTER: benchmark} GIT_READ = ("git", "--no-replace-objects", "--no-lazy-fetch") CONTROLLERS = (GUARD_SOURCE, "tools/skill_eval_compare.py", "tools/skill_eval_pilot.py", "tools/skill_eval_artifact_guidance.py", @@ -188,9 +191,9 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> "adapter", "execution_mode"}: raise ValueError("Unknown configuration field") adapter = config.get("adapter", "audit") - if adapter not in ("audit", diagnostic.ADAPTER, engineering.ADAPTER): + if adapter not in ("audit", diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): raise ValueError("Unknown experiment adapter") - if adapter in (diagnostic.ADAPTER, engineering.ADAPTER): + if adapter in (diagnostic.ADAPTER, *ARTIFACT_ADAPTERS): if config["mode"] != "update": raise ValueError("Diagnostic comparisons require paired old/new bundles (update mode)") if "grader" in config or "schema" in config: @@ -201,7 +204,7 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if generation_only and (not isinstance(config.get("client"), dict) or config["client"].get("kind") != FULL_CONTEXT_KIND): raise ValueError("Generation-only requires the bounded full-context provider") - if adapter == engineering.ADAPTER and "client" in config and not generation_only: + if adapter in ARTIFACT_ADAPTERS and "client" in config and not generation_only: raise ValueError("Engineering artifacts require local execution, not read/submit provider") if config["mode"] not in ("addition", "update"): raise ValueError("mode must be addition or update") @@ -236,8 +239,8 @@ def validate(config: dict, tasks: list[dict], *, default_grader: bool = True) -> if not tasks: raise ValueError("cases must be nonempty with unique IDs") for task in tasks: - if adapter == engineering.ADAPTER: - engineering.validate_task(task) + if adapter in ARTIFACT_ADAPTERS: + ARTIFACT_ADAPTERS[adapter].validate_task(task) elif adapter == diagnostic.ADAPTER: diagnostic.validate_task(task) else: @@ -326,7 +329,9 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: tasks = read_json(config_path.parent / config["cases"]) is_diagnostic = config.get("adapter") == diagnostic.ADAPTER is_engineering = config.get("adapter") == engineering.ADAPTER - grader = (Path(engineering.__file__) if is_engineering else + is_benchmark = config.get("adapter") == benchmark.ADAPTER + artifact_adapter = ARTIFACT_ADAPTERS.get(config.get("adapter")) + grader = (Path(artifact_adapter.__file__) if artifact_adapter else Path(diagnostic.__file__) if is_diagnostic else config_path.parent / config["grader"] if "grader" in config else ROOT / "tools/skill_eval_verifier.py") @@ -335,7 +340,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: require_git_version(subprocess.check_output(("git", "--version"), cwd=repo, text=True)) generation_only = config.get("execution_mode") == engineering.GENERATION_MODE schema = (engineering.GENERATION_SCHEMA if generation_only else - engineering.SCHEMA if is_engineering else + artifact_adapter.SCHEMA if artifact_adapter else diagnostic.SCHEMA if is_diagnostic else read_json(config_path.parent / config["schema"]) if "schema" in config else SCHEMA) if "client" in config: @@ -349,11 +354,19 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: if is_engineering: (output / "plan/starter").mkdir() (output / "plan/starter/parser.py").write_bytes(engineering.STARTER.encode("utf-8")) + if is_benchmark: + for task in tasks: + package = load_task(task["benchmark"]["task_id"]) + shutil.copytree(package.root, output / "plan/benchmark-tasks" / task["id"], + ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + starter.parent.mkdir(parents=True) + starter.write_bytes(benchmark.starter(task)) arms = {} with tempfile.TemporaryDirectory(prefix="se-") as temporary: for role, short in zip(ROLES, ("b", "c")): selected = config["snapshots"][role] - if is_diagnostic or is_engineering: + if is_diagnostic or artifact_adapter: closure = selected_snapshot( repo, selected["git_revision"], config["skill_root"], config["skill"], Path(temporary) / short, bundle_revision=selected.get("bundle_revision")) @@ -393,7 +406,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: "grader": files["validator/verify.py"]}, "same_bundle": before is not None and before["revision"] == after["revision"], "prompt": (engineering.GENERATION_PROMPT if generation_only else - engineering.PROMPT if is_engineering else + artifact_adapter.PROMPT if artifact_adapter else diagnostic.PROMPT if is_diagnostic else PROMPT), "files": files, "controllers": {name: sha256(ROOT / name) for name in CONTROLLERS}, "retry_policy": "none", "cost": None, @@ -403,14 +416,24 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: } if is_diagnostic: plan["adapter"] = diagnostic.ADAPTER - if is_engineering: - plan["adapter"] = engineering.ADAPTER - name = "src/inferrsi/harness/skill_eval_engineering.py" - plan["controllers"][name] = sha256(ROOT / name) + if artifact_adapter: + plan["adapter"] = artifact_adapter.ADAPTER local_source = "tools/skill_eval_local.py" plan["controllers"][local_source] = sha256(ROOT / local_source) plan["isolation"] += (" Artifact replay requires bubblewrap namespaces and fails closed. " - "Public tests, not blind evaluation; reconstructed CPU fixture.") + "Public CPU development checks, not blind evaluation.") + if is_benchmark: + dependencies = [ROOT / "src/inferrsi/harness/skill_eval_benchmark.py"] + dependencies += list((ROOT / "src/inferrsi/harness/benchmarks").glob("*.py")) + for task in tasks: + dependencies += [p for p in load_task(task["benchmark"]["task_id"]).root.rglob("*") + if p.is_file() and "__pycache__" not in p.parts + and p.suffix != ".pyc"] + for path in dependencies: + plan["controllers"][path.relative_to(ROOT).as_posix()] = sha256(path) + plan["isolation"] += ( + " Independent oracle stays in the parent; only inputs and implementation enter " + "the NumPy artifact sandbox. Correctness only; no performance or skill-benefit claim.") if "client" in config: client = AnthropicAgentClient(config["model"]["name"], config["client"], config["limits"]["timeout_seconds"], @@ -500,6 +523,20 @@ def load_plan(output: Path) -> dict: for name, path in frozen_exam.items(): if plan["exam"].get(name) != plan["files"].get(path): raise ValueError(f"Frozen exam hash disagrees with the frozen file list: {name}") + if plan.get("adapter") == benchmark.ADAPTER: + if (plan["mode"] != "update" or "client" in plan or "execution_mode" in plan + or plan["exam"]["grader"] != sha256(Path(benchmark.__file__)) + or read_json(output / "plan/schema.json") != benchmark.SCHEMA + or plan["prompt"] != benchmark.PROMPT): + raise ValueError("Frozen benchmark adapter contract changed") + for task in tasks: + benchmark.validate_task(task) + package = load_task(output / "plan/benchmark-tasks" / task["id"]) + if package.task_sha256 != task["benchmark"]["task_sha256"]: + raise ValueError("Frozen benchmark package disagrees with task identity") + starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT + if starter.read_bytes() != benchmark.starter(task): + raise ValueError("Frozen benchmark starter changed") return plan @@ -579,7 +616,7 @@ def execute(output: Path, authorization: str, *, client=None, stop_on_unknown_usage = True if plan.get("execution_mode") == engineering.GENERATION_MODE and "client" not in plan: raise ValueError("Generation-only requires the frozen bounded provider") - if plan.get("adapter") == engineering.ADAPTER and client is None: + if plan.get("adapter") in ARTIFACT_ADAPTERS and client is None: raise ValueError("Engineering execution requires a verified confined local client; " "the default danger-full-access Codex client is not qualified") if client is None: @@ -613,9 +650,12 @@ def execute(output: Path, authorization: str, *, client=None, with tempfile.TemporaryDirectory(prefix="se-trial-") as temporary: workspace = Path(temporary) write_json(workspace / "task.json", task) - is_engineering = plan.get("adapter") == engineering.ADAPTER - if is_engineering: + artifact_adapter = ARTIFACT_ADAPTERS.get(plan.get("adapter")) + if artifact_adapter is engineering: shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + elif artifact_adapter is benchmark: + shutil.copyfile(output / "plan/starters" / task["id"] / benchmark.ARTIFACT, + workspace / benchmark.ARTIFACT) prompt = plan["prompt"] + "\nTask operation: " + task["request"] if bundle is not None: folder = ".agents/skills" if task["split"] == "discovery" else "knowledge" @@ -627,7 +667,7 @@ def execute(output: Path, authorization: str, *, client=None, directory, output / "plan/schema.json", output / "validator/verify.py", prompt, deadline, timeout_seconds=plan["limits"]["timeout_seconds"], - engineering_artifact=is_engineering, + artifact_adapter=artifact_adapter, generation_only=(plan.get("execution_mode") == engineering.GENERATION_MODE)) result["evidence_sha256"] = {name: sha256(directory / name) for name in EVIDENCE @@ -656,6 +696,9 @@ def execute(output: Path, authorization: str, *, client=None, def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_mode=None) -> dict: + artifact_adapter = ARTIFACT_ADAPTERS.get(expected_adapter) + artifact_name = (getattr(artifact_adapter, "ARTIFACT", "parser.py") + if artifact_adapter else "parser.py") directory = trial_directory(output, entry) path = directory / "result.json" if not path.is_file(): @@ -664,10 +707,10 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m "evaluation_error": "interrupted_attempt" if interrupted else None, "missing": not interrupted, "checks": None, "output": None, "evidence": {}, "elapsed_seconds": None, "observations": {"usage": None}} - if expected_adapter == engineering.ADAPTER: + if artifact_adapter: # A retained tree is untrusted input too: reject special/symlink evidence # before legacy report readers open it. Producers must already be stopped. - names = ("result.json", "receipt.json", "answer.json", "parser.py", + names = ("result.json", "receipt.json", "answer.json", artifact_name, "author-execution.json", *EVIDENCE) try: for name in names: @@ -702,6 +745,9 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m # Select report fields; raw commands and private workspace paths stay in local artifacts. selected = {key: record[key] for key in ("task_id", "condition", "repeat", "split", "verdict", "checks", "elapsed_seconds", "observations", "inputs_unchanged")} + if artifact_adapter is benchmark: + selected["artifact_kind"] = record.get("artifact_kind") + selected["artifact_sha256"] = record.get("artifact_sha256") # Legacy local imports used [] for unparsed logs, not an observed zero. unparsed_author_log = (record.get("execution_state") == "author_reported" and record["observations"].get("completed_turns") is None) @@ -713,9 +759,8 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m } record_error = record.get("evaluation_error") # Artifact integrity failures take precedence over a replayable host error. - error = None if expected_adapter == engineering.ADAPTER else record_error - if (expected_adapter == engineering.ADAPTER - and record.get("artifact_kind") != engineering.ADAPTER): + error = None if artifact_adapter else record_error + if (artifact_adapter and record.get("artifact_kind") != expected_adapter): error = error or "missing_retained_artifact_identity" if execution_mode == engineering.GENERATION_MODE: if record.get("execution_mode") != execution_mode: @@ -757,14 +802,14 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m error = error or "retained_evidence_hash_mismatch" elif name == "author-execution.json": evidence[name] = artifact.relative_to(output).as_posix() - if record.get("artifact_kind") == engineering.ADAPTER and record.get("artifact_sha256") is None: + if artifact_adapter and record.get("artifact_sha256") is None: error = error or "missing_retained_artifact" if record.get("artifact_sha256") is not None: try: artifact_answer = json.loads(data) if data is not None else None except ValueError: artifact_answer = None - artifact = directory / "parser.py" + artifact = directory / artifact_name if artifact.is_symlink() or not artifact.is_file(): error = error or "missing_retained_artifact" elif sha256(artifact) != record["artifact_sha256"]: @@ -773,7 +818,7 @@ def retained_trial(output: Path, entry: dict, expected_adapter=None, execution_m or artifact_answer.get("artifact_source") != artifact.read_bytes().decode("utf-8")): error = error or "retained_artifact_disagrees_with_answer" else: - evidence["parser.py"] = artifact.relative_to(output).as_posix() + evidence[artifact_name] = artifact.relative_to(output).as_posix() error = error or record_error if (error or record["execution_state"] not in ("completed", "author_reported") or not record["inputs_unchanged"]): @@ -868,14 +913,14 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "fix", "regression", "unchanged_correct", "unchanged_incorrect", "missing_pair", "inconclusive", *( ("artifact_improved", "artifact_regressed", "artifacts_both_pass", "artifacts_both_fail") - if plan.get("adapter") == engineering.ADAPTER else ()))}, + if plan.get("adapter") in ARTIFACT_ADAPTERS else ()))}, "discovery": discovery, "discovery_conditions": condition_summary(discovery), "status": status, "claim": "Pipeline results only; fixed-output offline clients do not establish " "skill efficacy. Discovery is separate. Unknown usage and cost remain unknown.", } - if plan.get("adapter") == engineering.ADAPTER: - result["adapter"] = engineering.ADAPTER + if plan.get("adapter") in ARTIFACT_ADAPTERS: + result["adapter"] = plan["adapter"] if any(trial.get("author_reported_execution") for trial in trials): captured = sum(bool(trial.get("author_reported_execution")) for trial in trials) result["status"] = {"execution_state": "author_reported_unverified", @@ -887,6 +932,15 @@ def read_report(output: Path, *, replay_artifacts: bool = True) -> dict: "usage and costs are self-reported, not independently authenticated. Public tests " "may be overfit. Paired artifact differences do not prove skill causality, original " "engine behavior, hardware efficacy or whole-PR acceptance. Reviewer decides merge.") + if plan.get("adapter") == benchmark.ADAPTER: + result["claim"] = ( + "Shared CPU benchmark artifact correctness only. The trusted parent checks actual " + "outputs with the shared oracle; candidate source runs only in bubblewrap. " + "Public development inputs are not holdout. Authored controls and paired artifact " + "differences do not establish skill causality, reliable performance ranking, " + "hardware qualification or service gains. Author-reported provenance/usage remain " + "unverified; absent usage and cost remain unknown.") + result["benchmark_tasks"] = read_json(output / "plan/tasks.json") if plan.get("execution_mode") == engineering.GENERATION_MODE: result["execution_mode"] = engineering.GENERATION_MODE result["claim"] = ( @@ -917,6 +971,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: sha256(ROOT / "tools/skill_eval_verifier.py"): verify, sha256(Path(diagnostic.__file__)): diagnostic.verify, sha256(Path(engineering.__file__)): engineering.verify, + sha256(Path(benchmark.__file__)): benchmark.verify, }.get(plan["exam"]["grader"]) if trusted_verifier is None: for trial in trials: @@ -931,7 +986,7 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: trial["evaluation_error"] = "unverified_custom_grader" return for trial in trials: - manual_artifact = (trusted_verifier is engineering.verify + manual_artifact = (trusted_verifier in (engineering.verify, benchmark.verify) and trial.get("author_reported_execution") and trial.get("evaluation_error") in ( None, "reported_agent_failure", "artifact_sandbox_unavailable", @@ -950,7 +1005,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: if trial["verdict"] not in ("PASS", "FAIL") and not replay_artifact: continue try: - checks = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + verified = trusted_verifier(tasks[trial["task_id"]], json.loads(trial["output"])) + checks = {key: verified[key] for key in ("passed", "assertions")} except engineering.SandboxUnavailable: trial["verdict"] = "INCONCLUSIVE" trial["grader_trust"] = "unverified" @@ -972,6 +1028,8 @@ def verify_retained_grader(plan: dict, trials: list[dict], tasks: dict) -> None: # without rewriting capture failure or upgrading pair comparability. trial["artifact_checks"] = checks trial["artifact_verdict"] = "PASS" if checks["passed"] else "FAIL" + if trusted_verifier is benchmark.verify and "benchmark" in verified: + trial["benchmark"] = verified["benchmark"] trial["grader_trust"] = "reverified" diff --git a/tools/skill_eval_local.py b/tools/skill_eval_local.py index 44d85a222..0b26aa8ed 100644 --- a/tools/skill_eval_local.py +++ b/tools/skill_eval_local.py @@ -13,18 +13,27 @@ from pathlib import Path from inferrsi.harness import skill_eval_engineering as engineering -from inferrsi.harness.skill_evaluation import normalize_usage +from inferrsi.harness.skill_evaluation import normalize_usage, valid_checks from inferrsi.infra.artifacts import sha256, write_json +def _adapter(plan): + if plan.get("adapter") == engineering.ADAPTER: + return engineering + from inferrsi.harness import skill_eval_benchmark as benchmark + + if plan.get("adapter") == benchmark.ADAPTER: + return benchmark + raise ValueError("Local artifact handoff requires a supported engineering artifact adapter") + + def _selection(output, authorization, task_id, condition, repeat, current_head): from tools import skill_eval_compare as compare if authorization != sha256(output / "plan/freeze.json"): raise ValueError("Local handoff requires the exact frozen plan SHA-256") plan = compare.load_plan(output) - if plan.get("adapter") != engineering.ADAPTER: - raise ValueError("Local artifact handoff is only for the engineering adapter") + _adapter(plan) if current_head != plan["arms"]["candidate"]["git_revision"]: raise ValueError("Current PR head differs from the frozen candidate; revalidate first") selected = [entry for entry in plan["schedule"] if entry["task_id"] == task_id @@ -54,7 +63,7 @@ def _read_regular(path, bound=8 * 1024 * 1024): return data -def _hashes(workspace): +def _hashes(workspace, *, editable="parser.py"): result = {} for path in workspace.rglob("*"): if path.is_symlink(): @@ -62,7 +71,7 @@ def _hashes(workspace): if path.is_dir(): continue name = path.relative_to(workspace).as_posix() - if name != "parser.py": + if name != editable: result[name] = hashlib.sha256(_read_regular(path)).hexdigest() return result @@ -92,6 +101,8 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat workspace: Path, current_head: str): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") if directory.exists() or workspace.exists(): raise ValueError("Stage requires fresh trial and workspace; never overwrite an attempt") # Keep writable work separate from retained evidence/validator and the other arm. @@ -101,7 +112,9 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat # Frozen inputs are LF bytes on every host; never apply platform translation. (workspace / "task.json").write_bytes( (json.dumps(task, ensure_ascii=False, indent=2, allow_nan=False) + "\n").encode("utf-8")) - shutil.copyfile(output / "plan/starter/parser.py", workspace / "parser.py") + starter = (output / "plan/starter/parser.py" if adapter is engineering else + output / "plan/starters" / task["id"] / artifact_name) + shutil.copyfile(starter, workspace / artifact_name) bundle = plan["arms"][condition]["bundle"] prompt = _prompt(plan, task, condition) if bundle: @@ -109,11 +122,13 @@ def stage(output: Path, authorization: str, task_id: str, condition: str, repeat (workspace / "INSTRUCTIONS.txt").write_bytes(prompt.encode("utf-8")) directory.mkdir(parents=True) write_json(directory / "local-stage.json", { - "plan_sha256": authorization, "entry": entry, "input_hashes": _hashes(workspace), + "plan_sha256": authorization, "entry": entry, + "input_hashes": _hashes(workspace, editable=artifact_name), "provenance": "author-run; execution not observed by controller", }) return {"entry": entry, "plan_sha256": authorization, - "next": "Run approved local agent; only parser.py is writable. Keep logs separately. " + "next": f"Run approved local agent; only {artifact_name} is writable. " + "Keep logs separately. " "Capture does not authorize or pay for that run."} @@ -121,6 +136,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe workspace: Path, current_head: str, submission: Path): plan, entry, task, directory = _selection( output, authorization, task_id, condition, repeat, current_head) + adapter = _adapter(plan) + artifact_name = getattr(adapter, "ARTIFACT", "parser.py") stage_record = json.loads(_read_regular(directory / "local-stage.json")) if stage_record["plan_sha256"] != authorization or stage_record["entry"] != entry: raise ValueError("Stage receipt does not match the frozen trial") @@ -153,21 +170,22 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe error = None try: expected = _expected_inputs(plan, task, condition) - unchanged = (_hashes(workspace) == expected + unchanged = (_hashes(workspace, editable=artifact_name) == expected and stage_record["input_hashes"] == expected) except ValueError: unchanged = False if not unchanged: error = "inputs_changed" - artifact = workspace / "parser.py" + artifact = workspace / artifact_name source = None artifact_bytes = None - if artifact.is_symlink() or not artifact.is_file() or artifact.stat().st_size > 65536: + if (artifact.is_symlink() or not artifact.is_file() + or artifact.stat().st_size > adapter.MAX_SOURCE_BYTES): error = error or "missing_or_invalid_artifact" else: try: - artifact_bytes = _read_regular(artifact, 65536) - if len(artifact_bytes) > 65536: + artifact_bytes = _read_regular(artifact, adapter.MAX_SOURCE_BYTES) + if len(artifact_bytes) > adapter.MAX_SOURCE_BYTES: raise UnicodeError("artifact grew beyond bound") source = artifact_bytes.decode("utf-8") except (UnicodeError, ValueError): @@ -180,22 +198,28 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe (directory / "agent").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "agent" / name).write_bytes(submitted[name]) - answer = {"artifact": "parser.py", "summary": metadata["summary"], + answer = {"artifact": artifact_name, "summary": metadata["summary"], "artifact_source": source} write_json(directory / "answer.json", answer) if source is not None: - (directory / "parser.py").write_bytes(artifact_bytes) - checks = None + (directory / artifact_name).write_bytes(artifact_bytes) + checks, raw_checks = None, None if error is None: try: - checks = engineering.verify(task, answer) - except engineering.SandboxUnavailable: + raw_checks = adapter.verify(task, answer) + except adapter.SandboxUnavailable: error = "artifact_sandbox_unavailable" + else: + proposed = {key: raw_checks.get(key) for key in ("passed", "assertions")} + if valid_checks(proposed): + checks = proposed + else: + error = "invalid_verifier_result" (directory / "verification").mkdir() for name in ("stdout.log", "stderr.log"): (directory / "verification" / name).write_text("", encoding="utf-8") - if checks is not None: - write_json(directory / "verification/result.json", checks) + if raw_checks is not None: + write_json(directory / "verification/result.json", raw_checks) artifact_verdict = None if checks is None else "PASS" if checks["passed"] else "FAIL" if metadata["agent_exit_code"] != 0: error = error or "reported_agent_failure" @@ -211,8 +235,8 @@ def capture(output: Path, authorization: str, task_id: str, condition: str, repe "execution_state": "author_reported", "inputs_unchanged": unchanged, "evaluation_error": error, "input_hashes": stage_record["input_hashes"], "output_sha256": sha256(directory / "answer.json"), - "artifact_kind": engineering.ADAPTER, - "artifact_sha256": sha256(directory / "parser.py") if source is not None else None, + "artifact_kind": adapter.ADAPTER, + "artifact_sha256": sha256(directory / artifact_name) if source is not None else None, "provenance": "author-supplied; agent execution and costs not independently verified", "artifact_verdict": artifact_verdict, "evidence_sha256": {name: sha256(directory / name) for name in compare.EVIDENCE From c79739bdcf365bbe05f3a0d6ad8bdfbbda730bbe Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 08:14:41 +0800 Subject: [PATCH 07/20] ci: install isolated NumPy runtime outside user site Agent-assisted-by: Codex (review) Signed-off-by: dengyunyang <584797741@qq.com> --- .github/workflows/ci-ready.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index 000cd9feb..a794a53e8 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -82,7 +82,8 @@ jobs: run: | sudo apt-get update sudo apt-get install --yes --no-install-recommends bubblewrap - sudo /usr/bin/python3 -m pip install -e ".[dev]" + sudo /usr/bin/python3 -m pip install "numpy>=1.24" + /usr/bin/python3 -m pip install -e ".[dev]" /usr/bin/python3 -I -c "import numpy; print(numpy.__version__)" - name: Real NumPy sandbox and authored two-arm software controls env: From dfcc18557fd065dbe499f5cbaf77ad20cf6c5870 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 08:16:05 +0800 Subject: [PATCH 08/20] ci: run artifact controls directly from checkout Agent-assisted-by: Codex (validation) Signed-off-by: dengyunyang <584797741@qq.com> --- .github/workflows/ci-ready.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-ready.yml b/.github/workflows/ci-ready.yml index a794a53e8..63de236d4 100644 --- a/.github/workflows/ci-ready.yml +++ b/.github/workflows/ci-ready.yml @@ -83,7 +83,7 @@ jobs: sudo apt-get update sudo apt-get install --yes --no-install-recommends bubblewrap sudo /usr/bin/python3 -m pip install "numpy>=1.24" - /usr/bin/python3 -m pip install -e ".[dev]" + /usr/bin/python3 -m pip install "pytest>=7.4" "PyYAML>=6.0" /usr/bin/python3 -I -c "import numpy; print(numpy.__version__)" - name: Real NumPy sandbox and authored two-arm software controls env: From fd0f396c2bd7574a97af013120d4d0fb1733699f Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 08:24:54 +0800 Subject: [PATCH 09/20] docs: record verified SkillEval benchmark integration Agent-assisted-by: Codex (validation and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/README.md | 2 +- docs/plans/issue-252/formal-integration.md | 14 + .../issue-252/formal-validation-summary.json | 325 ++++++++++++++++++ 3 files changed, 340 insertions(+), 1 deletion(-) create mode 100644 docs/plans/issue-252/formal-validation-summary.json diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 13d665a1d..097066be7 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -52,7 +52,7 @@ 运行 `inferrsi benchmarks list`、`show rmsnorm-cpu`，将示例候选复制成独立 `implementation.py`，再按[使用文档](../../engineering-benchmarks.md)执行 `check`、`profile` 和 `compare`。共享接口优先，硬件尚未确定，所以目前只支持明确的 CPU 子集。 -当前证据见 [v2 协议验证](replay-validation.md)；历史证据见 [v1 集成记录](validation.md) 和 [v1 用途对照](usefulness.md)。本地测试、组件实现、正式系统接入与维护者验收分别记录。GPU/NPU、通信/容量任务、真实 agent/skill/search 收益和可靠性能排序均不能由当前 CPU 结果代替。 +正式两臂入口的当前软件验证见[接入与实际 CI 记录](formal-integration.md#已执行的软件验证)：49 项通过、无跳过，包含 5 个真实 Linux sandbox 案例。原 Workbench 测量证据见 [v2 协议验证](replay-validation.md)；历史证据见 [v1 集成记录](validation.md) 和 [v1 用途对照](usefulness.md)。本地测试、组件实现、正式系统接入与维护者验收分别记录。GPU/NPU、通信/容量任务、真实 agent/skill/search 收益和可靠性能排序均不能由当前 CPU 结果代替。 ## 跟踪与范围 diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md index f212f8328..eae79a960 100644 --- a/docs/plans/issue-252/formal-integration.md +++ b/docs/plans/issue-252/formal-integration.md @@ -145,6 +145,20 @@ python -m pytest tests/test_skill_eval_benchmark_compare.py tests/test_skill_eva 严格 Linux 软件对照设置 `INFERRSI_REQUIRE_BENCHMARK_SANDBOX=1`：缺少隔离环境时测试失败，不能用 skip 形成绿色资格结论。实际执行结果由本轮验证记录/CI 链接另行记录；本文的命令和测试设计本身不代表执行已经通过。 +## 已执行的软件验证 + +在 `dfcc18557fd065dbe499f5cbaf77ad20cf6c5870`，现有 ready/review CI 内新增的 [benchmark-artifact-controls](https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558/job/126297873523) 已实际通过：**49 tests、0 failures、0 errors、0 skipped**。这是 Ubuntu 22.04 系统 Python + 匹配 NumPy + bubblewrap 的软件对照，严格环境变量已开启，未调用模型。下载的原始 JUnit 与[可公开汇总](formal-validation-summary.json)记录所有测试名称、原件 SHA-256 和来源。 + +job 日志确认实际 checkout 为 PR 合并快照 `e74eb3d0a693f88599805faa8bc1afaf05544dbc`，与上述 head 比较为零文件差异；隔离运行时预检输出 NumPy 2.2.6。不会把 CI 的合并快照误称为直接 checkout 了 head。 + +其中 5 个 case 明确执行真实 sandbox：Softmax 与 RMSNorm 各自的正确/错误实现；同进程状态缓存重放拒绝；宿主 sentinel 与 oracle 文件不可见；正式 prepare/execute/report 的两臂文件交接。其余测试覆盖控制器、协议和拒绝路径，并不全部属于真实 sandbox 执行。 + +本地 Windows 新测试为 44 passed、5 个真实 Linux 控制 skipped。旧路径的两批回归分别为 66 passed/14 failed、133 passed/1 failed；全部 15 个失败报告在未修改基线复现，来自 symlink 权限、CRLF 字面量或 Windows 路径行为，该本地批次仍不绿。新 hosted 全量检查与上述专门软件控制分开记录。 + +同一 head 的 [ready/review 全流程](https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558) 已完成：Python 3.10 / 3.11 full-check、两套 installed-package 检查和 PR references 均通过；Draft 的既有 KV qualification 按原规则跳过。全量日志中的精确计数保存在上述汇总，真实 NumPy sandbox 的无跳过结果由专门 job 单独证明。之后仅文档/证据整理的提交不冒充这次原始执行版本。 + +首个 Linux CI 准备阶段曾因系统 pip 的 editable-build hook 失败，未进入 sandbox 测试。随后改为显式安装系统 NumPy 与测试依赖、直接从 checkout 运行，才得到上述成功记录；没有改变宿主 namespace 安全策略或加入 host fallback。该成功只验证当前软件环境中的接口、数值和隔离控制，不证明性能收益、真实技能收益或全硬件资格。 + ## 本轮交付结束后，仍然分开验收的内容 共享任务已经有正式两臂产物入口的设计与实现位置；是否在某个具体平台完成真实 sandbox 软件对照，以对应执行记录为准。性能排序、真实旧/新 skill 的 Agent 实验、生产者隔离资格、成本效果、改进控制器及目标硬件仍需各自证据。 diff --git a/docs/plans/issue-252/formal-validation-summary.json b/docs/plans/issue-252/formal-validation-summary.json new file mode 100644 index 000000000..f3f48a8f3 --- /dev/null +++ b/docs/plans/issue-252/formal-validation-summary.json @@ -0,0 +1,325 @@ +{ + "scope": "Software controls for the formal shared-benchmark artifact adapter; no model or skill-efficacy experiment", + "tested_head": "dfcc18557fd065dbe499f5cbaf77ad20cf6c5870", + "adapter": "engineering-benchmark-v1", + "ci": { + "run_id": 41745549558, + "job_id": 126297873523, + "url": "https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558/job/126297873523", + "job": "benchmark-artifact-controls", + "conclusion": "success", + "runner": "ubuntu-22.04", + "strict_environment": { + "INFERRSI_REQUIRE_BENCHMARK_SANDBOX": "1" + }, + "artifact": "benchmark-artifact-controls-41745549558-1", + "original_junit_sha256": "ac2d216425f07adfde4d599d14b0e1203bd5bbe258e23640857a0d56deb339d0", + "tests": 49, + "failures": 0, + "errors": 0, + "skipped": 0, + "checkout_merge_sha": "e74eb3d0a693f88599805faa8bc1afaf05544dbc", + "merge_tree_matches_tested_head": true, + "numpy_version": "2.2.6", + "artifact_zip_sha256": "37ced936f75faa9caad0b95051de4a1a5b2bca30ff01b2d7e09541298f46518c" + }, + "transformation": "Read-only extraction of suite counts and all testcase identities/outcomes from the original CI JUnit artifact. Hostname, timestamp, absolute paths and test durations omitted. Original XML stays in the linked CI artifact and local validation archive.", + "real_sandbox_cases": [ + "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", + "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", + "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", + "test_real_sandbox_cannot_read_host_sentinel_or_oracle", + "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable" + ], + "cases": [ + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_freezes_builtin_task_and_shared_grader[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_freezes_builtin_task_and_shared_grader[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[hash]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[holdout]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[path]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[request]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[extra]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[contract]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change0]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change1]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change2]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change3]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_no_sandbox_means_explicit_unavailable_never_host_execution", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_missing_numpy_is_explicit_unavailable", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[dtype]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[values]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[mutation]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[truncated]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[malformed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_workspace_freeze_excludes_only_implementation", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_runtime_version_mismatch_is_unavailable", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_worker_startup_is_distinct_from_artifact_failure[False]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_worker_startup_is_distinct_from_artifact_failure[True]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_cannot_read_host_sentinel_or_oracle", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_real_authored_processes_use_frozen_task_and_retain_both_sources", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[task]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[schema]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[starter]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_task_grader_and_oracle_dependencies_are_frozen_before_producer", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[changed-input-inputs_changed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[extra-file-inputs_changed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[missing-artifact-missing_or_invalid_artifact]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[oversized-missing_or_invalid_artifact]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[invalid-answer-invalid_artifact_answer]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_no_unsafe_default_launch_or_silent_sandbox_fallback", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_retained_source_tamper_invalidates_report_without_rerunning_producer", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_unknown_usage_stops_new_trials_without_inventing_cost", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change0]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change1]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change2]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change3]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable", + "outcome": "passed" + } + ], + "local_windows": { + "new_tests_passed": 44, + "real_sandbox_tests_skipped": 5, + "legacy_batches": [ + { + "passed": 66, + "failure_reports": 14, + "baseline_reproduced": 14 + }, + { + "passed": 133, + "failure_reports": 1, + "baseline_reproduced": 1 + } + ], + "legacy_baseline": "663f3b327998526a13aa31e72ad729025c18983c", + "legacy_batches_green": false + }, + "limits": [ + "Correctness-only; starter is already correct and retaining it can pass.", + "Authored producers are software controls, not real Agent or old/new skill efficacy evidence.", + "No timing, hardware or service-performance qualification.", + "Artifact sandbox protects the host boundary; public inputs and returned-call records are not tamper-proof invocation evidence.", + "Provider generation and skill_eval_improve are outside this integration." + ], + "hosted_full_checks": [ + { + "python": "3.10", + "job_id": 126297873658, + "conclusion": "success", + "summary": "3631 passed, 18 skipped, 1 warning, 284 subtests passed in 308.28s (0:05:08)" + }, + { + "python": "3.11", + "job_id": 126297873629, + "conclusion": "success", + "summary": "3631 passed, 18 skipped, 1 warning, 284 subtests passed in 281.39s (0:04:41)" + } + ], + "hosted_installed_package_checks": { + "3.10": "success", + "3.11": "success" + }, + "ci_run_conclusion": "success" +} From 7af6606bc22ec97d9127e818de36734b4ba1ac56 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 09:32:41 +0800 Subject: [PATCH 10/20] docs: clarify benchmark workflow scope and remaining integration Agent-assisted-by: Codex (documentation and scope review) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/04-integration.md | 31 ++++++++++++++++++++++++++ docs/plans/issue-252/README.md | 6 ++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index fd8abd751..d10b59bd8 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -12,6 +12,37 @@ 例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 +## 流程也是交付内容 + +本子项要交付的是能够使用任务、得到结果并据此行动的接入流程。区分以下三层，不能把其中一层完成写成全部完成： + +- **单次实验的执行与验收：** 固定比较条件、检查实际实现、测量、留存结果，交给现有 Workbench 或 SkillEval 消费。这是 #252 必须提供并验证的接入。 +- **主 Agent 的完整使用过程：** 从目标开始，运行基线和 profiling，提出假设，修改实现，检查与比较，记录保留、放弃或证据不足的理由。需要通过实际演练验证，而不仅是画流程图；当前还没有完整闭环验收。 +- **自动搜索策略：** 程序选择下一种改法、分配多轮预算、决定何时停止或接受候选。这类决策需与独立的 [Optimization Search Policy 提案](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306)对齐，提案本身也不负责模型生成候选。本次为其提供可消费的结果；不将该提案默认为已经实现，也不把它悄悄并入 #252 的验收范围。 + +### 从哪里开始，以及每一步由谁做 + +下表描述需要讲清并演练的使用过程，最后一列单独记录当前实现，不能据表中步骤推断已有自动编排器。 + +| 步骤 | 谁做、交出什么 | 当前实现与待补项 | +| --- | --- | --- | +| 1. 明确目标和验收依据 | 主 Agent 根据目标项目确定输入、环境、指标、质量约束及预算；适用时复用内置题包 | 内置题可加载；已有通用 Workbench 可接外部命令。尚无自动从用户目标生成合格任务的能力 | +| 2. 准备比较条件 | 调用者在搜索前固定基线、验证规则和工作负载；每个候选有独立身份 | 题包内容身份、执行快照与 SkillEval prepare 冻结已实现。普通 `plan` 仅预览，不等于保存了冻结的 campaign | +| 3. 运行基线并 profiling | 主 Agent 调用对应环境的工具，保留诊断并说明要改什么、依据是什么 | 有独立 CPU `profile` 操作及历史产物；不会自动完成目标设备诊断，也没有强制 Agent 先诊断再修改 | +| 4. 提交并检查实现 | Agent 修改允许范围；独立工具实际运行并检查产物 | 内置题 worker 和正式产物 verifier 已实现；正式 SkillEval 候选隔离执行，可信 Workbench 源码仍需审核 | +| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 已有 AB/BA、源快照和报告；可靠性能排序尚未建立。正式 SkillEval adapter 当前只判正确性 | +| 6. 使用结果 | 主 Agent 引用证据解释保留、放弃或继续诊断；下一候选重新标识 | 现有报告可供使用，已有部分用途对照；未完成真实 Agent 全链路演练，没有程序化跨轮次搜索/停止控制 | + +用户目标不在题库时，第一步使用目标项目自己的 benchmark 和测试，通过现有 `inferrsi run` 的 baseline/candidate/validation 命令接入 Workbench，不需要先扩充题库。缺少质量检查或可信测量时，应补齐对应实验条件；通用 runner 无法判断测试是否充分代表真实业务。当前 CPU `TaskPackage` 和正式内置题 adapter 也不能直接承担任意模型或服务的适配。 + +### 流程怎样约束结果，而不是只提醒 Agent + +现有程序已检查任务/产物身份漂移、缺失验证和不完整执行；内置 NumPy worker 先检查正确性，错误实现不产生可用于选择的性能指标。**通用 Workbench 的外部命令路径不能因此被描述为同一个先检查再计时的执行顺序**：它当前在测量后运行独立 validation，并由最终评价拒绝无效结果。 + +尚无程序强制宿主 Agent 按“profile → 假设 → 修改”顺序行动，也没有完整优化轮次的资格检查。后续若增加流程约束，应在既有入口校验证据和声明的完成条件：缺少该实验要求的诊断或验收，不得记录成完整优化成功。不能仅在提示词里写了顺序，就声称所有 Agent 都会遵守。 + +本子项后续验证需要保留一条可复查的演练链：诊断依据、候选修改、独立检查、比较结果、主 Agent 的决定。至少证明错误候选被拒绝、证据不足不会被当作改进，以及最终结论只覆盖实际验收的目标。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 + ## 理解 只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 097066be7..a5e488441 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -1,6 +1,10 @@ # #252：给 InferRSI 提供共享优化任务和独立评测 -**我们要交付一套可复用的优化任务和独立评测工具：Agent 交付实现，工具检查算得对不对、在固定条件下测得多快，主 Agent 再依据这些证据决定下一步。** +**我们要交付两部分：准备好的工程优化任务，以及这些任务从交付实现到检查、诊断、测量和报告的可运行流程。主 Agent 使用结果继续优化；SkillEval 使用相同任务比较不同 skill 条件下的产物。** + +流程也是交付内容。只有题目目录，或者几个可以分别调用的命令，都不足以证明结果已在 RSI 中发挥作用。第四个子任务需要说明谁准备实验、谁运行 profiling、谁修改代码、谁验收，以及结果怎样支持下一步，并保留实际演练证据。当前已接通单次 Workbench 比较和正式 SkillEval 正确性产物交接；完整优化演练、可靠性能排序和真实 Agent/skill 收益仍未完成。详见[流程范围与缺口](04-integration.md#流程也是交付内容)。 + +题库不是用户需求的白名单。用户任务不在内置题库时，主 Agent 可以将目标项目自己的 benchmark 和质量检查接到**已有的通用 Workbench**；本次两道 CPU 题不参与也可以运行。该通用入口不是本次新增功能，也不会自动替用户定义验收标准。当前 `TaskPackage` 仍限于明确支持的 CPU 算子，不能称为任意用户任务适配器。 本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。它的输入是选定的任务包和候选实现，输出是正确性结果、独立 profile、原始计时及对应版本。名称是本次交付提议，正式范围仍待维护者接受。 From 9107db37c6ef2d862ed46133628a67bde0382ac6 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:21:45 +0800 Subject: [PATCH 11/20] docs: research task sources beyond kernel benchmarks Agent-assisted-by: Codex (source research and task design) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/01-research.md | 2 + docs/plans/issue-252/03-tasks.md | 2 + docs/plans/issue-252/README.md | 2 + .../issue-252/task-expansion/01-operators.md | 36 +++++++ .../task-expansion/02-host-runtime.md | 77 +++++++++++++++ .../issue-252/task-expansion/03-memory.md | 64 +++++++++++++ .../task-expansion/04-communication.md | 69 ++++++++++++++ .../issue-252/task-expansion/05-serving.md | 78 ++++++++++++++++ docs/plans/issue-252/task-expansion/README.md | 93 +++++++++++++++++++ .../task-expansion/gso-source-audit.json | 44 +++++++++ 10 files changed, 467 insertions(+) create mode 100644 docs/plans/issue-252/task-expansion/01-operators.md create mode 100644 docs/plans/issue-252/task-expansion/02-host-runtime.md create mode 100644 docs/plans/issue-252/task-expansion/03-memory.md create mode 100644 docs/plans/issue-252/task-expansion/04-communication.md create mode 100644 docs/plans/issue-252/task-expansion/05-serving.md create mode 100644 docs/plans/issue-252/task-expansion/README.md create mode 100644 docs/plans/issue-252/task-expansion/gso-source-audit.json diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md index 74a58bfd9..c5aff2452 100644 --- a/docs/plans/issue-252/01-research.md +++ b/docs/plans/issue-252/01-research.md @@ -1,5 +1,7 @@ # 子项 01：选定第一批任务和复用方案 +2026-11-04 补充：[按优化方向扩展任务的开源调研](task-expansion/README.md)及五类任务卡。下面保留初始 CPU 算子试点的选型依据；它不代表整个 SearchSpace 的最终题目覆盖。新的重点是实际仓库 CPU 热路径、已有 KV 任务复用，以及设备明确后的通信/服务题源。 + **这项工作的交付是选型结论：先做 Softmax/RMSNorm CPU 两道任务，复用成熟项目的任务定义和评测分工，并说明为何适合当前 InferRSI。** | 要说清楚的问题 | 本子项的答案 | diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index 107f526eb..9e2581e6b 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -1,5 +1,7 @@ # 子项 03：把第一批任务做成可执行的包 +本文记录已实现的两道 CPU 试点。2026-11-04 的[五类任务扩展方案](task-expansion/README.md)提出下一批题源与设计；其中 repo-patch、launcher、通信等 adapter 尚未实现，不能按本文两题的已有测试结果替代其资格。 + **这项工作的交付是两道真正能运行的优化任务：Softmax 和 RMSNorm，每道题都带输入、性能基线、独立参考答案与测量规则。** | 要说清楚的问题 | 本子项的答案 | diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index a5e488441..d8149eed3 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -10,6 +10,8 @@ **第一次阅读请从[完整导读](walkthrough.md)开始**：沿 RMSNorm 例子解释四个子任务、每个流程框、profiling、实际命令和结果。想核对本轮发现及修复的问题，请读[审查记录](review.md)。 +**多类任务的后续选择见[开源调研与五类任务方案](task-expansion/README.md)**：除了算子，还比较真实仓库 CPU 热路径、内存与缓存、通信、服务配置/调度。下一项新增任务优先考虑 GSO 的 Tokenizers.encode；内存类先复用仓内已有 KV 回收任务。该文及五份任务卡是扩展设计，当前可执行交付仍以各入口已声明的范围为准。 + ## 实际交付给使用者什么 以当前 `rmsnorm-cpu` 为例： diff --git a/docs/plans/issue-252/task-expansion/01-operators.md b/docs/plans/issue-252/task-expansion/01-operators.md new file mode 100644 index 000000000..7d9b00886 --- /dev/null +++ b/docs/plans/issue-252/task-expansion/01-operators.md @@ -0,0 +1,36 @@ +# 算子计算：保留已有试点，设备明确后扩展 + +**题目例子：在给定输入和容差下，让 RMSNorm 实现更快，同时保留计算结果。** + +状态：Softmax/RMSNorm CPU 两题已存在；本轮没有新增算子、执行测量或建立 GPU/NPU 资格。当前可靠性能排序仍未完成。原始实现与证据见[子项 03](../03-tasks.md)及[正式正确性接入](../formal-integration.md)。 + +## 为什么保留，而不继续只加同类题 + +它们适合验证单文件候选、独立数值参考、输入变化与结果交接。增加更多 NumPy 数学函数会扩大数值样例，却不能自动覆盖 CPU 编码、状态回收、通信或服务调度。下一批开发优先补这些不同的工程问题。 + +在现有 SearchSpace 中关联 Operator 的 Optimize/Verify；CPU profile 只能提供本次 CPU 调用诊断。任务标签不证明 Agent 具有 Diagnose 能力，CPU 适配也不提升 Accelerator 资源层的实测成熟度。 + +## 固定来源与复用决定 + +| 来源 | 具体借鉴 | 适用边界 | +| --- | --- | --- | +| [KernelBench Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py)、[RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) | 数学题意；`423217d`，MIT，已有任务包保留许可证 | 当前缩小为 CPU/NumPy/float32 开发子集，不是上游 GPU workload | +| [AMD AgentKernelArena task v2](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) | semantics、candidate、baseline、reference、workload 与 evaluation 的分工 | 作为协议设计参考；不导入其 GPU/Agent 执行平台或声称当前 CPU 实现满足其设备资格 | +| [BackendBench](https://github.com/meta-pytorch/BackendBench/blob/2a8d7e19e2c13c789ff2dceb44883ea8ea04dab4/README.md) | 边界输入/布局检查与模型来源 shape 的区分 | 作为扩展输入的设计参考；未复制 OpInfo/FACTO 或 TorchBench 运行体系 | + +后两项本轮仅引用既有[源码调研](../01-research.md)，没有复制其代码；未来复制具体文件时按其文件及依赖许可保留归属。 + +## Agent 做什么、工具检查什么 + +- 当前改动面是 `implementation.py` 的 `compute(x, *, eps)`；不能修改 oracle、输入分布、容差或计时器。 +- 数值检查覆盖普通/极端值、不同 shape/layout、输入保持及多次变化输入；裁判与候选实现分开。 +- CPU profiling 单独运行，不将诊断开销计入优化分数。 +- Workbench 比较适用的已审核源码；正式 SkillEval adapter 隔离 replay，仅提供正确性结果。 + +## 下一步资格实验 + +先完成同码/no-op、错误但快的实现和参考优化对照，确认计时信号可区分；保留原始样本与版本，不把历史样本重绑到新 grader。 + +设备确定后，从实际模型 profile 选一个代表瓶颈，再决定是扩展 reduction、fusion、attention 还是其他算子。设备版需要独立定义同步、编译、warmup、dtype、数值检查和计时边界；不能只给 CPU manifest 换一个 backend 字符串。 + +该类任务的成功只支持相应算子/环境结论。模型或服务收益仍需对应工作负载的端到端验收。 diff --git a/docs/plans/issue-252/task-expansion/02-host-runtime.md b/docs/plans/issue-252/task-expansion/02-host-runtime.md new file mode 100644 index 000000000..fe239876b --- /dev/null +++ b/docs/plans/issue-252/task-expansion/02-host-runtime.md @@ -0,0 +1,77 @@ +# 候选类别 02：CPU 侧数据处理与运行时开销 + +状态：2026-11-04 来源检查与任务设计；尚未导入题包、构建环境或运行性能实验。 + +## 要测什么 + +让 Agent 在真实仓库中发现并减少推理前后的 CPU 开销，同时保持输入输出语义。 +首题建议选择 **大量 added tokens 时的 Tokenizer.encode**，而非再增加一道数学算子。 +它关联现有 SearchSpace 中的主机侧处理与运行时优化；具体矩阵归属仍需评审，不据此新增已验证覆盖。 +这类题测试真实 repo 的分析和补丁能力；单题结果不代表服务 TTFT 或整个 RSI 能力提升。 + +## 可核查的题源 + +| 对象 | 本轮核对结果 | +| --- | --- | +| benchmark | [GSO](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/README.md)，代码 commit `32d78b331d59ed689b1238789dc20d36576951c0`，MIT | +| 具体任务 | 官方数据中的 `huggingface__tokenizers-076319d`，API 为 `Tokenizer.encode`，4 个测试 | +| 目标仓库 | `huggingface/tokenizers`；目标版本 [LICENSE](https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/LICENSE) 为 Apache-2.0 | +| baseline | `fb837b4adb130dfc89104aa09b1ae223da120278`，由官方 expert commit 的 parent 核对 | +| expert | [`076319d542f87e9f72db379d7e47b29b1ce7bd87`](https://github.com/huggingface/tokenizers/commit/076319d542f87e9f72db379d7e47b29b1ce7bd87)，Aho-Corasick 处理大量 added tokens | +| 数据版本 | 已下载并读取 `gso-bench/gso` revision `c2e4f1a58427cccd15e0e542f136bd204fb19284` 的 `data/test-00000-of-00001.parquet`，102 行中该 task ID 唯一、包含 4 个测试；数据许可声明 MIT | +| 来源记录 | [下载与任务行身份](gso-source-audit.json)保留文件 SHA-256、行选择、canonical JSON 哈希及 base/expert 身份；这是源码检查，不是任务执行证据 | + +初始发现来自实时 datasets-server；随后以固定 revision 的 parquet 完成任务行复核。正式导入还需归档所选资产及许可，锁定最终派生题包身份。 +上游参考改动主要位于 `tokenizers/src/tokenizer/added_vocabulary.rs`，并增加了 Rust 测试。 +Agent 从 baseline 开始修改仓库并提交 patch；expert patch 用于评测资格对照，不默认提供给 Agent。 + +## 输入、正确性和允许改动 + +GSO 的公开问题脚本使用 GPT-2 tokenizer、约 4000 个 added tokens 和多个文本片段；4 个测试还含其他构造及 BERT tokenizer，不能把问题脚本等同于全部测试。性能区间为编码调用。 +当前脚本会下载浮动 `resolve/main/tokenizer.json` 和在线 Gutenberg 语料。导入时必须固定版本与内容哈希，提前准备输入；安装、下载不计入编码耗时。 +数据行的 setup 还包含通用系统安装命令及 `upt upgrade` 拼写问题；本轮未执行这些命令，需要单独整理并验证最小可重建环境。 +任务应固定 tokenizer、added-token 集、文本及线程条件；允许修改实现，不允许替换 tokenizer 定义、测试、输入或参考结果。 + +- 保留 GSO 对 token 序列、token IDs、输出数量及长度的比较。 +- 复用上游 [added_tokens.rs](https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/tokenizers/tests/added_tokens.rs)，覆盖 single_word、lstrip、rstrip、重叠匹配及 offsets。 +- 补充不同文本长度、Unicode、空输入、无命中与多命中的条件；具体检查须经 baseline/expert 双方验证。 +- 基线输出等价检查与固定语义断言并用。基线结果不是完全独立的实现 oracle，不能把相同输出测试称为全语义证明。 +- 同一进程连续处理不同文本，保留输入和结果证据，检查状态缓存或复用结果造成的假改进。 + +## profiling 与测量流程 + +1. 准备固定仓库、Rust/Python 构建工具链与数据，验证 baseline 能重建、测试能通过。 +2. 独立 profiling：定位 Python/Rust 编码调用的时间；需要原生栈时使用支持 Rust 的 profiler，不能仅凭 cProfile 判断 Rust 内部热点。 +3. Agent 写出诊断依据和改法，交付真实 patch；工具审核适用范围并独立验收。 +4. 在声明的 CPU、线程、编译选项和工作负载下重复比较，安装和构建时间单独记录。 +5. 报告每个工作负载的正确性、原始耗时、波动、版本及预算，再解释是否足以保留候选。 + +GSO 的 [执行脚本](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/src/gso/harness/grading/evalscript.py) 对该仓库使用一次外层迭代；不能直接把这一设置当作 RSI 的可靠性能资格。 +重复次数和可接受误差来自本次实验的冻结条件，不照搬排行榜分数、阈值或无测量支持的默认值。 + +## 先证明题目有效，再评 Agent + +| 对照 | 需要检查什么 | 本轮状态 | +| --- | --- | --- | +| baseline 对 baseline（A/A） | 同码波动是否让实验错误判断为改进 | 未运行 | +| baseline 对 expert patch | 已知优化在当前输入和机器上是否仍有稳定信号，并通过正确性检查 | 未运行 | +| 故意错误 patch | 省略 added-token 规则、错误 offsets 或复用前次结果会被拒绝 | 待编写、未运行 | +| Agent 候选 | 在前述资格建立后检查正确性与性能，不以代码改动或测试 PASS 代替优化成功 | 未运行 | + +[跨机器可靠性研究的数据](https://github.com/chenzhi-cz/performance-optimization-benchmark-reliability/blob/44a1cb86a9f765b0375b4cfb479641f3c19f0774/data/gso_cross_machine_valid_tasks.csv)包含此 task ID；这是作者报告的筛选线索，不是我们的本地资格证明。 + +## 怎样接入 InferRSI + +需要的是 **repo-patch adapter**：固定 baseline checkout，接收并留存 patch，构建候选，调用独立测试和测量脚本，产生 Workbench 可读结果。 +此 adapter **尚未实现**；现有 `numpy-axis1-v2` 与 `implementation.py` 入口不能直接承担它。 +优先复用现有 Workbench 的外部命令和证据机制；正式 SkillEval 的 patch 交接、执行隔离与报告复核需要另外适配，不将任意生成代码送入可信宿主 worker。 +Rust 工具链、历史依赖、容器及原生编译让它的接入成本高于当前 NumPy 题，但仍不要求 GPU；实际可重建性待验证。 + +## 借鉴与不采用的边界 + +- [SWE-fficiency TestSpec](https://github.com/swefficiency/swefficiency/blob/12d32a2d6800824a7d84bdb6797b5708e7b7957f/swefficiency/harness/test_spec.py)将 workload、guarding tests 和 profiling 分开，适合作为流程参考；不引入另一套排行榜总分。 +- GSO 的 `tokenizers-fc76ad4` 优化 UnigramTrainer.train，属于训练任务；不能因为仓库与推理相关就纳入本类别。 +- [ISO-Bench 固定数据说明](https://huggingface.co/datasets/ISO-Bench/ISO-Bench/blob/24ef5d04b4993aca5e0d2cfa64b51cb40fc4cb55/README.md)提供更直接的 vLLM/SGLang repo 任务，可作为后续引擎运行时题源。其数据声明 H100／多 GPU 条件，不能据 CPU 局部测试宣称复现。 +- 例如 vLLM [`22d33baca2c0c639cfd45c48e99803e56c3efa74`](https://github.com/vllm-project/vllm/commit/22d33baca2c0c639cfd45c48e99803e56c3efa74)优化单请求的 merge_async_iterators；本轮仅核对官方 diff 身份及实时数据的 H100 标记，未验证该题的性能与 lm-eval 命令。 + +完成标准是固定来源、可重建环境、可靠对照和真实消费者接入均有证据；本页只完成来源选择与设计。 diff --git a/docs/plans/issue-252/task-expansion/03-memory.md b/docs/plans/issue-252/task-expansion/03-memory.md new file mode 100644 index 000000000..69975a827 --- /dev/null +++ b/docs/plans/issue-252/task-expansion/03-memory.md @@ -0,0 +1,64 @@ +# 内存与 KV 管理：先复用回收任务，再增加分配任务 + +**任务例子：给定请求结束、保存完成和写入完成事件，Agent 改进待回收队列，既不能提前释放块，也不能遗漏已经可以释放的块。** + +当前状态：设计候选；本次只调查和说明来源，没有新增执行、硬件资格或收益证据。 + +## 放在现有 SearchSpace 的哪里 + +引用既有 [SearchSpace Matrix](../../../searchspace-matrix.md) 的资源与生命周期说明：队列元数据和 CPU 管理开销属于 host 侧；实际设备 KV 容量属于相应加速器资源。这里不创建另一份权威分类，也不因加入任务而提升矩阵成熟度。 + +先区分三个问题：回收逻辑是否正确、管理操作是否更快、真实模型的显存/容量是否改善。CPU 任务只能直接回答前两者；第三个问题需要目标设备和模型实验。 + +## 已有基础与固定来源 + +| 来源 | 可复用内容与身份 | +| --- | --- | +| InferRSI 既有 KV 任务 | [资格说明](../../../kv-reclamation-qualification.md)、[可见契约](../../../../evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md)和[原始证据](../../../../evidence/kv-reclamation-20261030/README.md)；属于既有 #248 工作，不算本次新造题 | +| vLLM | commit `af261bab7f717b6ece30f5ed3213c43974e4a6c4`；[Apache-2.0](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/LICENSE)；[BlockPool](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/vllm/v1/core/block_pool.py)、[延迟回收测试](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/tests/v1/core/test_deferred_block_free.py)提供后续真实语义参考 | +| Mooncake | commit `e5ef0e427a49cd2f8ad5b4356b09435095030439`；[Apache-2.0](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/LICENSE-APACHE)；[分配 benchmark](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/README.md)与[正确性测试](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/tests/offset_allocator_test.cpp)是下一道实际新题候选 | + +既有 KV verifier 的冻结文件 SHA-256 为 `163c21134a8155da2a6469d9dbddb6ea71cb162e14f53e1c4266fe8ce75126f7`。任务、工作负载、补充 oracle 和数值计划各有原记录；复用时逐项检查身份，不能重新解释或改写冻结字节。 + +历史本地隔离不可用，留存结果为 **INCONCLUSIVE**。可信控制直接导入通过与隔离回放是不同证据；之后的新环境结果要另存，不能覆盖旧结果或倒推旧环境通过。 + +## 第一阶段：复用既有合成 KV 回收题 + +- Agent 只修改 `candidate.py` 的 `Reclaimer`；输入事件、oracle、grader 和工作负载受保护。 +- 实现 `apply(event)` 和 `pending()`，保留事件结构校验、请求 generation、重复通知幂等性、无效输入原子性及确定的返回顺序。 +- 回收要求 finish、所需 save_complete 和 processed frontier 三项条件同时满足；允许优化查找与排队数据结构，不允许跳过事件或缓存预定答案。 +- **既有题明确不含共享块、引用计数、前缀匹配、取消、超时、真实线程/异步执行、字节内容或资源容量策略。** 不能借 vLLM 来源把这些写成已覆盖功能。 +- 它是围绕特定知识机制构造的合成近迁移任务，不是历史缺陷复现、自然抽样 holdout，也不是 vLLM 兼容性测试。 + +正确性沿用独立 temporal oracle，逐事件核对回收列表和 pending 状态；原有非法输入、generation 重用、提前释放等控制保持原身份。 + +现有计时是宿主测得的**整条 CPU 任务命令耗时**，包括解释器和 sandbox 启动；不能改称核心队列延迟或显存收益。若另测核心操作，应建立新版本边界和独立资格记录。 + +Profiling 单独分析可信控制的扫描、索引和队列开销，不混入计时，不推断设备 timeline。真实生产者和不可信产物的权限边界仍按原入口处理。 + +## 第二阶段：Mooncake 混合尺寸分配题 + +任务例子：固定内存池容量和 allocate/free 请求序列，改进分配器，使分配更快或更少因碎片失败，同时保持请求完成量和正确的容量不足语义，不发生越界或重叠。 + +- 借鉴 `allocation_strategy_bench` 的 `size_class_churn`、容量预填充和尺寸分布；Agent 修改 allocator/placement 数据结构，不能改请求序列、容量、统计或副本要求。 +- 正确性独立检查地址范围、活动分配不重叠、分配大小、释放恢复和容量守恒；不能靠拒绝困难请求或返回部分副本来获得速度优势。 +- 分别记录完整/部分/失败分配、碎片率、延迟分布和完成量；先定义质量/容量约束，再评价速度，不能只报最快的一项。 +- CPU profiling 查分配、锁和数据结构开销；实际 RSS、设备显存、RDMA 和端到端 KV 命中收益都是不同实验。 +- 上游 [CMake target](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/CMakeLists.txt)链接 `mooncake_store`、cachelib allocator、gflags/glog 等；尚未验证可独立构建，不能承诺零依赖或跨平台立即可跑。 + +## 怎样接入 InferRSI + +先复用 [现有 KV 资格入口](../../../../tools/qualify_kv_reclamation.py)及其独立 oracle/隔离回放，不新建另一套 KV runner。它不是当前 `numpy-axis1-v2` 的一种 operation，也未因列入本文就接上新的正式两臂 adapter。 + +Mooncake 待构建和测量资格明确后，用目标拥有的脚本输出结构化结果，接既有 RunSpec/Workbench；质量脚本和候选分开冻结。不能直接把不可信 C++/Python 交给可信执行入口。 + +若比较 skill/Agent，另固定模型、预算和任务条件，保留两边真实产物及全部尝试；已有合成控制不能证明 skill 收益。 + +## 最小资格实验与采用理由 + +1. KV：核对原冻结身份，在合格 Linux 隔离环境执行既有正/负控制与同码对照，保存新的完整 receipt；历史 INCONCLUSIVE 保持原样。 +2. Mooncake：先证明最小 target 可构建，再固定小规模混合分配 trace，用简单独立区间模型拒绝越界、重叠、漏释放和部分分配伪成功。 +3. 性能：固定容量、预填充、请求量和环境，先检验同码对照；正确但测量不可分辨时保持证据不足，不选速度赢家。 +4. 设备外推：只有实际模型/设备上的容量、质量和性能验收，才能补充显存或服务结论。 + +采用顺序是“复用已有 KV → 资格验证 Mooncake 新题”，因为两者有具体可改对象和明确正确性义务。暂不直接引入完整分布式 KV 服务：部署、网络、模型和存储因素太多，难以在硬件未定时隔离单次改动。 diff --git a/docs/plans/issue-252/task-expansion/04-communication.md b/docs/plans/issue-252/task-expansion/04-communication.md new file mode 100644 index 000000000..dd96f3377 --- /dev/null +++ b/docs/plans/issue-252/task-expansion/04-communication.md @@ -0,0 +1,69 @@ +# 通信：复用真实测量底座，单独定义可优化实现 + +**任务例子：在固定多卡拓扑、消息大小和数据类型下，Agent 改进 AllReduce 实现或允许的配置，保持所有 rank 的归约结果正确，并减少真实通信耗时。** + +当前状态：设计候选；本次没有新增编译、通信执行或设备资格。硬件未确定，因此只确定来源、候选边界和验收方法，不发布性能结论。 + +## 放在现有 SearchSpace 的哪里 + +引用既有 [SearchSpace Matrix](../../../searchspace-matrix.md) 的 Communication/Network 与相应加速器资源说明，不建立新的权威分类。实际归属取决于瓶颈：同机 GPU 互连实验不能自动充当跨节点网络覆盖。 + +区分通信原语性能、通信/计算重叠和完整服务收益。前者可从单个 collective 开始；重叠必须检查依赖及端到端阶段耗时；服务收益仍需要真实工作负载验收。 + +## 固定来源与采用范围 + +| 来源 | 固定身份、材料与本次决定 | +| --- | --- | +| NVIDIA nccl-tests | commit `afd59abab774a6a4b492eb344bd0c845a0b7bb4d`；[BSD-3-Clause](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/LICENSE.txt)；[README](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/README.md)、[AllReduce](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/src/all_reduce.cu)。作为正确性与真实性能测量底座，不能把跑一次硬件测试等同于给 Agent 一道优化题 | +| MSCCL++ | commit `f5643de98b0c174baa3af2b2e6126f93dd07dc06`；[MIT](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/LICENSE)；[AllReduce candidate/预期结果](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/mscclpp-test/allreduce_test.cu)、[测试框架](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/README.md)。作为可修改通信算法、channel 与 pipeline 的候选来源 | +| MSCCL-tools | commit `030a750fc56e03fb863ffe48268672d84d4cdc7b`；[MIT](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/LICENSE)；[DSL 及 Check 测试](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/tests/test_language.py)。可借鉴通信计划接口和 CPU 符号检查，不作为无设备的性能替身 | + +方法参考：[MSCCLang 论文](https://arxiv.org/abs/2201.11840)解释拓扑与 collective DSL；[MSCCL++ 论文](https://arxiv.org/abs/2504.09014)解释细粒度通信抽象。这里采用接口设计，不复述作者数字为本地结果。 + +## Agent 可以改什么 + +首题在设备明确后从以下两种候选边界选一种，并在实验开始前固定，不能一次混改所有层： + +- **配置题：** 使用固定通信库/二进制，Agent 提交被允许的算法、协议或调度配置。允许项必须经实际版本核对，不能修改 rank 数、消息量、正确性检查或测量命令。 +- **实现题：** 从 MSCCL++ 的指定实现出发，修改通信算法或 pipeline；baseline、输入、预期结果生成和报告器保持受保护。 + +不可用空操作、跳过 rank、减少有效数据、关闭校验或只提交“带宽提高”的文字作为结果。更换通信库、拓扑、数据布局或精度会改变实验条件，需显式新任务/比较身份。 + +## 正确性、测量与 profiling + +正确性需要所有参与 rank 完成，返回与受信参考相符的数据；覆盖不同消息尺寸、边界/非整块输入，以及该实现声明支持的 in-place/out-of-place 模式和数据类型。 + +nccl-tests 有独立校验路径与 `--check`；冻结时不得用零次检查获得合格结果。MSCCL++ AllReduce 测试在 host 生成 expected buffer，可作为适配来源；不能把候选自身返回的通过标记当 oracle。 + +设备测量固定进程/rank 映射、拓扑、软件版本、消息大小、dtype、warmup、重复次数、同步/graph 模式和计时边界。保留原始各次测量及失败，不挑最好一次。 + +报告 latency、algbw、busbw 时保留各自定义；busbw 是 collective 相关归一化，不是任意多 rank 带宽简单相加。CPU launch 时间与设备完成时间不能混用。 + +nccl-tests 此固定版本支持 JSON 和每次 CUDA event 计时；per-iteration 模式与 graph capture 的约束需按原文保留。上游可输出样本不等于我们已验证测量稳定性。 + +Profiling 单独查看各 rank 时间线、同步等待和暴露的通信时间。重叠实验应保留依赖、数据一致性及总阶段耗时，不能把各 stream 时长相加后声称加速。 + +## 无硬件时可以做什么 + +MSCCL-tools 的 CPU `Check()` 根据 chunk 来源和归约成员检查计划语义，可验证 AllGather/AllReduce 调度是否完成规定操作。 + +它不证明真实数据类型数值误差、GPU 执行、跨进程同步或链路性能。计划步数、传输量或模型估计必须标注为模型指标，不能换算成已实测 latency/bandwidth。 + +nccl-tests 的 CUDA/NCCL 运行以及 MSCCL++ 的 CUDA/ROCm 路径均需实际环境；多进程/跨节点场景还涉及 MPI、设备和网络。未具备这些条件时保留“未执行”，不提供 CPU fallback 性能分数。 + +## 怎样接入现有 InferRSI + +使用目标拥有的构建、启动和解析脚本，把正确性命令与 baseline/candidate 测量命令接入既有 RunSpec/Workbench。保留进程退出状态、超时、原始输出、二进制/源码/配置身份及环境记录。 + +通用 Workbench 不会替代多 rank 启停、远端部署、拓扑资格或输出质量检查；这些须由目标脚本明确承担。现有 `numpy-axis1-v2` 和正式内置题 adapter 也不能直接执行通信题。 + +若之后接 SkillEval，两臂使用同一题目和测量规则，保留实际候选及完整成本；设备产物的执行隔离与生产 Agent 权限分别验证，不沿用 CPU sandbox 通过来推断 GPU 隔离。 + +## 最小资格实验与选择理由 + +1. 环境就绪后，运行固定 baseline 及已知正确实现，核对全部 rank、输入/输出和测量边界。 +2. 以漏 rank、错误归约、遗漏尾部数据和缺失输出为负控制，确认独立验收拒绝；超时或设备不可用记证据不足。 +3. 先跑同码/同配置对照，检验噪声和重复性；再比较候选，留存失败和原始样本，不以通用阈值代替具体实验约定。 +4. 重叠与服务收益分别做额外实验，不能由单个 collective 的数字推导。 + +优先采用 nccl-tests 的成熟测量和 MSCCL++ 的明确可改对象，避免重建通信平台。暂不把旧 MSCCL runtime 整体移植为默认依赖，也不在硬件未定时同时实现 CUDA、ROCm、NPU 三条执行链。 diff --git a/docs/plans/issue-252/task-expansion/05-serving.md b/docs/plans/issue-252/task-expansion/05-serving.md new file mode 100644 index 000000000..57f3145ee --- /dev/null +++ b/docs/plans/issue-252/task-expansion/05-serving.md @@ -0,0 +1,78 @@ +# 第五类候选：服务配置与运行时优化 + +**首选任务：固定模型、精度、请求流和硬件资源，让 Agent 优化服务配置，在质量约束下改善服务指标。** +本页是 2026-11-04 的源码调研与任务设计，尚未适配或执行；硬件未定，不报告性能收益或 Agent 能力提升。 +它考察服务配置、批处理和运行时行为，与修改单个 kernel 的题目分开；采用时沿用已有 Workbench，不复制另一套 runner。 + +## 任务要解决什么问题 + +用户给定模型与典型请求流，原配置可能让长输入响应慢、长输出生成慢，或高并发时排队严重。 +Agent 先运行基线和 profiling，再调整声明允许的配置，提交可复现的 launcher；工具独立验收质量和服务性能。 +任务允许直接使用目标项目的 benchmark，不要求真实用户的模型先进入固定题库。 + +| 项目 | 拟定规则 | +| --- | --- | +| 首个场景 | 从长输入、长输出、高负载中选一个，以实际模型与可用环境决定，不一次覆盖全部 | +| 固定内容 | 模型与 tokenizer 身份、权重/精度、输入与到达序列、输出长度规则、质量数据、资源和评测版本 | +| Agent 产物 | 首选 `start_server.sh` 或等价 launcher；若明确允许修改运行时，则单列提交 repo patch | +| 可改范围 | 经任务声明的 batching、prefill、并发或缓存配置；配置题和源码策略题分开比较 | +| 不可改内容 | 受信评测器、请求工作量、质量标准和资源额度；不能少处理请求、截短输出或换模型来获得分数 | +| 基线 | 固定版本服务引擎的明确配置；必须经过相同质量检查和测量流程 | +| 主指标 | 按目标选择 TTFT、decode 吞吐或满足 SLO 的 goodput；方向、单位和阈值由该实验冻结 | +| 辅助指标 | 错误/超时率、完成请求数、实际输出长度、ITL/TPOT、尾延迟、资源使用及原始逐请求记录 | + +## 首选参考：InferenceBench + +固定源码为 `aisa-group/InferenceBench` @ `108ec747fab0bd92fd9de501f62157d074c50ed5`，Apache-2.0。 +[README](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/README.md) 明确：Agent 在固定时间预算内写 launcher，最终在 fresh container 测量，并与 PyTorch 参考比较答案质量。 +它区分输入密集、输出密集、高负载和综合场景，还提供相同时间预算的 random/TPE/SMAC 配置搜索基线。 +这比仅测算子速度更接近 InferRSI 要支持的服务目标；采用的是任务设计和验收分工。 + +本页首版固定模型精度、缩小可调配置，是拟定的派生任务边界。上游允许更开放的框架、量化和运行时选择；以后扩大这些范围时，需要相应质量与资源验收，不能把本页限制说成上游规则。 + +- [场景 A 定义](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/tasks/inference_scenario_a_input_heavy/scenario.json)：输入/输出长度、请求数、并发和负载模式均显式声明。 +- [工作区规则](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/tasks/_shared/workspace_conventions.txt)：launcher 可编辑，测试入口与评测产物由 benchmark 持有。 +- [质量检查](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/inference/quality_gate.py)：核对质量样本完整性、参考准确率与声明门槛；答案质量不是耗时指标的副产品。 +- [运行入口](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/inference/runner.py)：分别保留速度、质量及执行完整性结果；未合格结果不能作为有效改进。 + +原生环境需要 H100 80 GB 或 A100、模型和数据缓存、Apptainer；完整流程还有 Claude 裁判审查捷径。 +不把这些平台依赖、付费裁判或论文默认阈值整体搬入 InferRSI。模型权重与数据许可也须另行核对。 +development/evaluation seed 分开可减少调参泄漏，但同一公开任务换 seed 不自动形成独立能力验证集。 + +## 备用与辅助来源 + +| 来源、固定版本及许可 | 具体借鉴 | 不能据此声称什么 | +| --- | --- | --- | +| [llama.cpp / llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) @ `149f5641b931aaf33697d70f9abc90c3e5300247`；MIT | 硬件未定时的 CPU 备用：固定模型，调整 threads、batch、ubatch；复用 warmup、重复样本及 JSON 输出 | 性能 benchmark 自身不提供完整质量 oracle；采用前须补输出/logit 或任务质量验收，CPU 结果不代表加速器 | +| [vllm-evolve](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/README.md) @ `43cc82c2c649699fb166339f85de2c8f79f2db4e`；[LICENSE 正文 Apache-2.0](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/LICENSE) | 窄策略题、SLO goodput、多 seed 比较，以及候选策略实际被加载/调用的检查 | 当前原生后端是 vLLM；simulation、更多 KV/routing 目标仍是 roadmap，不能当成现成通用模拟器 | +| [NVIDIA/SWE-Serve](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/README.md) @ `84d8fd92739a4a76ceddba56e5a70605921c4ad2`；Apache-2.0 | 生产工程任务、独立 verifier、参考修复/no-op 控制；可辅助检验运行时状态和容量逻辑 | 53 题以工程正确性为目标；任务名称含 perf 不代表有速度评分，不能替代本类的性能验收 | + +vllm-evolve 的 [plugin_template.py](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/targets/scheduling/plugin_template.py) 主要按策略结果重排 FCFS waiting deque；priority heap 会跳过。 +`invoked` 只证明调用，`reordered` 才表明 admission 顺序改变；两者都不直接证明收益。其 [evaluator](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/src/vllm_evolve/bench/evaluator.py) 拒绝无法证明插件被调用的性能结果。 +SWE-Serve 的 [adaptive-allocation-safety 评分器](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/tests/score.py) 仅在全部 F2P/P2P 节点通过后给 1 分，无速度评分；CPU 任务也可能要求较大内存和容器空间。 + +## 与现有 RSI 的接入流程 + +1. 在目标环境固定任务、模型、输入、质量标准和资源；准备可审核的服务启动、停止、质量及测量脚本。 +2. 运行基线并独立 profiling：保留队列、prefill/decode 分段或设备 trace，说明所改配置针对哪个观测瓶颈。 +3. Agent 提交 launcher 或声明允许的 patch；记录真实产物身份和全部尝试，不接受文字自报成绩。 +4. 受信入口确认产物实际生效，执行质量检查，再组织正式性能比较；profiling 开销不混入计时。 +5. 将 baseline/candidate/validation 脚本接到已有 Workbench，保留样本和报告；主 Agent 引用证据决定下一步。 + +上述是待适配流程，不能据此声称现有通用 runner 自动先验质量再计时；它当前在测量后执行独立 validation。 +质量预检应显式接入受信脚本；最终质量门槛仍需保留。服务生命周期由审核后的目标脚本处理，不另建执行平台。 +独立执行 Agent 产物的环境与权限仍需资格验证，当前 NumPy 产物 sandbox 不能直接承担任意服务源码。 + +## 必须完成的验证与当前状态 + +| 验证 | 需要保留的证据 | +| --- | --- | +| 质量控制 | 正常基线通过；损坏输出、漏请求、超时或实际输出长度不足被拒绝；基线/参考缺失保持证据不足 | +| 生效检查 | 实际进程、引擎/模型/配置身份与候选匹配；改变 launcher 却没有生效的情况不能记作候选收益 | +| no-op 对照 | 同代码同配置重复比较，检查漂移与噪声；不能因一次测量差异就宣称改进 | +| treatment 对照 | 预先声明一个会改变执行行为的配置，确认行为变化，并报告性能结果；不预设该配置一定更快 | +| 结果使用 | 一份从 profile、假设、产物、质量、计时到主 Agent 决定的完整记录，保留负结果和不确定性 | +| 能力评测 | 另行固定 Agent/skill 条件、预算与任务集；单个 launcher 变快不能直接归因于 skill 或搜索策略 | + +目前完成源码、许可与接口设计调研；未选定模型、硬件、质量数据或性能门槛，未新增 adapter，未运行上述控制。 +下一实施顺序：确定一个可用环境和场景 → 资格验证基线与质量检查 → 接已有 Workbench → 运行上述控制 → 再做真实 Agent 对照。 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md new file mode 100644 index 000000000..aa6b6d3de --- /dev/null +++ b/docs/plans/issue-252/task-expansion/README.md @@ -0,0 +1,93 @@ +# 按优化方向扩展任务：开源调研与首批选择 + +2026-11-04。本轮是官方源码、任务文件、许可证及论文的调研与设计交付；没有执行新增题目、调用模型或运行设备实验。以下五类是阅读分组，映射到已有 [SearchSpace](../../../searchspace-matrix.md)，不另建权威分类，也不把“找到来源”记成实测覆盖。 + +## 本轮决定 + +**保留 Softmax/RMSNorm 作为算子接口试点；下一批重点增加真实仓库 CPU 热路径和内存管理任务，设备明确后再接通信及服务性能。** 每类先做一个完整任务，验证裁判、计时和结果用途后再扩充数量。 + +AgentKernelArena 适合学习 kernel 任务的提交与独立评测；整个 InferRSI 还需要仓库补丁、状态管理、资源配置和服务工作负载。不能把所有任务都变成 `compute(x)`，也不能把功能修复的通过率叫作性能提升。 + +| 类别与任务卡 | 给 Agent 的具体问题 | 优先来源 | 当前状态 / 次序 | +| --- | --- | --- | --- | +| [① 算子计算](01-operators.md) | 在既定输入和数值约束下优化 Softmax/RMSNorm 实现 | KernelBench、AgentKernelArena、BackendBench | 已有 CPU 试点；先修完测量资格，不急于再加同类算子 | +| [② CPU 热路径与运行时](02-host-runtime.md) | 大量 added tokens 时，保持分词结果不变，降低编码开销 | GSO 的 Tokenizers 历史优化任务；SWE-fficiency 的任务契约 | **下一项新增任务首选**；真实 repo-patch adapter 和历史环境尚未实现 | +| [③ 内存与缓存](03-memory.md) | 正确处理异步完成与 KV 回收；随后优化混合尺寸分配的容量和开销 | 已有 RSI KV 任务、vLLM、Mooncake | **先接已有任务**，保留原资格边界；Mooncake 为下一独立新题候选 | +| [④ 多卡通信](04-communication.md) | 固定拓扑/消息量下优化 collective 实现或配置 | nccl-tests、MSCCL-tools、MSCCL++ | 先固定任务契约，实际带宽/延迟等待目标设备；CPU 符号检查单独标记 | +| [⑤ 服务配置与调度](05-serving.md) | 固定模型、请求和质量要求下，优化首 token 延迟或有效吞吐 | InferenceBench、llama.cpp、vllm-evolve；Apple RSI-Inference Bench 作真实引擎扩展参考 | 硬件及模型明确后执行；CPU llama.cpp 也是候选，但仍需模型资产与质量裁判 | + +第一阶段从“多做两道数学题”转为“三种不同问题的试点”：算子计算、CPU 分词、状态回收。状态回收是既有工作，接入不算新造题；也不因此宣布内存容量、GPU kernel 或服务端到端全部覆盖。 + +## 哪些开源项目更适合借鉴 + +以下采用决定由本次调研提出，不表示维护者已接受扩展范围。固定版本链接用于复查源码；具体任务卡保留更多测试、指标和适配条件。 + +| 来源及已检查内容 | 固定版本 / 许可 | 采用什么；为什么不整体搬入 | +| --- | --- | --- | +| [GSO：任务与实际 grading](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/src/gso/harness/grading/evalscript.py) | `32d78b3`；代码 MIT，目标 Tokenizers 为 Apache-2.0 | 借真实仓库 base / expert / candidate 三方、输出等价检查与性能 workload；选择推理相关 encode 任务。历史工具链、数据资产与测量资格须重新验证 | +| [SWE-fficiency：TestSpec](https://github.com/swefficiency/swefficiency/blob/12d32a2d6800824a7d84bdb6797b5708e7b7957f/swefficiency/harness/test_spec.py) | `12d32a2`；代码 Apache-2.0，README 声明其他材料 CC-BY-4.0 | 借 correctness tests、performance workload、独立 profiling script 的分工。通用科学 Python 任务不全部适合推理，不导入整套 runner 或总分 | +| [SWE-Serve：adaptive allocation](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/task.toml) 与 [score](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/tests/score.py) | `84d8fd9`；Apache-2.0 | 借真实 SGLang 状态/容量修复题与 F2P/P2P。所读题为 correctness 评分，没有速度分；“CPU 任务”仍可能要求较大容器资源 | +| [ISO-Bench](https://github.com/ISO-Bench/ISO-Bench/blob/9cc08ddd1011a34c49b43c22671ae639f16a21d3/README.md) 与 [固定数据卡](https://huggingface.co/datasets/ISO-Bench/ISO-Bench/blob/24ef5d04b4993aca5e0d2cfa64b51cb40fc4cb55/README.md) | 代码 `9cc08dd` Apache-2.0；数据卡 `24ef5d0` CC-BY-4.0 | vLLM/SGLang 性能提交可作为后续真实引擎补丁题源。论文的 54 题与所读数据卡的 106 题是不同版本；具体行来自实时接口，尚未绑定固定 parquet，部分题命令读取失败，不称为已可复现 | +| [Apple RSI-Inference Bench](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/README.md) | `1a38072`；[Apple Sample Code License](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/LICENSE)，不是 MIT/Apache；README 的论文仍为 coming soon | 更贴近整个推理系统：模型接入、实现优化、性能回归修复。优先借真实引擎任务设计；1–8 张 H100/B200、大模型与长执行预算不适合当前轻量首批 | +| [InferenceBench：任务与评测入口](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/README.md) | `108ec74`；Apache-2.0 | 借服务 launcher、不同负载、独立质量门槛、同预算非 Agent 搜索对照。完整原生流程有指定 GPU、数据缓存及模型裁判，不直接移植平台 | +| [Mooncake：分配 benchmark](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/README.md) | `e5ef0e4`；Apache-2.0 | 借混合尺寸分配/释放 workload、分配失败与碎片指标，并绑定独立 allocator tests；先验证最小 C++ 构建，不把逻辑容量称为 HBM 节省 | +| [nccl-tests](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/README.md)、[MSCCL-tools](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/tests/test_language.py)、[MSCCL++](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/mscclpp-test/allreduce_test.cu) | `afd59ab` BSD-3-Clause；`030a750` / `f5643de` MIT | 分别提供真实测量、可检查的通信计划、自定义 collective。需要明确候选修改面；硬件测试工具本身不是 Agent 优化题 | +| [llama.cpp：llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) | `149f564`；MIT | 借 CPU prefill/decode、线程/batch 配置与逐次样本。固定模型资产后可先做 CPU；计时工具不自动提供质量 oracle | +| [vllm-evolve：plugin](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/targets/scheduling/plugin_template.py) | `43cc82c`；实际 LICENSE 为 Apache-2.0 | 借窄 admission 策略入口与实际生效证明。当前主要重排 FCFS waiting deque；不能按 roadmap 声称已有 KV eviction/routing 或模拟平台 | +| [AutoTuneBench](https://github.com/li-ch/autotunebench/blob/64cc8d7ca5d0ee40c4e2189b2c2c8dfb1a2a7c2f/README.md) 与仓内 [已核查参考](../../../../src/inferrsi/data/skill_library/methods/inference-benchmark-design/references/autotunebench-trial-validity.md) | `64cc8d7`；选定代码/数据 MIT，vendor 许可分开 | 借 treatment 是否真正送达、原始记录与资格先于收益的审计方式。既有审查已发现声明与部分执行路径不一致，不把 README 的完整流程当作已验证实现 | + +不优先使用 PIE/MLGym 扩充本轮题数：前者以竞赛程序与模拟 CPU 为主，后者主要优化 ML 质量/奖励，且许可与训练成本另有约束。它们的任务组织经验有价值，但不是当前推理工程性能题的直接替代。 + +## 两个尤其贴近 RSI 的题例 + +**真实仓库热路径：Tokenizers.encode。** 固定原始仓库和 tokenizer 资产，Agent 用 profile 找大量 added tokens 的开销，修改 Rust/Python 代码，提交 patch；裁判检查 token IDs、tokens、offsets 等语义，再比较编码时间。它直接验证 CPU 分词优化，不直接证明服务 TTFT。具体 base/expert 身份及资产冻结缺口见[任务卡](02-host-runtime.md)。 + +**真实引擎回归：SGLang 流式 decode 吞吐下降。** Apple 的 [instruction](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/regression_sglang_stream_decode_tps_c1/instruction.md) 与 [task.toml](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/regression_sglang_stream_decode_tps_c1/task.toml) 给出真实 baseline、流式 workload、输出保持、scored/guard 条件；并允许不同修复机制。这比指定“必须写某个 kernel”更接近 RSI 的瓶颈调查。但该题固定 4×B200 和大模型；其环境清理脚本有枚举 GPU 进程的操作，不能直接移植到共享机器。当前只借设计、保留来源，未执行或复制脚本。 + +Apple 的另一个 [vLLM 源码优化题](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/optimize_qwen3_6_27b_1xh100_open/instruction.md) 允许热路径、布局、融合等多种改法，但不计纯配置修改；InferenceBench 则允许更开放的服务配置选择。两者不是谁更“完整”，而是考察不同搜索范围。InferRSI 应分别声明代码任务和配置任务，避免同一排行榜混用规则。 + +## 论文怎样影响选择 + +| 一手来源 | 本次采用的结论 | 不由此推导什么 | +| --- | --- | --- | +| [GSO §2.1–2.2](https://arxiv.org/html/2505.23671v3#S2) | 从真实优化提交提取 base、workload、等价检查和 expert control；Agent 提交真实 patch | 历史提交更快不等于我们的环境更快；参考答案不是唯一合法算法 | +| [ISO-Bench](https://arxiv.org/abs/2602.19594) | 从真实推理引擎的优化 PR 中挑代表任务；执行指标与诊断/机制分析分别保留 | LLM 对补丁的相似性/意图评分不能替代数值、质量及性能实测；不要求 Agent 复制人类补丁 | +| [InferenceBench §3、§5](https://arxiv.org/html/2607.20468v1#S3) | 分开 prefill、decode、并发目标；固定质量标准；后续搜索实验加入同预算 random/TPE/SMAC 等对照，检查最终提交是否保留最好有效候选 | 论文中 Agent 或搜索基线的排名不能迁移到 InferRSI；本轮没有执行该实验 | +| [性能 benchmark 可靠性研究 §III–IV](https://arxiv.org/html/2607.01211v2#S3) | 在接入任务前重放参考优化、做同码对照，分开任务有效性与 Agent 成绩；报告分项结果，不急于做一个总分 | 作者跨机结果不代替我们的硬件资格；“能运行”不等于性能信号稳定 | + +软件测试、公开论文、源码检查、真实运行证据分别记录。上述论文/仓库的收益数值没有被用作本次本地结果或默认阈值。 + +## 对接口和流程的具体影响 + +不同类别共享“任务/环境/验收规则固定 → baseline/profile → 候选 → 独立检查 → 比较 → 结果与决定”的过程，候选形式不能只支持一种： + +| 候选形式 | 对应题型 | 与当前实现的关系 | +| --- | --- | --- | +| 一个受限实现文件 | 已有算子；已有 KV 的 `candidate.py` | NumPy adapter 与 KV replay 是两条既有路径，语义/计时不同；接入时保留各自裁判与冻结身份 | +| 固定仓库上的 patch | Tokenizers、Mooncake、引擎回归 | **待实现 repo-patch adapter**；独立构建、受保护测试、完整产物和对应 baseline；不让模型改裁判 | +| 声明的配置或 launcher | llama.cpp / 服务配置 | **待实现对应任务适配**；锁定模型、请求量和设备，检查实际启动/生效的配置及最终服务器 | +| collective 算法或候选库 | 多卡通信 | 待选设备后实现；符号验证、编译、真数据校验、计时分别记录 | + +复用现有 Workbench 的执行、AB/BA 和证据，不把 GSO、Pier、Harbor、InferenceBench 的控制器全部引入。上游 task 包是素材和契约来源；任何新执行环境或多文件候选都需要其自身的产物隔离与验收，现有单文件 sandbox 不能自动覆盖仓库 build/server。 + +Profiling 保留独立阶段与原始产物。若以后评估“给 Agent profiler 反馈是否有用”，还应保留反馈进入下一次 Agent 输入的证据；生成了一份 profile 本身不足以证明它发挥了作用。 + +## 新任务入库前怎样验证有用 + +1. **来源与可重现性：** 固定仓库、base、参考修复、数据/模型、编译环境和许可证；公开历史参考修复不提供给 Agent。公开题只能作为 development 或受限条件下评测，不能冒充天然未见 holdout。 +2. **裁判区分度：** 正确基线、参考优化、错误但快的变体都实际运行；错误结果不得获得可用于选择的性能分数。 +3. **测量资格：** 同码 A/A 和 no-op 不应被当作进步；参考优化在选定环境确有可分辨信号。保留失败、波动与全部原始样本，无法分辨就暂不用于性能排名。 +4. **目标覆盖：** 计时工作负载、非计时语义边界、guard 工作负载分开说明。不能缩短输入、少处理请求或减少分配来换取虚假加速。 +5. **真实接入：** 源码/编译产物/配置确实进入最终执行；profile 和结果可追溯。用一条演练证明报告支持“保留/放弃/证据不足”的决定。 +6. **能力评测另做：** 任务资格成立后，固定模型与预算做 Agent/skill/搜索策略对照；记录失败成本、有效尝试数、最终质量和性能。不把一个新任务的通过当成整个 RSI 变强。 + +每个方向分开报告“找到来源、已适配、正确性验证、性能资格、Agent 对照”的状态。第一版不将内存、延迟、带宽或功能通过率压成一个未经论证的总分。 + +## 与四个子项的关系及本轮产物 + +- **#334 调研：** 本文与五类任务卡补充原来的算子选型；首批扩展优先级已明确。 +- **#335 接口：** 识别 repo-patch / launcher / 多进程任务的真实差异；这些是下一步设计输入，尚未新增通用 adapter。 +- **#336 任务：** 保留原两题；CPU 分词是新增首选，KV 是既有任务复用，其他候选按任务卡分阶段实施。 +- **#337 接入验证：** 按上面六项资格步骤验证真实用途；现有正式 SkillEval correctness 接入仍只支持它已声明的内置题。 + +本轮新增的是可复查的来源选择和任务设计文档。没有修改历史证据、SearchSpace 成熟度、现有任务或执行协议，也没有宣布新类别已经能运行。范围扩展与具体硬件/模型验收仍需沿原 Issue 跟踪。 diff --git a/docs/plans/issue-252/task-expansion/gso-source-audit.json b/docs/plans/issue-252/task-expansion/gso-source-audit.json new file mode 100644 index 000000000..f40e4e18f --- /dev/null +++ b/docs/plans/issue-252/task-expansion/gso-source-audit.json @@ -0,0 +1,44 @@ +{ + "kind": "source-inspection-only", + "inspected_on": "2026-11-04", + "dataset": "gso-bench/gso", + "dataset_revision": "c2e4f1a58427cccd15e0e542f136bd204fb19284", + "dataset_file": "data/test-00000-of-00001.parquet", + "download_url": "https://huggingface.co/datasets/gso-bench/gso/resolve/c2e4f1a58427cccd15e0e542f136bd204fb19284/data/test-00000-of-00001.parquet", + "dataset_file_bytes": 3843463, + "dataset_file_sha256": "bda458a7b5437c252f6cefdbc896f5f2868de51479e7a221ceda3f3ab74879bc", + "dataset_rows": 102, + "selector": {"instance_id": "huggingface__tokenizers-076319d"}, + "matching_rows": 1, + "selected_row_sha256": "152c078497a9da3e93878655942aa7b0535c39daae3586f800e0b7de02fa3b5f", + "selected_row_hash_encoding": "UTF-8 json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(',', ':')), without trailing newline", + "reader": "pyarrow 20.0.0; parquet read_table(...).to_pylist()", + "task": { + "repo": "huggingface/tokenizers", + "api": "Tokenizer.encode", + "base_commit_expression": "076319d542f87e9f72db379d7e47b29b1ce7bd87^", + "resolved_base_commit": "fb837b4adb130dfc89104aa09b1ae223da120278", + "expert_commit": "076319d542f87e9f72db379d7e47b29b1ce7bd87", + "test_count": 4, + "declared_arch": "x86_64" + }, + "license": { + "dataset_declared": "MIT", + "dataset_card": "https://huggingface.co/datasets/gso-bench/gso/blob/c2e4f1a58427cccd15e0e542f136bd204fb19284/README.md", + "target_code": "Apache-2.0", + "target_license": "https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/LICENSE" + }, + "remaining": [ + "Freeze GPT-2 and BERT tokenizer assets currently fetched from resolve/main", + "Freeze the Gutenberg text assets and review redistribution scope", + "Reconstruct and verify the historical build environment instead of executing setup commands blindly", + "Implement the repository-patch adapter and its execution boundary", + "Run baseline/expert correctness, negative controls and same-code measurement qualification" + ], + "execution": { + "setup_commands_run": false, + "benchmark_run": false, + "model_run": false, + "performance_qualified": false + } +} From 00c41b24427f9caea945db576a0a641e1b2c4904 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:44:29 +0800 Subject: [PATCH 12/20] fix: load formal benchmark tasks from canonical resources Keep reviewed path loading available to Workbench while resolving formal descriptors, starters, oracle checks and prepare dependencies only from packaged task resources. Reject unsupported device adapters before the NumPy sandbox. Agent-assisted-by: Codex (implementation and tests) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/formal-integration.md | 12 ++++ src/inferrsi/harness/benchmarks/__init__.py | 4 +- src/inferrsi/harness/benchmarks/contract.py | 20 +++++- src/inferrsi/harness/skill_eval_benchmark.py | 21 ++++-- tests/test_skill_eval_benchmark.py | 72 +++++++++++++++++++- tests/test_skill_eval_benchmark_compare.py | 27 ++++++++ tools/skill_eval_compare.py | 5 +- 7 files changed, 146 insertions(+), 15 deletions(-) diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md index eae79a960..da8773966 100644 --- a/docs/plans/issue-252/formal-integration.md +++ b/docs/plans/issue-252/formal-integration.md @@ -18,6 +18,18 @@ starter 本身是正确实现，因此原样保留也能通过正确性检查。这层入口能检查优化过程有没有破坏语义、拒绝错误产物，并保留两边真实实现；它不能区分“正确但未优化”和“正确且更快”。性能选择必须另有经过资格验证的测量，不能把正确性 PASS 当作优化成功。 +## 规范题包来源与设备边界 + +[同名工作目录替代问题](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4462417808)暴露了“冻结内容”和“选对内容”的区别：旧版正式路径按名字调用可信 Workbench 的路径优先 loader，可能把工作目录中放宽容差的题包当作内置题，再一致地冻结和重验错误规则。 + +正式路径现在显式调用 `load_builtin_task`，只从已安装的 `inferrsi` 包资源加载题包。descriptor、starter、verifier、prepare 的题包复制及依赖收集使用同一入口；留存副本则与规范 descriptor 的哈希比较。可信 Workbench 的 `load_task(path)` 保留，不能用于替代正式内置题的来源。 + +回归覆盖同名 `softmax-cpu/` 将 `atol` 改成 1 并替换 starter 的情况，检查规范 task hash / starter、prepare 的依赖和冻结副本，以及父进程拒绝全零数组。模拟返回数组的测试不声称执行了候选；单独的真实 Linux sandbox 控制负责实际源码执行，不支持的本地平台明确跳过。 + +此正式产物 adapter 仍只支持 `numpy-axis1-v2` 的 CPU / NumPy / float32 契约。新增 NPU 内置题不自动进入该 NumPy sandbox；不支持的 adapter 或环境被拒绝。真实 NPU 的可信 Workbench 运行和后续 NPU 产物隔离需要各自的资格，不能借本页的 CPU 隔离结果声明已经覆盖。 + +本次 loader 修复改变 grader 身份；历史软件控制与计时样本保留原被测版本，不能重新标为修复后的执行。 + 本次设计沿用[既有调研](01-research.md)中的任务、候选、独立评测分工，并结合现有 RSI 接口作了以下选择： | 接入方式 | 当前决定与理由 | diff --git a/src/inferrsi/harness/benchmarks/__init__.py b/src/inferrsi/harness/benchmarks/__init__.py index e1113d79d..21a95e916 100644 --- a/src/inferrsi/harness/benchmarks/__init__.py +++ b/src/inferrsi/harness/benchmarks/__init__.py @@ -1,5 +1,5 @@ """Shared engineering task contracts; execution belongs to the existing harness.""" -from .contract import TaskPackage, builtin_tasks, load_task +from .contract import TaskPackage, builtin_tasks, load_builtin_task, load_task -__all__ = ["TaskPackage", "builtin_tasks", "load_task"] +__all__ = ["TaskPackage", "builtin_tasks", "load_builtin_task", "load_task"] diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py index e4dc5b9fd..0bbbb166b 100644 --- a/src/inferrsi/harness/benchmarks/contract.py +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -151,11 +151,25 @@ def runspec( def load_task(name: str | Path) -> TaskPackage: + """Load a reviewed path or a packaged name for the trusted Workbench API.""" root = Path(name) if not root.is_dir(): - if str(name) not in builtin_tasks(): - raise ValueError(f"Unknown task package: {name}") - root = Path(str(files("inferrsi").joinpath("data/engineering_benchmarks", str(name)))) + return load_builtin_task(str(name)) + return _load_task_root(root) + + +def load_builtin_task(name: str) -> TaskPackage: + """Resolve only packaged task resources, never a same-named cwd directory.""" + if not isinstance(name, str) or name not in builtin_tasks(): + raise ValueError(f"Unknown built-in task package: {name}") + root = Path(str(files("inferrsi").joinpath("data/engineering_benchmarks", name))) + package = _load_task_root(root) + if package.manifest["id"] != name: + raise ValueError("Built-in task identity does not match its resource name") + return package + + +def _load_task_root(root: Path) -> TaskPackage: if root.is_symlink(): raise ValueError("Task root must not be a symlink") root = root.resolve() diff --git a/src/inferrsi/harness/skill_eval_benchmark.py b/src/inferrsi/harness/skill_eval_benchmark.py index 97012b9b7..224a6c7e4 100644 --- a/src/inferrsi/harness/skill_eval_benchmark.py +++ b/src/inferrsi/harness/skill_eval_benchmark.py @@ -17,7 +17,7 @@ from . import skill_eval_engineering as engineering from .benchmarks import numpy_task -from .benchmarks.contract import builtin_tasks, load_task +from .benchmarks.contract import TaskPackage, load_builtin_task ADAPTER = "engineering-benchmark-v1" ARTIFACT = "implementation.py" @@ -44,10 +44,19 @@ ) +def task_package(name: str) -> TaskPackage: + """The artifact sandbox supports only the packaged NumPy CPU contract.""" + package = load_builtin_task(name) + manifest = package.manifest + if (manifest["adapter"] != "numpy-axis1-v2" + or manifest["environment"] != { + "backend": "cpu", "runtime": "numpy", "dtype": "float32"}): + raise ValueError("Formal benchmark artifacts require the NumPy CPU adapter") + return package + + def task_descriptor(name: str) -> dict: - if name not in builtin_tasks(): - raise ValueError("Only built-in shared CPU benchmark tasks are supported") - package = load_task(name) + package = task_package(name) manifest = package.manifest return { "id": name, @@ -81,7 +90,7 @@ def validate_task(task): def starter(task) -> bytes: validate_task(task) - return (load_task(task["benchmark"]["task_id"]).root / "candidate.py").read_bytes() + return (task_package(task["benchmark"]["task_id"]).root / "candidate.py").read_bytes() def input_hashes(workspace: Path) -> dict: @@ -268,7 +277,7 @@ def verify(task, output): return {"passed": False, "assertions": {"artifact_size": False}} np = _numpy() - package = load_task(task["benchmark"]["task_id"]) + package = task_package(task["benchmark"]["task_id"]) prepared, base, sequence = _prepare(package, np) request = { "eps": package.manifest["oracle"]["eps"], diff --git a/tests/test_skill_eval_benchmark.py b/tests/test_skill_eval_benchmark.py index eb30b9b84..78c9a7b85 100644 --- a/tests/test_skill_eval_benchmark.py +++ b/tests/test_skill_eval_benchmark.py @@ -3,6 +3,8 @@ import base64 import json import os +import shutil +from dataclasses import replace from pathlib import Path from unittest import mock @@ -12,7 +14,7 @@ from inferrsi.harness import skill_eval_benchmark as benchmark from inferrsi.harness import skill_eval_engineering as engineering from inferrsi.harness.benchmarks import numpy_task -from inferrsi.harness.benchmarks.contract import load_task +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task READY = {"ready": True, "numpy": np.__version__, "python": "3.10.0", "machine": "x86_64"} @@ -25,7 +27,7 @@ def answer(name="rmsnorm-cpu", source=None): def oracle_transport(task): """Synthetic transport exercises parent grading only, never executes source.""" - package = load_task(task["benchmark"]["task_id"]) + package = load_builtin_task(task["benchmark"]["task_id"]) prepared, _, _ = benchmark._prepare(package, np) outputs = [{"output": {"shape": list(x.shape), "dtype": "float32", "data": base64.b64encode(expected.tobytes()).decode()}, @@ -47,6 +49,62 @@ def test_descriptor_freezes_builtin_task_and_shared_grader(name): assert benchmark.starter(task) == (package.root / "candidate.py").read_bytes() +def shadow_softmax(directory): + """A reviewed-path task is valid but must never replace the formal rubric.""" + canonical = load_builtin_task("softmax-cpu") + shadow = directory / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + (shadow / "task.json").write_text(json.dumps(manifest), encoding="utf-8") + (shadow / "candidate.py").write_text( + "import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n", + encoding="utf-8", + ) + return canonical, shadow + + +def test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + original, _ = answer("softmax-cpu") + monkeypatch.chdir(tmp_path) + # Trusted callers can still deliberately load an external reviewed task. + assert load_task("softmax-cpu").root == shadow + assert load_task(shadow).manifest["oracle"]["atol"] == 1 + assert load_builtin_task("softmax-cpu").root == canonical.root + task, output = answer("softmax-cpu") + assert task == original + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + assert benchmark.starter(task) == (canonical.root / "candidate.py").read_bytes() + assert benchmark.starter(task) != (shadow / "candidate.py").read_bytes() + # Synthetic returned arrays test parent grading, not sandbox source execution. + transport = oracle_transport(task) + for record in transport["outputs"]: + record["output"]["data"] = base64.b64encode( + bytes(len(base64.b64decode(record["output"]["data"]))) + ).decode() + with mock.patch.object(benchmark, "_invoke", side_effect=[READY, transport]): + result = benchmark.verify(task, output) + assert not result["passed"] + assert not result["assertions"]["normal.values"] + assert "metrics" not in result + + +@pytest.mark.parametrize("change", ["adapter", "environment"]) +def test_formal_artifact_path_rejects_new_backend_packages(change, monkeypatch): + package = load_builtin_task("rmsnorm-cpu") + manifest = package.manifest + if change == "adapter": + manifest["adapter"] = "torch-npu-rmsnorm-v1" + else: + manifest["environment"] = {"backend": "ascend", "runtime": "torch_npu", + "dtype": "float32"} + unsupported = replace(package, frozen_manifest=json.dumps(manifest)) + monkeypatch.setattr(benchmark, "load_builtin_task", lambda _: unsupported) + with pytest.raises(ValueError, match="NumPy CPU adapter"): + benchmark.task_descriptor("rmsnorm-npu") + + @pytest.mark.parametrize("change", ["hash", "holdout", "path", "request", "extra", "contract"]) def test_descriptor_drift_and_noncanonical_tasks_rejected(change): task, _ = answer() @@ -227,6 +285,16 @@ def test_real_sandbox_correct_and_wrong_authored_controls(name): assert not rejected["passed"] and "metrics" not in rejected +def test_real_sandbox_rejects_zero_output_despite_cwd_shadow(tmp_path, monkeypatch): + canonical, shadow = shadow_softmax(tmp_path) + monkeypatch.chdir(tmp_path) + task, wrong = answer("softmax-cpu", source=(shadow / "candidate.py").read_text()) + assert task["benchmark"]["task_sha256"] == canonical.task_sha256 + result = run_real(task, wrong) + assert not result["passed"] and not result["assertions"]["normal.values"] + assert "metrics" not in result + + def test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay(): source = """import numpy as np cache = {} diff --git a/tests/test_skill_eval_benchmark_compare.py b/tests/test_skill_eval_benchmark_compare.py index 219a4190c..46f38cbce 100644 --- a/tests/test_skill_eval_benchmark_compare.py +++ b/tests/test_skill_eval_benchmark_compare.py @@ -8,12 +8,14 @@ import hashlib import json import os +import shutil import subprocess import sys import pytest from inferrsi.harness import skill_eval_benchmark as benchmark +from inferrsi.harness.benchmarks.contract import load_builtin_task, load_task from inferrsi.infra.artifacts import sha256, write_json from tests.fixtures.skill_eval_compare_support import configuration from tools import skill_eval_compare as compare @@ -160,6 +162,31 @@ def modified_digest(path): assert client.calls == [] +def test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow(tmp_path, monkeypatch): + canonical = load_builtin_task("softmax-cpu") + shadow = tmp_path / "softmax-cpu" + shutil.copytree(canonical.root, shadow, ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) + manifest = canonical.manifest + manifest["oracle"]["atol"] = 1 + write_json(shadow / "task.json", manifest) + (shadow / "candidate.py").write_text("# substituted starter\n", encoding="utf-8") + monkeypatch.chdir(tmp_path) + output, _, _ = prepared(tmp_path, names=("softmax-cpu",)) + plan = compare.load_plan(output) + frozen = load_task(output / "plan/benchmark-tasks/softmax-cpu") + assert frozen.task_sha256 == canonical.task_sha256 + assert frozen.manifest["oracle"] == canonical.manifest["oracle"] + assert (output / "plan/starters/softmax-cpu/implementation.py").read_bytes() == ( + canonical.root / "candidate.py").read_bytes() + name = "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json" + assert plan["controllers"][name] == sha256(canonical.root / "task.json") + assert not any(name.startswith("softmax-cpu/") for name in plan["controllers"]) + # A later cwd change cannot make the frozen package validate against a new rubric. + manifest["oracle"]["atol"] = 2 + write_json(shadow / "task.json", manifest) + assert compare.load_plan(output) == plan + + @pytest.mark.parametrize("mode,error", [ ("changed-input", "inputs_changed"), ("extra-file", "inputs_changed"), diff --git a/tools/skill_eval_compare.py b/tools/skill_eval_compare.py index 7f8c27a11..be75dfea8 100644 --- a/tools/skill_eval_compare.py +++ b/tools/skill_eval_compare.py @@ -356,7 +356,7 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: (output / "plan/starter/parser.py").write_bytes(engineering.STARTER.encode("utf-8")) if is_benchmark: for task in tasks: - package = load_task(task["benchmark"]["task_id"]) + package = benchmark.task_package(task["benchmark"]["task_id"]) shutil.copytree(package.root, output / "plan/benchmark-tasks" / task["id"], ignore=shutil.ignore_patterns("__pycache__", "*.pyc")) starter = output / "plan/starters" / task["id"] / benchmark.ARTIFACT @@ -426,7 +426,8 @@ def prepare(config_path: Path, repo: Path, output: Path) -> str: dependencies = [ROOT / "src/inferrsi/harness/skill_eval_benchmark.py"] dependencies += list((ROOT / "src/inferrsi/harness/benchmarks").glob("*.py")) for task in tasks: - dependencies += [p for p in load_task(task["benchmark"]["task_id"]).root.rglob("*") + package = benchmark.task_package(task["benchmark"]["task_id"]) + dependencies += [p for p in package.root.rglob("*") if p.is_file() and "__pycache__" not in p.parts and p.suffix != ".pyc"] for path in dependencies: From 2008e80f95d09765285cad665f04b23deb175664 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:50:18 +0800 Subject: [PATCH 13/20] docs: make NPU RMSNorm the current benchmark pilot Define the affine device task and qualification flow; retain CPU results as historical software evidence and keep NPU SkillEval isolation separate. Agent-assisted-by: Codex (documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/01-research.md | 26 ++++-- docs/plans/issue-252/02-contract.md | 22 +++-- docs/plans/issue-252/03-tasks.md | 59 ++++++------- docs/plans/issue-252/04-integration.md | 30 ++++--- docs/plans/issue-252/README.md | 70 +++++++-------- docs/plans/issue-252/npu-pilot.md | 86 +++++++++++++++++++ .../issue-252/task-expansion/01-operators.md | 27 +++--- .../task-expansion/02-host-runtime.md | 4 +- docs/plans/issue-252/task-expansion/README.md | 30 +++---- 9 files changed, 223 insertions(+), 131 deletions(-) create mode 100644 docs/plans/issue-252/npu-pilot.md diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md index c5aff2452..9c958d4c2 100644 --- a/docs/plans/issue-252/01-research.md +++ b/docs/plans/issue-252/01-research.md @@ -1,16 +1,16 @@ # 子项 01：选定第一批任务和复用方案 -2026-11-04 补充：[按优化方向扩展任务的开源调研](task-expansion/README.md)及五类任务卡。下面保留初始 CPU 算子试点的选型依据；它不代表整个 SearchSpace 的最终题目覆盖。新的重点是实际仓库 CPU 热路径、已有 KV 任务复用，以及设备明确后的通信/服务题源。 +2026-11-04 范围更新：用户已指定 NPU 试点，当前优先交付 [RMSNorm NPU 方案](npu-pilot.md)。[五类任务研究](task-expansion/README.md)继续保留，但 Tokenizers.encode 不再作为当前试点。下面的初始 CPU 选型作为历史设计依据，不代表本轮设备验收或整个 SearchSpace 的最终覆盖。 -**这项工作的交付是选型结论：先做 Softmax/RMSNorm CPU 两道任务，复用成熟项目的任务定义和评测分工，并说明为何适合当前 InferRSI。** +**这项工作的交付是选型结论：用真实 NPU affine RMSNorm 跑通任务、独立检查、profiling 和比较，复用已有任务契约与 Workbench，而不是再建一个执行平台。** | 要说清楚的问题 | 本子项的答案 | | --- | --- | | 具体做什么 | 阅读论文与固定版本源码，比较任务、正确性参考、测量和执行边界，确定首批范围 | | 交付什么 | 下文的一手来源表、方案比较、采用/不采用理由和任务选择 | | 谁使用 | 子项 02 的接口开发者、子项 03 的任务开发者，以及范围评审者 | -| 实际作用 | 让实现围绕明确的两个优化问题展开，复用现有 Workbench/SkillEval，而不是从头搭另一个平台 | -| 当前状态 | 调研和方案已完成；维护者对首批范围的接受仍待记录 | +| 实际作用 | 明确 NPU 试点问题、基线与参考候选，复用已有 Workbench；CPU SkillEval 资格不自动延伸到设备 | +| 当前状态 | 既有研究保留，NPU 选择与资格计划明确；设备结果和维护者验收分别记录 | 它是一次设计交付，运行时不会执行“调研模块”。具体成果落实在后面三个子项。 @@ -40,7 +40,21 @@ | AIDE | [§3.1–3.2](https://arxiv.org/html/2502.13138v1) 显式区分 evaluator h、search policy π、coding operator f、solution tree T | search policy 是独立模块。本次提供任务评估，不实现搜索树或停止规则 | | InferRSI 当前源码 | RunSpec 要求 baseline/candidate；runner 已有 AB/BA、超时和快照；SkillEval run_trial 已有独立 verifier；生产 engineering verifier 有 Linux bubblewrap 边界 | 生成真实 RunSpec，诊断用 execute_trial；保留产物接入现有 verifier。生产 sandbox 和执行权限不变 | -## 方案比较与结论 +## 当前 NPU 选择 + +官方算子与测试固定在 op-plugin `9f23f00b1ebe54cca612b0537bc582dba409a0af`，NPU profiler 使用方式参考 Ascend/pytorch `9065e614f02639f5ddde0cf370bb9d38de7bd8a9`；[NPU 来源表](npu-pilot.md#为什么选择-rmsnorm)给出具体文件与许可。它们提供 API 和任务设计依据，不替代安装环境验证。 + +1. 首个试点选 **affine RMSNorm**：二维 contiguous float32 输入、非全一权重，基线使用 NPU 上的基础运算组合，参考候选使用 `torch_npu.npu_rms_norm`。独立 CPU scalar double / `math.fsum` 路径计算参考结果。官方 NPU 算子的语义与运行时支持需单独核对，不直接继承旧 CPU 四维无 affine 任务。 +2. 保留 `check / profile / measure` 与现有 RunSpec / Workbench 接口，由 NPU adapter 明确依赖、设备身份、同步、输入准备和计时边界。实际设备不支持时拒绝执行，不静默退回 CPU。 +3. 先用正确/错误候选、A/A 和基础算子/融合候选对照验证任务能否产生可信决策，再扩充题数或评估 Agent。融合候选是参考控制，不预设其必然加速。 +4. 真实 CPU/NPU profiling 单独运行；记录观察如何影响改法或最终决定。仅生成 profile 文件不能证明 Agent 使用了诊断。 +5. 正式 SkillEval 的现有 Linux sandbox 是 CPU 路径。NPU Workbench 先执行已审核可信源码；NPU 产物隔离与真实 skill 收益单独跟踪。 + +具体契约、验收矩阵和未完成项见 [NPU 试点](npu-pilot.md)。后续 Tokenizers、KV、通信与服务题仍按各自来源、环境和裁判重新资格，不因已有 CPU 或 NPU 任务完成而宣布全部覆盖。 + +## 历史 CPU 方案比较与采用理由 + +以下记录最初硬件未定时的 CPU 决策，不作为当前优先级。原设计和失败证据保留便于审查，本轮已由上面的 NPU 决定更新交付目标。 | 方案 | 优点 | 当前问题 | 决策 | | --- | --- | --- | --- | @@ -63,4 +77,4 @@ - correctness/profile 有独立执行记录，不需要伪造 baseline；只有 measurement 构成 A/B。 - 没有把某 cell 一条失败记录解释成搜索耗尽，也没有增加停止控制器。 -本方案针对硬件未确定的当前约束；下一步设备和任务类别应由实际 campaign 决定。它不是维护者对 #252 的最终验收决定。 +以上 CPU 选择针对当时硬件未定的约束；当前设备方向已明确为 NPU。无论哪一阶段，来源调研和贡献者方案都不代替维护者对 #252 的最终验收。 diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md index f6ccead72..bccbbc078 100644 --- a/docs/plans/issue-252/02-contract.md +++ b/docs/plans/issue-252/02-contract.md @@ -8,13 +8,21 @@ | 交付什么 | `TaskPackage`、`check/profile/measure` 操作及生成既有 RunSpec 的接口 | | 谁使用 | Workbench 适配代码、离线 verifier 和后续任务作者 | | 实际作用 | 一次比较中的题目和裁判固定；源码或规则改变时会被识别，两个调用者不用分别维护答案 | -| 当前状态 | 已实现并验证内容漂移、错误配置和未支持环境的拒绝；只承诺已实现的 CPU adapter | +| 当前状态 | CPU 接口与软件验证保留；本轮增加 NPU 设备语义、同步计时与环境身份，实际资格单独记录 | -例如同一个 RMSNorm 候选在两条路径中都按 axis=1、相同 epsilon 和输入序列检查。接口不决定候选怎么生成。 +NPU RMSNorm 的入口为 `compute(x, weight, *, eps)`，采用二维 contiguous float32 与非全一权重；旧 CPU 题的 `compute(x, *, eps)`、四维 axis=1、无 affine 语义保持原身份。不能因同名算子就混用题目或裁判。接口不决定候选怎么生成。 + +## 本轮 NPU 契约 + +[NPU 试点](npu-pilot.md)列出当前目标：独立 adapter 在实际设备上验证输入/输出设备、dtype、数值与输入保持，使用 CPU scalar double / `math.fsum` oracle；数据准备与传输、oracle、校验在性能计时外完成。同步边界必须覆盖实际完成的 NPU 计算。独立 CPU/NPU profile 不进入性能分数。 + +运行时身份需绑定实际设备与 torch / torch_npu / CANN 版本，不沿用只包含 Python/NumPy 的 CPU 身份。任务、候选、oracle、输入和计时规则继续使用共享内容身份；缺依赖、无设备或不支持语义不能回退 CPU。具体已支持字段和环境以最终 manifest 及实测记录为准。 + +Workbench 接入先运行已审核可信源码。正式 `engineering-benchmark-v1` 目前只接受已声明的 CPU 任务；NPU 题出现在 catalog 中，不代表正式 SkillEval 具备设备隔离或产物重验能力。adapter 的能力检查必须显式拒绝不支持的组合。 ## 理解 -任务包负责题目事实。消费者负责执行组织或 agent 对照。任务接口不应依赖硬件最终选择，也不应承担搜索调度。 +任务包负责题目事实。消费者负责执行组织或 agent 对照。共享操作接口保持可复用，具体 adapter 必须声明硬件语义与支持范围，不承担搜索调度。 ## 计划 @@ -26,22 +34,22 @@ ## 验收 -同一包身份被两个消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。当前变化输入协议及实际消费者验证见 [v2 验证](replay-validation.md)；早期安装包检查和 v1 运行保留在[历史验证记录](validation.md)，不作为当前协议的执行结果。 +同一包身份被支持它的消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。本轮 NPU 需证明真实设备检查、同步计时和 Workbench 证据，不能由 CPU 消费者测试替代。已有 CPU 变化输入协议见 [v2 验证](replay-validation.md)；早期安装包检查和 v1 运行保留在[历史验证记录](validation.md)，不作为当前 NPU 的执行结果。 -## 已实施 +## 已有软件接口 代码在 `src/inferrsi/harness/benchmarks/contract.py`，CLI 入口为 `inferrsi benchmarks`。 | 对象 | 实际字段或操作 | 用途 | | --- | --- | --- | | TaskPackage | schema_version、adapter、source、environment、candidate、oracle、measurement、workloads、coverage | 消费者读取同一份题目事实 | -| 冻结身份 | task_sha256、grader_sha256、candidate_sha256、runtime_sha256 | 记录本次真正使用的包、实现和解释器/NumPy 版本 | +| 冻结身份 | task_sha256、grader_sha256、candidate_sha256、runtime_sha256 | 记录实际包、实现与运行时；NPU 需扩充实际设备环境身份 | | 单次操作 | check、measure、profile | 都生成结构化结果，只有 measure 提供性能指标 | | Workbench 绑定 | runspec(candidate, repo, output_dir, repetitions, max_minutes, min_improvement_pct) | 使用已有 AB/BA、快照、截止时间和报告 | | 工程评估绑定 | grade_artifact；skill_verifier.task_descriptor | 用实际实现与独立 oracle 生成 SkillEval checks | 选择 JSON 是为了保持核心包零第三方依赖并便于命令快照。任务包描述题目，不保存密钥或用户运行配置。执行预算和增益门槛仍由 CLI/Python 调用方提供。 -首个 adapter 明确限定 CPU/NumPy/float32；未知后端直接拒绝。未来 backend 实现新 adapter，不改变上述消费者接口。当前没有承诺任意任务 manifest 都能执行，也没有提供空的通信/显存适配。 +已有 `numpy-axis1-v2` 明确限定 CPU/NumPy/float32；本轮 NPU 使用独立 adapter，不改变上述操作分工。未知后端直接拒绝。没有承诺任意任务 manifest 都能执行，也没有提供空的通信/显存适配。 真实 payload 仍由已有 `execute_trial/execute_command` 运行，独立 worker 仅执行一道题。`profile` 不需要 RunSpec，`compare` 才使用 RunSpec。保留缺依赖、超时、候选/任务漂移等失败路径。哈希冻结不是恶意代码隔离。 diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index 9e2581e6b..b52056793 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -1,52 +1,43 @@ -# 子项 03：把第一批任务做成可执行的包 +# 子项 03：把 NPU 试点做成可执行任务 -本文记录已实现的两道 CPU 试点。2026-11-04 的[五类任务扩展方案](task-expansion/README.md)提出下一批题源与设计；其中 repo-patch、launcher、通信等 adapter 尚未实现，不能按本文两题的已有测试结果替代其资格。 - -**这项工作的交付是两道真正能运行的优化任务：Softmax 和 RMSNorm，每道题都带输入、性能基线、独立参考答案与测量规则。** +**这项工作的当前交付是 `rmsnorm-npu`：给出可以在真实 Ascend NPU 上运行的基线、候选、输入、独立正确性检查、profiling 和测量规则。** 用户已经指定 NPU，不能再以 CPU 两题可运行作为试点验收。 | 要说清楚的问题 | 本子项的答案 | | --- | --- | -| 具体做什么 | 将上游数学任务适配为 CPU float32 任务，补充布局、数值和逐次变化输入 | -| 交付什么 | 两个任务目录中的 `task.json`、基线、示例候选、独立 oracle 适配及来源许可 | -| 谁使用 | 负责修改实现的 Agent/开发者，以及执行检查的 Workbench/verifier | -| 实际作用 | Agent 有明确的代码修改目标，评测能抓住错误轴、遗漏 epsilon、溢出、输入修改与旧输出重放 | -| 当前状态 | 已实现并通过实际计算验证；当前计时还不能保证可靠性能排序 | - -例如省略 RMSNorm 的 epsilon 看似减少了一步计算，但完整任务会发现错误并拒绝给出可用性能指标。 +| 具体做什么 | 将 affine RMSNorm 的数值语义接到 PyTorch / torch_npu，独立定义设备同步与计时 | +| 交付什么 | 任务包、基础算子 baseline、融合候选、数值 oracle、变化输入、错误控制与原始结果 | +| 谁使用 | 修改实现的 Agent/开发者，以及 Workbench 的任务操作 | +| 实际作用 | 验证候选确实在 NPU 计算且算对，并比较真实完成一次计算的成本 | +| 当前状态 | NPU 适配与资格实验是当前实施目标，实际结果按 [NPU 试点](npu-pilot.md)更新；CPU 旧结果不替代它 | -## 理解 +## 任务定义 -首批任务应足够小，能在硬件未定时跑通共享契约，又包含会出错的真实计算。CPU 适配必须公开与上游 workload/backend 的差异。 +首版使用二维 contiguous float32 输入 `[rows, features]` 与一维 float32 `weight`，沿 feature 轴计算 `x / sqrt(mean(x²) + eps) * weight`。这一版本明确采用真实 affine 语义，权重不是全一。二维的 axis=1 与末轴一致；不能直接将旧 CPU 四维 axis=1、无 affine 任务送给按末维归一化的融合算子。 -## 计划 +基线由在 NPU 上执行的 PyTorch 基础运算组成；候选验证 `torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0]`。接口为 `compute(x, weight, *, eps)`。具体形状、epsilon、容差与变化输入协议在最终 manifest 中冻结；不从 CPU 结果推导设备支持范围。融合算子是可检验的参考候选，不要求所有合法实现都调用它。 -- 从成熟算子题目选择两个小任务，核对源码语义、许可和形状。 -- 公开 CPU 子集的 dtype、轴、形状、容差、布局和异常输入范围。 -- baseline 和候选可独立替换，oracle 使用不同实现路径。 -- 覆盖普通输入、数值边界、非连续布局、错误轴、NaN、输入修改等反例。 -- 实际执行正确性、计时和 CPU profiling，保存每个工作负载的样本。 +独立参考计算在 CPU 使用 scalar double / `math.fsum` 做归约，再乘权重并转换为 float32。输入与权重准备、传到设备、oracle、复制回 host 检查均在性能计时区间之外。正确性包括输出形状、dtype、设备、有限性、数值误差和输入/权重内容不被改变。普通输入之外需覆盖全零、小值及不同形状，防止省略 epsilon、忽略权重或硬编码结果通过。 -## 验收 +## 测量与诊断 -正确实现能通过，常见错误能被识别；测量执行相同候选函数并核对输出；不要求某个实现必须加速。GPU/NPU、低精度、通信和容量边界保持未验证。 +- 输入在调用前准备并绑定摘要；变化输入避免前次正确输出被重放成“优化”。不得将缓存后的答案当作实际计算时间。 +- NPU 是异步设备；计时边界必须明确等待调用前后的设备工作完成。仅统计 Python 发起调用的时间不能作为 NPU 完成延迟。 +- 预热、首次编译或初始化、输入准备、oracle 与输出校验不计入正式样本；具体次数及策略来自冻结任务与实验计划。 +- profiling 独立于正式计时，保留 CPU/NPU 活动与 trace 身份。若设备 trace 不可用，明确缺口，不能用 CPU cProfile 替代。 +- 运行同码 A/A、错误但快的候选、基础算子与融合候选；保留全部有效和失败记录，避免只报告最快样本。 -## 已实施 +完整流程、控制组和证据清单见 [NPU 试点](npu-pilot.md)。 -任务位于 `src/inferrsi/data/engineering_benchmarks/softmax-cpu` 和 `rmsnorm-cpu`。每包包括 task.json、baseline.py、candidate.py、README.md、LICENSE。原始 KernelBench 来源固定到完整 commit，MIT 许可随包分发。 - -| 任务 | 算什么 | 为什么选它 | -| --- | --- | --- | -| Softmax | 沿 axis=1 的稳定指数归一化 | exp 溢出、错误轴、分母和形状都可能出错 | -| RMSNorm | x / sqrt(mean(x², axis=1) + eps)，无 affine 参数 | 四维且各轴长度不同，能识别把 feature 轴写成末轴的常见错误 | +## 验收 -每题六个公开 workload，包括普通随机、非连续布局、全零、大数、小数和单元素归约轴。普通 Softmax 为 16×257；普通 RMSNorm 为 4×17×9×7。这是刻意缩小的 CPU 开发集，不是原题性能结果。 +任务能在明确设备与运行时上重现；基线和合法候选计算正确；错误候选被拒绝且没有可用性能分数；真实 NPU profile 与同步计时进入 Workbench 证据。是否有可分辨加速另行判断，不能预先承诺融合必快。 -NumPy 基线与使用临时 buffer 的示例候选都由本次实现提供。参考计算使用标量 double、math.exp/math.sqrt/math.fsum，独立于 NumPy 归约。每题检查形状/dtype、有限输出、输入保持、数值误差四类断言。 +这只是算子级 NPU 验收，不证明模型质量、服务吞吐、其他 dtype/形状、通信或显存容量。正式 SkillEval NPU 候选隔离仍需另外设计和验证，不能把可信 Workbench 源码执行包装为该能力。 -当前 v2 计时使用 perf_counter_ns，仅包围同步 compute 调用。任务明确五个计时 case，各预热 2 次、保留 5 次计时；全零 case 保留为正确性检查，不进入性能分数。单个 trial 指标为五个 case 中位数的和，所有原始样本与对应输入哈希均保留。所有调用的变化输入和 oracle 在执行前生成并保持存活；预检、预热和计时的输入字节不重复，双方使用相同的固定序列。这些是任务版本的测量协议，非全仓库性能门槛。外部 trial 次数和预算另行冻结。 +## 保留的 CPU 工作 -变异测试覆盖全零错误答案、NaN、dtype 错误、shape 错误、不稳定 exp、修改输入、错误轴，以及前六次正确之后变错。失败候选不输出可用性能指标；中途失败保留失败调用和已有样本。timeout/缺依赖与完整数值拒绝分别记录。 +`softmax-cpu` 和 `rmsnorm-cpu` 仍是软件控制与历史任务，保留原 KernelBench 来源和 MIT 许可。它们的 NumPy 基线、标量 double / math.fsum oracle、变化输入和错误反例可以检验共享软件接口；不向 NPU 迁移其计时结果或环境资格。 -新增缓存重放回归：按 shape/layout 返回旧输出会在变化输入上被拒绝；按 id/完整内容 memoize 的正确候选可以正常计算，但验证其在本次序列没有缓存命中，不能从预检借用答案。公开序列仍不是隐藏测试或恶意代码隔离。 +旧 `numpy-axis1-v2` 每题一次完整 trial 为 41 次调用、164 项断言、25 个计时样本；CPU profile 仅解释 CPU 调用。v1 的 360 个样本与 v2 的 300 个样本保留各自 task/grader 身份，[旧协议验证](replay-validation.md)和[用途对照](usefulness.md)中的噪声、失败与限制不改写。 -Profiling 实际运行同一候选，生成可读 CPU 调用表，完全独立于正式计时。它能帮助检查 Python 调用开销；它不能回答 GPU/NPU 空洞或设备 kernel 利用率。 +[五类扩展方案](task-expansion/README.md)仍保留后续题源。当前优先跑通 NPU RMSNorm；Tokenizers、KV 复用、通信和服务任务依次独立评估，不因本任务完成而标记整类已覆盖。 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index d10b59bd8..30eb47707 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -2,13 +2,15 @@ **这项工作的交付是调用链和实际证据：让现有 Workbench 执行任务，让现有 verifier 按同一题目检查实现，并明确这些结果支持哪些决定。** +本轮以 [NPU RMSNorm](npu-pilot.md) 为试点验收对象：真实 NPU 基线、profiling、候选、独立检查、A/A 与性能比较必须在同一条证据链中。已有 CPU Workbench 与正式 SkillEval 结果保持历史身份；正式 SkillEval NPU 产物隔离尚未建立，不能因设备任务能在 Workbench 运行就宣称该能力完成。 + | 要说清楚的问题 | 本子项的答案 | | --- | --- | | 具体做什么 | 把任务操作绑定到既有执行、比较、验证和报告入口，实际运行正确与错误候选 | -| 交付什么 | Workbench RunSpec 绑定、SkillEval 低层 verifier、正式两臂产物 adapter、复现工具和结果记录 | +| 交付什么 | NPU Workbench RunSpec 绑定、真实设备演练和结果记录；保留已有 CPU 正式产物路径 | | 谁使用 | 主 Agent 的引擎优化流程、评测开发者、PR 评审者 | | 实际作用 | 工作台能够执行比较并保留证据；错误实现被共同裁判拒绝；结果可供 RSI 决定保留、放弃或继续诊断 | -| 当前状态 | Workbench、底层 verifier、正式 SkillEval 正确性产物交接已实现；真实技能收益与可靠性能排序未完成 | +| 当前状态 | CPU 接入已有；NPU Workbench 演练与性能资格为当前实施目标，设备结果待单独记录；真实技能收益未测 | 例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 @@ -27,13 +29,13 @@ | 步骤 | 谁做、交出什么 | 当前实现与待补项 | | --- | --- | --- | | 1. 明确目标和验收依据 | 主 Agent 根据目标项目确定输入、环境、指标、质量约束及预算；适用时复用内置题包 | 内置题可加载；已有通用 Workbench 可接外部命令。尚无自动从用户目标生成合格任务的能力 | -| 2. 准备比较条件 | 调用者在搜索前固定基线、验证规则和工作负载；每个候选有独立身份 | 题包内容身份、执行快照与 SkillEval prepare 冻结已实现。普通 `plan` 仅预览，不等于保存了冻结的 campaign | -| 3. 运行基线并 profiling | 主 Agent 调用对应环境的工具，保留诊断并说明要改什么、依据是什么 | 有独立 CPU `profile` 操作及历史产物；不会自动完成目标设备诊断，也没有强制 Agent 先诊断再修改 | -| 4. 提交并检查实现 | Agent 修改允许范围；独立工具实际运行并检查产物 | 内置题 worker 和正式产物 verifier 已实现；正式 SkillEval 候选隔离执行，可信 Workbench 源码仍需审核 | -| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 已有 AB/BA、源快照和报告；可靠性能排序尚未建立。正式 SkillEval adapter 当前只判正确性 | +| 2. 准备比较条件 | 调用者在搜索前固定基线、验证规则和工作负载；每个候选有独立身份 | 复用题包身份和执行快照，NPU 需记录设备/运行时。普通 `plan` 仅预览，不等于保存了冻结的 campaign | +| 3. 运行基线并 profiling | 主 Agent 调用对应环境工具，保留诊断并说明改法依据 | 本轮需要真实 CPU/NPU profile；CPU 旧产物不替代。没有强制 Agent 先诊断再修改 | +| 4. 提交并检查实现 | Agent 修改允许范围；工具实际运行并独立检查产物 | NPU 先执行已审核可信源码；正式 SkillEval 隔离路径仍限受支持的 CPU 任务 | +| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 复用 AB/BA、快照和报告；本轮需验证 NPU 同步计时及 A/A，可靠性能资格待实测 | | 6. 使用结果 | 主 Agent 引用证据解释保留、放弃或继续诊断；下一候选重新标识 | 现有报告可供使用，已有部分用途对照；未完成真实 Agent 全链路演练，没有程序化跨轮次搜索/停止控制 | -用户目标不在题库时，第一步使用目标项目自己的 benchmark 和测试，通过现有 `inferrsi run` 的 baseline/candidate/validation 命令接入 Workbench，不需要先扩充题库。缺少质量检查或可信测量时，应补齐对应实验条件；通用 runner 无法判断测试是否充分代表真实业务。当前 CPU `TaskPackage` 和正式内置题 adapter 也不能直接承担任意模型或服务的适配。 +用户目标不在题库时，第一步使用目标项目自己的 benchmark 和测试，通过现有 `inferrsi run` 的 baseline/candidate/validation 命令接入 Workbench，不需要先扩充题库。缺少质量检查或可信测量时，应补齐对应实验条件；通用 runner 无法判断测试是否充分代表真实业务。当前具体任务 adapter 不能直接承担任意模型或服务的适配。 ### 流程怎样约束结果，而不是只提醒 Agent @@ -41,27 +43,27 @@ 尚无程序强制宿主 Agent 按“profile → 假设 → 修改”顺序行动，也没有完整优化轮次的资格检查。后续若增加流程约束，应在既有入口校验证据和声明的完成条件：缺少该实验要求的诊断或验收，不得记录成完整优化成功。不能仅在提示词里写了顺序，就声称所有 Agent 都会遵守。 -本子项后续验证需要保留一条可复查的演练链：诊断依据、候选修改、独立检查、比较结果、主 Agent 的决定。至少证明错误候选被拒绝、证据不足不会被当作改进，以及最终结论只覆盖实际验收的目标。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 +本子项当前 NPU 验证需要保留一条可复查的演练链：诊断依据、候选修改、独立检查、比较结果、主 Agent 的决定。至少证明错误候选被拒绝、证据不足不会被当作改进，以及最终结论只覆盖实际验收的目标。如果融合候选是预先选定的 authored control，应如实记录，不能写成 Agent 从 profile 自主发现。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 ## 理解 只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 -[RSI 接入审计](rsi-integration.md)区分早期的 `run_trial/verifier` 低层接线与后续[正式两臂接入](formal-integration.md)：新 `engineering-benchmark-v1` 复用 `skill_eval_compare` 的冻结、产物留存与配对报告，使用共享 oracle 在隔离进程外检查候选实际输出。`skill_eval_improve` 仍不支持这些公开 development 任务；系统收益仍需要真实受控 Agent 实验。 +[RSI 接入审计](rsi-integration.md)区分早期的 `run_trial/verifier` 低层接线与后续[正式 CPU 两臂接入](formal-integration.md)：`engineering-benchmark-v1` 复用 `skill_eval_compare` 的冻结、产物留存与配对报告，使用共享 oracle 在隔离进程外检查候选实际输出。该 CPU 边界不自动支持 NPU；`skill_eval_improve` 仍不支持这些公开 development 任务，系统收益仍需要真实受控 Agent 实验。 ## 计划 - Workbench 接入复用 run_experiment、AB/BA、超时、源快照、结果和报告。 - 离线工程评估接入复用现有 SkillEval 的独立 checks/verdict 消费契约，明确本地 authored control 与真实 agent 输出的区别。 -- 用正确、错误、改变冻结文件三种情况验证两端一致性。 -- 用实际工作负载保留原始结果和 CPU profile，检查来自真实执行而非预制分数。 +- 用 NPU 正确、错误、同码及融合候选验证实际运算、裁判与测量资格；改变冻结文件时拒绝沿用原比较。 +- 保留真实 CPU/NPU profile 与原始结果，检查来自设备执行而非预制分数。 - 验证非 editable wheel 安装，以及现有相关回归；最终完整测试结果如实记录。 ## 验收 -两个消费者共享 task/grader 身份，正确性结论一致；错误候选不能获得可用性能证据；缺失 profiling 不冒充已有诊断。付费模型、生产 SkillEval 执行授权和设备资格不由本次接入改变。 +当前 NPU 验收以真实 Workbench 演练为边界：错误候选没有可用性能证据，缺失 device profiling 不冒充已有诊断，同码噪声不自动成为改进。用户授权了本轮 NPU 执行；这不代表付费模型、正式 SkillEval NPU 隔离或真实 Agent 收益已验证。既有 CPU 两消费者的一致性仍按原软件记录保留。 -## 已实施 +## 已有 CPU 软件接入与历史证据 - `inferrsi benchmarks plan/compare` 生成并执行真正的既有 RunSpec；不改原 runner 的统计或阶段规则。 - `grade_artifact` 将独立正确性结果转成既有 SkillEval checks 结构。 @@ -72,6 +74,6 @@ 消费链为“同一题目与 oracle → Workbench 性能比较 / SkillEval 独立正确性验证”。Workbench 和早期离线 verifier 执行已审核可信源码；正式两臂 adapter 接收真实产物，复用既有 bubblewrap 边界，仅让候选计算在子进程运行，参考答案留在父进程。它隔离产物 replay，不自动认证或限制生产者 Agent 的全部权限。是否由技能帮助产出更好代码，需要真实 Agent、匹配预算和对照条件的实验。 -运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，当前协议的实际结果见 [v2 验证](replay-validation.md)。早期安装包检查和 v1 集成结果另存于[历史验证记录](validation.md)。 +CPU 运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，已有 CPU 协议结果见 [v2 验证](replay-validation.md)。早期安装包检查和 v1 集成结果另存于[历史验证记录](validation.md)。当前 NPU 演练的接口、步骤和验收矩阵见 [NPU 试点](npu-pilot.md)。 补充[实际工程用途对照](usefulness.md)把预先固定的 no-epsilon、einsum 和 unchanged 实现接入同一任务，证明普通冒烟漏检会被完整任务拒绝。它也暴露了计时稳定性限制：同一代码的比较出现大幅差异，可靠性能排序仍待后续协议验证，不能作为已完成的验收项。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index d8149eed3..645e3921a 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -1,67 +1,57 @@ -# #252：给 InferRSI 提供共享优化任务和独立评测 +# #252：在 NPU 上跑通共享优化任务和独立评测 -**我们要交付两部分：准备好的工程优化任务，以及这些任务从交付实现到检查、诊断、测量和报告的可运行流程。主 Agent 使用结果继续优化；SkillEval 使用相同任务比较不同 skill 条件下的产物。** +**当前交付目标是：让 Agent 面对一个真实 NPU 优化任务，交付实际实现，由 InferRSI 检查正确性、保留 profiling、在固定条件下比较性能，并据此决定是否保留候选。** 用户已明确将试点指定为 NPU；已有 CPU 两题保留为历史验证与软件控制，不作为这轮试点完成依据。 -流程也是交付内容。只有题目目录，或者几个可以分别调用的命令，都不足以证明结果已在 RSI 中发挥作用。第四个子任务需要说明谁准备实验、谁运行 profiling、谁修改代码、谁验收，以及结果怎样支持下一步，并保留实际演练证据。当前已接通单次 Workbench 比较和正式 SkillEval 正确性产物交接；完整优化演练、可靠性能排序和真实 Agent/skill 收益仍未完成。详见[流程范围与缺口](04-integration.md#流程也是交付内容)。 +从 [NPU 试点方案](npu-pilot.md)开始。首个任务是 `rmsnorm-npu`：在 Ascend NPU 上比较 PyTorch 基础算子组成的 affine RMSNorm 与 `torch_npu.npu_rms_norm` 候选。融合实现是否更快，要由实际结果决定。NPU 接入、设备执行与性能资格分别验收，不能用设计或 CPU 测试代替。 -题库不是用户需求的白名单。用户任务不在内置题库时，主 Agent 可以将目标项目自己的 benchmark 和质量检查接到**已有的通用 Workbench**；本次两道 CPU 题不参与也可以运行。该通用入口不是本次新增功能，也不会自动替用户定义验收标准。当前 `TaskPackage` 仍限于明确支持的 CPU 算子，不能称为任意用户任务适配器。 - -本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。它的输入是选定的任务包和候选实现，输出是正确性结果、独立 profile、原始计时及对应版本。名称是本次交付提议，正式范围仍待维护者接受。 - -**第一次阅读请从[完整导读](walkthrough.md)开始**：沿 RMSNorm 例子解释四个子任务、每个流程框、profiling、实际命令和结果。想核对本轮发现及修复的问题，请读[审查记录](review.md)。 - -**多类任务的后续选择见[开源调研与五类任务方案](task-expansion/README.md)**：除了算子，还比较真实仓库 CPU 热路径、内存与缓存、通信、服务配置/调度。下一项新增任务优先考虑 GSO 的 Tokenizers.encode；内存类先复用仓内已有 KV 回收任务。该文及五份任务卡是扩展设计，当前可执行交付仍以各入口已声明的范围为准。 +本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。输入是任务包和候选实现，输出是正确性结果、独立 profile、原始计时及版本身份。它服务主 Agent 的优化工作，也为后续 SkillEval 比较不同 skill 条件下的产物提供任务。名称与范围是本次交付提议，维护者验收另行记录。 ## 实际交付给使用者什么 -以当前 `rmsnorm-cpu` 为例： +以 NPU RMSNorm 试点为例： -- **任务是什么：** 在规定的 CPU float32 输入上计算沿 axis=1 的 RMSNorm，保留 epsilon 和输入内容。 -- **Agent 可以改什么：** 独立 `implementation.py` 中的 `compute(x, *, eps)` 实现。 -- **Agent 交付什么：** 一份候选源码；输入、oracle、容差和计时规则来自受保护的任务包。 -- **谁检查：** `check` 使用独立参考答案验证；`profile` 生成单独诊断；`compare` 交给现有 Workbench 比较原版与候选版。 -- **结果如何发挥作用：** 错误实现不能获得可用性能分数；主 Agent 根据正确实现的计时和完整性证据判断是否保留。若计时无法分辨，继续记录“证据不足”，不能宣布优化成功。 +- **任务：** 在明确的 NPU、dtype、输入与容差下计算 affine RMSNorm，保留 epsilon、权重和输入内容。 +- **Agent 修改：** `implementation.py` 的 `compute(x, weight, *, eps)`；不修改输入、oracle、容差或计时器。 +- **基线与候选：** 相同 NPU 输入上的基础算子组合与融合算子调用；独立 CPU double 参考结果负责数值检查，不计入设备性能。 +- **诊断与测量：** 独立记录 CPU/NPU profile；计时必须等待 NPU 运算完成，并排除输入准备、传输和正确性检查。 +- **结果用途：** 错误实现不能获得可用性能分数。同码对照与真实候选比较支持“保留、放弃或证据不足”的决定；测不清就不宣布优化成功。 -一次真实对照中，省略 epsilon 的实现通过了普通输入冒烟，却被完整任务和 SkillEval 底层 verifier 一致拒绝。这证明了错误识别用途。该对照同时暴露计时噪声；可靠性能排序仍未建立，见[用途验证](usefulness.md)。之后还修复了重复输入导致缓存重放可被计时的问题，见 [v2 验证](replay-validation.md)。 +流程也是交付内容。只有题目目录或独立命令不够；第四项需要保留基线、profiling、改法依据、候选、验收结果和最终决定的一条实际演练链。见[流程范围与缺口](04-integration.md#流程也是交付内容)。 ## 四个子任务分别做什么 -**这四项是交付步骤，不是四个运行时模块。** 第一项决定做什么，第二项规定规则，第三项交付可执行任务，第四项接入已有系统。 +这四项是交付分工，不是四个运行时模块；一个完整 NPU 试点会同时使用它们。 -| 子任务 | 要交付的具体东西 | 谁使用、实际发挥什么作用 | 当前状态 | +| 子任务 | 具体交付 | 实际作用 | 当前状态 | | --- | --- | --- | --- | -| **① 选定第一批任务和复用方案** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [详细说明](01-research.md) | 论文/开源对照、任务选择、复用和适配决定 | 开发者据此确定先做哪几题、复用哪些设计，以及当前硬件范围 | 调研与方案已完成，范围待维护者接受 | -| **② 规定任务怎样提交、运行和判定** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [详细说明](02-contract.md) | 统一任务格式、候选入口、check/profile/measure 接口和版本记录 | Workbench 与评估工具读取同一题目和裁判；题目或源码变更后不能继续当作原比较 | 已实现共享接口和内容冻结 | -| **③ 把第一批任务做成可执行的包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [详细说明](03-tasks.md) | Softmax、RMSNorm CPU 任务；每题带输入、基线、独立参考答案和测量规则 | Agent 有明确修改对象，工具有真实可运行的正确性、诊断和测量任务 | 已实现并实测；可靠性能排序未建立 | -| **④ 接入现有 RSI，并验证结果能被使用** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [详细说明](04-integration.md) | Workbench 绑定、正式两臂产物 adapter、复现入口和证据记录 | 现有实验流程执行任务；两臂分别交付实现，由同一裁判检查并保留证据 | Workbench、底层 verifier、正式 SkillEval 正确性产物入口已实现；真实技能收益和性能排序仍未完成 | +| **① 选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [说明](01-research.md) | 论文/源码比较、NPU 选型、来源与边界 | 决定优化什么、复用哪些接口和评测设计 | 既有研究保留；本轮 NPU 方案见试点文档 | +| **② 统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [说明](02-contract.md) | 任务、候选、独立 oracle、NPU 同步计时、profile 与版本身份 | 固定一次比较中的工作和裁判，拒绝 CPU 回退或错误设备结果 | CPU 接口已有；NPU 适配与实测需单独验收 | +| **③ 交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [说明](03-tasks.md) | `rmsnorm-npu` 基线、候选、输入、数值检查、测量规则 | Agent 有明确修改对象，工具能在真实 NPU 执行它 | 当前交付目标；设备运行结果按试点记录更新 | +| **④ 接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [说明](04-integration.md) | NPU Workbench 演练、正确/错误控制、A/A 与候选比较、报告 | 证明真实产物进入执行，结果能支持下一步 | NPU 演练待实测记录；真实 skill 收益未测 | -## 放在我们 RSI 的哪里 +## 在 RSI 中的位置 -同一套任务服务两种不同的比较： - -| RSI 要回答的问题 | 现有调用者 | #252 的作用 | +| RSI 要回答的问题 | 调用者 | 本次作用与边界 | | --- | --- | --- | -| 这个引擎实现是否正确、有可验证的改进？ | 主 Agent + Workbench | 提供共同工作负载、独立正确性检查和测量规则 | -| 新 skill/知识是否帮助 Agent 产出更好的实现？ | SkillEval 两臂比较；后续衔接已有改进流程 | 两边产出的实现使用同一套任务和裁判，不随技能条件更换答案 | - -第一条路径已有实际执行。第二条现已增加 `skill_eval_compare` 的正式产物交接：两臂分别交付 `implementation.py`，候选在隔离进程运行，父进程以共享 oracle 检查，报告重验留存源码。接入方法及平台验证边界见[正式接入说明](formal-integration.md)；尚未比较旧/新技能对真实 Agent 交付的影响。完整调用关系和分层验收见 [RSI 接入审计](rsi-integration.md)。 +| 这个实现是否正确、是否值得保留？ | 主 Agent + Workbench | 本轮优先在真实 NPU 跑通任务检查、profiling 和比较 | +| 新 skill/知识是否帮助 Agent 写出更好的实现？ | SkillEval 两臂比较 | 当前正式产物 adapter 仅支持已声明的 CPU 任务；NPU 产物隔离与真实 Agent 对照没有完成 | -当前由主 Agent 决定下一步尝试什么；任务包决定要算什么、怎样检查和测量。Optimization Search Policy 的提案讨论如何选择下一次尝试，本次没有实现该策略。Profiling 是独立诊断，不冒充速度比较。#252 提供可调用的评测操作，不负责强制主 Agent 遵守整个流程或另建搜索控制器。 +当前正式 `engineering-benchmark-v1` adapter 的 Linux bubblewrap 控制只能作为 CPU 软件边界证据，不能直接挂载设备就声称 NPU 候选已安全隔离。NPU 试点先运行已审核的可信源码；接入方法和既有 CPU 证据见[正式接入说明](formal-integration.md)。不以本次 Workbench NPU 演练冒充正式 SkillEval NPU 支持。 -## 参考 AMD 项目时具体学什么 +题库不是用户需求的白名单。题库外目标可以将自己的 benchmark 和质量检查接入**已有通用 Workbench**；它不会自动生成验收标准。当前任务 adapter 只支持明确声明的语义和环境，不等于任意模型或服务适配器。 -之前调研的 AMD 项目是 [AgentKernelArena](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/README.md)。它明确描述：给 Agent GPU 优化任务，在工作区产出实现，由独立 evaluator 检查、计时并产生结果；外部系统负责训练或策略更新。 +主 Agent 决定下一次尝试；任务工具检查与测量。Optimization Search Policy 讨论如何选择改法、分配预算和停止，本次没有实现该控制器，也没有程序强制所有 Agent 先 profile 再修改。 -本次借鉴它的[任务契约](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md)：明确计算语义、可编辑实现、性能基线、正确性参考、工作负载和评测操作。我们把这层能力接到已有 InferRSI；当前交付范围是共享任务及其接口，不等同于 Arena 完整的 GPU/Agent 执行平台。 +## 后续类别与历史证据 -## 怎么开始使用和检查结果 +[五类任务方案](task-expansion/README.md)保留算子、CPU 热路径、内存与缓存、通信、服务配置/调度的研究。**次序现调整为先完成 NPU RMSNorm 试点。** Tokenizers.encode 保留为后续 CPU 仓库补丁候选；KV 保留既有任务复用方向。NCCL/MSCCL 等 CUDA 题源不能直接作为 Ascend 通信实现，需另选 NPU 对应机制和资格条件。 -运行 `inferrsi benchmarks list`、`show rmsnorm-cpu`，将示例候选复制成独立 `implementation.py`，再按[使用文档](../../engineering-benchmarks.md)执行 `check`、`profile` 和 `compare`。共享接口优先，硬件尚未确定，所以目前只支持明确的 CPU 子集。 +既有 CPU Softmax/RMSNorm 的错误识别、输出重放修复、计时噪声和失败记录不改写：[v1 集成](validation.md)、[v1 用途](usefulness.md)、[v2 协议](replay-validation.md)。正式 CPU 产物软件控制见[既有 CI 记录](formal-integration.md#已执行的软件验证)。这些记录不重标为 NPU 执行、可靠速度排序或真实 skill 收益。 -正式两臂入口的当前软件验证见[接入与实际 CI 记录](formal-integration.md#已执行的软件验证)：49 项通过、无跳过，包含 5 个真实 Linux sandbox 案例。原 Workbench 测量证据见 [v2 协议验证](replay-validation.md)；历史证据见 [v1 集成记录](validation.md) 和 [v1 用途对照](usefulness.md)。本地测试、组件实现、正式系统接入与维护者验收分别记录。GPU/NPU、通信/容量任务、真实 agent/skill/search 收益和可靠性能排序均不能由当前 CPU 结果代替。 +旧的 [CPU 完整导读](walkthrough.md)解释已实现软件接口；当前交付与启动方式以 [NPU 试点](npu-pilot.md)及本轮实际结果为准。 ## 跟踪与范围 -实现及文档在 [Draft PR #338](https://github.com/InferMatrixLab/InferRSI/pull/338)，四个子任务保持打开。原生父子关系写入受账号权限限制，目前通过相互引用和父任务索引追踪。 +实现及文档在 [Draft PR #338](https://github.com/InferMatrixLab/InferRSI/pull/338)，四个子任务保持打开。原生父子关系写入受账号权限限制，目前通过相互引用和父任务索引追踪。用户的 NPU 试点委托授权了本轮实施、运行和发布；它不代替维护者对整个 #252 的接受或关闭决定。 -初始基线为 `663f3b327998526a13aa31e72ad729025c18983c`，工作分支为 `codex/issue252-engineering-benchmarks`。本地实施与提交依据为用户的明确委托；这不等价于维护者已接受整个 #252。父 Issue 的 [Search Policy RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306) 按[维护者范围意见](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922)保持独立。 +父 Issue 的 [Search Policy RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306)按[维护者范围意见](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922)保持独立。 diff --git a/docs/plans/issue-252/npu-pilot.md b/docs/plans/issue-252/npu-pilot.md new file mode 100644 index 000000000..c4865f9fc --- /dev/null +++ b/docs/plans/issue-252/npu-pilot.md @@ -0,0 +1,86 @@ +# NPU 试点：用 RMSNorm 跑通一次真实优化比较 + +2026-11-04，用户将试点明确指定为 NPU。本页定义当前实施与验收目标；下面的执行状态在取得实际日志、退出码和产物前保持待验证。已有 CPU 测试和样本仅作历史软件证据。 + +## 这次究竟要跑通什么 + +给 Agent 一个具体问题：**在 Ascend NPU 上保持 affine RMSNorm 计算结果，检查基础算子组合能否由更合适的实现替代。** Agent 交付真实源码；InferRSI 负责独立检查、诊断和测量，再用证据决定是否保留。首个参考候选采用 `torch_npu.npu_rms_norm`，是否更快由实际实验判断。 + +```mermaid +flowchart LR + A["固定题目、环境和验收"] --> B["运行 NPU 基线并 profiling"] + B --> C["记录改法依据并交付候选"] + C --> D["独立正确性和错误控制"] + D --> E["同码对照与 NPU 同步计时"] + E --> F["Workbench 报告与保留决定"] +``` + +这些步骤由主 Agent 调用已有接口完成；本次没有新增跨轮次搜索控制器，也没有通过图示保证任何 Agent 都自动遵守顺序。留存的诊断、代码与决定才是演练证据。 + +## 为什么选择 RMSNorm + +RMSNorm 是推理中明确的归约与逐元素计算。基础运算组合和 NPU 融合算子提供两条可实际比较的实现路径，且遗漏 epsilon、忽略权重、错误 dtype 或旧结果重放都有可构造的反例。相比先接入整个服务，它能在一个受限问题上验证设备执行、数值裁判、同步计时和 profiler 是否真正接通。 + +已有 [KernelBench 语义与评测研究](01-research.md)提供任务/基线/裁判分离的来源；[AgentKernelArena task contract](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md)提供任务责任划分。它们的 GPU/CPU 结果不能证明 Ascend 资格；本轮任务采用独立的 NPU affine 语义，torch_npu API、运行时兼容性和设备行为必须以源码核对与真实执行补证。 + +| NPU 一手来源 | 采用与验证边界 | +| --- | --- | +| [op-plugin RmsNormKernelOpApi.cpp](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) 与 [对应测试](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py) | 固定 `9f23f00`，核对 affine 权重、epsilon 和输出接口；[BSD-3-Clause 许可](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/LICENSE)。实际安装版本的支持范围仍需本轮执行验证 | +| [Ascend PyTorch profiler tests](https://github.com/Ascend/pytorch/blob/9065e614f02639f5ddde0cf370bb9d38de7bd8a9/test/profiler/test_npu_profiler.py) | 固定 `9065e61`，核对 NPU profiler 使用方式；trace 成功和实际设备活动须检查最终产物，不能只凭 API 存在 | + +本次使用原生算子 API 和自编任务适配，不搬入上游整套执行器。源码参考、许可、版本与实测分开记录。 + +## 任务接口与冻结内容 + +| 项目 | 本轮约定 | +| --- | --- | +| 任务 | `rmsnorm-npu`；单个实现文件 `implementation.py` | +| 候选入口 | `compute(x, weight, *, eps)`，输入输出是目标 NPU 上的 tensor | +| 数值语义 | 二维 contiguous float32 `[rows, features]`；`x / sqrt(mean(x²) + eps) * weight`，非全一的一维 float32 权重 | +| baseline | 在 NPU 上执行的 PyTorch 基础运算组合 | +| 参考候选 | `torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0]`；取规范化输出而非辅助结果 | +| oracle | CPU scalar double / `math.fsum` 归约后乘权重并转 float32；不复用设备归约或将 baseline 当唯一真值 | +| 环境 | 冻结实际设备型号、torch / torch_npu / CANN 版本、设备选择与源码身份；缺少 NPU 时明确失败，不回退 CPU | +| 输入范围 | 形状、分布、epsilon、容差与变化输入序列写入 manifest；仅宣称实际检查范围 | +| 修改权限 | 候选源码可改；输入、oracle、容差、计时规则和报告不可由候选改写 | +| 性能边界 | 输入/权重和 oracle 在计时外准备；使用声明的设备同步边界统计完成调用；预热/初始化与正式样本分开 | +| profiler | 单独运行并保存 CPU/NPU trace 或表格、配置与文件摘要，不把 profiler 开销混入性能分数 | + +共享接口保持 `check / profile / measure / plan / compare` 的分工；NPU adapter 负责自身语义、依赖、设备检查、同步与运行时身份。不能只把 CPU manifest 的 backend 改名就称为设备适配。 + +普通 `plan` 只是命令/RunSpec 预览。正式实验仍要在搜索前保存任务、基线、候选、oracle、环境与验收条件；重复次数、时间预算及判断门槛放在本次实验计划与 acceptance issue，不能写成仓库通用常量。 + +## 实际执行顺序 + +1. **确认环境。** 在已有 NPU 容器内核对个人身份、设备映射和共享调度器。所有 torch_npu 导入、设备诊断、测试与基准在有效设备预约内执行；不改共享安装，不切换正在被其他任务使用的源码。 +2. **固定本轮输入与实现。** 保留源码、task/grader、运行时及 manifest 身份；记录工作目录和实际导入位置，避免只改变目录却导入其他 checkout。 +3. **运行基线与独立 profiling。** 先确认数值正确；保留真实 NPU 活动，记录观察到的算子、调用或同步行为。没有 device trace 时不得声称已诊断设备瓶颈。 +4. **记录假设并运行候选。** 将观察与候选改动关联；如果融合候选来自预先选定的参考实现，明确是 authored control，不能声称 Agent 从 profile 自主发现它。 +5. **执行正确性和错误控制。** 检查合法候选、全零错误、遗漏 epsilon、忽略权重等语义错误；失败不得获得可用性能分数。实际被计时的输出也要核对。 +6. **执行 A/A 和候选比较。** 相同设备与固定条件下保留逐次样本；使用现有 Workbench 组织 AB/BA。诊断、H2D/D2H、oracle 和初始化开销单独记录。 +7. **记录决定。** 引用数值、trace 和原始样本，说明保留、放弃或证据不足。信号无法区分、运行不完整或环境漂移时，不得写成优化成功。 + +## 验收矩阵与待补证据 + +| 要回答的问题 | 必须执行的控制/记录 | 本轮状态 | +| --- | --- | --- | +| 是否确实在 NPU 运算？ | 输入/输出设备、实际运行时、设备预约、真实设备活动、退出码 | 待执行并记录 | +| 是否算对？ | baseline 与参考候选对独立 oracle；声明范围内的边界输入、输入与权重保持 | 待执行并记录 | +| 错误但快是否会被拒绝？ | 全零、遗漏 epsilon、忽略权重等候选；无可用性能指标 | 待执行并记录 | +| 计时是否覆盖完成的设备计算？ | 同步边界、变化输入、预热与正式样本、实际输出校验 | 待执行并记录 | +| 同码是否被误判成进步？ | A/A 或 no-op 对照，保留波动与原始 verdict | 待执行并记录 | +| 融合候选是否值得保留？ | composed baseline 对 fused 候选的 AB/BA、分工作负载样本和结论 | 待执行并记录 | +| profiling 是否发挥作用？ | 真实 profile、据此作出的具体判断及后续动作；预定控制与自主发现分开 | 待执行并记录 | +| 结果是否接入 RSI？ | Workbench 的 RunSpec、实际实现身份、检查/比较报告与最终决定 | 待执行并记录 | +| 正式 SkillEval 是否能隔离 NPU 产物？ | 设备访问、父子边界、可信 oracle、重验与攻击控制 | 本轮尚未建立，不能用 CPU sandbox 替代 | +| 是否提高了 Agent/skill 或服务能力？ | 受控 Agent 两臂或目标模型/服务实验 | 未测，不由本试点推导 | + +“有用”首先指可以识别错误、记录真实设备行为，并支持有根据的保留决定。算子完成延迟有改善，只能支持对应 NPU、dtype 和 workload 的结论；没有模型/服务验收就不声称端到端收益。 + +## 产物与边界 + +保留任务/源码/环境摘要、独立检查结果、全部计时样本、profile 文件及其摘要、Workbench 报告、日志和退出码。公开前移除私有主机、账号与个人路径，完整原始日志留在受控执行归档中；任何删减或转换说明来源和范围。 + +首轮运行已审核可信源码。现有正式 `engineering-benchmark-v1` 只支持 CPU 任务，Linux bubblewrap 控制没有验证 NPU 设备隔离；不得通过隐式宿主执行或 CPU fallback 绕过不支持状态。NPU 产物隔离需要后续单独验收。 + +旧 CPU 任务继续服务回归和历史记录。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，在本次 NPU 通路成立后按各自任务资格推进。四个 issue 和 Draft PR 保持原有追踪关系，NPU 演练完成不自动关闭整个 #252。 diff --git a/docs/plans/issue-252/task-expansion/01-operators.md b/docs/plans/issue-252/task-expansion/01-operators.md index 7d9b00886..dba2ebb77 100644 --- a/docs/plans/issue-252/task-expansion/01-operators.md +++ b/docs/plans/issue-252/task-expansion/01-operators.md @@ -1,20 +1,21 @@ -# 算子计算：保留已有试点，设备明确后扩展 +# 算子计算：当前在 NPU 上跑通 RMSNorm -**题目例子：在给定输入和容差下，让 RMSNorm 实现更快，同时保留计算结果。** +**当前题目：在 Ascend NPU 的给定输入、权重和容差下，比较 affine RMSNorm 的基础运算与融合实现，同时保留计算结果。** -状态：Softmax/RMSNorm CPU 两题已存在；本轮没有新增算子、执行测量或建立 GPU/NPU 资格。当前可靠性能排序仍未完成。原始实现与证据见[子项 03](../03-tasks.md)及[正式正确性接入](../formal-integration.md)。 +状态：用户已指定 NPU 试点，`rmsnorm-npu` 是当前实施目标；真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较以 [NPU 试点记录](../npu-pilot.md)为准，不能从方案推断完成。Softmax/RMSNorm CPU 两题保留为软件控制；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 -## 为什么保留，而不继续只加同类题 +## 为什么先在 NPU 做这个任务 -它们适合验证单文件候选、独立数值参考、输入变化与结果交接。增加更多 NumPy 数学函数会扩大数值样例，却不能自动覆盖 CPU 编码、状态回收、通信或服务调度。下一批开发优先补这些不同的工程问题。 +RMSNorm 有清晰的数值语义、基础运算基线和 torch_npu 融合候选，能在一个受限问题上检验真实设备执行、独立 oracle、同步计时和结果交接。先完成这条 NPU 通路，再扩展题类；更多 CPU 数学样例不能替代设备验收。 -在现有 SearchSpace 中关联 Operator 的 Optimize/Verify；CPU profile 只能提供本次 CPU 调用诊断。任务标签不证明 Agent 具有 Diagnose 能力，CPU 适配也不提升 Accelerator 资源层的实测成熟度。 +在现有 SearchSpace 中关联 Operator 的 Optimize/Verify。NPU profile 必须实际生成并解释；任务标签不证明 Agent 具有 Diagnose 能力。设备结果只提升该环境和工作负载的证据，不能自动提高整个 Accelerator 层的成熟度。 ## 固定来源与复用决定 | 来源 | 具体借鉴 | 适用边界 | | --- | --- | --- | -| [KernelBench Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py)、[RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) | 数学题意；`423217d`，MIT，已有任务包保留许可证 | 当前缩小为 CPU/NumPy/float32 开发子集，不是上游 GPU workload | +| [Ascend op-plugin RMSNorm](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) 与 [对应测试](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py) | 真实 affine 算子 API；`9f23f00`，BSD-3-Clause | 本轮先声明二维 contiguous float32，运行时支持与性能另做设备验证；不导入上游 runner | +| [KernelBench Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py)、[RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) | 既有 CPU 任务的数学题意；`423217d`，MIT，原包保留许可证 | 新 NPU 题单独定义二维 affine RMSNorm，不继承旧四维 axis=1、无 affine 或上游 GPU 资格 | | [AMD AgentKernelArena task v2](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) | semantics、candidate、baseline、reference、workload 与 evaluation 的分工 | 作为协议设计参考；不导入其 GPU/Agent 执行平台或声称当前 CPU 实现满足其设备资格 | | [BackendBench](https://github.com/meta-pytorch/BackendBench/blob/2a8d7e19e2c13c789ff2dceb44883ea8ea04dab4/README.md) | 边界输入/布局检查与模型来源 shape 的区分 | 作为扩展输入的设计参考；未复制 OpInfo/FACTO 或 TorchBench 运行体系 | @@ -22,15 +23,15 @@ ## Agent 做什么、工具检查什么 -- 当前改动面是 `implementation.py` 的 `compute(x, *, eps)`；不能修改 oracle、输入分布、容差或计时器。 -- 数值检查覆盖普通/极端值、不同 shape/layout、输入保持及多次变化输入；裁判与候选实现分开。 -- CPU profiling 单独运行，不将诊断开销计入优化分数。 -- Workbench 比较适用的已审核源码；正式 SkillEval adapter 隔离 replay，仅提供正确性结果。 +- NPU 改动面是 `implementation.py` 的 `compute(x, weight, *, eps)`；不能修改 oracle、输入分布、容差或计时器。 +- 首版明确二维 contiguous float32，输入/权重保持及变化输入需检查；独立 CPU double oracle 不复用设备归约。 +- CPU/NPU profiling 单独运行，不将诊断开销计入优化分数；缺少设备 trace 如实记录。 +- Workbench 比较已审核可信源码；正式 SkillEval adapter 当前只支持 CPU 隔离 replay，不接收 NPU 题。 ## 下一步资格实验 -先完成同码/no-op、错误但快的实现和参考优化对照，确认计时信号可区分；保留原始样本与版本，不把历史样本重绑到新 grader。 +在真实 NPU 完成同码/no-op、错误但快的实现和基础算子/融合候选对照，确认计时信号可区分；保留原始样本与版本，不把 CPU 历史样本重绑到新 grader。 -设备确定后，从实际模型 profile 选一个代表瓶颈，再决定是扩展 reduction、fusion、attention 还是其他算子。设备版需要独立定义同步、编译、warmup、dtype、数值检查和计时边界；不能只给 CPU manifest 换一个 backend 字符串。 +本轮先验证选定 NPU 任务的执行与测量协议。后续从实际模型 profile 选择代表瓶颈，再决定扩展 reduction、fusion、attention 或其他算子。每个设备任务独立定义同步、编译、warmup、dtype、数值检查和计时边界；不能只给 CPU manifest 换一个 backend 字符串。 该类任务的成功只支持相应算子/环境结论。模型或服务收益仍需对应工作负载的端到端验收。 diff --git a/docs/plans/issue-252/task-expansion/02-host-runtime.md b/docs/plans/issue-252/task-expansion/02-host-runtime.md index fe239876b..a5db83890 100644 --- a/docs/plans/issue-252/task-expansion/02-host-runtime.md +++ b/docs/plans/issue-252/task-expansion/02-host-runtime.md @@ -1,11 +1,11 @@ # 候选类别 02：CPU 侧数据处理与运行时开销 -状态：2026-11-04 来源检查与任务设计；尚未导入题包、构建环境或运行性能实验。 +状态：2026-11-04 来源检查与任务设计；尚未导入题包、构建环境或运行性能实验。用户已将当前试点指定为 NPU，先完成 [RMSNorm NPU 通路](../npu-pilot.md)；本页保留为后续 CPU 仓库补丁方向，不再是下一项必须交付的试点。 ## 要测什么 让 Agent 在真实仓库中发现并减少推理前后的 CPU 开销，同时保持输入输出语义。 -首题建议选择 **大量 added tokens 时的 Tokenizer.encode**，而非再增加一道数学算子。 +若后续开展这一类别，首题建议选择 **大量 added tokens 时的 Tokenizer.encode**。 它关联现有 SearchSpace 中的主机侧处理与运行时优化；具体矩阵归属仍需评审，不据此新增已验证覆盖。 这类题测试真实 repo 的分析和补丁能力；单题结果不代表服务 TTFT 或整个 RSI 能力提升。 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md index aa6b6d3de..6a2dec942 100644 --- a/docs/plans/issue-252/task-expansion/README.md +++ b/docs/plans/issue-252/task-expansion/README.md @@ -1,22 +1,22 @@ # 按优化方向扩展任务：开源调研与首批选择 -2026-11-04。本轮是官方源码、任务文件、许可证及论文的调研与设计交付；没有执行新增题目、调用模型或运行设备实验。以下五类是阅读分组，映射到已有 [SearchSpace](../../../searchspace-matrix.md)，不另建权威分类，也不把“找到来源”记成实测覆盖。 +2026-11-04。本页记录官方源码、任务文件、许可证及论文的调研设计，来源检查不等于执行结果。用户随后明确指定 NPU 试点，当前实施顺序已按下文调整；实际设备证据另见 [NPU 试点](../npu-pilot.md)。以下五类是阅读分组，映射到已有 [SearchSpace](../../../searchspace-matrix.md)，不另建权威分类，也不把“找到来源”记成实测覆盖。 ## 本轮决定 -**保留 Softmax/RMSNorm 作为算子接口试点；下一批重点增加真实仓库 CPU 热路径和内存管理任务，设备明确后再接通信及服务性能。** 每类先做一个完整任务，验证裁判、计时和结果用途后再扩充数量。 +**先在真实 NPU 上跑通 affine RMSNorm 的基线、profiling、候选、独立检查与 Workbench 比较。** 原 CPU Softmax/RMSNorm 保留作软件控制和历史证据；Tokenizers、KV、通信与服务来源保留为后续扩展。每类先完成一个任务的执行资格，再扩充数量。 AgentKernelArena 适合学习 kernel 任务的提交与独立评测；整个 InferRSI 还需要仓库补丁、状态管理、资源配置和服务工作负载。不能把所有任务都变成 `compute(x)`，也不能把功能修复的通过率叫作性能提升。 | 类别与任务卡 | 给 Agent 的具体问题 | 优先来源 | 当前状态 / 次序 | | --- | --- | --- | --- | -| [① 算子计算](01-operators.md) | 在既定输入和数值约束下优化 Softmax/RMSNorm 实现 | KernelBench、AgentKernelArena、BackendBench | 已有 CPU 试点；先修完测量资格，不急于再加同类算子 | -| [② CPU 热路径与运行时](02-host-runtime.md) | 大量 added tokens 时，保持分词结果不变，降低编码开销 | GSO 的 Tokenizers 历史优化任务；SWE-fficiency 的任务契约 | **下一项新增任务首选**；真实 repo-patch adapter 和历史环境尚未实现 | -| [③ 内存与缓存](03-memory.md) | 正确处理异步完成与 KV 回收；随后优化混合尺寸分配的容量和开销 | 已有 RSI KV 任务、vLLM、Mooncake | **先接已有任务**，保留原资格边界；Mooncake 为下一独立新题候选 | -| [④ 多卡通信](04-communication.md) | 固定拓扑/消息量下优化 collective 实现或配置 | nccl-tests、MSCCL-tools、MSCCL++ | 先固定任务契约，实际带宽/延迟等待目标设备；CPU 符号检查单独标记 | -| [⑤ 服务配置与调度](05-serving.md) | 固定模型、请求和质量要求下，优化首 token 延迟或有效吞吐 | InferenceBench、llama.cpp、vllm-evolve；Apple RSI-Inference Bench 作真实引擎扩展参考 | 硬件及模型明确后执行；CPU llama.cpp 也是候选，但仍需模型资产与质量裁判 | +| [① 算子计算](01-operators.md) | 在 NPU 上保持 affine RMSNorm 结果并比较基础运算与融合实现 | torch_npu 的实际算子；KernelBench、AgentKernelArena、BackendBench 的评测设计 | **当前试点**；真实设备正确性、profile、同步计时与 A/A 均需验收 | +| [② CPU 热路径与运行时](02-host-runtime.md) | 大量 added tokens 时，保持分词结果不变，降低编码开销 | GSO 的 Tokenizers 历史优化任务；SWE-fficiency 的任务契约 | 后续 CPU 仓库补丁候选，不再作为当前试点；adapter 和历史环境尚未实现 | +| [③ 内存与缓存](03-memory.md) | 正确处理异步完成与 KV 回收；随后优化混合尺寸分配的容量和开销 | 已有 RSI KV 任务、vLLM、Mooncake | 本轮 NPU 通路之后优先评估复用已有任务，保留原资格边界；Mooncake 为独立新题候选 | +| [④ 多卡通信](04-communication.md) | 固定拓扑/消息量下优化 collective 实现或配置 | nccl-tests、MSCCL-tools、MSCCL++ 的任务与测量设计 | 后续需要 NPU 对应通信机制及实际拓扑；CUDA 实现不能直接作为 Ascend 任务 | +| [⑤ 服务配置与调度](05-serving.md) | 固定模型、请求和质量要求下，优化首 token 延迟或有效吞吐 | InferenceBench、llama.cpp、vllm-evolve；Apple RSI-Inference Bench 作引擎扩展参考 | NPU 算子试点后另定模型、服务和质量裁判；已有 GPU/CPU 题源不自动支持 NPU | -第一阶段从“多做两道数学题”转为“三种不同问题的试点”：算子计算、CPU 分词、状态回收。状态回收是既有工作，接入不算新造题；也不因此宣布内存容量、GPU kernel 或服务端到端全部覆盖。 +此前提出的“算子、CPU 分词、状态回收”扩展方向继续保留，执行次序已调整为 NPU 优先。状态回收是既有工作，接入不算新造题；RMSNorm 设备试点完成也不代表内存容量、多卡通信或服务端到端已覆盖。 ## 哪些开源项目更适合借鉴 @@ -63,10 +63,10 @@ Apple 的另一个 [vLLM 源码优化题](https://github.com/apple-aiml-research | 候选形式 | 对应题型 | 与当前实现的关系 | | --- | --- | --- | -| 一个受限实现文件 | 已有算子；已有 KV 的 `candidate.py` | NumPy adapter 与 KV replay 是两条既有路径，语义/计时不同；接入时保留各自裁判与冻结身份 | +| 一个受限实现文件 | 当前 NPU RMSNorm；已有 CPU 算子和 KV 的 `candidate.py` | NPU 需要自身设备、权重、oracle 与同步协议；NumPy / KV 既有资格不迁移 | | 固定仓库上的 patch | Tokenizers、Mooncake、引擎回归 | **待实现 repo-patch adapter**；独立构建、受保护测试、完整产物和对应 baseline；不让模型改裁判 | | 声明的配置或 launcher | llama.cpp / 服务配置 | **待实现对应任务适配**；锁定模型、请求量和设备，检查实际启动/生效的配置及最终服务器 | -| collective 算法或候选库 | 多卡通信 | 待选设备后实现；符号验证、编译、真数据校验、计时分别记录 | +| collective 算法或候选库 | 多卡通信 | NPU 方向已定，具体机制/拓扑与多进程适配待研究；符号验证、编译、数据校验、计时分别记录 | 复用现有 Workbench 的执行、AB/BA 和证据，不把 GSO、Pier、Harbor、InferenceBench 的控制器全部引入。上游 task 包是素材和契约来源；任何新执行环境或多文件候选都需要其自身的产物隔离与验收，现有单文件 sandbox 不能自动覆盖仓库 build/server。 @@ -85,9 +85,9 @@ Profiling 保留独立阶段与原始产物。若以后评估“给 Agent profil ## 与四个子项的关系及本轮产物 -- **#334 调研：** 本文与五类任务卡补充原来的算子选型；首批扩展优先级已明确。 -- **#335 接口：** 识别 repo-patch / launcher / 多进程任务的真实差异；这些是下一步设计输入，尚未新增通用 adapter。 -- **#336 任务：** 保留原两题；CPU 分词是新增首选，KV 是既有任务复用，其他候选按任务卡分阶段实施。 -- **#337 接入验证：** 按上面六项资格步骤验证真实用途；现有正式 SkillEval correctness 接入仍只支持它已声明的内置题。 +- **#334 调研：** 保留五类来源对照，当前优先级调整为真实 NPU RMSNorm。 +- **#335 接口：** 当前先实现 NPU 的设备/数值/同步/诊断契约；repo-patch / launcher / 多进程为后续设计，尚未新增通用 adapter。 +- **#336 任务：** 当前交付目标为 `rmsnorm-npu`，原 CPU 两题保留历史身份；其他类别按任务卡分阶段实施。 +- **#337 接入验证：** 按六项资格步骤验证 NPU Workbench 的真实用途；正式 SkillEval correctness 仍仅支持它已声明的 CPU 任务，NPU 隔离未建立。 -本轮新增的是可复查的来源选择和任务设计文档。没有修改历史证据、SearchSpace 成熟度、现有任务或执行协议，也没有宣布新类别已经能运行。范围扩展与具体硬件/模型验收仍需沿原 Issue 跟踪。 +本页保留可复查的来源选择与任务设计，不改写历史证据或 SearchSpace 成熟度。当前 NPU 实施与实测沿 [试点文档](../npu-pilot.md)及原 Issue 跟踪；本页列出的后续来源不因此变成已能运行的新类别。 From 167f52d19a471f3bd5b1ccb0cdd6fa15b7ecb242 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:54:50 +0800 Subject: [PATCH 14/20] feat: add Ascend RMSNorm task and device qualification controls Execute trusted candidates on an allocated NPU with an independent scalar oracle, synchronized eager timing, separate device profiling and Workbench A/A and reference controls. Agent-assisted-by: Codex (implementation and review) Signed-off-by: dengyunyang <584797741@qq.com> --- .../rmsnorm-npu/LICENSE | 41 ++++ .../rmsnorm-npu/README.md | 50 +++++ .../rmsnorm-npu/baseline.py | 7 + .../rmsnorm-npu/candidate.py | 7 + .../rmsnorm-npu/task.json | 103 +++++++++ src/inferrsi/harness/benchmarks/cli.py | 6 +- src/inferrsi/harness/benchmarks/contract.py | 37 ++- src/inferrsi/harness/benchmarks/npu_task.py | 211 ++++++++++++++++++ src/inferrsi/harness/benchmarks/worker.py | 5 +- tests/check_installed.py | 2 +- tests/test_engineering_benchmarks.py | 2 +- tests/test_npu_engineering_benchmark.py | 129 +++++++++++ tools/engineering_benchmark_validate.py | 3 + tools/npu_benchmark_validate.py | 177 +++++++++++++++ 14 files changed, 766 insertions(+), 14 deletions(-) create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py create mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json create mode 100644 src/inferrsi/harness/benchmarks/npu_task.py create mode 100644 tests/test_npu_engineering_benchmark.py create mode 100644 tools/npu_benchmark_validate.py diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE new file mode 100644 index 000000000..f74af3c84 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/LICENSE @@ -0,0 +1,41 @@ +rom PyTorch: + +Copyright (c) 2020, Huawei Technologies Co., Ltd +Copyright (c) 2016- Facebook, Inc (Adam Paszke) +Copyright (c) 2014- Facebook, Inc (Soumith Chintala) +Copyright (c) 2011-2014 Idiap Research Institute (Ronan Collobert) +Copyright (c) 2012-2014 Deepmind Technologies (Koray Kavukcuoglu) +Copyright (c) 2011-2012 NEC Laboratories America (Koray Kavukcuoglu) +Copyright (c) 2011-2013 NYU (Clement Farabet) +Copyright (c) 2006-2010 NEC Laboratories America (Ronan Collobert, Leon Bottou, Iain Melvin, Jason Weston) +Copyright (c) 2006 Idiap Research Institute (Samy Bengio) +Copyright (c) 2001-2004 Idiap Research Institute (Ronan Collobert, Samy Bengio, Johnny Mariethoz) + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the names of Facebook, Deepmind Technologies, NYU, NEC Laboratories America + and IDIAP Research Institute nor the names of its contributors may be + used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md new file mode 100644 index 000000000..6fb0d2a68 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md @@ -0,0 +1,50 @@ +# Affine RMSNorm on Ascend NPU + +Implement `compute(x, weight, *, eps) -> y` for contiguous float32 NPU tensors +`x[rows, features]` and `weight[features]`: +`y = x / sqrt(mean(x*x, axis=-1, keepdims=True) + eps) * weight`. +Preserve input and weight bytes, shape, dtype and device. The public, seeded +suite includes nonuniform signed weights, ordinary, zero, large and tiny inputs. +Only the decode/prefill shapes are timed. The independent CPU oracle uses scalar +double arithmetic and `math.fsum`; CPU reference computation is never a timed +candidate or a device fallback. + +The baseline composes eager PyTorch NPU operations. The example candidate uses +the existing fused `torch_npu.npu_rms_norm` implementation; it is a reference +control, not a newly written kernel or evidence of Agent efficacy. + +## Execution boundary + +Run reviewed source only, inside a caller-owned single-device allocation. The +worker requires exactly one `ASCEND_RT_VISIBLE_DEVICES` entry and uses logical +device 0 without changing the scheduler's mapping. Discovery needs no PyTorch +import; execution requires an installed compatible torch/torch_npu/CANN runtime. +The existing formal SkillEval CPU sandbox explicitly rejects this adapter. + +Input/weight generation, transfer, oracle preparation and output validation are +outside timing. Each timed call has distinct retained input objects and bytes. +The clock surrounds the eager call and its completion synchronization, following +an untimed pre-synchronization. It includes Python dispatch and synchronization +cost, so **synchronized eager-call latency is not pure device kernel time**. +Output copies and checks happen after the clock stops. Every measured output is +checked; incorrect output has no usable performance metric. + +`profile` uses the Ascend PyTorch CPU/NPU profiler after warmup, separately from +measurement. Its raw local trace includes verification copies and must be +reviewed before publication; it is diagnostic, not a speed score. An empty +device-kernel trace is an error rather than successful profiling. + +## Sources and limits + +API/shape semantics follow the [official op-plugin API](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/docs/en/custom_APIs/torch_npu/%28beta%29torch_npu-npu_rms_norm.md), +[implementation](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) +and [tests](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py). +The source BSD-3-Clause license is retained in `LICENSE`. Workloads, scalar +oracle and adapters here are independently authored; upstream performance +numbers are not imported. + +First qualification is limited to the recorded device and installed runtime, +float32, contiguous 2D forward execution and this public suite. It establishes +neither BF16/FP16, gradients, graph replay, hostile-producer isolation, arbitrary +NPU qualification nor service/skill/search benefit. A/A and reference controls +and issue-scoped predeclared gates are needed before choosing by speed. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py new file mode 100644 index 000000000..af36f4051 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/baseline.py @@ -0,0 +1,7 @@ +"""Eager affine RMSNorm baseline; all arithmetic executes on the input NPU.""" + +import torch + + +def compute(x, weight, *, eps): + return x * torch.rsqrt(torch.mean(x * x, dim=-1, keepdim=True) + eps) * weight diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py new file mode 100644 index 000000000..7b492f940 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/candidate.py @@ -0,0 +1,7 @@ +"""Authored reference optimization, not an Agent/skill efficacy result.""" + +import torch_npu + + +def compute(x, weight, *, eps): + return torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0] diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json new file mode 100644 index 000000000..3ea4889d5 --- /dev/null +++ b/src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/task.json @@ -0,0 +1,103 @@ +{ + "schema_version": 1, + "id": "rmsnorm-npu", + "adapter": "torch-npu-rmsnorm-v1", + "description": "Optimize affine RMSNorm on contiguous 2D float32 NPU tensors; preserve input and nonuniform weight.", + "searchspace": "docs/searchspace-matrix.md — Operator / Diagnose, Optimize, Verify; explicit Ascend workload qualification", + "source": { + "url": "https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py", + "revision": "9f23f00b1ebe54cca612b0537bc582dba409a0af", + "license": "BSD-3-Clause", + "adaptation": "Official affine RMSNorm semantics; independently authored smaller public workloads, scalar double oracle and trusted Workbench adapter. Baseline composes eager NPU operations, reference candidate calls existing fused operator. No upstream scores imported." + }, + "environment": { + "backend": "ascend", + "runtime": "torch_npu", + "dtype": "float32" + }, + "candidate": { + "entrypoint": "compute", + "editable": [ + "implementation.py" + ] + }, + "oracle": { + "operation": "rmsnorm", + "axis": 1, + "eps": 1e-05, + "rtol": 1e-05, + "atol": 1e-06 + }, + "measurement": { + "key": "metrics.suite_latency_ms", + "unit": "ms", + "boundary": "device-synchronized-eager-call", + "aggregation": "sum-of-per-case-medians", + "warmups": 10, + "samples": 25, + "input_sequence": "seeded-distinct-per-call-v1", + "case_ids": [ + "decode", + "prefill" + ] + }, + "workloads": [ + { + "id": "decode", + "shape": [ + 1, + 4096 + ], + "seed": 252, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "prefill", + "shape": [ + 64, + 4096 + ], + "seed": 253, + "distribution": "normal", + "layout": "contiguous" + }, + { + "id": "zeros", + "shape": [ + 2, + 4096 + ], + "seed": 254, + "distribution": "zeros", + "layout": "contiguous" + }, + { + "id": "large", + "shape": [ + 4, + 4096 + ], + "seed": 255, + "distribution": "large", + "layout": "contiguous" + }, + { + "id": "tiny", + "shape": [ + 4, + 4096 + ], + "seed": 256, + "distribution": "tiny", + "layout": "contiguous" + } + ], + "coverage": [ + "Public development workloads; not holdout or Agent/skill/search efficacy", + "Single allocated Ascend NPU, contiguous float32 2D forward only; device/runtime qualification recorded separately", + "No BF16/FP16, gradients, graph replay, strided inputs, communication, capacity or service/E2E claim", + "Trusted reviewed source only; formal SkillEval NPU isolation not implemented", + "Distinct retained per-call inputs and weights; CPU oracle and transfers excluded from synchronized eager-call latency; profiling separate" + ] +} diff --git a/src/inferrsi/harness/benchmarks/cli.py b/src/inferrsi/harness/benchmarks/cli.py index ac523e730..490a74961 100644 --- a/src/inferrsi/harness/benchmarks/cli.py +++ b/src/inferrsi/harness/benchmarks/cli.py @@ -11,7 +11,7 @@ def add_parser(commands): - root = commands.add_parser("benchmarks", help="Shared engineering tasks (trusted CPU sources)") + root = commands.add_parser("benchmarks", help="Shared engineering tasks (trusted sources)") actions = root.add_subparsers(dest="benchmark_action", required=True) actions.add_parser("list") show = actions.add_parser("show") @@ -62,8 +62,8 @@ def dispatch(args) -> dict | list: spec, { "benchmark": identity, - "backend": "cpu", - "qualification": "local CPU development workload only", + "backend": task.manifest["environment"]["backend"], + "qualification": "public development workload; see task coverage and run evidence", }, ) atomic_write_json(run_dir / "task.lock.json", task.manifest) diff --git a/src/inferrsi/harness/benchmarks/contract.py b/src/inferrsi/harness/benchmarks/contract.py index 0bbbb166b..4acac0f3a 100644 --- a/src/inferrsi/harness/benchmarks/contract.py +++ b/src/inferrsi/harness/benchmarks/contract.py @@ -3,6 +3,7 @@ import hashlib import json import math +import os import platform import re import sys @@ -34,12 +35,19 @@ def grader_digest() -> str: return tree_digest(Path(__file__).parent) -def runtime_digest() -> str: +def runtime_digest(environment=None) -> str: try: numpy_version = version("numpy") except PackageNotFoundError: numpy_version = "unavailable" facts = [platform.python_version(), platform.system(), platform.machine(), numpy_version] + if environment and environment["backend"] == "ascend": + for package in ("torch", "torch-npu"): + try: + facts.append(version(package)) + except PackageNotFoundError: + facts.append("unavailable") + facts.append(os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "unallocated")) return hashlib.sha256(json.dumps(facts).encode()).hexdigest() @@ -89,7 +97,7 @@ def identity(self, candidate: Path) -> dict: "task_sha256": self.task_sha256, "grader_sha256": self.grader_sha256, "candidate_sha256": digest(candidate), - "runtime_sha256": runtime_digest(), + "runtime_sha256": runtime_digest(self.manifest["environment"]), } def command(self, action: str, candidate: Path) -> tuple[str, ...]: @@ -194,8 +202,11 @@ def _load_task_root(root: Path) -> TaskPackage: or type(data["schema_version"]) is not int ): raise ValueError("Invalid task fields") - if data["schema_version"] != 1 or data["adapter"] != "numpy-axis1-v2": + if data["schema_version"] != 1 or data["adapter"] not in { + "numpy-axis1-v2", "torch-npu-rmsnorm-v1" + }: raise ValueError("Unsupported task schema or adapter") + npu = data["adapter"] == "torch-npu-rmsnorm-v1" if not isinstance(data["id"], str) or not re.fullmatch(r"[a-z0-9][a-z0-9-]*", data["id"]): raise ValueError("Invalid task id") for field in ("description", "searchspace"): @@ -208,7 +219,11 @@ def _load_task_root(root: Path) -> TaskPackage: if not re.fullmatch(r"[0-9a-f]{40}", source["revision"]): raise ValueError("Source revision must be a full commit") env = data["environment"] - if env != {"backend": "cpu", "runtime": "numpy", "dtype": "float32"}: + expected_env = ( + {"backend": "ascend", "runtime": "torch_npu", "dtype": "float32"} if npu + else {"backend": "cpu", "runtime": "numpy", "dtype": "float32"} + ) + if env != expected_env: raise ValueError("Unsupported environment; no implicit device fallback") if data["candidate"] != {"entrypoint": "compute", "editable": ["implementation.py"]}: raise ValueError("Unsupported candidate contract") @@ -223,6 +238,8 @@ def _load_task_root(root: Path) -> TaskPackage: raise ValueError("Unsupported oracle") for key in ("eps", "rtol", "atol"): _positive(oracle[key], "oracle." + key) + if npu and oracle["operation"] != "rmsnorm": + raise ValueError("NPU adapter only supports affine RMSNorm") metric = data["measurement"] if ( not isinstance(metric, dict) @@ -239,7 +256,9 @@ def _load_task_root(root: Path) -> TaskPackage: } or metric["key"] != "metrics.suite_latency_ms" or metric["unit"] != "ms" - or metric["boundary"] != "synchronous-compute-call" + or metric["boundary"] != ( + "device-synchronized-eager-call" if npu else "synchronous-compute-call" + ) or metric["aggregation"] != "sum-of-per-case-medians" or metric["input_sequence"] != "seeded-distinct-per-call-v1" ): @@ -275,7 +294,9 @@ def _load_task_root(root: Path) -> TaskPackage: or any(type(n) is not int or n < 1 for n in case["shape"]) or math.prod(case["shape"]) > 1048576 ): - raise ValueError("Invalid or oversized CPU workload") + raise ValueError("Invalid or oversized workload") + if npu and (len(case["shape"]) != 2 or case["layout"] != "contiguous"): + raise ValueError("NPU task requires contiguous 2D inputs") if type(case["seed"]) is not int or case["seed"] < 0: raise ValueError("Seed must be a nonnegative integer") if case["distribution"] not in {"normal", "zeros", "large", "tiny"}: @@ -291,8 +312,8 @@ def _load_task_root(root: Path) -> TaskPackage: prepared_elements += (metric["warmups"] + metric["samples"]) * sum( math.prod(case["shape"]) for case in measured_cases ) - if prepared_elements > 1048576: - raise ValueError("Prepared input sequence exceeds the CPU workload bound") + if prepared_elements > (16777216 if npu else 1048576): + raise ValueError("Prepared input sequence exceeds the workload bound") if ( not isinstance(data["coverage"], list) or not data["coverage"] diff --git a/src/inferrsi/harness/benchmarks/npu_task.py b/src/inferrsi/harness/benchmarks/npu_task.py new file mode 100644 index 000000000..2303c534b --- /dev/null +++ b/src/inferrsi/harness/benchmarks/npu_task.py @@ -0,0 +1,211 @@ +"""Trusted affine RMSNorm workload on an explicitly allocated Ascend NPU. + +The independent CPU oracle and host/device transfers are outside timing. Timing +is synchronized eager-call latency, not pure kernel time. No CPU fallback and no +untrusted-code sandbox are provided by this Workbench adapter. +""" + +import hashlib +import json +import math +import os +import platform +import statistics +import time +from pathlib import Path + +from .numpy_task import input_record, inputs + + +def reference(x, weight, eps, np): + """Scalar double arithmetic, independent of the device reduction/fused op.""" + expected = np.empty_like(x) + for index, row in enumerate(x): + values = [float(value) for value in row] + scale = math.sqrt(math.fsum(value * value for value in values) / len(values) + eps) + expected[index] = [value / scale * float(w) for value, w in zip(values, weight)] + return expected + + +def environment(torch, torch_npu, np): + from torch_npu.npu.utils import get_cann_version + + return { + "backend": "ascend", + "device": torch.npu.get_device_name(0), + "logical_device": 0, + "python": platform.python_version(), + "torch": torch.__version__, + "torch_npu": torch_npu.__version__, + "numpy": np.__version__, + "cann": str(get_cann_version()), + "os": platform.system(), + "machine": platform.machine(), + "allocation": "caller-owned; one visible device", + } + + +def check_output(output, x, weight, original, original_weight, expected, oracle, torch, np): + tensor = isinstance(output, torch.Tensor) + shape_dtype_device = ( + tensor and tuple(output.shape) == original.shape + and output.dtype == torch.float32 and output.device == torch.device("npu:0") + ) + def preserved(tensor, original): + return (tuple(tensor.shape) == original.shape and tensor.dtype == torch.float32 + and tensor.device == torch.device("npu:0") and tensor.is_contiguous() + and tensor.cpu().numpy().tobytes() == original.tobytes()) + + # Reading results and checking preservation happens after device synchronization. + values = output.detach().cpu().numpy() if shape_dtype_device else None + return { + "shape_dtype_device": shape_dtype_device, + "finite": values is not None and bool(np.isfinite(values).all()), + "input_preserved": preserved(x, original), + "weight_preserved": preserved(weight, original_weight), + "values": values is not None + and bool(np.allclose(values, expected, rtol=oracle["rtol"], atol=oracle["atol"])), + } + + +def device_kernels(document): + """Read actual Ascend hardware records, not CPU launches/flow arrows.""" + events = document.get("traceEvents", []) if isinstance(document, dict) else document + hardware = {event["pid"] for event in events + if event.get("name") == "process_name" + and event.get("args", {}).get("name") == "Ascend Hardware"} + return [event for event in events + if event.get("ph") == "X" and event.get("pid") in hardware + and str(event.get("args", {}).get("Task Type", "")).startswith("KERNEL_")] + + +def evaluate(task, candidate: Path, action: str, result_dir: Path) -> dict: + # The caller owns resource admission. This adapter does not reserve devices + # or reinterpret the scheduler's selected physical device IDs. + allocated = os.environ.get("ASCEND_RT_VISIBLE_DEVICES", "") + if not allocated or len(allocated.split(",")) != 1: + raise RuntimeError("One explicitly allocated visible NPU is required") + import numpy as np + import torch + import torch_npu + + if not torch.npu.is_available() or torch.npu.device_count() != 1: + raise RuntimeError("One NPU must be available; there is no CPU fallback") + torch.npu.set_device(0) + manifest = task.manifest + oracle, measurement = manifest["oracle"], manifest["measurement"] + namespace = {"__name__": "benchmark_candidate", "__file__": str(candidate)} + exec(compile(candidate.read_bytes(), "implementation.py", "exec"), namespace) + compute = namespace.get("compute") + if not callable(compute): + raise ValueError("Candidate must define compute(x, weight, *, eps)") + + prepared, records, seen = [], [], set() + for case in manifest["workloads"]: + count = 1 + if case["id"] in measurement["case_ids"]: + count += measurement["warmups"] + measurement["samples"] + for index in range(count): + original = inputs(case, np, sequence=index) + rng = np.random.default_rng(np.random.SeedSequence([case["seed"], index, 7])) + original_weight = rng.uniform(-1.5, 1.5, case["shape"][-1]).astype(np.float32) + expected = reference(original, original_weight, oracle["eps"], np) + record = input_record(case, original) + record["weight_sha256"] = hashlib.sha256(original_weight.tobytes()).hexdigest() + record["call_index"] = index + record["phase"] = ( + "precheck" if index == 0 + else "warmup" if index <= measurement["warmups"] else "measurement" + ) + # x itself varies: a weight-only change cannot evade x-output replay checks. + if record["input_sha256"] in seen: + raise ValueError("Repeated input bytes in NPU sequence") + seen.add(record["input_sha256"]) + x = torch.from_numpy(original.copy()).to("npu:0") + weight = torch.from_numpy(original_weight.copy()).to("npu:0") + prepared.append((case, x, weight, original, original_weight, expected, record)) + records.append(record) + torch.npu.synchronize() + checks, executed, samples = {}, [], {key: [] for key in measurement["case_ids"]} + result = { + "status": "completed", "action": action, "passed": False, + "assertions": checks, "environment": environment(torch, torch_npu, np), + "coverage": manifest["coverage"], "input_sequence": records, "executed_calls": executed, + } + + def call(entry, *, timed=False): + case, x, weight, original, original_weight, expected, record = entry + torch.npu.synchronize() + started = time.perf_counter_ns() if timed else None + with torch.inference_mode(): + output = compute(x, weight, eps=oracle["eps"]) + torch.npu.synchronize() + elapsed = (time.perf_counter_ns() - started) / 1e6 if timed else None + assertions = check_output( + output, x, weight, original, original_weight, expected, oracle, torch, np + ) + checks.update({f"{case['id']}.call-{record['call_index']}.{k}": v + for k, v in assertions.items()}) + executed.append(record) + if not all(assertions.values()): + result["failed_call"] = {**record, "usable_measurement": False} + return False + if timed: + if not math.isfinite(elapsed) or elapsed <= 0: + raise ValueError("Timer did not resolve positive latency") + samples[case["id"]].append({"elapsed_ms": elapsed, **record}) + return True + + # All correctness cases are checked before any timed input. + for entry in prepared: + if entry[-1]["phase"] == "precheck" and not call(entry): + return result + + if action == "profile": + # Warm all measured shapes before attaching the profiler. Profile one + # fresh input per shape; ordinary measurement never enables instrumentation. + for entry in prepared: + if entry[-1]["phase"] == "warmup" and not call(entry): + return result + selected = [entry for entry in prepared if entry[-1]["phase"] == "measurement" + and entry[-1]["call_index"] == measurement["warmups"] + 1] + with torch_npu.profiler.profile( + activities=[torch_npu.profiler.ProfilerActivity.CPU, + torch_npu.profiler.ProfilerActivity.NPU], + record_shapes=True, profile_memory=False, with_stack=False, + ) as profiler: + for entry in selected: + with torch.profiler.record_function("rmsnorm-candidate-call"): + if not call(entry): + return result + trace = result_dir / "npu-trace.json" + profiler.export_chrome_trace(str(trace)) + # Raw profiler output may contain local identifiers: retain locally; + # publication should use a reviewed, path-free kernel summary. + document = json.loads(trace.read_text(encoding="utf-8")) + kernels = device_kernels(document) + if not kernels: + raise RuntimeError("NPU profiler produced no device kernel events") + result["diagnostic"] = { + "kind": "ascend-torch-profiler", "artifact": "npu-trace.json", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + "performance_verdict": None, + "kernels": [{"name": event["name"], "duration_us": event.get("dur")} + for event in kernels], + "limitation": "Instrumented diagnostic; includes verification copies; not a score", + } + else: + for entry in prepared: + if entry[-1]["phase"] == "precheck": + continue + if not call(entry, timed=action == "measure" and entry[-1]["phase"] == "measurement"): + return result + if action == "measure": + cases = [{"id": key, "samples_ms": [v["elapsed_ms"] for v in values], + "sample_input_sha256": [v["input_sha256"] for v in values], + "median_ms": statistics.median(v["elapsed_ms"] for v in values)} + for key, values in samples.items()] + result.update(cases=cases, measurement=measurement, + metrics={"suite_latency_ms": sum(case["median_ms"] for case in cases)}) + result["passed"] = all(checks.values()) + return result diff --git a/src/inferrsi/harness/benchmarks/worker.py b/src/inferrsi/harness/benchmarks/worker.py index ffa05a4c2..d1aca05bf 100644 --- a/src/inferrsi/harness/benchmarks/worker.py +++ b/src/inferrsi/harness/benchmarks/worker.py @@ -36,7 +36,10 @@ def main(): } if task.identity(candidate) != expected: raise ValueError("Frozen task, grader or candidate changed") - from .numpy_task import evaluate + if task.manifest["adapter"] == "torch-npu-rmsnorm-v1": + from .npu_task import evaluate + else: + from .numpy_task import evaluate result = evaluate(task, candidate, args.action, destination.parent) if task.identity(candidate) != expected: diff --git a/tests/check_installed.py b/tests/check_installed.py index 067fa09fb..1c44bd195 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -15,7 +15,7 @@ assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) -assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu"} +assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu", "rmsnorm-npu"} for benchmark_id in builtin_tasks(): benchmark = load_task(benchmark_id) benchmark.assert_unchanged() diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py index ea5f3c842..12f661ae8 100644 --- a/tests/test_engineering_benchmarks.py +++ b/tests/test_engineering_benchmarks.py @@ -221,7 +221,7 @@ def test_descriptor_does_not_mutate_task(): manifest = task.manifest manifest["oracle"]["axis"] = 0 assert task.manifest["oracle"]["axis"] == 1 - assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu"} + assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu", "rmsnorm-npu"} @pytest.mark.parametrize( diff --git a/tests/test_npu_engineering_benchmark.py b/tests/test_npu_engineering_benchmark.py new file mode 100644 index 000000000..a7a5a4c4a --- /dev/null +++ b/tests/test_npu_engineering_benchmark.py @@ -0,0 +1,129 @@ +"""Device-free contract controls; actual NPU execution is explicitly opted in.""" + +import json +import os +import shutil +import subprocess +import sys +from types import SimpleNamespace + +import pytest + +from inferrsi.harness.benchmarks import load_builtin_task, load_task +from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.npu_task import check_output, device_kernels, reference +from inferrsi.harness.skill_eval_benchmark import task_descriptor + + +def test_discovery_has_no_torch_dependency_and_formal_cpu_path_refuses_npu(): + task = load_builtin_task("rmsnorm-npu") + assert task.manifest["environment"]["backend"] == "ascend" + subprocess.run([sys.executable, "-c", "import sys; " + "from inferrsi.harness.benchmarks import load_builtin_task; " + "load_builtin_task('rmsnorm-npu'); " + "assert 'torch' not in sys.modules and 'torch_npu' not in sys.modules"], + check=True) + with pytest.raises(ValueError, match="CPU"): + task_descriptor("rmsnorm-npu") + + +@pytest.mark.parametrize("field,value", [ + ("environment", {"backend": "cpu", "runtime": "numpy", "dtype": "float32"}), + ("oracle.operation", "softmax"), ("measurement.boundary", "synchronous-compute-call"), + ("workloads.0.layout", "strided"), ("workloads.0.shape", [2, 3, 4]), +]) +def test_npu_contract_rejects_wrong_execution_semantics(field, value, tmp_path): + root = tmp_path / "task" + shutil.copytree(load_builtin_task("rmsnorm-npu").root, root) + manifest = json.loads((root / "task.json").read_text(encoding="utf-8")) + current = manifest + parts = field.split(".") + for key in parts[:-1]: + current = current[int(key)] if isinstance(current, list) else current[key] + current[parts[-1]] = value + (root / "task.json").write_text(json.dumps(manifest)) + with pytest.raises(ValueError): + load_task(root) + + +def test_unallocated_device_is_inconclusive_without_import_or_fallback(tmp_path, monkeypatch): + monkeypatch.delenv("ASCEND_RT_VISIBLE_DEVICES", raising=False) + task = load_builtin_task("rmsnorm-npu") + grade = grade_artifact(task, task.root / "candidate.py", tmp_path / "run", timeout_seconds=20) + assert grade["verdict"] == "INCONCLUSIVE" + result = json.loads((tmp_path / "run" / "result.json").read_text(encoding="utf-8")) + assert result["error_type"] == "RuntimeError" and "metrics" not in result + + +def test_scalar_affine_oracle_preserves_epsilon_and_weight(): + np = pytest.importorskip("numpy") + x = np.array([[0, 0], [1e-7, -1e-7]], dtype=np.float32) + weight = np.array([2, -3], dtype=np.float32) + y = reference(x, weight, 1e-5, np) + np.testing.assert_allclose(y[0], [0, 0]) + np.testing.assert_allclose(y[1], [2e-7 / (1e-5 + 1e-14)**0.5, + 3e-7 / (1e-5 + 1e-14)**0.5], rtol=1e-6) + + +def test_profile_requires_hardware_kernel_records(): + launch = {"pid": 1, "ph": "X", "name": "npu_rms_norm", "cat": "cpu_op"} + metadata = {"pid": 2, "name": "process_name", "args": {"name": "Ascend Hardware"}} + flow = {"pid": 2, "ph": "f", "cat": "async_npu"} + kernel = {"pid": 2, "ph": "X", "name": "RmsNorm", "dur": 5.0, + "args": {"Task Type": "KERNEL_AIVEC"}} + assert device_kernels([launch, metadata, flow]) == [] + assert device_kernels({"traceEvents": [launch, metadata, flow, kernel]}) == [kernel] + + +def test_changed_input_shape_and_broadcast_output_cannot_pass(): + np = pytest.importorskip("numpy") + + class Tensor: + dtype = "float32" + device = "npu:0" + + def __init__(self, array): + self.array, self.shape = array, array.shape + + def cpu(self): + return self + + def detach(self): + return self + + def numpy(self): + return self.array + + def is_contiguous(self): + return True + + torch = SimpleNamespace(Tensor=Tensor, float32="float32", device=lambda value: value) + original = np.array([[1., 2., 3.]], dtype=np.float32) + weight = np.ones(3, dtype=np.float32) + # Identical bytes, wrong metadata, and values that allclose could broadcast. + squeezed = Tensor(original.reshape(3)) + checks = check_output(squeezed, squeezed, Tensor(weight), original, weight, original, + {"rtol": 1e-5, "atol": 1e-6}, torch, np) + assert checks["shape_dtype_device"] is False + assert checks["input_preserved"] is False + assert checks["values"] is False + + +@pytest.mark.skipif(os.environ.get("INFERRSI_REQUIRE_NPU_BENCHMARK") != "1", + reason="Requires explicitly scheduled Ascend NPU execution") +def test_real_npu_baseline_fused_wrong_output_and_profile(tmp_path): + task = load_builtin_task("rmsnorm-npu") + for name in ("baseline", "candidate"): + report = grade_artifact(task, task.root / f"{name}.py", tmp_path / name, + timeout_seconds=180) + assert report["verdict"] == "PASS" + wrong = tmp_path / "wrong.py" + wrong.write_text("import torch\ndef compute(x, weight, *, eps): return torch.zeros_like(x)\n") + result = run_action(task, wrong, "measure", tmp_path / "wrong", timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert not record["passed"] and "metrics" not in record + result = run_action(task, task.root / "candidate.py", "profile", tmp_path / "profile", + timeout_seconds=180) + record = json.loads(open(result["result_path"]).read()) + assert record["passed"] and "metrics" not in record + assert any("RmsNorm" in kernel["name"] for kernel in record["diagnostic"]["kernels"]) diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py index b43c966eb..951eab850 100644 --- a/tools/engineering_benchmark_validate.py +++ b/tools/engineering_benchmark_validate.py @@ -64,6 +64,9 @@ def main(): output.mkdir(parents=True, exist_ok=False) records = [] for name in builtin_tasks(): + # This historical replay utility has a NumPy-only candidate protocol. + if load_task(name).manifest["adapter"] != "numpy-axis1-v2": + continue directory = output / name directory.mkdir() original = load_task(name) diff --git a/tools/npu_benchmark_validate.py b/tools/npu_benchmark_validate.py new file mode 100644 index 000000000..8d57a0c68 --- /dev/null +++ b/tools/npu_benchmark_validate.py @@ -0,0 +1,177 @@ +"""Qualify a trusted NPU task with controls, profiling and existing Workbench. + +Run inside a caller-owned NPU allocation. All policy values are explicit CLI +inputs, frozen before execution. This is an authored software-control exercise, +not a model-generated optimization campaign or formal NPU SkillEval sandbox. +""" + +import argparse +import json +import math +import shutil +import statistics +import subprocess +from pathlib import Path + +from inferrsi.harness.benchmarks import load_builtin_task +from inferrsi.harness.benchmarks.consumers import run_action +from inferrsi.harness.runner import run_experiment +from inferrsi.infra.artifacts import atomic_write_json + + +def read(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--repo", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--repetitions", type=int, required=True) + parser.add_argument("--min-improvement-pct", type=float, required=True) + parser.add_argument("--max-aa-difference-pct", type=float, required=True) + parser.add_argument("--max-minutes", type=float, required=True) + parser.add_argument("--timeout-seconds", type=float, required=True) + args = parser.parse_args() + if args.repetitions < 2 or any(not math.isfinite(v) or v <= 0 for v in ( + args.min_improvement_pct, args.max_aa_difference_pct, + args.max_minutes, args.timeout_seconds, + )): + parser.error("Use at least two pairs and positive finite policy values") + output = args.output_dir.resolve() + output.mkdir(parents=True, exist_ok=False) + for directory in ("plan", "validator", "patch", "results"): + (output / directory).mkdir() + task = load_builtin_task("rmsnorm-npu") + baseline = task.root / "baseline.py" + candidate = task.root / "candidate.py" + controls = { + "zeros": "return torch.zeros_like(x)", + "missing-epsilon": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True)) * weight", + "missing-weight": "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps)", + "wrong-axis": "return x * torch.rsqrt((x*x).mean(0,keepdim=True) + eps) * weight", + "input-shape": "x.squeeze_(0); " + "return x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight", + } + wrong_paths = {} + for name, body in controls.items(): + path = output / "patch" / f"{name}.py" + path.write_text(f"import torch\ndef compute(x, weight, *, eps):\n {body}\n") + wrong_paths[name] = path + # A correct precheck followed by stale output must also fail before scoring. + replay = output / "patch" / "replay.py" + replay.write_text( + "import torch\ncache = {}\n" + "def compute(x, weight, *, eps):\n" + " key = tuple(x.shape)\n" + " if key not in cache:\n" + " cache[key] = x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight\n" + " return cache[key]\n" + ) + wrong_paths["replay"] = replay + policy = {key: getattr(args, key) for key in ( + "repetitions", "min_improvement_pct", "max_aa_difference_pct", + "max_minutes", "timeout_seconds", + )} + frozen = { + "scope": "NPU software controls; no model/skill benefit or service claim", + "source_head": subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=args.repo, text=True + ).strip(), + "policy": policy, + "implementations": {name: task.identity(path) for name, path in { + "baseline": baseline, "reference": candidate, **wrong_paths + }.items()}, + } + atomic_write_json(output / "plan" / "frozen.json", frozen) + atomic_write_json(output / "validator" / "task.json", task.manifest) + for path in (baseline, candidate): + shutil.copyfile(path, output / "patch" / path.name) + actions = {} + + def action(name, path, mode): + execution = run_action(task, path, mode, output / "results" / name, + timeout_seconds=args.timeout_seconds) + result = read(execution["result_path"]) + record = {"execution_status": execution["status"], + "returncode": execution["returncode"], "result": result} + actions[name] = record + print(json.dumps({"action": name, "status": execution["status"], + "passed": result.get("passed")}), flush=True) + return record + + for name, path in (("baseline", baseline), ("reference", candidate)): + action(name + "-check", path, "check") + action(name + "-profile", path, "profile") + for name, path in wrong_paths.items(): + action(name, path, "measure") + comparisons = {} + # Same implementation in both arms is the explicit no-op/A-A control. + for name, path in (("aa", baseline), ("reference", candidate)): + spec = task.runspec(path, repo=args.repo, output_dir=output / "results" / name, + repetitions=args.repetitions, max_minutes=args.max_minutes, + min_improvement_pct=args.min_improvement_pct) + run = run_experiment(spec, {"benchmark": task.identity(path), + "backend": "ascend", "qualification": "pending"}) + evaluation = read(run / "evaluation.json") + trials = {side: [read(p) for p in sorted((run / side).glob("trial-*/result.json"))] + for side in ("baseline", "candidate")} + comparisons[name] = {"run": run.relative_to(output).as_posix(), + "evaluation": evaluation, "trials": trials} + print(json.dumps({"comparison": name, "verdict": evaluation["verdict"]}), flush=True) + + def correct(name): + record = actions[name] + return (record["execution_status"] == "completed" and record["returncode"] == 0 + and record["result"].get("status") == "completed" + and record["result"].get("passed") is True) + + checks = {"correctness": all(correct(name + "-check") for name in ("baseline", "reference")), + "device_profiles": all(correct(name + "-profile") + for name in ("baseline", "reference")), + "reject_wrong_outputs": all(actions[name]["execution_status"] == "failed" + and actions[name]["returncode"] == 1 + and actions[name]["result"].get("status") == "completed" + and actions[name]["result"].get("passed") is False + and "metrics" not in actions[name]["result"] for name in wrong_paths)} + all_results = [record["result"] for record in actions.values()] + all_results.extend(trial for comparison in comparisons.values() + for trials in comparison["trials"].values() for trial in trials) + environments = [result.get("environment") for result in all_results] + checks["fixed_environment"] = bool(environments) and all( + env and env == environments[0] for env in environments + ) + task.assert_unchanged() + shape_comparisons = {} + for name, comparison in comparisons.items(): + by_shape = {} + valid = all(len(trials) == args.repetitions + and all(trial.get("passed") and "cases" in trial for trial in trials) + for trials in comparison["trials"].values()) + for case in task.manifest["measurement"]["case_ids"] if valid else []: + medians = {side: statistics.median( + next(c["median_ms"] for c in trial["cases"] if c["id"] == case) + for trial in trials) for side, trials in comparison["trials"].items()} + medians["improvement_pct"] = (1 - medians["candidate"] / medians["baseline"]) * 100 + by_shape[case] = medians + shape_comparisons[name] = by_shape + checks[name + "_qualified"] = valid and bool(by_shape) and ( + all(abs(values["improvement_pct"]) <= args.max_aa_difference_pct + for values in by_shape.values()) and comparison["evaluation"]["verdict"] == "FAIL" + if name == "aa" else all(values["improvement_pct"] >= args.min_improvement_pct + for values in by_shape.values()) + and comparison["evaluation"]["verdict"] == "PASS" + ) + summary = {"scope": frozen["scope"], "frozen": frozen, "checks": checks, + "software_qualification": "PASS" if all(checks.values()) else "INCONCLUSIVE", + "campaign_acceptance": "not established", "agent_benefit": "not measured", + "shape_comparisons": shape_comparisons, + "actions": actions, "comparisons": comparisons} + atomic_write_json(output / "summary.json", summary) + print(json.dumps({"checks": checks, + "software_qualification": summary["software_qualification"]})) + return 0 if all(checks.values()) else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) From cae3050581833a9364ef15cea970de0d1c1a268c Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:56:17 +0800 Subject: [PATCH 15/20] fix: require final validation in NPU qualification controls A no-op comparison may fail either its speed gate or correctness. Accept the no-op control only after successful final quality validation, and compare its actual environment with every other result. Agent-assisted-by: Codex (implementation and review) Signed-off-by: dengyunyang <584797741@qq.com> --- tests/test_npu_engineering_benchmark.py | 10 ++++++++++ tools/npu_benchmark_validate.py | 20 ++++++++++++++++++-- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tests/test_npu_engineering_benchmark.py b/tests/test_npu_engineering_benchmark.py index a7a5a4c4a..979eee2e3 100644 --- a/tests/test_npu_engineering_benchmark.py +++ b/tests/test_npu_engineering_benchmark.py @@ -13,6 +13,7 @@ from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action from inferrsi.harness.benchmarks.npu_task import check_output, device_kernels, reference from inferrsi.harness.skill_eval_benchmark import task_descriptor +from tools.npu_benchmark_validate import validation_passed def test_discovery_has_no_torch_dependency_and_formal_cpu_path_refuses_npu(): @@ -75,6 +76,15 @@ def test_profile_requires_hardware_kernel_records(): assert device_kernels({"traceEvents": [launch, metadata, flow, kernel]}) == [kernel] +@pytest.mark.parametrize("status,code,passed,expected", [ + ("completed", 0, True, True), ("failed", 1, False, False), + ("timeout", None, True, False), ("completed", 0, False, False), +]) +def test_noop_qualification_requires_final_quality_pass(status, code, passed, expected): + assert validation_passed({"execution_status": status, "returncode": code, + "result": {"status": "completed", "passed": passed}}) is expected + + def test_changed_input_shape_and_broadcast_output_cannot_pass(): np = pytest.importorskip("numpy") diff --git a/tools/npu_benchmark_validate.py b/tools/npu_benchmark_validate.py index 8d57a0c68..b17ac8e17 100644 --- a/tools/npu_benchmark_validate.py +++ b/tools/npu_benchmark_validate.py @@ -23,6 +23,13 @@ def read(path): return json.loads(Path(path).read_text(encoding="utf-8")) +def validation_passed(validation): + result = validation.get("result", {}) + return (validation.get("execution_status") == "completed" + and validation.get("returncode") == 0 + and result.get("status") == "completed" and result.get("passed") is True) + + def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--repo", type=Path, required=True) @@ -114,10 +121,18 @@ def action(name, path, mode): run = run_experiment(spec, {"benchmark": task.identity(path), "backend": "ascend", "qualification": "pending"}) evaluation = read(run / "evaluation.json") + validation_dir = run / "validation" + execution = (read(validation_dir / "execution.json") + if (validation_dir / "execution.json").is_file() else {}) + validated = (read(validation_dir / "result.json") + if (validation_dir / "result.json").is_file() else {}) + validation = {"execution_status": execution.get("status"), + "returncode": execution.get("returncode"), "result": validated} trials = {side: [read(p) for p in sorted((run / side).glob("trial-*/result.json"))] for side in ("baseline", "candidate")} comparisons[name] = {"run": run.relative_to(output).as_posix(), - "evaluation": evaluation, "trials": trials} + "evaluation": evaluation, "trials": trials, + "validation": validation} print(json.dumps({"comparison": name, "verdict": evaluation["verdict"]}), flush=True) def correct(name): @@ -137,6 +152,7 @@ def correct(name): all_results = [record["result"] for record in actions.values()] all_results.extend(trial for comparison in comparisons.values() for trials in comparison["trials"].values() for trial in trials) + all_results.extend(comparison["validation"]["result"] for comparison in comparisons.values()) environments = [result.get("environment") for result in all_results] checks["fixed_environment"] = bool(environments) and all( env and env == environments[0] for env in environments @@ -145,7 +161,7 @@ def correct(name): shape_comparisons = {} for name, comparison in comparisons.items(): by_shape = {} - valid = all(len(trials) == args.repetitions + valid = validation_passed(comparison["validation"]) and all(len(trials) == args.repetitions and all(trial.get("passed") and "cases" in trial for trial in trials) for trials in comparison["trials"].values()) for case in task.manifest["measurement"]["case_ids"] if valid else []: From 6308268fc6d1fbf90e9ab97b5da8af79152f39ea Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 10:57:41 +0800 Subject: [PATCH 16/20] fix: verify frozen identities in NPU qualification reports Match all action, trial and final validation identities against the pre-execution freeze, including incorrect controls stored outside the task package. Agent-assisted-by: Codex (implementation and review) Signed-off-by: dengyunyang <584797741@qq.com> --- tools/npu_benchmark_validate.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/tools/npu_benchmark_validate.py b/tools/npu_benchmark_validate.py index b17ac8e17..831de7d25 100644 --- a/tools/npu_benchmark_validate.py +++ b/tools/npu_benchmark_validate.py @@ -80,15 +80,15 @@ def main(): "repetitions", "min_improvement_pct", "max_aa_difference_pct", "max_minutes", "timeout_seconds", )} + implementation_paths = {"baseline": baseline, "reference": candidate, **wrong_paths} frozen = { "scope": "NPU software controls; no model/skill benefit or service claim", "source_head": subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=args.repo, text=True ).strip(), "policy": policy, - "implementations": {name: task.identity(path) for name, path in { - "baseline": baseline, "reference": candidate, **wrong_paths - }.items()}, + "implementations": {name: task.identity(path) + for name, path in implementation_paths.items()}, } atomic_write_json(output / "plan" / "frozen.json", frozen) atomic_write_json(output / "validator" / "task.json", task.manifest) @@ -100,7 +100,9 @@ def action(name, path, mode): execution = run_action(task, path, mode, output / "results" / name, timeout_seconds=args.timeout_seconds) result = read(execution["result_path"]) - record = {"execution_status": execution["status"], + record = {"implementation": next(key for key, value in implementation_paths.items() + if value == path), + "execution_status": execution["status"], "returncode": execution["returncode"], "result": result} actions[name] = record print(json.dumps({"action": name, "status": execution["status"], @@ -157,6 +159,19 @@ def correct(name): checks["fixed_environment"] = bool(environments) and all( env and env == environments[0] for env in environments ) + identity_checks = [record["result"].get("identity") + == frozen["implementations"][record["implementation"]] + for record in actions.values()] + for name, comparison in comparisons.items(): + selected = "baseline" if name == "aa" else "reference" + for side, trials in comparison["trials"].items(): + expected = frozen["implementations"]["baseline" if side == "baseline" else selected] + identity_checks.extend(trial.get("identity") == expected for trial in trials) + identity_checks.append(comparison["validation"]["result"].get("identity") + == frozen["implementations"][selected]) + identity_checks.extend(task.identity(path) == frozen["implementations"][name] + for name, path in implementation_paths.items()) + checks["frozen_identities"] = all(identity_checks) task.assert_unchanged() shape_comparisons = {} for name, comparison in comparisons.items(): From a91b913e7a71ba2d8460ea3b8f2b8964de7dafe1 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 11:22:58 +0800 Subject: [PATCH 17/20] docs: record verified Ascend RMSNorm pilot evidence Publish the NPU workflow, all timing samples and scoped keep decision; distinguish the trusted device pilot from formal artifact isolation and Agent efficacy. Agent-assisted-by: Codex (documentation and evidence review) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/architecture.md | 4 +- docs/engineering-benchmarks.md | 80 +- docs/operations.md | 2 +- docs/plans/issue-252/02-contract.md | 2 +- docs/plans/issue-252/03-tasks.md | 2 +- docs/plans/issue-252/04-integration.md | 12 +- docs/plans/issue-252/README.md | 8 +- docs/plans/issue-252/npu-pilot.md | 20 +- .../issue-252/npu-validation-summary.json | 12575 ++++++++++++++++ docs/plans/issue-252/npu-validation.md | 60 + .../issue-252/task-expansion/01-operators.md | 2 +- docs/plans/issue-252/task-expansion/README.md | 2 +- 12 files changed, 12729 insertions(+), 40 deletions(-) create mode 100644 docs/plans/issue-252/npu-validation-summary.json create mode 100644 docs/plans/issue-252/npu-validation.md diff --git a/docs/architecture.md b/docs/architecture.md index 1f47bfdaf..9377407d8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -4,7 +4,9 @@ The [shared engineering benchmark adapter](engineering-benchmarks.md) lives in `harness/benchmarks`, with task data in `data/engineering_benchmarks`. It binds task-owned oracles and workloads to the existing Workbench runner and offline SkillEval verifier contract. It adds neither a search controller nor another -execution platform; its first qualified scope is the locally checked CPU subset. +execution platform. The current device pilot runs trusted affine RMSNorm on +Ascend through Workbench; the formal artifact sandbox remains CPU NumPy only. +Each task's actual qualification and limits are recorded separately. The local experiment path now has concrete Harness, Model and Infra modules. The CLI, RunSpec, knowledge retrieval and existing command contracts remain stable. diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index 32b833288..d692d7908 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -1,12 +1,61 @@ # Shared engineering benchmarks The `benchmarks` command exposes versioned task packages for Workbench and -offline artifact evaluation. The `numpy-axis1-v2` adapter supplies -Softmax and RMSNorm CPU adaptations of attributed KernelBench tasks. Packages -own semantics, workload, oracle and metric. Consumers own comparison conditions; -search policy and agent execution remain separate. +offline artifact evaluation. The current device pilot is **Ascend affine +RMSNorm**, with a trusted NPU worker, independent scalar CPU oracle, synchronized +eager-call timing and separate device profiling. Existing CPU tasks remain +software controls. Packages own semantics, workload, oracle and metric; +consumers own comparison conditions. Search policy and agent execution remain +separate. + +| Adapter | Task and submission | Execution boundary | +| --- | --- | --- | +| `torch-npu-rmsnorm-v1` | `rmsnorm-npu`; `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | +| `numpy-axis1-v2` | `softmax-cpu`, `rmsnorm-cpu`; `compute(x, *, eps)` | Trusted CPU worker; formal SkillEval also has a separate NumPy artifact sandbox | + +## Run the NPU pilot + +Discovery imports neither torch nor torch_npu. Execution requires the target's +compatible installed torch/torch_npu/CANN stack; this project does not install +or replace the device runtime. Enter an explicit single-NPU allocation before +executing any device command. The worker requires one visible device, uses its +logical index 0 and never falls back to CPU. -## Use the tasks +```bash +inferrsi benchmarks show rmsnorm-npu +inferrsi benchmarks check rmsnorm-npu --candidate --output-dir --timeout-seconds +inferrsi benchmarks profile rmsnorm-npu --candidate --output-dir --timeout-seconds +inferrsi benchmarks compare rmsnorm-npu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +``` + +The task is contiguous 2D float32 affine RMSNorm with nonuniform weights. Inputs, +weights, scalar double/math.fsum reference results and transfers are prepared +outside timing. The measured clock covers the eager call and completion +synchronization after an untimed pre-synchronization. Dispatch and synchronization +costs are included: this is **not pure kernel latency**. Every output is checked +outside the timer, including shape/dtype/device and input/weight preservation. + +`profile` records real Ascend Hardware kernel events in `npu-trace.json`, separately +from latency sampling. Its local raw trace may contain private paths/identifiers; +review it before publication. The result includes a path-free kernel summary, +not a performance metric. Profiling includes verification copies, which must be +distinguished from compute kernels during diagnosis. + +To qualify the path with correct, wrong, replay, same-code and fused-reference +controls through the existing Workbench: + +```bash +python -m tools.npu_benchmark_validate --repo --output-dir --repetitions --min-improvement-pct --max-aa-difference-pct --max-minutes --timeout-seconds +``` + +The utility freezes policy and implementation identities before execution, +preserves every sample and checks actual device/CANN/runtime consistency across +actions, trials and final validation. The general runtime fingerprint binds +package versions and selected device IDs; it alone does not qualify device/CANN +equivalence or host contention. Qualification is an authored-control exercise, +not Agent/skill efficacy or service acceptance. See the [NPU plan and results](plans/issue-252/npu-pilot.md). + +## Use the CPU regression tasks Install with `pip install -e ".[dev,benchmarks]"`. Discovery and planning do not import NumPy; execution requires it and never silently switches backend. @@ -39,7 +88,7 @@ editing. The package and grader remain protected comparison inputs. This adapter executes trusted source and is **not an untrusted-code sandbox**. Hash checks detect drift, not malicious same-process code. Do not connect raw model output. -## Measurement and evidence +## CPU measurement and shared evidence `TaskPackage` validates the schema/adapter/environment and freezes manifest, package and grader identities. Commands bind candidate and runtime fingerprints. @@ -85,14 +134,17 @@ The shared oracle independently grades actual implementation behavior. This offline handoff does not modify production engineering sandbox or paid execution. Live artifact production and skill/search benefit need separate controlled runs. -For formal paired artifact evaluation, `tools.skill_eval_compare` now supports -`engineering-benchmark-v1`: freeze the common public task and two skill bundles, +For formal paired artifact evaluation, `tools.skill_eval_compare` supports +`engineering-benchmark-v1` for the CPU NumPy tasks only: freeze the common public task and two skill bundles, retain each arm's actual `implementation.py`, and independently check its output outside the artifact's Linux bubblewrap sandbox. Local stage/capture uses the same controller. This adapter evaluates correctness only; it does not accept child-reported timing, launch an unqualified default agent, or fall back to host execution when the sandbox/NumPy runtime is unavailable. It is separate from the -trusted-source worker above. See [setup and handoff](plans/issue-252/formal-integration.md). +trusted-source worker above. Formal loading uses canonical packaged resources, +independent of same-named cwd task folders; the trusted Workbench retains explicit +path loading. NPU tasks are explicitly rejected by the formal adapter until +device isolation is separately qualified. See [setup and handoff](plans/issue-252/formal-integration.md). The offline harness-improvement controller and provider generation mode do not yet support these tasks. See [the RSI integration audit](plans/issue-252/rsi-integration.md) @@ -102,7 +154,7 @@ Exceptions, missing dependencies, drift and timeouts leave incomplete evidence. Wrong results emit no usable performance metric. Workbench statistics remain descriptive; a local verdict is not service gain or campaign acceptance. -## Reproduce the integration +## Reproduce the historical CPU integration ```bash python tools/engineering_benchmark_validate.py --repo /path/to/checkout --output-dir /path/to/new-validation --repetitions 3 @@ -115,8 +167,8 @@ correct and deliberately wrong **authored controls**. No model is called. bundles contain resolved paths; review/redact a copy before sharing. See [#252 research and subtask plans](plans/issue-252/README.md). Formal Issue -acceptance, independent review, accelerator qualification, communication and -memory-capacity coverage remain distinct from this local CPU subset. +acceptance, independent review, communication and memory-capacity coverage remain +distinct from each task's scoped qualification. -For a Chinese walkthrough of each component and an RMSNorm example, start with -[the reading guide](plans/issue-252/walkthrough.md). +Start with the current [Chinese NPU pilot guide](plans/issue-252/npu-pilot.md). +The earlier [CPU walkthrough](plans/issue-252/walkthrough.md) retains its historical scope. diff --git a/docs/operations.md b/docs/operations.md index 181e08f4b..ac0c7c0bc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -3,7 +3,7 @@ Shared engineering tasks use `inferrsi benchmarks list|show|plan` for read-only inspection and `check|measure|profile|grade|compare` for explicit local execution. See the [engineering benchmark contract](engineering-benchmarks.md) for trusted -source requirements, task identities, CPU coverage and offline SkillEval reuse. +source requirements, task identities, Ascend/CPU coverage and offline SkillEval reuse. One concise map from an agent's intent to the supported entry point. This guide is navigation only: detailed contracts stay with their owning documents, and it diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md index bccbbc078..22f460601 100644 --- a/docs/plans/issue-252/02-contract.md +++ b/docs/plans/issue-252/02-contract.md @@ -16,7 +16,7 @@ NPU RMSNorm 的入口为 `compute(x, weight, *, eps)`，采用二维 contiguous [NPU 试点](npu-pilot.md)列出当前目标：独立 adapter 在实际设备上验证输入/输出设备、dtype、数值与输入保持，使用 CPU scalar double / `math.fsum` oracle；数据准备与传输、oracle、校验在性能计时外完成。同步边界必须覆盖实际完成的 NPU 计算。独立 CPU/NPU profile 不进入性能分数。 -运行时身份需绑定实际设备与 torch / torch_npu / CANN 版本，不沿用只包含 Python/NumPy 的 CPU 身份。任务、候选、oracle、输入和计时规则继续使用共享内容身份；缺依赖、无设备或不支持语义不能回退 CPU。具体已支持字段和环境以最终 manifest 及实测记录为准。 +运行时身份需绑定实际设备与 torch / torch_npu / CANN 版本，不沿用只包含 Python/NumPy 的 CPU 身份。当前命令指纹包含 torch/torch_npu 版本与分配的可见设备；worker 另记录实际型号和 CANN，资格工具检查所有 action/trial/final validation 的实际环境一致。不能把命令指纹单独称为完整硬件资格。任务、候选、oracle、输入和计时规则继续使用共享内容身份；缺依赖、无设备或不支持语义不能回退 CPU。已支持范围及执行证据见[本轮记录](npu-validation.md)。 Workbench 接入先运行已审核可信源码。正式 `engineering-benchmark-v1` 目前只接受已声明的 CPU 任务；NPU 题出现在 catalog 中，不代表正式 SkillEval 具备设备隔离或产物重验能力。adapter 的能力检查必须显式拒绝不支持的组合。 diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index b52056793..c2b9297c8 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -8,7 +8,7 @@ | 交付什么 | 任务包、基础算子 baseline、融合候选、数值 oracle、变化输入、错误控制与原始结果 | | 谁使用 | 修改实现的 Agent/开发者，以及 Workbench 的任务操作 | | 实际作用 | 验证候选确实在 NPU 计算且算对，并比较真实完成一次计算的成本 | -| 当前状态 | NPU 适配与资格实验是当前实施目标，实际结果按 [NPU 试点](npu-pilot.md)更新；CPU 旧结果不替代它 | +| 当前状态 | 已在 Ascend 910B1 完成 [NPU 资格实验](npu-validation.md)，保留全部 800 样本与错误控制；CPU 旧结果不替代它 | ## 任务定义 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index 30eb47707..ba31f9979 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -10,7 +10,7 @@ | 交付什么 | NPU Workbench RunSpec 绑定、真实设备演练和结果记录；保留已有 CPU 正式产物路径 | | 谁使用 | 主 Agent 的引擎优化流程、评测开发者、PR 评审者 | | 实际作用 | 工作台能够执行比较并保留证据；错误实现被共同裁判拒绝；结果可供 RSI 决定保留、放弃或继续诊断 | -| 当前状态 | CPU 接入已有；NPU Workbench 演练与性能资格为当前实施目标，设备结果待单独记录；真实技能收益未测 | +| 当前状态 | NPU Workbench 参考控制演练与本轮软件资格已完成，见[实测与决定](npu-validation.md)；真实技能收益未测 | 例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 @@ -19,7 +19,7 @@ 本子项要交付的是能够使用任务、得到结果并据此行动的接入流程。区分以下三层，不能把其中一层完成写成全部完成： - **单次实验的执行与验收：** 固定比较条件、检查实际实现、测量、留存结果，交给现有 Workbench 或 SkillEval 消费。这是 #252 必须提供并验证的接入。 -- **主 Agent 的完整使用过程：** 从目标开始，运行基线和 profiling，提出假设，修改实现，检查与比较，记录保留、放弃或证据不足的理由。需要通过实际演练验证，而不仅是画流程图；当前还没有完整闭环验收。 +- **主 Agent 的使用过程：** 从目标开始，运行基线和 profiling，提出假设，修改实现，检查与比较，记录保留、放弃或证据不足的理由。本轮已完成预定融合参考实现的诊断核对、正确性、对照和保留决定；没有将这次控制演练当成从 profile 自主搜索新实现或技能收益的验收。 - **自动搜索策略：** 程序选择下一种改法、分配多轮预算、决定何时停止或接受候选。这类决策需与独立的 [Optimization Search Policy 提案](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306)对齐，提案本身也不负责模型生成候选。本次为其提供可消费的结果；不将该提案默认为已经实现，也不把它悄悄并入 #252 的验收范围。 ### 从哪里开始，以及每一步由谁做 @@ -30,10 +30,10 @@ | --- | --- | --- | | 1. 明确目标和验收依据 | 主 Agent 根据目标项目确定输入、环境、指标、质量约束及预算；适用时复用内置题包 | 内置题可加载；已有通用 Workbench 可接外部命令。尚无自动从用户目标生成合格任务的能力 | | 2. 准备比较条件 | 调用者在搜索前固定基线、验证规则和工作负载；每个候选有独立身份 | 复用题包身份和执行快照，NPU 需记录设备/运行时。普通 `plan` 仅预览，不等于保存了冻结的 campaign | -| 3. 运行基线并 profiling | 主 Agent 调用对应环境工具，保留诊断并说明改法依据 | 本轮需要真实 CPU/NPU profile；CPU 旧产物不替代。没有强制 Agent 先诊断再修改 | +| 3. 运行基线并 profiling | 主 Agent 调用对应环境工具，保留诊断并说明改法依据 | 本轮真实 CPU/NPU trace 已核对基础 kernel 与融合 kernel 路径。没有强制 Agent 先诊断再修改 | | 4. 提交并检查实现 | Agent 修改允许范围；工具实际运行并独立检查产物 | NPU 先执行已审核可信源码；正式 SkillEval 隔离路径仍限受支持的 CPU 任务 | -| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 复用 AB/BA、快照和报告；本轮需验证 NPU 同步计时及 A/A，可靠性能资格待实测 | -| 6. 使用结果 | 主 Agent 引用证据解释保留、放弃或继续诊断；下一候选重新标识 | 现有报告可供使用，已有部分用途对照；未完成真实 Agent 全链路演练，没有程序化跨轮次搜索/停止控制 | +| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 本轮 NPU 同步计时、A/A、AB/BA、最终 validation 与 800 样本已完成；范围见实测记录 | +| 6. 使用结果 | 主 Agent 引用证据解释保留、放弃或继续诊断；下一候选重新标识 | 本轮证据支持保留融合参考实现；没有程序化跨轮次搜索/停止控制或真实两臂收益结果 | 用户目标不在题库时，第一步使用目标项目自己的 benchmark 和测试，通过现有 `inferrsi run` 的 baseline/candidate/validation 命令接入 Workbench，不需要先扩充题库。缺少质量检查或可信测量时，应补齐对应实验条件；通用 runner 无法判断测试是否充分代表真实业务。当前具体任务 adapter 不能直接承担任意模型或服务的适配。 @@ -43,7 +43,7 @@ 尚无程序强制宿主 Agent 按“profile → 假设 → 修改”顺序行动，也没有完整优化轮次的资格检查。后续若增加流程约束，应在既有入口校验证据和声明的完成条件：缺少该实验要求的诊断或验收，不得记录成完整优化成功。不能仅在提示词里写了顺序，就声称所有 Agent 都会遵守。 -本子项当前 NPU 验证需要保留一条可复查的演练链：诊断依据、候选修改、独立检查、比较结果、主 Agent 的决定。至少证明错误候选被拒绝、证据不足不会被当作改进，以及最终结论只覆盖实际验收的目标。如果融合候选是预先选定的 authored control，应如实记录，不能写成 Agent 从 profile 自主发现。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 +本子项的[本轮 NPU 记录](npu-validation.md)已保留可复查的诊断、候选、独立检查、比较与保留决定：错误控制被拒绝，A/A 不被当作改进，结论只覆盖实际任务。融合候选是预先选定的 authored control，不是 Agent 从 profile 自主发现的证明。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 ## 理解 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 645e3921a..4604f7b4a 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -2,7 +2,7 @@ **当前交付目标是：让 Agent 面对一个真实 NPU 优化任务，交付实际实现，由 InferRSI 检查正确性、保留 profiling、在固定条件下比较性能，并据此决定是否保留候选。** 用户已明确将试点指定为 NPU；已有 CPU 两题保留为历史验证与软件控制，不作为这轮试点完成依据。 -从 [NPU 试点方案](npu-pilot.md)开始。首个任务是 `rmsnorm-npu`：在 Ascend NPU 上比较 PyTorch 基础算子组成的 affine RMSNorm 与 `torch_npu.npu_rms_norm` 候选。融合实现是否更快，要由实际结果决定。NPU 接入、设备执行与性能资格分别验收，不能用设计或 CPU 测试代替。 +从 [NPU 试点方案](npu-pilot.md)和[实际执行结果](npu-validation.md)开始。首个任务是 `rmsnorm-npu`：在 Ascend NPU 上比较 PyTorch 基础算子组成的 affine RMSNorm 与 `torch_npu.npu_rms_norm` 候选。已在 Ascend 910B1 完成独立检查、真实 profile、错误控制、A/A 与 Workbench 对照；七项软件资格检查通过，全部 800 个样本保留。项目验收、正式 NPU SkillEval 隔离和真实 Agent/skill 收益仍未完成。 本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。输入是任务包和候选实现，输出是正确性结果、独立 profile、原始计时及版本身份。它服务主 Agent 的优化工作，也为后续 SkillEval 比较不同 skill 条件下的产物提供任务。名称与范围是本次交付提议，维护者验收另行记录。 @@ -25,9 +25,9 @@ | 子任务 | 具体交付 | 实际作用 | 当前状态 | | --- | --- | --- | --- | | **① 选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [说明](01-research.md) | 论文/源码比较、NPU 选型、来源与边界 | 决定优化什么、复用哪些接口和评测设计 | 既有研究保留；本轮 NPU 方案见试点文档 | -| **② 统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [说明](02-contract.md) | 任务、候选、独立 oracle、NPU 同步计时、profile 与版本身份 | 固定一次比较中的工作和裁判，拒绝 CPU 回退或错误设备结果 | CPU 接口已有；NPU 适配与实测需单独验收 | -| **③ 交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [说明](03-tasks.md) | `rmsnorm-npu` 基线、候选、输入、数值检查、测量规则 | Agent 有明确修改对象，工具能在真实 NPU 执行它 | 当前交付目标；设备运行结果按试点记录更新 | -| **④ 接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [说明](04-integration.md) | NPU Workbench 演练、正确/错误控制、A/A 与候选比较、报告 | 证明真实产物进入执行，结果能支持下一步 | NPU 演练待实测记录；真实 skill 收益未测 | +| **② 统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [说明](02-contract.md) | 任务、候选、独立 oracle、NPU 同步计时、profile 与版本身份 | 固定一次比较中的工作和裁判，拒绝 CPU 回退或错误设备结果 | NPU adapter 与冻结身份已在真实试点核对 | +| **③ 交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [说明](03-tasks.md) | `rmsnorm-npu` 基线、候选、输入、数值检查、测量规则 | Agent 有明确修改对象，工具能在真实 NPU 执行它 | 已实测；限本次设备/runtime/float32/workload | +| **④ 接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [说明](04-integration.md) | NPU Workbench 演练、正确/错误控制、A/A 与候选比较、报告 | 证明真实产物进入执行，结果能支持下一步 | 参考控制演练已完成，证据支持本轮保留决定；真实 skill 收益未测 | ## 在 RSI 中的位置 diff --git a/docs/plans/issue-252/npu-pilot.md b/docs/plans/issue-252/npu-pilot.md index c4865f9fc..715eb3f81 100644 --- a/docs/plans/issue-252/npu-pilot.md +++ b/docs/plans/issue-252/npu-pilot.md @@ -1,6 +1,6 @@ # NPU 试点：用 RMSNorm 跑通一次真实优化比较 -2026-11-04，用户将试点明确指定为 NPU。本页定义当前实施与验收目标；下面的执行状态在取得实际日志、退出码和产物前保持待验证。已有 CPU 测试和样本仅作历史软件证据。 +2026-11-04，用户将试点明确指定为 NPU。**本轮已在 Ascend 910B1 完成实际运行，七项软件资格检查通过**，见[实测解释与保留决定](npu-validation.md)和[全部样本及身份](npu-validation-summary.json)。本页保留任务接口与执行方法。已有 CPU 测试和样本仅作历史软件证据；项目接受、NPU 产物隔离与真实 Agent 收益另行验收。 ## 这次究竟要跑通什么 @@ -60,18 +60,18 @@ RMSNorm 是推理中明确的归约与逐元素计算。基础运算组合和 NP 6. **执行 A/A 和候选比较。** 相同设备与固定条件下保留逐次样本；使用现有 Workbench 组织 AB/BA。诊断、H2D/D2H、oracle 和初始化开销单独记录。 7. **记录决定。** 引用数值、trace 和原始样本，说明保留、放弃或证据不足。信号无法区分、运行不完整或环境漂移时，不得写成优化成功。 -## 验收矩阵与待补证据 +## 验收矩阵与实际证据 | 要回答的问题 | 必须执行的控制/记录 | 本轮状态 | | --- | --- | --- | -| 是否确实在 NPU 运算？ | 输入/输出设备、实际运行时、设备预约、真实设备活动、退出码 | 待执行并记录 | -| 是否算对？ | baseline 与参考候选对独立 oracle；声明范围内的边界输入、输入与权重保持 | 待执行并记录 | -| 错误但快是否会被拒绝？ | 全零、遗漏 epsilon、忽略权重等候选；无可用性能指标 | 待执行并记录 | -| 计时是否覆盖完成的设备计算？ | 同步边界、变化输入、预热与正式样本、实际输出校验 | 待执行并记录 | -| 同码是否被误判成进步？ | A/A 或 no-op 对照，保留波动与原始 verdict | 待执行并记录 | -| 融合候选是否值得保留？ | composed baseline 对 fused 候选的 AB/BA、分工作负载样本和结论 | 待执行并记录 | -| profiling 是否发挥作用？ | 真实 profile、据此作出的具体判断及后续动作；预定控制与自主发现分开 | 待执行并记录 | -| 结果是否接入 RSI？ | Workbench 的 RunSpec、实际实现身份、检查/比较报告与最终决定 | 待执行并记录 | +| 是否确实在 NPU 运算？ | 输入/输出设备、实际运行时、设备预约、真实设备活动、退出码 | Ascend 910B1 已运行，退出 0；实际 kernel trace 已检查 | +| 是否算对？ | baseline 与参考候选对独立 oracle；声明范围内的边界输入、输入与权重保持 | 两者各 75 次调用、375 项断言通过 | +| 错误但快是否会被拒绝？ | 全零、遗漏 epsilon、忽略权重等候选；无可用性能指标 | 六类错误控制均拒绝；另有预检后重放控制 | +| 计时是否覆盖完成的设备计算？ | 同步边界、变化输入、预热与正式样本、实际输出校验 | 同步 eager 计时，保留全部 800 样本 | +| 同码是否被误判成进步？ | A/A 或 no-op 对照，保留波动与原始 verdict | 两个 shape 差异小于 0.4%；未达到收益门槛 | +| 融合候选是否值得保留？ | composed baseline 对 fused 候选的 AB/BA、分工作负载样本和结论 | 本轮两个 workload 达到预定门槛，保留参考实现 | +| profiling 是否发挥作用？ | 真实 profile、据此作出的具体判断及后续动作；预定控制与自主发现分开 | 核对每输入 6 个基础 kernel → 1 个 RmsNorm；非自主发现证明 | +| 结果是否接入 RSI？ | Workbench 的 RunSpec、实际实现身份、检查/比较报告与最终决定 | 实际 A/A 与参考 AB/BA、最终 validation 和身份重验已完成 | | 正式 SkillEval 是否能隔离 NPU 产物？ | 设备访问、父子边界、可信 oracle、重验与攻击控制 | 本轮尚未建立，不能用 CPU sandbox 替代 | | 是否提高了 Agent/skill 或服务能力？ | 受控 Agent 两臂或目标模型/服务实验 | 未测，不由本试点推导 | diff --git a/docs/plans/issue-252/npu-validation-summary.json b/docs/plans/issue-252/npu-validation-summary.json new file mode 100644 index 000000000..1f1ea6e8c --- /dev/null +++ b/docs/plans/issue-252/npu-validation-summary.json @@ -0,0 +1,12575 @@ +{ + "scope": "NPU software controls; no model/skill benefit or service claim", + "tested_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", + "acceptance_plan": "https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6621316099", + "software_qualification": "PASS", + "campaign_acceptance": "not established", + "checks": { + "correctness": true, + "device_profiles": true, + "reject_wrong_outputs": true, + "fixed_environment": true, + "frozen_identities": true, + "aa_qualified": true, + "reference_qualified": true + }, + "agent_benefit": "not measured", + "execution": { + "qualification_exit_code": 0, + "postcheck_replay_exit_code": 0, + "scheduled_devices": 1, + "user": "non-root personal UID with existing device-group membership", + "release": "scheduler completed and allocated device was available afterward" + }, + "source": { + "summary_sha256": "c2baefb9bb9624a9542e4c3ec03d42db09025511cbb8d526604a3f01b37e38e2", + "archive_sha256": "c39bb59536b59cb6a9c743d29290f7e0a4d9f16d157b26a3ad2ea9b849b35d91", + "full_raw_archive": "retained locally, not embedded; contains private paths" + }, + "transformation": "Read-only extraction. All 800 timing samples, original evaluations, frozen identities and profile kernel summaries retained. One identical planned input sequence/environment stored once; repeated assertions summarized by counts and every failed key. Every executed call retains id/index/phase mapping to that sequence. Raw result and trace SHA-256 identify local originals; raw traces/logs are not included and hashes do not substitute for those files.", + "frozen": { + "scope": "NPU software controls; no model/skill benefit or service claim", + "source_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", + "policy": { + "repetitions": 4, + "min_improvement_pct": 10.0, + "max_aa_difference_pct": 10.0, + "max_minutes": 8.0, + "timeout_seconds": 180.0 + }, + "implementations": { + "baseline": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "reference": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "zeros": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "1615e05c422cfc824b5d85b589d751289a14fd92268b96ba92895adc4887564a", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "missing-epsilon": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "025fc929dd888697240d78b8d58030c7a19df0e32b7bffdc86ed4297be55d73b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "missing-weight": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "a7c77754e4e305fc6c88b5ebaec06538489d024cead945ef67ad64c87beae7af", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "wrong-axis": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "f522d375639326991e4f4f8bdb8d2a6e761e51f2470c116162f359916837eebf", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "input-shape": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "9ed9d0539bb07913db3d117311e12b971eaf324746b600c3b9a9f6cee7157efd", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "replay": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "d2307fb34cac2e6e127c288578322679ecb04459c92ae30c8214a2831afded62", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + } + } + }, + "environment": { + "backend": "ascend", + "device": "Ascend910B1", + "logical_device": 0, + "python": "3.12.13", + "torch": "2.10.0+cpu", + "torch_npu": "2.10.0.post4", + "numpy": "2.5.3", + "cann": "9.1.0", + "os": "Linux", + "machine": "aarch64", + "allocation": "caller-owned; one visible device" + }, + "input_sequence": [ + { + "id": "decode", + "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "input_sha256": "67e5405e42f45a9022c5d100d708f939a21a1db991f7d88a09cd2badfbc73faf", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "bbcce025be7b34405e19e3e2c51ae1a9ee54556c8a2c50f8310709b5645b19e4", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "1b9d2b1b11300da022b73370314a92f7338fb4f519e106b2c7b440b7fff30992", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "5c47fe8288727110b526fb241677814b361c013e48feb52ebe7d062773ef6e81", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "adf128f85e21b7a1b635450bcb9bb2f478e813a352325f15f98811d678002f03", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "94e4f4a69f42e13540f6d0d85492ce1a7c12f66e8d71a0f863e8b957590691c6", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "02e8ebb8c6143b3d1121af61856ea948e4eae5cabe8dd2f37ecf3e96c7d72c73", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "ec0adadf85c19c1c0c70947b33ce97c4fa4e045d2db1d8e7ed91d3fb6754a7a3", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "c06ae630eb37f04c3469f46b288eb6718f2c11253edf73fdd2f1796d0254ca4b", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "35579c2e878a529f9858865aacc2b6c29a18fe2b14adf3a99fe613b2bee48316", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "f02bc6ff8c5b21941045b08b339451b0155fbb02298454097235356807b7d7b5", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "31cefd7002d981c70ff2a9f029843fba0a844863022103fb664383d33ec89136", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "e14628f15ec7e8a972ed4e537affe9113682f6ec7eb2c9857a62b42f50f8f06f", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "cc92864419bb9e875efb43f8532e6aa7f6813caa7c294be27ca15612b2ae2689", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "77325d78aad1312351252a5ba9449831ae7c2c0a69b707ed139fa2290b6a9edf", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "0979c82f4f16d80bc3c4dccb42048fdf0a1ca091406c4686f2157ad497dd9338", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "32ce39afe09617927c5690adfbf682f2c3b7bdbf2a1305df2c85466673be74b9", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "a97206ff3175de40cb7f18fc72f13704925019ef94a39f2981f76df097780ea2", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "bd55b85cf2b3feebb78c06fdfdcc02edfc005211414f6a9d88772fb5716498a2", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "6365fe4e41f62c29341869a54d536730e55ed2ff071580ba3ce9bda2730584ab", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "input_sha256": "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "7bf20a4d149fffd1227663f4ea610b29150eee58eeac96a6535d3ea49c0bd7da", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "fd2e53466459343f302f3ca60ccc5d725af8d8bdd1a328a9e69e9e4b6da6a84a", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "6f84d8ecfafe81a7e6fa6b1adff784e695e255a8f9c8cdd25dded5b0c566cf27", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "ef24320331639ec55deb6318ebad0ae20d33d9d9d78ca55a04f72b944ba6576e", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "260b7bf862444f39605bb1d8b866e3fab3f80d50ae4d200cca17dfbc4d52cac3", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "734e4b02cb2ce77cf33d5541ac91a70ba4a3d337110575c06fd8c841710e01f7", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "852852f78bf3e5dd994dd0a0ee6c7d9d8884d9a179b12a110c500e8ebf7bb29e", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "1db290780ae85fd2f7e956a75c2e1112cf3ffadeca8765d512072dbad7092606", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "fbc2e40841970fc9f0032765a8846c1f561ecf81d7968ff224eb2877d339e2be", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "d87af900bb9847a6d803262c1424f46ceed2a4e9114e3f3f3b474bfb79f5912e", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "6aa37f0d4d395fc7b1a6e1ac363655c7522e016f9af9ed6eea2ece4861546a9d", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "8b5d6e7fda0a0cd7ef01fa541c9106001ef33003986da4702fb8d930ec0ee1fb", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "30ae9e7b8c82f09b8090370c9b26635e0c9938df8afdc25ea1cc91f9e49acc7a", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "716c54f925401dcae62b378adfc416ad6ec2bcfd5baa1450550564f13c160d82", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "052d8abdfa0f612878b5942053f335668867391a8d5a1d526f949d6015701bde", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "f54464abd4d7e88456f7af0f383a8f87a272ff14f056a5871ff7287b0897b7e4", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "cdb0a669d90aab982f348db84aa3352a9712a92e8e671a35115e492c8e546bc0", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "5f5195ff2db02fa76d087bda9a4088ae84b05513e380979dabab6e1a0debf175", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "d3970bb0c428d50034061239e065b20729b0a3ab4aa448c3b3042493eafad9ed", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "ba5097c249135a071fe4d4951d7282a55f3a08fad85ccb445140993b26446d08", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "ece71d95686d8b6e860d1db955705edad6f7f82fa994f7509ebf5a9d0e9468f1", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "a522dfb1ff7a38d77b3616e757e76a745075d3bbec3a7cdfc6d55d8ce3371ac0", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "69a8f334773a595947f42776377f2e602c43dcd55108d649b8bd6f1135744141", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "b72bfd4faaa5decc5ec4a1933801ad95134cbf97afd777e15b7e51e80baef7aa", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "input_sha256": "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "e19ebeab7be1a93cf416bb8cbff53efcb13e9b6b466ed283b96efeb7511fdc9f", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "308e41ee69b40074c40bd3b2eee74773baba275bc1b47284f0c200d1fff310b4", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "d024b54d7dac98b718d67fb80a4cafcbba7df2e001eeb45e9811c1d9f8eb74db", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "input_sha256": "9ed37ae2d2d07b85395e3d8eae98af70654c2fc76c9213cfba7bf213639b560c", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "5e14f8a1d2c798eebc415fa78392b013ff15549dbea9a6c98ffbd38b37129418", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "6c877742ace1d75b6858f58f5627ffd6b0b51e1c8420e16eed426ac7ce1d7723", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "1568551c28137ddbc29e25b2340e63b7d579c13ad1887c82ac3a6523d90fa88b", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "8dcf06509296931f3f807d51c8f4e35a527da84af9d0b48e7367aae90aff4229", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "f9c76c905fc246b5535009c9bb16301721abbde6ac0045a9eb7b84f17a85dbef", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "c5595da004e11308e95cf7ce9ac80368ed0bcc5b7ea433f3ae22a836ef29b4dd", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "4dd078d9682e3baedaac103f0a49f720d7333ed35a2c0dfa989a9d8bc1e0114b", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "e3bb7889247dc9ea493a0da78053a5da4b34ea7a4504728d7e5af50d3552c0e4", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "2851e139d0494ab60090270665e522d47a12b59e9964df9c2088dcb834d2cae3", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "b393f65f71eaad27bf958332fd70fdaa5c2271a22124bcf7544836a8155eadad", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "e0914bd356fbbbdfa8d132cab980fdadf443c8f710ef1e338c878008644f7cf1", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "3664ee501755106b5bc0e9a904c65ad1ff609226ba55245d336d8ac5b8a83d33", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "336ddcb67c73f5c1434edcf5bf89e6e21e63613b6ba2e956e7cdb15e2911e416", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "829455e1b93dad13a73ff070e426bb0e2069c0e67770a135c3c06eadc1fa5c96", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "54074b89e4b968ce56413d46520140563de8516c0e7fa6ab62c34bcd5fe93b47", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "56401b4db5df9b3f6677d071a2d17e9b75c583a0ab68544a58c9e2c698a8006d", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "e6ea20b07896b49ca3a21e1b285c020f9ab2cb0d9923213a777c72a1bf8a924f", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "7d037f31a44526b4be24c48f34ca2666770808c50b58c5a27e053b72a7f01460", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "c27e5de79f782c022a34d2127e08d07fb4b65ca25879170524d44f51b6f5ca80", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "input_sha256": "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "b2ed3add5da5efe3af5628096d5ff8b1fbe33f28e6a8c985e72ff5510ad8e500", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "6dfdb3ae75e8821c2a5b3f6526755f40b593a77d732855e01dfd7ee6a8f9d346", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "9ae2936b3322d8bf03ce38f0ee4fa91df43abc6e923b5156155f85a8ce4eae44", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "cc1f2a67b8e4664c723cc3eb0fed4d3e01c073bbab7a9ffd1f3d4bfeaaf0ba16", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "4c22aceffc31159f51e1ccd41fed6d9b8e752d2dc00812ed9ff99edbd4a8b058", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "0446209c7a1cf2cf58d516ae074e13c4112769460581d5d4422dbc78adffcb7a", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "2ca61fd77bdaf215122a3c01b02734ce0d874d66a8cdeb35201818386dceaf89", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "2865af940cde618eed38bbd4732ebbbc9d7f7917677178203ae8045ac51ea2d3", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "2c1d1b19e0d5d3168369723c30348ce95aa77a16be0e2236d8580181386175bc", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "8f21a4e0eab656713ddfad4bd876bf2325062a5ffc8da57ef29c64a67430697f", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "4444006264a044c02eef87b9bcf7edd648c4ea6f2cfb8add3f260b2327d853c6", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "12c29f46c17552f1d848c345d62c6895e4938f95d4c2b3ffc8ac0e6d203122da", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "882b2bd2626047f2bfa81bd8378e1ff762f4883540cad696087623d4f8c0bb4e", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "61f8c98043d2576ed42ceafb1c173e652c2f957eb385dc62a3e8919ecd433e3c", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "f3fb6d9580316932f297013ba11d18584dbde77092015d22956f6c5615d7d50a", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "3253836502182f819a81c83e8a205fe515d791b69a0266109c2010e69935f2c5", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "f4a28650d4372458fa5f4ca2cdcdb38cd90b0a9e75d38a098e3e608e8e023b29", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "e5a8309d80d6f7d699f9ff3c74c258fbb06f4e777b955c56603319c069c8b3fc", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "62b0a4928daf34cb9c2d9d5cf2f3666e91e9a4b28848b9b15f5dc0c59279e96b", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "df29a68e0b10e1987dec3fdfa32d0bb89f97838ae67b1330df0cbc548cada712", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "247a9bfbe32f0fa31b8fcf46c713e8bed2a6d7aabe22dbae268270ad4a64cd8e", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "4dbc040b0adce5d633006cf20f1d6f09f3c56a641d8d8696896ae7d4315b784c", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "7c5d608b6615654e860046f3df610938c7575db80def275a56c138a1f797068e", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "ab674012075f97a469bf85f53e85c80bee1ae4fafa62823d1544fe38ff1b8a6e", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "input_sha256": "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf", + "shape": [ + 64, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "de454aebaf27a9b49554e330f720ed86a51fed964a201aec50a4e998c4cd7259", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "zeros", + "input_sha256": "c35020473aed1b4642cd726cad727b63fff2824ad68cedd7ffb73c7cbd890479", + "shape": [ + 2, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "7e7c7ad1a45dd9288204324e4f98ea57aca3cbce4b06ed2c7700f09fcba6a40f", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "input_sha256": "a8e8f83cff88a89a0c7019c1246f7c087896fd77c6f5c44f22a2c3bd03f04312", + "shape": [ + 4, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "0629c729a9360829ccac972a3021cea3f534ced7b0d2d81e9f6894d394093bdc", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "input_sha256": "73a7d73ac83fe1d628b5c39d1469bc2393d3715f9702ace8e6d8e69187c7e215", + "shape": [ + 4, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "22366ccbb58dd678cff742c25370137369c5fc27a6f345ac80e9a34688977e45", + "call_index": 0, + "phase": "precheck" + } + ], + "shape_comparisons": { + "aa": { + "decode": { + "baseline": 0.280772, + "candidate": 0.2797225, + "improvement_pct": 0.3737908338438456 + }, + "prefill": { + "baseline": 0.311018, + "candidate": 0.310098, + "improvement_pct": 0.29580281527115115 + } + }, + "reference": { + "decode": { + "baseline": 0.278377, + "candidate": 0.18149500000000002, + "improvement_pct": 34.80244416744198 + }, + "prefill": { + "baseline": 0.31132800000000005, + "candidate": 0.216051, + "improvement_pct": 30.603415047795266 + } + } + }, + "timed_samples": 800, + "actions": { + "baseline-check": { + "implementation": "baseline", + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "check", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "4cd62c65c5c13a7c65729bd77473e6563ceaf3a89d51c218d7ffa208efe578f9", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ] + } + }, + "baseline-profile": { + "implementation": "baseline", + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "profile", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "67fbbee82b21ab6c364c0dc68d61db1c9fd88851fddd414ed681438853e515ea", + "assertions": { + "total": 135, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + } + ], + "diagnostic": { + "kind": "ascend-torch-profiler", + "artifact": "npu-trace.json", + "sha256": "e4dfa493b9066a8bcb4194d77928de9780e5f946dce3cbbadd3f4402f2607950", + "performance_verdict": null, + "kernels": [ + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 1.28 + }, + { + "name": "aclnnMean_ReduceMeanAiCore_ReduceMean", + "duration_us": 2.9 + }, + { + "name": "aclnnAdds_AddAiCore_Add", + "duration_us": 1.12 + }, + { + "name": "aclnnRsqrt_RsqrtAiCore_Rsqrt", + "duration_us": 0.94 + }, + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 1.4 + }, + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 1.38 + }, + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 3.1 + }, + { + "name": "aclnnMean_ReduceMeanAiCore_ReduceMean", + "duration_us": 8.5 + }, + { + "name": "aclnnAdds_AddAiCore_Add", + "duration_us": 1.1 + }, + { + "name": "aclnnRsqrt_RsqrtAiCore_Rsqrt", + "duration_us": 0.88 + }, + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 5.26 + }, + { + "name": "aclnnMul_MulAiCore_Mul", + "duration_us": 5.66 + } + ], + "limitation": "Instrumented diagnostic; includes verification copies; not a score" + } + } + }, + "reference-check": { + "implementation": "reference", + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "check", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "1455ed6031f651524506adb1e834d4d069c48c6742c0f8a1dd27641dec35ea8f", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ] + } + }, + "reference-profile": { + "implementation": "reference", + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "profile", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "4926cb61be9031760239f7b269cef7b519f21f719e600a522cf2e11d01bc4ebd", + "assertions": { + "total": 135, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + } + ], + "diagnostic": { + "kind": "ascend-torch-profiler", + "artifact": "npu-trace.json", + "sha256": "3a3303354707d65cc2dccf438cdac1c0a9b39920bfcbad64f7b8cd714cd870be", + "performance_verdict": null, + "kernels": [ + { + "name": "RmsNorm", + "duration_us": 2.02 + }, + { + "name": "RmsNorm", + "duration_us": 6.44 + } + ], + "limitation": "Instrumented diagnostic; includes verification copies; not a score" + } + } + }, + "zeros": { + "implementation": "zeros", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "1615e05c422cfc824b5d85b589d751289a14fd92268b96ba92895adc4887564a", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "2387d984bd57af12b739482198340c79ec9578a0bb98b550d2c6f8c74b1853f7", + "assertions": { + "total": 5, + "failed": [ + "decode.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "decode", + "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + }, + "missing-epsilon": { + "implementation": "missing-epsilon", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "025fc929dd888697240d78b8d58030c7a19df0e32b7bffdc86ed4297be55d73b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "5c2e2a13a0e81a2420fec1b769693712b73cc7b7e50291d83a84c3cf5bfa1ce3", + "assertions": { + "total": 15, + "failed": [ + "zeros.call-0.finite", + "zeros.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "zeros", + "input_sha256": "c35020473aed1b4642cd726cad727b63fff2824ad68cedd7ffb73c7cbd890479", + "shape": [ + 2, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "7e7c7ad1a45dd9288204324e4f98ea57aca3cbce4b06ed2c7700f09fcba6a40f", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + }, + "missing-weight": { + "implementation": "missing-weight", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "a7c77754e4e305fc6c88b5ebaec06538489d024cead945ef67ad64c87beae7af", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "079737c082a1b4a768755c81f636dd43fbc2478a2c1db1fe2da494675e371a77", + "assertions": { + "total": 5, + "failed": [ + "decode.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "decode", + "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + }, + "wrong-axis": { + "implementation": "wrong-axis", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "f522d375639326991e4f4f8bdb8d2a6e761e51f2470c116162f359916837eebf", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "c02bc2a197e9ad4c287f2f7e9a645f6bdc1efade6ec03d601486f6eb165b5efe", + "assertions": { + "total": 5, + "failed": [ + "decode.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "decode", + "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + }, + "input-shape": { + "implementation": "input-shape", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "9ed9d0539bb07913db3d117311e12b971eaf324746b600c3b9a9f6cee7157efd", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "b260a08297f3fd9a0cfe9806cc9653c8e7b9275586941d399e632b01f05413e9", + "assertions": { + "total": 5, + "failed": [ + "decode.call-0.shape_dtype_device", + "decode.call-0.finite", + "decode.call-0.input_preserved", + "decode.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "decode", + "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + }, + "replay": { + "implementation": "replay", + "execution_status": "failed", + "returncode": 1, + "result": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "d2307fb34cac2e6e127c288578322679ecb04459c92ae30c8214a2831afded62", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "cf91847bdedc0b7377a609e9e474b92a8e0f829081abd2a813b716df311c08ac", + "assertions": { + "total": 25, + "failed": [ + "tiny.call-0.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + } + ], + "failed_call": { + "id": "tiny", + "input_sha256": "73a7d73ac83fe1d628b5c39d1469bc2393d3715f9702ace8e6d8e69187c7e215", + "shape": [ + 4, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "22366ccbb58dd678cff742c25370137369c5fc27a6f345ac80e9a34688977e45", + "call_index": 0, + "phase": "precheck", + "usable_measurement": false + } + } + } + }, + "comparisons": { + "aa": { + "run": "results/aa/20261104T030249Z-4f3c7e153a", + "evaluation": { + "verdict": "FAIL", + "reasons": [ + "measured gain 0.493466% must be positive and meet 10% threshold" + ], + "metric_key": "metrics.suite_latency_ms", + "metric_unit": "ms", + "comparison_scope": "rmsnorm-npu:device-synchronized-eager-call", + "direction": "minimize", + "min_improvement_pct": 10.0, + "baseline": { + "values": [ + 0.562494, + 0.596805, + 0.594275, + 0.591216 + ], + "median": 0.5927454999999999, + "min": 0.562494, + "max": 0.596805, + "range_pct_of_median": 5.78848763929882, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.5784050000000001, + 0.6017159999999999, + 0.586865, + 0.592776 + ], + "median": 0.5898205, + "min": 0.5784050000000001, + "max": 0.6017159999999999, + "range_pct_of_median": 3.952219361653225, + "excluded_trials": 0 + }, + "improvement_pct": 0.4934664202427443, + "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", + "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", + "source_integrity": { + "verified": true, + "reason": "Git source snapshots unchanged", + "command_checks": 9 + }, + "statistical_significance": "not established" + }, + "validation": { + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "check", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "4cd62c65c5c13a7c65729bd77473e6563ceaf3a89d51c218d7ffa208efe578f9", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ] + } + }, + "trials": { + "baseline": [ + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "e6c4f930c7013e4fdfadde5df57f90cc79a3ec5a1359b50cd0bc898d559b479a", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.270847, + 0.266096, + 0.270057, + 0.271407, + 0.271567, + 0.264917, + 0.268437, + 0.265727, + 0.264737, + 0.270917, + 0.263607, + 0.269307, + 0.265517, + 0.268677, + 0.276737, + 0.263507, + 0.280237, + 0.276237, + 0.278567, + 0.274337, + 0.266127, + 0.277277, + 0.273037, + 0.288907, + 0.270107 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.270107 + }, + { + "id": "prefill", + "samples_ms": [ + 0.284087, + 0.292387, + 0.308548, + 0.306467, + 0.311088, + 0.281727, + 0.299718, + 0.323208, + 0.291927, + 0.310288, + 0.283168, + 0.290687, + 0.311578, + 0.359399, + 0.307678, + 0.287588, + 0.290848, + 0.281768, + 0.283218, + 0.361129, + 0.285997, + 0.283188, + 0.292227, + 0.303428, + 0.302648 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.292387 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.562494 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "daa8ed909c7317ec7945e8782ca553fb5ec64a81b9d4cb3486f2b6148e28e0a6", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.281927, + 0.279458, + 0.276987, + 0.284948, + 0.278717, + 0.279757, + 0.276777, + 0.278908, + 0.279247, + 0.281868, + 0.275567, + 0.280217, + 0.276387, + 0.305428, + 0.292888, + 0.279097, + 0.271337, + 0.292507, + 0.289458, + 0.292607, + 0.276958, + 0.290837, + 0.292118, + 0.293278, + 0.289917 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.280217 + }, + { + "id": "prefill", + "samples_ms": [ + 0.299068, + 0.325109, + 0.306588, + 0.307958, + 0.315788, + 0.295298, + 0.321088, + 0.337188, + 0.363559, + 0.348829, + 0.308698, + 0.318308, + 0.325238, + 0.318338, + 0.317199, + 0.304188, + 0.304588, + 0.329579, + 0.307348, + 0.340608, + 0.299978, + 0.311048, + 0.317938, + 0.307848, + 0.316588 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.316588 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.596805 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "e6ecc746fcee67b464c4d56ffd40e07f998aac1022e813f1b484f4c93196411a", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.282347, + 0.280247, + 0.276077, + 0.275798, + 0.277887, + 0.279207, + 0.284948, + 0.292547, + 0.275967, + 0.281327, + 0.280788, + 0.284397, + 0.294678, + 0.280827, + 0.275487, + 0.295578, + 0.296628, + 0.295377, + 0.293608, + 0.273857, + 0.293438, + 0.294307, + 0.278057, + 0.295888, + 0.268257 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.281327 + }, + { + "id": "prefill", + "samples_ms": [ + 0.348389, + 0.310048, + 0.37866, + 0.310458, + 0.302858, + 0.313628, + 0.316648, + 0.311838, + 0.310128, + 0.342249, + 0.319818, + 0.305708, + 0.317698, + 0.312948, + 0.308288, + 0.345349, + 0.306908, + 0.336739, + 0.298448, + 0.291778, + 0.329348, + 0.326248, + 0.308348, + 0.293687, + 0.340649 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.312948 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.594275 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "44552c11f81550c1c1150499927f0409a71fd1033b999bc6ed9f4809702072aa", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.279918, + 0.277137, + 0.276218, + 0.283837, + 0.278237, + 0.285907, + 0.279707, + 0.277067, + 0.301048, + 0.278927, + 0.282128, + 0.286027, + 0.277818, + 0.295238, + 0.282007, + 0.291728, + 0.276038, + 0.286067, + 0.289208, + 0.292457, + 0.290978, + 0.270327, + 0.289688, + 0.286017, + 0.281787 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.282128 + }, + { + "id": "prefill", + "samples_ms": [ + 0.300278, + 0.290167, + 0.324408, + 0.305528, + 0.302278, + 0.309088, + 0.332289, + 0.322138, + 0.316509, + 0.307378, + 0.289108, + 0.312208, + 0.327008, + 0.308968, + 0.304528, + 0.286548, + 0.310698, + 0.301538, + 0.313818, + 0.313968, + 0.293087, + 0.318148, + 0.310828, + 0.345019, + 0.300628 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.309088 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.591216 + } + } + ], + "candidate": [ + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "43d40b486033db5eb73393ce1c96073cbbdd94701c4d6f0ace5738c672095ab7", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.268957, + 0.274997, + 0.269927, + 0.270507, + 0.274437, + 0.269557, + 0.270717, + 0.274387, + 0.276287, + 0.274607, + 0.270547, + 0.285797, + 0.270807, + 0.275297, + 0.275727, + 0.265337, + 0.296247, + 0.288358, + 0.284787, + 0.280047, + 0.266397, + 0.288228, + 0.281347, + 0.270727, + 0.275027 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.274607 + }, + { + "id": "prefill", + "samples_ms": [ + 0.283128, + 0.328019, + 0.316228, + 0.310008, + 0.303258, + 0.368039, + 0.311428, + 0.299017, + 0.305118, + 0.299138, + 0.282677, + 0.310678, + 0.292498, + 0.296418, + 0.293128, + 0.286247, + 0.313318, + 0.303798, + 0.304588, + 0.276517, + 0.309998, + 0.313139, + 0.314248, + 0.299268, + 0.283177 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.303798 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.5784050000000001 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "c69faca2121e76d949e80543a81da9807a2a67515270985f5ea08db136b12f80", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.289408, + 0.291968, + 0.287238, + 0.287267, + 0.281397, + 0.280077, + 0.282908, + 0.293968, + 0.281697, + 0.284557, + 0.283707, + 0.287858, + 0.282117, + 0.278748, + 0.302808, + 0.279927, + 0.296038, + 0.297638, + 0.291787, + 0.294068, + 0.279767, + 0.287098, + 0.289248, + 0.288517, + 0.283037 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.287238 + }, + { + "id": "prefill", + "samples_ms": [ + 0.311598, + 0.318548, + 0.314478, + 0.319678, + 0.308078, + 0.340629, + 0.316158, + 0.316878, + 0.327178, + 0.298378, + 0.332238, + 0.39341, + 0.329178, + 0.298788, + 0.289857, + 0.331209, + 0.305818, + 0.318578, + 0.301828, + 0.307138, + 0.310748, + 0.353699, + 0.313078, + 0.306628, + 0.285477 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.314478 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.6017159999999999 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "707d18426bc85667b32bc24cb95094f4370f9a1508af27fdedefa0544a3bcf0b", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.276737, + 0.275457, + 0.277717, + 0.273337, + 0.269667, + 0.276157, + 0.279028, + 0.279767, + 0.274217, + 0.275427, + 0.274507, + 0.279858, + 0.272408, + 0.270557, + 0.274467, + 0.294408, + 0.289007, + 0.288408, + 0.275858, + 0.281447, + 0.276027, + 0.283317, + 0.276807, + 0.268507, + 0.281448 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.276157 + }, + { + "id": "prefill", + "samples_ms": [ + 0.310398, + 0.318938, + 0.308028, + 0.296927, + 0.310728, + 0.324799, + 0.298108, + 0.312448, + 0.293347, + 0.316298, + 0.328679, + 0.310708, + 0.328508, + 0.305748, + 0.304708, + 0.35932, + 0.293017, + 0.327178, + 0.287658, + 0.334258, + 0.310108, + 0.315598, + 0.313748, + 0.290037, + 0.308377 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.310708 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.586865 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "2f2735d06ad4d7e6e091dbc618a6bec04adc35d3611ed4fc577e298c7041e350", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.277157, + 0.286788, + 0.286737, + 0.279377, + 0.283347, + 0.279617, + 0.280617, + 0.276967, + 0.279507, + 0.284868, + 0.280347, + 0.288298, + 0.280207, + 0.277538, + 0.287137, + 0.277607, + 0.289497, + 0.287818, + 0.292537, + 0.278067, + 0.312748, + 0.292008, + 0.283288, + 0.288028, + 0.278667 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.283288 + }, + { + "id": "prefill", + "samples_ms": [ + 0.305378, + 0.318388, + 0.310698, + 0.306658, + 0.301188, + 0.329069, + 0.322308, + 0.324909, + 0.304208, + 0.295508, + 0.311948, + 0.309128, + 0.311028, + 0.35008, + 0.297178, + 0.309488, + 0.326808, + 0.309538, + 0.352869, + 0.296298, + 0.300757, + 0.310768, + 0.299508, + 0.299807, + 0.290638 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.309488 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.592776 + } + } + ] + } + }, + "reference": { + "run": "results/reference/20261104T030558Z-3dea96ead1", + "evaluation": { + "verdict": "PASS", + "reasons": [], + "metric_key": "metrics.suite_latency_ms", + "metric_unit": "ms", + "comparison_scope": "rmsnorm-npu:device-synchronized-eager-call", + "direction": "minimize", + "min_improvement_pct": 10.0, + "baseline": { + "values": [ + 0.576635, + 0.600085, + 0.576815, + 0.601486 + ], + "median": 0.5884499999999999, + "min": 0.576635, + "max": 0.601486, + "range_pct_of_median": 4.223128558076295, + "excluded_trials": 0 + }, + "candidate": { + "values": [ + 0.38543000000000005, + 0.391621, + 0.418691, + 0.403471 + ], + "median": 0.397546, + "min": 0.38543000000000005, + "max": 0.418691, + "range_pct_of_median": 8.366578961931431, + "excluded_trials": 0 + }, + "improvement_pct": 32.44183872886396, + "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", + "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", + "source_integrity": { + "verified": true, + "reason": "Git source snapshots unchanged", + "command_checks": 9 + }, + "statistical_significance": "not established" + }, + "validation": { + "execution_status": "completed", + "returncode": 0, + "result": { + "status": "completed", + "action": "check", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "1455ed6031f651524506adb1e834d4d069c48c6742c0f8a1dd27641dec35ea8f", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ] + } + }, + "trials": { + "baseline": [ + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "0d08d29a995bcf420bb0d6aef000c66b9974ba9cf4a4b608ce68cdaa46fe332b", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.267757, + 0.265677, + 0.256887, + 0.268697, + 0.268847, + 0.257436, + 0.267587, + 0.268607, + 0.262027, + 0.268957, + 0.279077, + 0.271568, + 0.263547, + 0.274217, + 0.273097, + 0.258777, + 0.265856, + 0.273887, + 0.277548, + 0.264257, + 0.262926, + 0.278037, + 0.266266, + 0.278607, + 0.270247 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.268607 + }, + { + "id": "prefill", + "samples_ms": [ + 0.299667, + 0.346879, + 0.313618, + 0.309108, + 0.303218, + 0.292688, + 0.312638, + 0.313888, + 0.309178, + 0.308868, + 0.290317, + 0.308028, + 0.306678, + 0.305498, + 0.396641, + 0.290727, + 0.324258, + 0.299758, + 0.327229, + 0.296688, + 0.287847, + 0.310688, + 0.300957, + 0.324979, + 0.296638 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.308028 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.576635 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "f693b899f85c79dc16d47fb4904ed471e4ffb565b0c50188d0072e9051e39830", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.284207, + 0.286007, + 0.282617, + 0.282627, + 0.282278, + 0.280157, + 0.278847, + 0.278977, + 0.283977, + 0.283128, + 0.281907, + 0.294708, + 0.282538, + 0.282777, + 0.289068, + 0.296658, + 0.296707, + 0.284458, + 0.296647, + 0.294347, + 0.293258, + 0.278618, + 0.295507, + 0.293528, + 0.288477 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.284207 + }, + { + "id": "prefill", + "samples_ms": [ + 0.296308, + 0.310748, + 0.315878, + 0.311748, + 0.318928, + 0.303837, + 0.321958, + 0.318509, + 0.345599, + 0.314638, + 0.318058, + 0.322178, + 0.323429, + 0.316218, + 0.303997, + 0.294498, + 0.315908, + 0.309688, + 0.327929, + 0.308798, + 0.308478, + 0.319508, + 0.325189, + 0.308988, + 0.298388 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.315878 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.600085 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "a72aa38bca6d047312aeb27cd5a4cdb4b1aa1942580475059b39bd3c728f3f8d", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.285008, + 0.272907, + 0.272557, + 0.269076, + 0.272347, + 0.289628, + 0.270177, + 0.271747, + 0.270787, + 0.266397, + 0.286838, + 0.272547, + 0.268597, + 0.268837, + 0.265257, + 0.302148, + 0.280647, + 0.276927, + 0.274677, + 0.262726, + 0.295267, + 0.278137, + 0.269807, + 0.275467, + 0.263237 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.272547 + }, + { + "id": "prefill", + "samples_ms": [ + 0.310008, + 0.303008, + 0.322589, + 0.304388, + 0.300998, + 0.304848, + 0.323119, + 0.293837, + 0.346969, + 0.295828, + 0.311328, + 0.290568, + 0.304638, + 0.294747, + 0.294827, + 0.334908, + 0.305898, + 0.321639, + 0.295658, + 0.282977, + 0.312888, + 0.301587, + 0.304268, + 0.292718, + 0.301867 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.304268 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.576815 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "2535a0b44525b0dfa72952f9d16d9a779fee5f30337ad2f758f8a502e07346cb", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.305708, + 0.283368, + 0.283817, + 0.287737, + 0.284228, + 0.288518, + 0.279467, + 0.287807, + 0.280578, + 0.282327, + 0.285267, + 0.287397, + 0.283177, + 0.284328, + 0.283157, + 0.286858, + 0.285507, + 0.299728, + 0.294537, + 0.320048, + 0.293948, + 0.284608, + 0.294488, + 0.300218, + 0.297227 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.286858 + }, + { + "id": "prefill", + "samples_ms": [ + 0.304908, + 0.309648, + 0.323928, + 0.314628, + 0.318128, + 0.313288, + 0.38072, + 0.320378, + 0.314718, + 0.322139, + 0.317569, + 0.291758, + 0.325328, + 0.326618, + 0.312038, + 0.319998, + 0.312948, + 0.315088, + 0.300598, + 0.304678, + 0.297107, + 0.294867, + 0.313938, + 0.308348, + 0.317828 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.314628 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.601486 + } + } + ], + "candidate": [ + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "f10e62ed2cea8643f3117c0eb973ead40196e5c67b753c5bbdc8d4fdc2a3955e", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.181515, + 0.178065, + 0.179005, + 0.182205, + 0.176414, + 0.175604, + 0.181525, + 0.174984, + 0.176575, + 0.179895, + 0.182755, + 0.177965, + 0.175754, + 0.181194, + 0.176764, + 0.176855, + 0.179594, + 0.175994, + 0.175415, + 0.178925, + 0.174664, + 0.176605, + 0.177355, + 0.177714, + 0.176105 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.177355 + }, + { + "id": "prefill", + "samples_ms": [ + 0.221756, + 0.217806, + 0.232246, + 0.226976, + 0.211076, + 0.216186, + 0.230906, + 0.205366, + 0.206545, + 0.208616, + 0.204075, + 0.207695, + 0.204405, + 0.204425, + 0.205185, + 0.216805, + 0.208755, + 0.206635, + 0.203855, + 0.208075, + 0.199936, + 0.204866, + 0.210575, + 0.210916, + 0.204786 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.208075 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.38543000000000005 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "eca6c01839b942f61805159230039382e439563d64cf837c5c27ccd7872e36dd", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.179844, + 0.182995, + 0.179665, + 0.176995, + 0.179974, + 0.179614, + 0.183784, + 0.175794, + 0.176565, + 0.192985, + 0.182935, + 0.182015, + 0.178125, + 0.176615, + 0.183165, + 0.176175, + 0.176875, + 0.176795, + 0.175054, + 0.174945, + 0.197355, + 0.181685, + 0.174924, + 0.178985, + 0.173115 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.178985 + }, + { + "id": "prefill", + "samples_ms": [ + 0.217126, + 0.217286, + 0.219296, + 0.225376, + 0.212636, + 0.211676, + 0.212565, + 0.210786, + 0.212675, + 0.225246, + 0.224156, + 0.217836, + 0.216165, + 0.209086, + 0.210585, + 0.213206, + 0.214775, + 0.209486, + 0.208496, + 0.208235, + 0.211406, + 0.213915, + 0.211525, + 0.211666, + 0.208085 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.212636 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.391621 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "f5410d252698f682a3f329db06602aeebd84a9bce2f80801dddd4e5761b224c4", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.190515, + 0.179955, + 0.192655, + 0.183875, + 0.183085, + 0.183675, + 0.183365, + 0.184325, + 0.184784, + 0.187925, + 0.186285, + 0.182764, + 0.183875, + 0.187485, + 0.185304, + 0.184644, + 0.188105, + 0.180804, + 0.184775, + 0.181704, + 0.194745, + 0.183125, + 0.179784, + 0.188125, + 0.182044 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.184325 + }, + { + "id": "prefill", + "samples_ms": [ + 0.233746, + 0.235126, + 0.239556, + 0.235576, + 0.231426, + 0.234366, + 0.229576, + 0.235086, + 0.229916, + 0.232946, + 0.233096, + 0.238186, + 0.236866, + 0.235816, + 0.239157, + 0.235706, + 0.231616, + 0.237136, + 0.238596, + 0.243846, + 0.232456, + 0.232976, + 0.231946, + 0.233166, + 0.231306 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.234366 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.418691 + } + }, + { + "status": "completed", + "action": "measure", + "passed": true, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "a71402152841bb12eb1d6d9d7bd4416b94fcf716835ca2fc13e26c9b4fd19936", + "assertions": { + "total": 375, + "failed": [] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "decode", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "decode", + "call_index": 35, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 1, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 2, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 3, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 4, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 5, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 6, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 7, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 8, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 9, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 10, + "phase": "warmup" + }, + { + "id": "prefill", + "call_index": 11, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 12, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 13, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 14, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 15, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 16, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 17, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 18, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 19, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 20, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 21, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 22, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 23, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 24, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 25, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 26, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 27, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 28, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 29, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 30, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 31, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 32, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 33, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 34, + "phase": "measurement" + }, + { + "id": "prefill", + "call_index": 35, + "phase": "measurement" + } + ], + "cases": [ + { + "id": "decode", + "samples_ms": [ + 0.187135, + 0.190325, + 0.182625, + 0.189474, + 0.192355, + 0.181384, + 0.183195, + 0.181024, + 0.189715, + 0.182675, + 0.181085, + 0.190195, + 0.181104, + 0.184575, + 0.181755, + 0.186085, + 0.185894, + 0.186874, + 0.178724, + 0.199585, + 0.186425, + 0.184005, + 0.183164, + 0.181894, + 0.180184 + ], + "sample_input_sha256": [ + "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", + "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", + "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", + "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", + "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", + "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", + "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", + "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", + "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", + "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", + "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", + "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", + "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", + "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", + "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", + "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", + "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", + "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", + "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", + "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", + "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", + "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", + "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", + "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", + "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" + ], + "median_ms": 0.184005 + }, + { + "id": "prefill", + "samples_ms": [ + 0.220516, + 0.222816, + 0.226316, + 0.215386, + 0.221336, + 0.219466, + 0.218496, + 0.217926, + 0.220756, + 0.225896, + 0.212485, + 0.215786, + 0.220526, + 0.232436, + 0.212225, + 0.213636, + 0.223476, + 0.220956, + 0.248296, + 0.218096, + 0.214235, + 0.216275, + 0.218026, + 0.222556, + 0.213775 + ], + "sample_input_sha256": [ + "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", + "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", + "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", + "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", + "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", + "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", + "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", + "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", + "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", + "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", + "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", + "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", + "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", + "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", + "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", + "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", + "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", + "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", + "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", + "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", + "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", + "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", + "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", + "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", + "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" + ], + "median_ms": 0.219466 + } + ], + "measurement": { + "aggregation": "sum-of-per-case-medians", + "boundary": "device-synchronized-eager-call", + "case_ids": [ + "decode", + "prefill" + ], + "input_sequence": "seeded-distinct-per-call-v1", + "key": "metrics.suite_latency_ms", + "samples": 25, + "unit": "ms", + "warmups": 10 + }, + "metrics": { + "suite_latency_ms": 0.403471 + } + } + ] + } + } + }, + "postcheck_replay": { + "status": "completed", + "action": "measure", + "passed": false, + "identity": { + "task_id": "rmsnorm-npu", + "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", + "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", + "candidate_sha256": "6e8d6bc8603515a3a7f3ecf697775bc2fa530d894a0fdb9d5b332d4a4936601b", + "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" + }, + "raw_result_sha256": "8040f42798852733230ed28ad8d945fbfc8421fd0139c75a98a095128b53afaf", + "assertions": { + "total": 30, + "failed": [ + "decode.call-1.values" + ] + }, + "executed_calls": [ + { + "id": "decode", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "prefill", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "zeros", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "large", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "tiny", + "call_index": 0, + "phase": "precheck" + }, + { + "id": "decode", + "call_index": 1, + "phase": "warmup" + } + ], + "failed_call": { + "id": "decode", + "input_sha256": "67e5405e42f45a9022c5d100d708f939a21a1db991f7d88a09cd2badfbc73faf", + "shape": [ + 1, + 4096 + ], + "dtype": "float32", + "strides": [ + 16384, + 4 + ], + "weight_sha256": "bbcce025be7b34405e19e3e2c51ae1a9ee54556c8a2c50f8310709b5645b19e4", + "call_index": 1, + "phase": "warmup", + "usable_measurement": false + }, + "candidate_source": "import torch\nn = 0\ncache = {}\ndef compute(x, weight, *, eps):\n global n\n n += 1\n key = tuple(x.shape)\n if n <= 5:\n cache[key] = x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight\n return cache[key]\n" + }, + "software_ci": { + "url": "https://github.com/InferMatrixLab/InferRSI/actions/runs/41764076901", + "tested_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", + "checkout_merge_sha": "6f1506bcb366c10836991701f8beaf7ac89e9a11", + "head_and_merge_tree": "5c22811e28f1c8c0182852916f43804c18b35edf", + "full_suites": { + "3.10": { + "passed": 3649, + "skipped": 20, + "subtests_passed": 284 + }, + "3.11": { + "passed": 3649, + "skipped": 20, + "subtests_passed": 284 + } + }, + "installed_packages": "passed on 3.10 and 3.11", + "cpu_artifact_controls": { + "passed": 54, + "skipped": 0, + "junit_sha256": "e724dce4b3f7ddfe753832bea4150b2ab6aa490d4e410847b34679cb8fe13dd2", + "cases": [ + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_freezes_builtin_task_and_shared_grader[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_freezes_builtin_task_and_shared_grader[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_formal_artifact_path_rejects_new_backend_packages[adapter]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_formal_artifact_path_rejects_new_backend_packages[environment]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[hash]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[holdout]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[path]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[request]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[extra]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[contract]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change0]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change1]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change2]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_invalid_artifact_does_not_start_source[change3]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_no_sandbox_means_explicit_unavailable_never_host_execution", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_missing_numpy_is_explicit_unavailable", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[dtype]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[values]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[mutation]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[truncated]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[malformed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_workspace_freeze_excludes_only_implementation", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_runtime_version_mismatch_is_unavailable", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_worker_startup_is_distinct_from_artifact_failure[False]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_worker_startup_is_distinct_from_artifact_failure[True]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_rejects_zero_output_despite_cwd_shadow", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark", + "name": "test_real_sandbox_cannot_read_host_sentinel_or_oracle", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_real_authored_processes_use_frozen_task_and_retain_both_sources", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[task]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[schema]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_changed_frozen_input_prevents_any_producer_call[starter]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_task_grader_and_oracle_dependencies_are_frozen_before_producer", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[changed-input-inputs_changed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[extra-file-inputs_changed]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[missing-artifact-missing_or_invalid_artifact]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[oversized-missing_or_invalid_artifact]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_invalid_handoff_is_not_a_clean_comparison[invalid-answer-invalid_artifact_answer]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_no_unsafe_default_launch_or_silent_sandbox_fallback", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_retained_source_tamper_invalidates_report_without_rerunning_producer", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_unknown_usage_stops_new_trials_without_inventing_cost", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change0]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change1]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change2]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change3]", + "outcome": "passed" + }, + { + "class": "tests.test_skill_eval_benchmark_compare", + "name": "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable", + "outcome": "passed" + } + ] + }, + "npu_in_hosted_ci": "not executed; hosted device test skipped, remote evidence above is separate" + } +} diff --git a/docs/plans/issue-252/npu-validation.md b/docs/plans/issue-252/npu-validation.md new file mode 100644 index 000000000..3f9ddc814 --- /dev/null +++ b/docs/plans/issue-252/npu-validation.md @@ -0,0 +1,60 @@ +# NPU 试点实测：RMSNorm 已接通检查、profiling 和 Workbench + +2026-11-04。**本轮已在一张 Ascend 910B1 上跑通完整的软件控制试点。** 正确实现通过，错误实现被拒绝，实际设备 profile 可读，同码对照没有被选为优化，融合参考实现在两个规定 workload 上达到执行前固定的改善门槛。七项资格检查均通过；项目验收、真实 Agent/skill 收益和服务收益仍未建立。 + +执行前的[范围、源码和判断门槛](https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6621316099)先于本次资格实验发表。被测源码为 `6308268fc6d1fbf90e9ab97b5da8af79152f39ea`；后续本记录仅整理证据，不将结果改标为新实现的运行。 + +[可公开证据与全部 800 个计时样本](npu-validation-summary.json)保留原 verdict、冻结身份、环境、逐项错误和 profile kernel 摘要。完整 raw archive 留在执行归档；公开转换及每份原始结果、trace、JUnit 的摘要也在该文件中。 + +## 实际做了什么，它有什么用 + +| 步骤 | 本轮实际结果 | 在 RSI 中发挥的作用 | +| --- | --- | --- | +| 固定任务 | `rmsnorm-npu`，二维 contiguous float32，`compute(x, weight, *, eps)`；5 个基础 case，非全一权重 | baseline 和候选做同样的工作，不能减少工作量换分 | +| 独立检查 | baseline、fused reference 各 75 次调用、375 项断言通过；oracle 使用 scalar double / math.fsum | 检查数值、shape/dtype/device、输入与权重保持 | +| profiling | 两个受测 shape 上，baseline 共 12 个计算 kernel，fused 共 2 个 `RmsNorm` kernel | 验证这份候选确实走融合设备路径；辅助解释调用延迟差异 | +| 错误控制 | 全零、漏 epsilon、漏权重、错误轴、原地改 shape、缓存旧输出均失败且无可用 metric | 阻止错误但看似很快的实现进入性能选择 | +| 同码 A/A | 两侧都运行 baseline；各 shape 中位数差小于 0.4%；Workbench 原始 verdict 为 FAIL（没有达到收益门槛） | 没有将 no-op 当作优化；这里的 FAIL 是预期控制结果 | +| 参考优化比较 | composed baseline → 既有 fused 实现；两个 shape 分别降低约 34.80%、30.60% 的同步调用延迟 | 在本轮范围内支持保留融合参考实现 | +| 留存与重验 | 16 个 trial / 800 个计时样本、最终 validation、完整环境和冻结 identity 均核对通过 | 决定可以回溯到真实源码、实际执行和原始数据 | + +另外执行了**预检后才重放**的控制：前 5 个基础 case 的 25 项断言全通过，首个变化输入 warmup 被拒绝，无性能 metric。它补充证明检查没有停在最初的正确输出。该额外控制的源码、身份、失败 phase 和结果摘要也保留在公开证据中。 + +## NPU 与计时条件 + +实际环境为 Ascend 910B1、Linux aarch64、Python 3.12.13、torch 2.10.0+cpu、torch_npu 2.10.0.post4、NumPy 2.5.3、CANN 9.1.0。PyTorch 的 `+cpu` 包版本后缀不表示在 CPU 计算：设备 tensor、实际 `Ascend Hardware` kernel 事件和 torch_npu 执行分别核对。使用个人非 root UID、已有设备组和共享调度器单卡预约，主资格与额外重放控制退出码均为 0，完成后设备已释放。 + +计时为 **同步 eager 完成调用延迟**：预同步后开始计时，调用实现，再等待 NPU 完成才停止。包含 Python dispatch 和完成同步成本；输入/权重准备、H2D、独立 oracle、D2H 校验、profiling 均不进入该计时。因此下表不能称作纯 kernel 时间，更不能直接转换成模型或服务收益。 + +每个 shape 先做 10 次变化输入预热，再保留 25 个样本；每个比较使用 4 对 AB/BA。下表为各臂 4 个 trial 的对应 shape 中位数再取中位数。所有慢样本和原始 Workbench 聚合均保留，没有按结果筛选。 + +| 对照 | shape | baseline（ms） | 对侧（ms） | 同步调用延迟降低 | +| --- | --- | ---: | ---: | ---: | +| A/A，同实现 | `[1,4096]` | 0.280772 | 0.2797225 | 0.374% | +| A/A，同实现 | `[64,4096]` | 0.311018 | 0.310098 | 0.296% | +| baseline → fused | `[1,4096]` | 0.278377 | 0.181495 | 34.802% | +| baseline → fused | `[64,4096]` | 0.311328 | 0.216051 | 30.603% | + +这证明了本轮固定环境和 workload 下的区分能力；仍是有限次描述性比较，没有建立跨机器稳定性、置信区间或服务增益。门槛来自执行前的 Issue 计划，不是新增仓库默认性能门槛。 + +## Profiling 怎样影响判断 + +baseline 在每个选定输入上出现 3 次 Mul、1 次 ReduceMean、1 次 Adds、1 次 Rsqrt；融合路径每个输入只有 1 次 `RmsNorm`。因此这份实现确实减少了设备算子调用，性能差异不是把结果移到 CPU 或返回另一个形状造成的。 + +本轮候选来自预先选择的官方 fused reference。主 Agent 根据 profile 核对机制，再根据独立检查、A/A 与固定条件对照决定保留它；没有声称 Agent 从 trace 自主发明或搜索出新 kernel。profile 属于诊断证据，不能用其中的 instrumented 时间替换上表。 + +## 软件回归与 review 修复 + +同一 source head 的 [Linux ready/review CI](https://github.com/InferMatrixLab/InferRSI/actions/runs/41764076901)通过：Python 3.10 和 3.11 各 **3649 passed / 20 skipped / 284 subtests passed**，两个安装包检查通过。托管 CI 中没有 NPU；设备验证来自上面的独立远端运行。 + +正式 CPU artifact job **54 passed / 0 skipped**，其中新增的 `test_real_sandbox_rejects_zero_output_despite_cwd_shadow` 真正执行，错误零输出在同名 cwd 包放宽容差后仍被规范 oracle 拒绝。另有 descriptor/starter/prepare/dependency 的来源回归。CI checkout merge `6f1506bcb366c10836991701f8beaf7ac89e9a11` 与被测 head 的 tree 均为 `5c22811e28f1c8c0182852916f43804c18b35edf`。这对应已实现的 P2 修复；原 reviewer 的审查结论仍待复审，不由作者标为接受。 + +新增 NPU 无设备回归本地最终 **14 passed / 1 device test skipped**。开发首轮的 6 项失败分别来自新测试的文本编码和输出目录使用错误，修复后重跑；没有归因于既有 baseline。独立源码审查还发现并修复了输入 shape 原地修改、A/A 最终 validation 失败混同无收益、冻结控制身份未重验三处问题。 + +早期探测的容器用户名初始化失败，以及一次 profile 后调用不存在的 `key_averages()` 导致退出 1，保留在原始日志中。正式 adapter 不使用该方法，直接核对硬件 trace；失败探测不参与 800 个性能样本。 + +## 如何复现和继续 + +使用 [NPU 试点的操作说明](npu-pilot.md)和 [共享接口](../../engineering-benchmarks.md)，在已预约的兼容 NPU 环境中运行 `python -m tools.npu_benchmark_validate`，显式传入本轮 Issue 的重复次数、预算与门槛。工具先写入 `plan/frozen.json`、`validator/task.json` 与 `patch/`，随后通过既有 Workbench 保留 `results/`、报告及 `summary.json`。 + +当前可保留的是这个 float32 NPU 参考候选及共享评测通路。BF16/FP16、更多形状、真实模型/服务、NPU 产物隔离和真实 Agent 两臂仍需分别验收。四个 Issue 与 PR #338 保持打开/Draft，NPU 软件控制通过不自动完成整个 #252。 diff --git a/docs/plans/issue-252/task-expansion/01-operators.md b/docs/plans/issue-252/task-expansion/01-operators.md index dba2ebb77..cfcfcb33c 100644 --- a/docs/plans/issue-252/task-expansion/01-operators.md +++ b/docs/plans/issue-252/task-expansion/01-operators.md @@ -2,7 +2,7 @@ **当前题目：在 Ascend NPU 的给定输入、权重和容差下，比较 affine RMSNorm 的基础运算与融合实现，同时保留计算结果。** -状态：用户已指定 NPU 试点，`rmsnorm-npu` 是当前实施目标；真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较以 [NPU 试点记录](../npu-pilot.md)为准，不能从方案推断完成。Softmax/RMSNorm CPU 两题保留为软件控制；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 +状态：`rmsnorm-npu` 已在 Ascend 910B1 完成真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较，见[实测记录](../npu-validation.md)。资格限已记录 runtime、float32 与 workload，不能外推整类算子或其他硬件。Softmax/RMSNorm CPU 两题保留为软件控制；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 ## 为什么先在 NPU 做这个任务 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md index 6a2dec942..6b1458e62 100644 --- a/docs/plans/issue-252/task-expansion/README.md +++ b/docs/plans/issue-252/task-expansion/README.md @@ -32,7 +32,7 @@ AgentKernelArena 适合学习 kernel 任务的提交与独立评测；整个 Inf | [InferenceBench：任务与评测入口](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/README.md) | `108ec74`；Apache-2.0 | 借服务 launcher、不同负载、独立质量门槛、同预算非 Agent 搜索对照。完整原生流程有指定 GPU、数据缓存及模型裁判，不直接移植平台 | | [Mooncake：分配 benchmark](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/README.md) | `e5ef0e4`；Apache-2.0 | 借混合尺寸分配/释放 workload、分配失败与碎片指标，并绑定独立 allocator tests；先验证最小 C++ 构建，不把逻辑容量称为 HBM 节省 | | [nccl-tests](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/README.md)、[MSCCL-tools](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/tests/test_language.py)、[MSCCL++](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/mscclpp-test/allreduce_test.cu) | `afd59ab` BSD-3-Clause；`030a750` / `f5643de` MIT | 分别提供真实测量、可检查的通信计划、自定义 collective。需要明确候选修改面；硬件测试工具本身不是 Agent 优化题 | -| [llama.cpp：llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) | `149f564`；MIT | 借 CPU prefill/decode、线程/batch 配置与逐次样本。固定模型资产后可先做 CPU；计时工具不自动提供质量 oracle | +| [llama.cpp：llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) | `149f564`；MIT | 借 CPU prefill/decode、线程/batch 配置与逐次样本。固定模型资产后可作为后续 CPU 服务任务来源；计时工具不自动提供质量 oracle | | [vllm-evolve：plugin](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/targets/scheduling/plugin_template.py) | `43cc82c`；实际 LICENSE 为 Apache-2.0 | 借窄 admission 策略入口与实际生效证明。当前主要重排 FCFS waiting deque；不能按 roadmap 声称已有 KV eviction/routing 或模拟平台 | | [AutoTuneBench](https://github.com/li-ch/autotunebench/blob/64cc8d7ca5d0ee40c4e2189b2c2c8dfb1a2a7c2f/README.md) 与仓内 [已核查参考](../../../../src/inferrsi/data/skill_library/methods/inference-benchmark-design/references/autotunebench-trial-validity.md) | `64cc8d7`；选定代码/数据 MIT，vendor 许可分开 | 借 treatment 是否真正送达、原始记录与资格先于收益的审计方式。既有审查已发现声明与部分执行路径不一致，不把 README 的完整流程当作已验证实现 | From 125b06451a487e76a1c20c45abe4e22cd8631d68 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 15:45:58 +0800 Subject: [PATCH 18/20] docs: consolidate task library and evaluation integration plans Keep profiling optional, distinguish existing task tools from pending real-Agent integration, and preserve scoped evidence and unresolved review findings. Agent-assisted-by: Codex (documentation and scope review) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/plans/issue-252/01-research.md | 6 +- docs/plans/issue-252/02-contract.md | 4 +- docs/plans/issue-252/03-tasks.md | 32 +++++- docs/plans/issue-252/04-integration.md | 99 ++++++++----------- docs/plans/issue-252/README.md | 69 +++++++------ docs/plans/issue-252/formal-integration.md | 2 +- docs/plans/issue-252/npu-pilot.md | 4 +- docs/plans/issue-252/npu-validation.md | 6 +- docs/plans/issue-252/review.md | 2 + docs/plans/issue-252/rsi-integration.md | 8 +- docs/plans/issue-252/task-expansion/README.md | 16 +-- docs/plans/issue-252/walkthrough.md | 2 + 12 files changed, 132 insertions(+), 118 deletions(-) diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md index 9c958d4c2..2c41e70fe 100644 --- a/docs/plans/issue-252/01-research.md +++ b/docs/plans/issue-252/01-research.md @@ -1,4 +1,6 @@ -# 子项 01：选定第一批任务和复用方案 +# 研究支持文档：选定任务和复用方案 + +原独立 Issue #334 的范围已并入 [#336 任务库建设](03-tasks.md)和 [#337 流程接入](04-integration.md)。本文保留论文、固定源码与历史采用理由；研究仍随每批任务和接口工作进行，不再作为独立交付 Issue。旧 Issue 关闭不代表所有研究候选已实现或获验收。 2026-11-04 范围更新：用户已指定 NPU 试点，当前优先交付 [RMSNorm NPU 方案](npu-pilot.md)。[五类任务研究](task-expansion/README.md)继续保留，但 Tokenizers.encode 不再作为当前试点。下面的初始 CPU 选型作为历史设计依据，不代表本轮设备验收或整个 SearchSpace 的最终覆盖。 @@ -12,7 +14,7 @@ | 实际作用 | 明确 NPU 试点问题、基线与参考候选，复用已有 Workbench；CPU SkillEval 资格不自动延伸到设备 | | 当前状态 | 既有研究保留，NPU 选择与资格计划明确；设备结果和维护者验收分别记录 | -它是一次设计交付，运行时不会执行“调研模块”。具体成果落实在后面三个子项。 +它是设计依据，运行时不会执行“调研模块”。具体成果落实在任务库和流程接入两项工作中。 ## 理解 diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md index 22f460601..6e8e47f91 100644 --- a/docs/plans/issue-252/02-contract.md +++ b/docs/plans/issue-252/02-contract.md @@ -1,4 +1,6 @@ -# 子项 02：规定任务怎样提交、运行和判定 +# 接口支持文档：规定任务怎样提交、运行和判定 + +原独立 Issue #335 的范围已并入 [#337 流程接入](04-integration.md)；任务作者需要的契约也由 [#336](03-tasks.md)引用。本文继续维护共享接口与已支持范围，不因旧 Issue 关闭而把待补的 NPU 产物执行边界或新 adapter 记为完成。 **这项工作的交付是一套共同规则：同一道优化任务交给 Workbench 或评估工具，都使用同一份输入、答案、计时边界和版本。** diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index c2b9297c8..cdbc026a1 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -1,4 +1,28 @@ -# 子项 03：把 NPU 试点做成可执行任务 +# #336：持续建设优化任务库 + +**本项长期维护可复用的优化任务，每批独立研究、实现和验收。** 首题是下面的 `rmsnorm-npu`；后续任务沿现有接口增加，不因为题库存在就限制用户只能做内置题。真实 Agent 与外部项目的接入归 [#337](04-integration.md)。 + +原 #334 的跨任务选题、来源核查与复用决定并入本项；原 #335 中任务作者需要的共享接口说明继续使用 [02-contract.md](02-contract.md)。两项旧 Issue 的关闭仅表示范围迁移，不表示所有来源或接口已实现。 + +## 每批任务交付什么 + +- 明确任务目标、允许的修改范围及对应工程问题，不按 skill 或 Agent 建专用题库。 +- 调研论文和固定版本开源源码，记录许可证、采用/不采用理由与适配变化；研究条目不计为执行覆盖。 +- 提供任务输入、基线、独立正确性检查、计时方法、设备/运行时条件和版本身份；profiling 按需提供，不作为所有任务的默认门槛。 +- 用正确、错误和同码等适用控制验证裁判与测量，再记录实际支持范围、失败和复现方法。 +- 复用既有 Workbench 与 #337 的接入。需要新任务形式时先明确 adapter 缺口，不增加空实现来充覆盖。 + +## 持续跟踪 + +- [x] 首个 NPU RMSNorm 题包及工具控制实验已在 Draft #338 交付，尚未合并或完成项目验收。 +- [x] 保留 [已有选型研究](01-research.md)与[五类来源对照](task-expansion/README.md)，供后续按实际需求选择。 +- [ ] 修复并复审 #338 的单次/整体超时执行缺口；原始样本不改标为修复后的运行。 +- [ ] 由 #337 验证真实 Agent 消费首题及外部项目任务；题包可运行不等于这一步完成。 +- [ ] 后续每批任务分别记录来源、实现 PR、实际执行环境、结果和剩余限制。 + +本 Issue 不因首题通过而关闭，也不要求一次覆盖所有类别。后续加题按任务或紧密相关的一批任务提交 PR；当前首轮基础 PR 为 #338，后续端到端接入 PR 尚未创建。 + +## 首题：NPU RMSNorm **这项工作的当前交付是 `rmsnorm-npu`：给出可以在真实 Ascend NPU 上运行的基线、候选、输入、独立正确性检查、profiling 和测量规则。** 用户已经指定 NPU，不能再以 CPU 两题可运行作为试点验收。 @@ -23,10 +47,10 @@ - 输入在调用前准备并绑定摘要；变化输入避免前次正确输出被重放成“优化”。不得将缓存后的答案当作实际计算时间。 - NPU 是异步设备；计时边界必须明确等待调用前后的设备工作完成。仅统计 Python 发起调用的时间不能作为 NPU 完成延迟。 - 预热、首次编译或初始化、输入准备、oracle 与输出校验不计入正式样本；具体次数及策略来自冻结任务与实验计划。 -- profiling 独立于正式计时，保留 CPU/NPU 活动与 trace 身份。若设备 trace 不可用，明确缺口，不能用 CPU cProfile 替代。 +- 调用 profiling 时独立于正式计时，保留 CPU/NPU 活动与 trace 身份。若设备 trace 不可用，不能声称已有设备诊断；未调用 profile 不默认判任务失败。 - 运行同码 A/A、错误但快的候选、基础算子与融合候选；保留全部有效和失败记录，避免只报告最快样本。 -完整流程、控制组和证据清单见 [NPU 试点](npu-pilot.md)。 +已执行的工具资格试点按冻结计划包含 profiling、A/A 与错误控制，完整流程和证据清单见 [NPU 试点](npu-pilot.md)。这些工具控制不要求每次真实 Agent 优化都重复执行。 ## 验收 @@ -40,4 +64,4 @@ 旧 `numpy-axis1-v2` 每题一次完整 trial 为 41 次调用、164 项断言、25 个计时样本；CPU profile 仅解释 CPU 调用。v1 的 360 个样本与 v2 的 300 个样本保留各自 task/grader 身份，[旧协议验证](replay-validation.md)和[用途对照](usefulness.md)中的噪声、失败与限制不改写。 -[五类扩展方案](task-expansion/README.md)仍保留后续题源。当前优先跑通 NPU RMSNorm；Tokenizers、KV 复用、通信和服务任务依次独立评估，不因本任务完成而标记整类已覆盖。 +[五类扩展方案](task-expansion/README.md)仍保留后续题源。NPU RMSNorm 工具试点已有记录，真实 Agent 接入归 #337；Tokenizers、KV 复用、通信和服务任务分别按实际需求独立评估，不因本任务完成而标记整类已覆盖。 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md index ba31f9979..428708939 100644 --- a/docs/plans/issue-252/04-integration.md +++ b/docs/plans/issue-252/04-integration.md @@ -1,79 +1,60 @@ -# 子项 04：接入现有 RSI，并验证结果能被使用 +# #337：将工程优化任务接入现有执行与评测流程 -**这项工作的交付是调用链和实际证据：让现有 Workbench 执行任务，让现有 verifier 按同一题目检查实现，并明确这些结果支持哪些决定。** +**交付目标：让已有 Agent + RSI 工作流接收题库或外部项目任务，留存 Agent 实际修改的代码，独立验收，并关联最终效果和整次执行的消耗。** 首轮使用一种真实 Agent，在 NPU 上验收一个内置任务和一个外部目标。 -本轮以 [NPU RMSNorm](npu-pilot.md) 为试点验收对象：真实 NPU 基线、profiling、候选、独立检查、A/A 与性能比较必须在同一条证据链中。已有 CPU Workbench 与正式 SkillEval 结果保持历史身份；正式 SkillEval NPU 产物隔离尚未建立，不能因设备任务能在 Workbench 运行就宣称该能力完成。 +这是一项尚未完成的接入工作。现有 Workbench、SkillEval、Agent 执行与日志能力继续复用；Draft #338 提供任务与工具基础，预写参考候选的 NPU 演练不代替真实 Agent 验收。原 #334 的接入调研和 #335 的接口工作归本项跟踪，不另建平台。 -| 要说清楚的问题 | 本子项的答案 | -| --- | --- | -| 具体做什么 | 把任务操作绑定到既有执行、比较、验证和报告入口，实际运行正确与错误候选 | -| 交付什么 | NPU Workbench RunSpec 绑定、真实设备演练和结果记录；保留已有 CPU 正式产物路径 | -| 谁使用 | 主 Agent 的引擎优化流程、评测开发者、PR 评审者 | -| 实际作用 | 工作台能够执行比较并保留证据；错误实现被共同裁判拒绝；结果可供 RSI 决定保留、放弃或继续诊断 | -| 当前状态 | NPU Workbench 参考控制演练与本轮软件资格已完成，见[实测与决定](npu-validation.md)；真实技能收益未测 | +## 已有什么，还需补什么 -例如主 Agent 交付一个候选后，可以检查它、查看 profile、取得比较证据。要证明新技能让 Agent 更好，还需要旧/新技能各自产出实现的正式两臂实验；预先写好的候选回放不能替代它。 - -## 流程也是交付内容 - -本子项要交付的是能够使用任务、得到结果并据此行动的接入流程。区分以下三层，不能把其中一层完成写成全部完成： - -- **单次实验的执行与验收：** 固定比较条件、检查实际实现、测量、留存结果，交给现有 Workbench 或 SkillEval 消费。这是 #252 必须提供并验证的接入。 -- **主 Agent 的使用过程：** 从目标开始，运行基线和 profiling，提出假设，修改实现，检查与比较，记录保留、放弃或证据不足的理由。本轮已完成预定融合参考实现的诊断核对、正确性、对照和保留决定；没有将这次控制演练当成从 profile 自主搜索新实现或技能收益的验收。 -- **自动搜索策略：** 程序选择下一种改法、分配多轮预算、决定何时停止或接受候选。这类决策需与独立的 [Optimization Search Policy 提案](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306)对齐，提案本身也不负责模型生成候选。本次为其提供可消费的结果；不将该提案默认为已经实现，也不把它悄悄并入 #252 的验收范围。 - -### 从哪里开始，以及每一步由谁做 - -下表描述需要讲清并演练的使用过程，最后一列单独记录当前实现，不能据表中步骤推断已有自动编排器。 - -| 步骤 | 谁做、交出什么 | 当前实现与待补项 | +| 环节 | 仓库或 Draft PR 中已有 | 本项待补 | | --- | --- | --- | -| 1. 明确目标和验收依据 | 主 Agent 根据目标项目确定输入、环境、指标、质量约束及预算；适用时复用内置题包 | 内置题可加载；已有通用 Workbench 可接外部命令。尚无自动从用户目标生成合格任务的能力 | -| 2. 准备比较条件 | 调用者在搜索前固定基线、验证规则和工作负载；每个候选有独立身份 | 复用题包身份和执行快照，NPU 需记录设备/运行时。普通 `plan` 仅预览，不等于保存了冻结的 campaign | -| 3. 运行基线并 profiling | 主 Agent 调用对应环境工具，保留诊断并说明改法依据 | 本轮真实 CPU/NPU trace 已核对基础 kernel 与融合 kernel 路径。没有强制 Agent 先诊断再修改 | -| 4. 提交并检查实现 | Agent 修改允许范围；工具实际运行并独立检查产物 | NPU 先执行已审核可信源码；正式 SkillEval 隔离路径仍限受支持的 CPU 任务 | -| 5. 测量并报告 | Workbench 在声明的比较条件下运行、保留原始样本和结论 | 本轮 NPU 同步计时、A/A、AB/BA、最终 validation 与 800 样本已完成；范围见实测记录 | -| 6. 使用结果 | 主 Agent 引用证据解释保留、放弃或继续诊断；下一候选重新标识 | 本轮证据支持保留融合参考实现；没有程序化跨轮次搜索/停止控制或真实两臂收益结果 | - -用户目标不在题库时，第一步使用目标项目自己的 benchmark 和测试，通过现有 `inferrsi run` 的 baseline/candidate/validation 命令接入 Workbench，不需要先扩充题库。缺少质量检查或可信测量时，应补齐对应实验条件；通用 runner 无法判断测试是否充分代表真实业务。当前具体任务 adapter 不能直接承担任意模型或服务的适配。 +| 任务与条件 | 既有 Workbench 可接外部 benchmark/validation 命令；#338 提供共享题包和版本身份 | 让两种来源接入同一轮任务记录，事先确定基线、输入、检查、环境和预算 | +| Agent 执行 | 已有客户端、日志和部分受控编辑机制 | 选一个适用入口，接实际任务、工具操作和真实代码交付；不能把现有专用 CPU actor 当通用 NPU 执行器 | +| 独立评测 | #338 已有 NPU 正确性、可选 profile、同步计时与 Workbench 比较 | 对 Agent 实际产物建立不可混淆的版本绑定和最终重验，不接收自报分数作为结论 | +| 产物执行边界 | CPU 正式产物路径已有隔离；NPU worker 当前执行已审核可信源码 | 确定并验证适合 NPU 产物的受限执行或审核交接方式；不因能访问设备就视为已经隔离 | +| 完成判定 | 已有单次执行和比较 verdict | 检查事先声明的实际产物、质量、性能和必要记录，漏检、超时或中断不记成完整成功 | +| 整轮消耗 | 既有 Agent 耗时、部分 token/调用记录及 Workbench 执行记录 | 按同一任务身份关联全过程时间、可获取的用量、调用和设备实验消耗；不能获取的值保持未知 | +| 实际验收 | #338 已运行参考、错误和同码控制 | 真实 Agent 运行并交付，覆盖两种任务来源；实际重复执行，验证记录可以供调用方比较 | -### 流程怎样约束结果，而不是只提醒 Agent - -现有程序已检查任务/产物身份漂移、缺失验证和不完整执行；内置 NumPy worker 先检查正确性，错误实现不产生可用于选择的性能指标。**通用 Workbench 的外部命令路径不能因此被描述为同一个先检查再计时的执行顺序**：它当前在测量后运行独立 validation，并由最终评价拒绝无效结果。 +## 流程也是交付内容 -尚无程序强制宿主 Agent 按“profile → 假设 → 修改”顺序行动，也没有完整优化轮次的资格检查。后续若增加流程约束，应在既有入口校验证据和声明的完成条件：缺少该实验要求的诊断或验收，不得记录成完整优化成功。不能仅在提示词里写了顺序，就声称所有 Agent 都会遵守。 +1. **准备任务。** 接收内置题包，或外部项目的仓库、基线、允许修改范围、工作负载、测试和 benchmark。缺少可靠验收方法时先补齐，不能凭目标文字生成成功判定。 +2. **记录运行条件并启动。** 复用已有入口保存任务、Agent/RSI 配置、版本、环境和预算；保留实际执行记录。 +3. **Agent 自主优化。** 在允许的工具和预算内选择方法、修改和测试，可以尝试多个候选。 +4. **留存实际产物。** 保存最终代码及其身份、过程和失败记录，不能替换成预写参考候选。 +5. **独立验收。** 工具按执行前确定的规则检查实际产物，并在固定条件下重测基线与候选。结论不采用 Agent 自报的加速数字。 +6. **汇总效果与消耗。** 将正确性、性能、实际执行状态、时间和可获取用量关联，生成可追溯记录。 -本子项的[本轮 NPU 记录](npu-validation.md)已保留可复查的诊断、候选、独立检查、比较与保留决定：错误控制被拒绝，A/A 不被当作改进，结论只覆盖实际任务。融合候选是预先选定的 authored control，不是 Agent 从 profile 自主发现的证明。该演练与真实 skill 收益实验、目标服务端到端验收分别记录。 +Profiling 是可选诊断，不是通用必做步骤，不因没有 profile 默认判失败，也不要求另行说明或申请跳过。只有具体任务在执行前明确要求诊断产物时，才检查该项。已经完成的 NPU 工具资格试点仍保留原冻结的 profiling 检查和证据。 -## 理解 +评测层不识别 indexer、知识库开关等组件语义。调用方负责切换条件、控制变量、重复执行和解释差异；本项保存配置身份、实际产物与统一结果。普通用户优化一次任务不需要额外运行对照 Agent。 -只有 schema 和几道题并不能完成 #252。需要证明同一任务包能用于现有 Workbench 和工程评估；该证明不等价于 agent 获益实验。 +## 外部项目怎样独立验收 -[RSI 接入审计](rsi-integration.md)区分早期的 `run_trial/verifier` 低层接线与后续[正式 CPU 两臂接入](formal-integration.md)：`engineering-benchmark-v1` 复用 `skill_eval_compare` 的冻结、产物留存与配对报告，使用共享 oracle 在隔离进程外检查候选实际输出。该 CPU 边界不自动支持 NPU；`skill_eval_improve` 仍不支持这些公开 development 任务，系统收益仍需要真实受控 Agent 实验。 +以 vLLM 项目的一次优化为例，调用方先明确目标、模型、输入、硬件和工作负载，优先复用项目已有测试与 benchmark，必要时增加专项检查。改动语义、模型质量和目标性能按任务要求分别检查；算子级结果不能代替模型或服务结果。验收规则、容差和输入不随候选一起降低。 -## 计划 +本项提供产物交接、执行、身份和结果关联，不提供适用于所有项目的万能 oracle，也不承诺把任意自然语言目标自动转为可靠验收方案。首轮外部目标应有明确、可复现的质量检查和 benchmark，再按实际选择完善适配。 -- Workbench 接入复用 run_experiment、AB/BA、超时、源快照、结果和报告。 -- 离线工程评估接入复用现有 SkillEval 的独立 checks/verdict 消费契约，明确本地 authored control 与真实 agent 输出的区别。 -- 用 NPU 正确、错误、同码及融合候选验证实际运算、裁判与测量资格；改变冻结文件时拒绝沿用原比较。 -- 保留真实 CPU/NPU profile 与原始结果，检查来自设备执行而非预制分数。 -- 验证非 editable wheel 安装，以及现有相关回归；最终完整测试结果如实记录。 +## 首轮完成条件 -## 验收 +- [ ] 一种真实 Agent 从受支持入口完成内置 `rmsnorm-npu` 任务，实际代码被留存并由独立工具重验；不得回放预写 reference 充当 Agent 产物。 +- [ ] 一个选定的外部项目 NPU 目标复用自己的测试与 benchmark，进入同一结果与消耗记录链路；项目、模型与 workload 在执行前确定。 +- [ ] NPU 产物执行边界、任务/候选版本绑定、单次与整轮预算、中断清理有相应检查；缺环境与失败保留原因。 +- [ ] 缺失产物、错误结果、超时或不完整测量不能被记录成优化成功；profiling 未调用不是默认失败原因。 +- [ ] 实际重复执行并保留全部记录，验证效果与代价可对照；缺失 usage/费用保持未知，不伪造精确总成本。 +- [ ] 文档说明已有入口如何调用、输出如何解释以及如何接外部任务；实现和验收证据随同后续 Draft PR 交付。 -当前 NPU 验收以真实 Workbench 演练为边界：错误候选没有可用性能证据，缺失 device profiling 不冒充已有诊断，同码噪声不自动成为改进。用户授权了本轮 NPU 执行；这不代表付费模型、正式 SkillEval NPU 隔离或真实 Agent 收益已验证。既有 CPU 两消费者的一致性仍按原软件记录保留。 +重复次数、模型、预算与性能门槛属于具体实验计划，执行前固定，不在本页添加仓库默认值。一个任务没有取得改善可以是完整的负结果；工具可靠性与 RSI 收益是不同结论。 -## 已有 CPU 软件接入与历史证据 +## 与其他工作和 PR 的关系 -- `inferrsi benchmarks plan/compare` 生成并执行真正的既有 RunSpec；不改原 runner 的统计或阶段规则。 -- `grade_artifact` 将独立正确性结果转成既有 SkillEval checks 结构。 -- `skill_verifier.py` 接入现有 `skill_evaluation.run_trial`，任务描述锁定 controller 审核后的保留产物，答复只提交该产物哈希。 -- 后续 `engineering-benchmark-v1` 正式 adapter 接入 `skill_eval_compare` 和 `skill_eval_local`，冻结题包/两臂 skill、留存真实 `implementation.py`、隔离运行候选、在父进程复用 oracle，并在报告中重验；方法与平台记录见[正式接入说明](formal-integration.md)。 -- `tools/engineering_benchmark_validate.py` 冻结任务和候选，运行两个真实任务、CPU profile、AB/BA，再通过真实 SkillEval trial 处理正确/错误的 authored control。没有生成模型请求，也没有伪造 token 使用。 -- `tests/check_installed.py` 检查 wheel 中的任务数据、许可和源码；新增回归覆盖真实子进程与失败路径。 +- #336 持续提供可复用任务；原 #334/#335 的研究和接口随具体任务或本接入工作完成。 +- [#338](https://github.com/InferMatrixLab/InferRSI/pull/338) 是基础交付，保持 Draft；后续计划一个 Draft PR 完成本页的实际接入与验收，尚未创建。更多题目按批次另交 PR。 +- [#248](https://github.com/InferMatrixLab/InferRSI/issues/248) 负责整体 Agent 收益实验；本项给它或其他调用方提供可用的 NPU 任务和记录，不接管其全部实验或重启旧运行。 +- #249 / Optimization Search Policy 的并行搜索、候选选择与停止策略不作为本项交付。也不将所有 provider、所有 Agent、自动知识晋升或 `skill_eval_improve` 支持列为首轮前置条件。 -消费链为“同一题目与 oracle → Workbench 性能比较 / SkillEval 独立正确性验证”。Workbench 和早期离线 verifier 执行已审核可信源码；正式两臂 adapter 接收真实产物，复用既有 bubblewrap 边界，仅让候选计算在子进程运行，参考答案留在父进程。它隔离产物 replay，不自动认证或限制生产者 Agent 的全部权限。是否由技能帮助产出更好代码，需要真实 Agent、匹配预算和对照条件的实验。 +## 现有证据与基础缺口 -CPU 运行方法和数据目录说明见 [使用文档](../../engineering-benchmarks.md)，已有 CPU 协议结果见 [v2 验证](replay-validation.md)。早期安装包检查和 v1 集成结果另存于[历史验证记录](validation.md)。当前 NPU 演练的接口、步骤和验收矩阵见 [NPU 试点](npu-pilot.md)。 +[已执行 NPU 工具试点](npu-validation.md)与[800 个样本及身份](npu-validation-summary.json)保留正确/错误控制、真实 profile、A/A 和参考 AB/BA；候选是已有官方融合算子，未调用真实 Agent 生成实现。正式 CPU 产物接入见 [formal-integration.md](formal-integration.md)，不能外推 NPU 支持。 -补充[实际工程用途对照](usefulness.md)把预先固定的 no-epsilon、einsum 和 unchanged 实现接入同一任务，证明普通冒烟漏检会被完整任务拒绝。它也暴露了计时稳定性限制：同一代码的比较出现大幅差异，可靠性能排序仍待后续协议验证，不能作为已完成的验收项。 +[Canonical loader finding 已由原 reviewer 清除](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)。[NPU action/total-timeout finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决，归 #338 修复，不因 Issue 合并而消失。GitHub 的整份 review 状态和项目验收仍分别记录。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md index 4604f7b4a..5fd4079eb 100644 --- a/docs/plans/issue-252/README.md +++ b/docs/plans/issue-252/README.md @@ -1,57 +1,54 @@ -# #252：在 NPU 上跑通共享优化任务和独立评测 +# #252：共享优化任务与现有执行、评测流程的接入 -**当前交付目标是：让 Agent 面对一个真实 NPU 优化任务，交付实际实现，由 InferRSI 检查正确性、保留 profiling、在固定条件下比较性能，并据此决定是否保留候选。** 用户已明确将试点指定为 NPU；已有 CPU 两题保留为历史验证与软件控制，不作为这轮试点完成依据。 +**目标是让真实 Agent 使用已有 RSI 流程完成优化，交付实际代码，由事先确定的测试和 benchmark 验收，并将优化效果与整次执行的消耗关联起来。** 任务可以来自题库，也可以来自外部项目；首轮设备验收使用 NPU。 -从 [NPU 试点方案](npu-pilot.md)和[实际执行结果](npu-validation.md)开始。首个任务是 `rmsnorm-npu`：在 Ascend NPU 上比较 PyTorch 基础算子组成的 affine RMSNorm 与 `torch_npu.npu_rms_norm` 候选。已在 Ascend 910B1 完成独立检查、真实 profile、错误控制、A/A 与 Workbench 对照；七项软件资格检查通过，全部 800 个样本保留。项目验收、正式 NPU SkillEval 隔离和真实 Agent/skill 收益仍未完成。 +当前已实现的是 Draft PR #338 的共享任务与评测基础，以及预写参考候选的 Ascend RMSNorm 试点。真实 Agent 产物到 NPU 验收、外部任务贯通与整轮消耗汇总仍未完成。不能把本页的目标能力当成已交付能力。 -本次模块称为 **Engineering Benchmarks（共享工程评测任务）**。输入是任务包和候选实现，输出是正确性结果、独立 profile、原始计时及版本身份。它服务主 Agent 的优化工作，也为后续 SkillEval 比较不同 skill 条件下的产物提供任务。名称与范围是本次交付提议，维护者验收另行记录。 +## 两项工作、首轮两个 PR -## 实际交付给使用者什么 +用户明确委托将原四项工作收敛为以下两项。#252 保留总索引，不形成第三条实施工作流。 -以 NPU RMSNorm 试点为例： - -- **任务：** 在明确的 NPU、dtype、输入与容差下计算 affine RMSNorm，保留 epsilon、权重和输入内容。 -- **Agent 修改：** `implementation.py` 的 `compute(x, weight, *, eps)`；不修改输入、oracle、容差或计时器。 -- **基线与候选：** 相同 NPU 输入上的基础算子组合与融合算子调用；独立 CPU double 参考结果负责数值检查，不计入设备性能。 -- **诊断与测量：** 独立记录 CPU/NPU profile；计时必须等待 NPU 运算完成，并排除输入准备、传输和正确性检查。 -- **结果用途：** 错误实现不能获得可用性能分数。同码对照与真实候选比较支持“保留、放弃或证据不足”的决定；测不清就不宣布优化成功。 +| 工作 Issue | 交付与边界 | 说明 | +| --- | --- | --- | +| [#337：接入现有执行与评测流程](https://github.com/InferMatrixLab/InferRSI/issues/337) | 题库/外部任务 → 真实 Agent → 实际产物 → 独立验收 → 效果与执行消耗记录 | [已有能力、待补工作与验收](04-integration.md) | +| [#336：持续建设优化任务库](https://github.com/InferMatrixLab/InferRSI/issues/336) | 按实际需求研究、实现和验证任务；首题为 `rmsnorm-npu`，后续逐批交付 | [任务建设与首题](03-tasks.md) | -流程也是交付内容。只有题目目录或独立命令不够；第四项需要保留基线、profiling、改法依据、候选、验收结果和最终决定的一条实际演练链。见[流程范围与缺口](04-integration.md#流程也是交付内容)。 +#334 的研究与来源选择、#335 的共享接口分别并入这两项，不再独立跟踪。关闭旧 Issue 表示范围迁移，不能解释为任务已全部实现或验收。原[调研](01-research.md)和[接口说明](02-contract.md)继续作为支持文档。 -## 四个子任务分别做什么 +| 首轮 PR | 范围 | 当前状态 | +| --- | --- | --- | +| [#338](https://github.com/InferMatrixLab/InferRSI/pull/338) | 共享题包、独立检查/测量、可选诊断、NPU RMSNorm 首题；保留 CPU 软件控制与既有产物适配 | 已有 Draft，尚未合并，仍有超时 review 待解决 | +| 后续一个 Draft PR | 在现有入口接真实 Agent，完成产物交接、NPU 执行边界、外部任务适配、消耗关联与实际验收 | 计划项，尚未创建；验收随实现交付 | -这四项是交付分工，不是四个运行时模块；一个完整 NPU 试点会同时使用它们。 +后续加题按任务或紧密相关的一批任务提交 PR，归 #336 跟踪，不预设总数量。 -| 子任务 | 具体交付 | 实际作用 | 当前状态 | -| --- | --- | --- | --- | -| **① 选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) · [说明](01-research.md) | 论文/源码比较、NPU 选型、来源与边界 | 决定优化什么、复用哪些接口和评测设计 | 既有研究保留；本轮 NPU 方案见试点文档 | -| **② 统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) · [说明](02-contract.md) | 任务、候选、独立 oracle、NPU 同步计时、profile 与版本身份 | 固定一次比较中的工作和裁判，拒绝 CPU 回退或错误设备结果 | NPU adapter 与冻结身份已在真实试点核对 | -| **③ 交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) · [说明](03-tasks.md) | `rmsnorm-npu` 基线、候选、输入、数值检查、测量规则 | Agent 有明确修改对象，工具能在真实 NPU 执行它 | 已实测；限本次设备/runtime/float32/workload | -| **④ 接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) · [说明](04-integration.md) | NPU Workbench 演练、正确/错误控制、A/A 与候选比较、报告 | 证明真实产物进入执行，结果能支持下一步 | 参考控制演练已完成，证据支持本轮保留决定；真实 skill 收益未测 | +## 完成后新增什么能力 -## 在 RSI 中的位置 +仓库已经有 Workbench、SkillEval、Agent 日志与部分用量记录。本次补齐的是这些能力在真实工程优化任务上的接入。 -| RSI 要回答的问题 | 调用者 | 本次作用与边界 | -| --- | --- | --- | -| 这个实现是否正确、是否值得保留？ | 主 Agent + Workbench | 本轮优先在真实 NPU 跑通任务检查、profiling 和比较 | -| 新 skill/知识是否帮助 Agent 写出更好的实现？ | SkillEval 两臂比较 | 当前正式产物 adapter 仅支持已声明的 CPU 任务；NPU 产物隔离与真实 Agent 对照没有完成 | +| 当前缺口 | 完成后的目标能力 | +| --- | --- | +| NPU 工具试点运行的是预写参考候选 | 接收真实 Agent 产生的代码，独立检查与测量 | +| 外部目标已有通用命令入口，但产物和评测记录需手工拼接 | 有明确测试和 benchmark 的外部项目任务也能进入同一条链路 | +| Agent 消耗与 NPU 代码性能尚未完整关联 | 每次任务记录实际产物、正确性、性能、整体耗时及可获取的 token/调用/设备消耗；未知值保持未知 | +| 现有对照评测尚未覆盖新增 NPU 任务 | 调用方控制变量并重复执行，利用同一格式的记录分析效果与代价 | -当前正式 `engineering-benchmark-v1` adapter 的 Linux bubblewrap 控制只能作为 CPU 软件边界证据，不能直接挂载设备就声称 NPU 候选已安全隔离。NPU 试点先运行已审核的可信源码；接入方法和既有 CPU 证据见[正式接入说明](formal-integration.md)。不以本次 Workbench NPU 演练冒充正式 SkillEval NPU 支持。 +评测层不需要理解调用方切换的是 indexer、Agent 还是知识库；它保存配置身份和真实结果。整体实验的设计、重复安排与归因沿 [#248](https://github.com/InferMatrixLab/InferRSI/issues/248) 或调用方的职责进行，不在这里另建组件实验平台。并行搜索与优化策略仍有独立范围。 -题库不是用户需求的白名单。题库外目标可以将自己的 benchmark 和质量检查接入**已有通用 Workbench**；它不会自动生成验收标准。当前任务 adapter 只支持明确声明的语义和环境，不等于任意模型或服务适配器。 +## 流程与独立验收 -主 Agent 决定下一次尝试；任务工具检查与测量。Optimization Search Policy 讨论如何选择改法、分配预算和停止，本次没有实现该控制器，也没有程序强制所有 Agent 先 profile 再修改。 +目标链路是：**提供任务和运行条件 → Agent 自主优化 → 留存实际产物 → 工具独立验收 → 效果与消耗报告。** Agent 在允许的工具和预算内自行选择方法；profiling 保留为可选工具，不是必做步骤，也不因没有 profile 默认判失败。 -## 后续类别与历史证据 +独立验收来自具体任务的测试与 benchmark，不是万能裁判。题库任务预先提供这些内容；外部项目优先复用项目自身测试，必要时补充专项检查，并在优化前固定输入、质量规则、测量条件和允许修改范围。比如 vLLM 优化，要按实际目标检查改动语义、模型质量及固定负载性能；单个算子变快不能代替模型或服务验收。当前工具不会自动从任意一句目标生成可靠裁判。 -[五类任务方案](task-expansion/README.md)保留算子、CPU 热路径、内存与缓存、通信、服务配置/调度的研究。**次序现调整为先完成 NPU RMSNorm 试点。** Tokenizers.encode 保留为后续 CPU 仓库补丁候选；KV 保留既有任务复用方向。NCCL/MSCCL 等 CUDA 题源不能直接作为 Ascend 通信实现，需另选 NPU 对应机制和资格条件。 +## 已有证据及尚未解决的问题 -既有 CPU Softmax/RMSNorm 的错误识别、输出重放修复、计时噪声和失败记录不改写：[v1 集成](validation.md)、[v1 用途](usefulness.md)、[v2 协议](replay-validation.md)。正式 CPU 产物软件控制见[既有 CI 记录](formal-integration.md#已执行的软件验证)。这些记录不重标为 NPU 执行、可靠速度排序或真实 skill 收益。 +已在 Ascend 910B1 执行 [RMSNorm 工具资格试点](npu-validation.md)，[公开摘要](npu-validation-summary.json)保留 800 个计时样本、独立检查、错误控制、A/A 和参考比较。该试点使用已有融合算子作为参考候选，证明的是限定任务下的工具行为，不是 Agent 自主发现、技能收益或服务收益。 -旧的 [CPU 完整导读](walkthrough.md)解释已实现软件接口；当前交付与启动方式以 [NPU 试点](npu-pilot.md)及本轮实际结果为准。 +已执行试点的 profiling、冻结规则和原始 verdict 保持原样。当前 [NPU 单次/整体超时 review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决：不能把脚本记录的门槛等同于所有配对 trial 和整体预算均有执行保障。历史样本不因文档整理改标为新运行。 -## 跟踪与范围 +[Canonical task loader 的具体 finding 已获原 reviewer 清除](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)，这不等于整个 PR 批准或 #252 验收。正式 SkillEval 当前仍只支持 CPU NumPy 产物，新 NPU worker 运行已审核源码；NPU 产物执行边界仍需在 #337 接入中解决。 -实现及文档在 [Draft PR #338](https://github.com/InferMatrixLab/InferRSI/pull/338)，四个子任务保持打开。原生父子关系写入受账号权限限制，目前通过相互引用和父任务索引追踪。用户的 NPU 试点委托授权了本轮实施、运行和发布；它不代替维护者对整个 #252 的接受或关闭决定。 +已有 CPU 任务和失败记录保留原身份：[v1 集成](validation.md)、[v1 用途](usefulness.md)、[v2 协议](replay-validation.md)、[正式 CPU 接入](formal-integration.md)。[五类来源研究](task-expansion/README.md)供后续选题使用，不等于已实现五类设备任务。 -父 Issue 的 [Search Policy RFC](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495138306)按[维护者范围意见](https://github.com/InferMatrixLab/InferRSI/issues/252#issuecomment-6495720922)保持独立。 +用户已授权本次整理和关闭被合并的旧 Issue；这不代替维护者对剩余交付的项目验收。#336、#337 与父 #252 保持打开，#338 保持 Draft。原生父子关系此前受写权限限制，继续通过引用和父索引跟踪。 diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md index da8773966..5c5bcda70 100644 --- a/docs/plans/issue-252/formal-integration.md +++ b/docs/plans/issue-252/formal-integration.md @@ -177,4 +177,4 @@ job 日志确认实际 checkout 为 PR 合并快照 `e74eb3d0a693f88599805faa8bc Profiling 仍由主 Agent 根据实际瓶颈调用：对已审核实现可使用 `benchmarks profile`，真实设备需对应 profiler。当前 formal adapter 不会自动做 profile、选择下一次候选或晋升 skill。它提供“同题、两臂、真实产物、独立检查”的接入层，供这些现有流程使用。 -在当前 RSI 中继续推进一次实现优化时，主 Agent 从报告的 `implementation.py` 证据链接取得候选，审核其源码与适用范围后，再用现有 Workbench 做 profile 和配对测量。formal adapter 的通过结果不会自动把未经审核的代码送入可信 worker，也不会自动批准该实现。 +在当前 RSI 中继续推进一次实现优化时，主 Agent 从报告的 `implementation.py` 证据链接取得候选，审核其源码与适用范围后，再用现有 Workbench 做配对测量，需要诊断时可单独调用 profile。Profiling 不作为通用完成条件。formal adapter 的通过结果不会自动把未经审核的代码送入可信 worker，也不会自动批准该实现。 diff --git a/docs/plans/issue-252/npu-pilot.md b/docs/plans/issue-252/npu-pilot.md index 715eb3f81..35b0675d2 100644 --- a/docs/plans/issue-252/npu-pilot.md +++ b/docs/plans/issue-252/npu-pilot.md @@ -2,6 +2,8 @@ 2026-11-04，用户将试点明确指定为 NPU。**本轮已在 Ascend 910B1 完成实际运行，七项软件资格检查通过**，见[实测解释与保留决定](npu-validation.md)和[全部样本及身份](npu-validation-summary.json)。本页保留任务接口与执行方法。已有 CPU 测试和样本仅作历史软件证据；项目接受、NPU 产物隔离与真实 Agent 收益另行验收。 +以下图示、步骤和验收矩阵描述本轮已执行的参考控制与工具资格试点，其 profiling 检查继续按原冻结计划保留。通用 Agent 优化流程中，profiling 是按需使用的工具，不是必做步骤，也不因缺少 profile 默认判失败；见[当前流程说明](04-integration.md#流程也是交付内容)。 + ## 这次究竟要跑通什么 给 Agent 一个具体问题：**在 Ascend NPU 上保持 affine RMSNorm 计算结果，检查基础算子组合能否由更合适的实现替代。** Agent 交付真实源码；InferRSI 负责独立检查、诊断和测量，再用证据决定是否保留。首个参考候选采用 `torch_npu.npu_rms_norm`，是否更快由实际实验判断。 @@ -83,4 +85,4 @@ RMSNorm 是推理中明确的归约与逐元素计算。基础运算组合和 NP 首轮运行已审核可信源码。现有正式 `engineering-benchmark-v1` 只支持 CPU 任务，Linux bubblewrap 控制没有验证 NPU 设备隔离；不得通过隐式宿主执行或 CPU fallback 绕过不支持状态。NPU 产物隔离需要后续单独验收。 -旧 CPU 任务继续服务回归和历史记录。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，在本次 NPU 通路成立后按各自任务资格推进。四个 issue 和 Draft PR 保持原有追踪关系，NPU 演练完成不自动关闭整个 #252。 +旧 CPU 任务继续服务回归和历史记录。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，按各自任务资格推进。当前跟踪已收敛为 #336 任务库和 #337 接入两项，见[总览](README.md)；本页的试点记录不因 Issue 调整而改变，NPU 演练完成不自动关闭整个 #252。 diff --git a/docs/plans/issue-252/npu-validation.md b/docs/plans/issue-252/npu-validation.md index 3f9ddc814..35efb4345 100644 --- a/docs/plans/issue-252/npu-validation.md +++ b/docs/plans/issue-252/npu-validation.md @@ -2,6 +2,8 @@ 2026-11-04。**本轮已在一张 Ascend 910B1 上跑通完整的软件控制试点。** 正确实现通过，错误实现被拒绝，实际设备 profile 可读，同码对照没有被选为优化，融合参考实现在两个规定 workload 上达到执行前固定的改善门槛。七项资格检查均通过；项目验收、真实 Agent/skill 收益和服务收益仍未建立。 +后续审阅与范围更新：原 reviewer 已[清除 canonical-loader 的具体 finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)，另一个 [NPU 单次/整体超时 finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决。下文的脚本检查结果和性能样本保持原身份，不能据此声称所有声明的单次/总时限均已强制执行。Issue 已收敛为 #336 与 #337 两项，见[当前总览](README.md)。 + 执行前的[范围、源码和判断门槛](https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6621316099)先于本次资格实验发表。被测源码为 `6308268fc6d1fbf90e9ab97b5da8af79152f39ea`；后续本记录仅整理证据，不将结果改标为新实现的运行。 [可公开证据与全部 800 个计时样本](npu-validation-summary.json)保留原 verdict、冻结身份、环境、逐项错误和 profile kernel 摘要。完整 raw archive 留在执行归档；公开转换及每份原始结果、trace、JUnit 的摘要也在该文件中。 @@ -47,7 +49,7 @@ baseline 在每个选定输入上出现 3 次 Mul、1 次 ReduceMean、1 次 Add 同一 source head 的 [Linux ready/review CI](https://github.com/InferMatrixLab/InferRSI/actions/runs/41764076901)通过：Python 3.10 和 3.11 各 **3649 passed / 20 skipped / 284 subtests passed**，两个安装包检查通过。托管 CI 中没有 NPU；设备验证来自上面的独立远端运行。 -正式 CPU artifact job **54 passed / 0 skipped**，其中新增的 `test_real_sandbox_rejects_zero_output_despite_cwd_shadow` 真正执行，错误零输出在同名 cwd 包放宽容差后仍被规范 oracle 拒绝。另有 descriptor/starter/prepare/dependency 的来源回归。CI checkout merge `6f1506bcb366c10836991701f8beaf7ac89e9a11` 与被测 head 的 tree 均为 `5c22811e28f1c8c0182852916f43804c18b35edf`。这对应已实现的 P2 修复；原 reviewer 的审查结论仍待复审，不由作者标为接受。 +正式 CPU artifact job **54 passed / 0 skipped**，其中新增的 `test_real_sandbox_rejects_zero_output_despite_cwd_shadow` 真正执行，错误零输出在同名 cwd 包放宽容差后仍被规范 oracle 拒绝。另有 descriptor/starter/prepare/dependency 的来源回归。CI checkout merge `6f1506bcb366c10836991701f8beaf7ac89e9a11` 与被测 head 的 tree 均为 `5c22811e28f1c8c0182852916f43804c18b35edf`。这对应已实现的 P2 修复；原 reviewer 后续仅清除了该具体 finding，未批准整个 PR 或项目验收。 新增 NPU 无设备回归本地最终 **14 passed / 1 device test skipped**。开发首轮的 6 项失败分别来自新测试的文本编码和输出目录使用错误，修复后重跑；没有归因于既有 baseline。独立源码审查还发现并修复了输入 shape 原地修改、A/A 最终 validation 失败混同无收益、冻结控制身份未重验三处问题。 @@ -57,4 +59,4 @@ baseline 在每个选定输入上出现 3 次 Mul、1 次 ReduceMean、1 次 Add 使用 [NPU 试点的操作说明](npu-pilot.md)和 [共享接口](../../engineering-benchmarks.md)，在已预约的兼容 NPU 环境中运行 `python -m tools.npu_benchmark_validate`，显式传入本轮 Issue 的重复次数、预算与门槛。工具先写入 `plan/frozen.json`、`validator/task.json` 与 `patch/`，随后通过既有 Workbench 保留 `results/`、报告及 `summary.json`。 -当前可保留的是这个 float32 NPU 参考候选及共享评测通路。BF16/FP16、更多形状、真实模型/服务、NPU 产物隔离和真实 Agent 两臂仍需分别验收。四个 Issue 与 PR #338 保持打开/Draft，NPU 软件控制通过不自动完成整个 #252。 +当前可保留的是这个 float32 NPU 参考候选及共享评测通路。BF16/FP16、更多形状、真实模型/服务、NPU 产物隔离和真实 Agent 两臂仍需分别验收。#336、#337 和父 #252 继续跟踪未完成工作，PR #338 保持 Draft；#334/#335 的范围迁移不构成这些能力的验收。 diff --git a/docs/plans/issue-252/review.md b/docs/plans/issue-252/review.md index a3c8d501b..3eed3f280 100644 --- a/docs/plans/issue-252/review.md +++ b/docs/plans/issue-252/review.md @@ -2,6 +2,8 @@ 本页固定记录 `9454672f` 这一轮审查。之后用户要求继续正式接入，新增实现与验证见[正式 SkillEval 接入](formal-integration.md)；下文“未接入”是当时状态，不能代替当前接入说明。 +旧四项分工及 Issue 状态同样保留为该轮历史；当前已按用户委托收敛为 #336 任务库和 #337 接入，见[当前总览](README.md)。历史审查记录不表示旧 Issue 仍需独立跟踪。 + 本轮继续审查 PR #338，并准备供首次接触 InferRSI 的读者使用的[完整导读](walkthrough.md)。结论是：发现并修复一个 CLI 失败状态传播缺陷，补齐公开可检查的既有 v2 样本/profile，修正文档的历史版本指向。PR 仍保持 Draft；这些工作不代表维护者已经接受范围或批准合并。 ## 审查对象和覆盖 diff --git a/docs/plans/issue-252/rsi-integration.md b/docs/plans/issue-252/rsi-integration.md index c4150e0f6..e66bb270d 100644 --- a/docs/plans/issue-252/rsi-integration.md +++ b/docs/plans/issue-252/rsi-integration.md @@ -18,7 +18,7 @@ | 入口 | 现在已经接通什么 | 尚未接通什么 | | --- | --- | --- | | `benchmarks plan/compare` → `TaskPackage.runspec` → `run_experiment` | 同一任务生成既有 RunSpec，由 Workbench 执行配对比较并保留证据 | 自动从用户目标/profile 选出适用任务；服务与目标硬件验收 | -| `benchmarks check/profile/measure` | 分开的数值校验、CPU 诊断、测量操作 | 强制宿主 Agent 按整个优化流程执行 | +| `benchmarks check/profile/measure` | 分开的数值校验、按需 CPU/NPU 诊断、测量操作 | 真实 Agent 产物到最终评测的完整接入；profiling 不作为默认门禁 | | `skill_verifier.task_descriptor` → `skill_evaluation.run_trial` | 对 controller 已审核的保留实现运行真实 verifier；答案提交该实现的 SHA | Agent 根据不同技能实际产出两份实现的比较 | | `tools.skill_eval_compare` | 新增 `engineering-benchmark-v1`：共享题目/两臂 skill 冻结、实际文件留存、隔离计算与父进程 oracle、配对报告重验；支持现有 local stage/capture | 当前仅正确性；真实受控 Agent/skill 收益、性能排序、provider generation 模式未接入 | | `tools.skill_eval_improve` | 已有面向支持的离线 audit 任务的选择、重跑和 holdout 流程 | 共享工程基准尚未接入该控制器的任务/grader 契约 | @@ -27,9 +27,9 @@ ## 一次引擎优化中应该怎样使用 -1. **从目标和基线 profile 开始。** 主 Agent 明确模型、硬件、工作负载和质量要求，再确认该任务是否覆盖所需语义、dtype、布局和环境。当前 CPU RMSNorm 任务不能因为名称相同就代表真实 GPU/NPU 服务。 +1. **从目标和验收条件开始。** 主 Agent 明确模型、硬件、工作负载、基线和质量要求，再确认该任务是否覆盖所需语义、dtype、布局和环境。名称相同的算子任务不自动代表真实模型或服务。 2. **冻结任务与实验条件。** 在候选尝试前固定任务、oracle、输入序列、版本、比较条件和预算。候选只改允许的实现；不同时修改裁判或基线。 -3. **利用知识形成可检验假设。** Agent 读取适用技能、论文和历史失败记录，说明该变更如何作用于已观测的瓶颈。CPU cProfile 无法确认的内存瓶颈继续保持未知。 +3. **在允许的工具和预算内自主优化。** Agent 可阅读代码、查阅允许使用的知识、运行 profiling、修改和测试，自行选择适用方法。评测不强制每次检索知识或调用 profiler；实际调用和消耗应留存，未做诊断时不声称已经观测到瓶颈。 4. **先挡住错误，再测量。** 数值不正确的候选不能获得可用于选择的性能分数。计时必须测到实际计算；预检不能给后续同输入缓存重放预热。测量规则的修改产生新的任务/grader 身份，不重写旧证据。 5. **区分原始报告与优化决定。** 保留 Workbench 原始 PASS/FAIL/INCONCLUSIVE；主 Agent 同时判断证据是否足以支持保留/放弃。若同码对照无法分辨，决策是“测量不足，先修协议”，不能把某次 FAIL 归纳成整个优化机制无效。 6. **记录对下一轮有用的事实。** 保留候选、环境、任务、正确性失败、原始样本及未解决的诊断。经验整理与晋升走现有流程；不从一次算子结果自动推广成服务收益或通用技能。 @@ -44,7 +44,7 @@ ### B. 证明结果确实进入优化决定 -最小演练应沿已有入口保留“profile → 假设 → 产物 → check → compare → 决策”链，并至少包含：错误产物被拒绝、正确但证据不足的产物不被当作改进、负结果留下适用条件。既有 [RMSNorm 对照](usefulness.md) 提供了部分输入与结果；当前没有程序化的全流程强制或真实 agent 对照，不能据此宣布完整闭环验收。 +最小演练应沿已有入口保留“任务与条件 → Agent 实际优化 → 产物 → check → compare → 结果与消耗记录”链，并至少包含：错误产物被拒绝、正确但证据不足的产物不被当作改进、负结果留下适用条件。Profiling 是可选诊断，不是通用链路的必经阶段；已执行工具资格试点的冻结规则不因此改变。既有 [RMSNorm 对照](usefulness.md) 提供了部分输入与结果；真实 Agent 到最终产物评测和整轮消耗的接入仍未完成，不能据此宣布完整闭环验收。 ### C. 接到已有 SkillEval 两臂入口 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md index 6b1458e62..e6005213f 100644 --- a/docs/plans/issue-252/task-expansion/README.md +++ b/docs/plans/issue-252/task-expansion/README.md @@ -59,7 +59,7 @@ Apple 的另一个 [vLLM 源码优化题](https://github.com/apple-aiml-research ## 对接口和流程的具体影响 -不同类别共享“任务/环境/验收规则固定 → baseline/profile → 候选 → 独立检查 → 比较 → 结果与决定”的过程，候选形式不能只支持一种： +不同类别共享“任务/环境/验收规则固定 → baseline → 候选 → 独立检查 → 比较 → 结果与决定”的过程，profiling 按需使用，候选形式不能只支持一种： | 候选形式 | 对应题型 | 与当前实现的关系 | | --- | --- | --- | @@ -70,7 +70,7 @@ Apple 的另一个 [vLLM 源码优化题](https://github.com/apple-aiml-research 复用现有 Workbench 的执行、AB/BA 和证据，不把 GSO、Pier、Harbor、InferenceBench 的控制器全部引入。上游 task 包是素材和契约来源；任何新执行环境或多文件候选都需要其自身的产物隔离与验收，现有单文件 sandbox 不能自动覆盖仓库 build/server。 -Profiling 保留独立阶段与原始产物。若以后评估“给 Agent profiler 反馈是否有用”，还应保留反馈进入下一次 Agent 输入的证据；生成了一份 profile 本身不足以证明它发挥了作用。 +按需调用 profiling 时，将诊断与正式计时分开，并保留原始产物。若以后评估“给 Agent profiler 反馈是否有用”，还应保留反馈进入下一次 Agent 输入的证据；生成了一份 profile 本身不足以证明它发挥了作用。 ## 新任务入库前怎样验证有用 @@ -78,16 +78,16 @@ Profiling 保留独立阶段与原始产物。若以后评估“给 Agent profil 2. **裁判区分度：** 正确基线、参考优化、错误但快的变体都实际运行；错误结果不得获得可用于选择的性能分数。 3. **测量资格：** 同码 A/A 和 no-op 不应被当作进步；参考优化在选定环境确有可分辨信号。保留失败、波动与全部原始样本，无法分辨就暂不用于性能排名。 4. **目标覆盖：** 计时工作负载、非计时语义边界、guard 工作负载分开说明。不能缩短输入、少处理请求或减少分配来换取虚假加速。 -5. **真实接入：** 源码/编译产物/配置确实进入最终执行；profile 和结果可追溯。用一条演练证明报告支持“保留/放弃/证据不足”的决定。 +5. **真实接入：** 源码/编译产物/配置确实进入最终执行，结果可追溯；调用了 profile 则保留诊断。用一条演练证明报告支持“保留/放弃/证据不足”的决定。 6. **能力评测另做：** 任务资格成立后，固定模型与预算做 Agent/skill/搜索策略对照；记录失败成本、有效尝试数、最终质量和性能。不把一个新任务的通过当成整个 RSI 变强。 每个方向分开报告“找到来源、已适配、正确性验证、性能资格、Agent 对照”的状态。第一版不将内存、延迟、带宽或功能通过率压成一个未经论证的总分。 -## 与四个子项的关系及本轮产物 +## 与两项工作的关系及本轮产物 -- **#334 调研：** 保留五类来源对照，当前优先级调整为真实 NPU RMSNorm。 -- **#335 接口：** 当前先实现 NPU 的设备/数值/同步/诊断契约；repo-patch / launcher / 多进程为后续设计，尚未新增通用 adapter。 -- **#336 任务：** 当前交付目标为 `rmsnorm-npu`，原 CPU 两题保留历史身份；其他类别按任务卡分阶段实施。 -- **#337 接入验证：** 按六项资格步骤验证 NPU Workbench 的真实用途；正式 SkillEval correctness 仍仅支持它已声明的 CPU 任务，NPU 隔离未建立。 +- **#336 任务库：** 承接原 #334 的选题与复用研究；首题 `rmsnorm-npu` 已有工具实测，其他类别按需求逐批研究、实现和验收，原 CPU 两题保留历史身份。 +- **#337 流程接入：** 承接原 #335 的共享接口工作，将实际 Agent 产物接入既有评测并关联消耗；外部项目不必先入题库。正式 SkillEval 仍仅支持它已声明的 CPU 产物，NPU 执行边界尚待解决。 + +repo-patch / launcher / 多进程等形式仍需具体适配；原 #334/#335 关闭表示范围迁移，不表示已经实现这些能力。当前分工与首轮两 PR 安排见[总览](../README.md)。 本页保留可复查的来源选择与任务设计，不改写历史证据或 SearchSpace 成熟度。当前 NPU 实施与实测沿 [试点文档](../npu-pilot.md)及原 Issue 跟踪；本页列出的后续来源不因此变成已能运行的新类别。 diff --git a/docs/plans/issue-252/walkthrough.md b/docs/plans/issue-252/walkthrough.md index cdc57f306..a81360146 100644 --- a/docs/plans/issue-252/walkthrough.md +++ b/docs/plans/issue-252/walkthrough.md @@ -2,6 +2,8 @@ 这份导读先讲用途，再沿一个 RMSNorm 例子走到代码与结果。总览见 [README](README.md)，本轮审查结论见 [review](review.md)。 +本页保留早期 CPU 实现、四项分工与当时状态，供追溯使用；当前两项工作、NPU 范围和 PR 安排以[总览](README.md)为准。当前通用流程中 profiling 是可选工具，不按下文演示顺序强制 Agent 执行。 + **我们给 InferRSI 增加的是 Engineering Benchmarks（共享工程评测任务）：一份题目、一份候选实现，交给固定的独立评测，得到可追溯的正确性、诊断和计时结果。** 主 Agent 根据这些证据决定修改、保留或放弃实现。 ## 1. 先理解它解决哪一个问题 From 9a755f97554d71a1a4b1d59200eddc30a11bdf01 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 16:17:52 +0800 Subject: [PATCH 19/20] chore: remove the built-in CPU RMSNorm task Remove the CPU RMSNorm package and its dedicated historical probe. Keep Softmax regression coverage, preserve source-pinned experiment records, and retain the NPU task unchanged. Agent-assisted-by: Codex (implementation, test migration, and documentation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/engineering-benchmarks.md | 4 +- docs/plans/issue-252/03-tasks.md | 6 +- docs/plans/issue-252/formal-integration.md | 8 +- docs/plans/issue-252/npu-pilot.md | 2 +- .../issue-252/task-expansion/01-operators.md | 2 +- docs/plans/issue-252/task-expansion/README.md | 2 +- docs/plans/issue-252/usefulness.md | 2 +- docs/plans/issue-252/walkthrough.md | 8 +- docs/skill-eval-comparison.md | 2 +- .../rmsnorm-cpu/LICENSE | 22 -- .../rmsnorm-cpu/README.md | 31 -- .../rmsnorm-cpu/baseline.py | 8 - .../rmsnorm-cpu/candidate.py | 11 - .../rmsnorm-cpu/task.json | 129 ------- tests/check_installed.py | 2 +- .../engineering_benchmark_usefulness.py | 341 ------------------ tests/test_engineering_benchmarks.py | 46 +-- tests/test_skill_eval_benchmark.py | 22 +- tests/test_skill_eval_benchmark_compare.py | 30 +- 19 files changed, 60 insertions(+), 618 deletions(-) delete mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE delete mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md delete mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py delete mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py delete mode 100644 src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json delete mode 100644 tests/fixtures/engineering_benchmark_usefulness.py diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index d692d7908..1aabb6dd9 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -11,7 +11,7 @@ separate. | Adapter | Task and submission | Execution boundary | | --- | --- | --- | | `torch-npu-rmsnorm-v1` | `rmsnorm-npu`; `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | -| `numpy-axis1-v2` | `softmax-cpu`, `rmsnorm-cpu`; `compute(x, *, eps)` | Trusted CPU worker; formal SkillEval also has a separate NumPy artifact sandbox | +| `numpy-axis1-v2` | `softmax-cpu`; `compute(x, *, eps)` | Trusted CPU worker; formal SkillEval also has a separate NumPy artifact sandbox | ## Run the NPU pilot @@ -160,7 +160,7 @@ descriptive; a local verdict is not service gain or campaign acceptance. python tools/engineering_benchmark_validate.py --repo /path/to/checkout --output-dir /path/to/new-validation --repetitions 3 ``` -The tool freezes copied tasks and candidate identities, executes both CPU tasks, +The tool freezes copied tasks and candidate identities, executes the retained CPU task, retains profiles and AB/BA samples, and runs the actual SkillEval trial path with correct and deliberately wrong **authored controls**. No model is called. `summary.json` separates integration checks from campaign acceptance. Local raw diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md index cdbc026a1..ece3e86b1 100644 --- a/docs/plans/issue-252/03-tasks.md +++ b/docs/plans/issue-252/03-tasks.md @@ -24,7 +24,7 @@ ## 首题：NPU RMSNorm -**这项工作的当前交付是 `rmsnorm-npu`：给出可以在真实 Ascend NPU 上运行的基线、候选、输入、独立正确性检查、profiling 和测量规则。** 用户已经指定 NPU，不能再以 CPU 两题可运行作为试点验收。 +**这项工作的当前交付是 `rmsnorm-npu`：给出可以在真实 Ascend NPU 上运行的基线、候选、输入、独立正确性检查、profiling 和测量规则。** 用户已经指定 NPU，不能以 CPU 软件检查作为试点验收。 | 要说清楚的问题 | 本子项的答案 | | --- | --- | @@ -60,7 +60,9 @@ ## 保留的 CPU 工作 -`softmax-cpu` 和 `rmsnorm-cpu` 仍是软件控制与历史任务，保留原 KernelBench 来源和 MIT 许可。它们的 NumPy 基线、标量 double / math.fsum oracle、变化输入和错误反例可以检验共享软件接口；不向 NPU 迁移其计时结果或环境资格。 +`softmax-cpu` 保留作软件控制，保留原 KernelBench 来源和 MIT 许可。其 NumPy 基线、标量 double / math.fsum oracle、变化输入和错误反例可以检验共享软件接口；不向 NPU 迁移其计时结果或环境资格。 + +按用户要求移除 `rmsnorm-cpu` 题包及其专用用途对照 fixture；当前内置题只有 `softmax-cpu` 和 `rmsnorm-npu`。通用接口与正式产物测试使用 Softmax，旧 CPU RMSNorm 的实测记录仍按原版本保留，复现须使用对应历史 checkout。 旧 `numpy-axis1-v2` 每题一次完整 trial 为 41 次调用、164 项断言、25 个计时样本；CPU profile 仅解释 CPU 调用。v1 的 360 个样本与 v2 的 300 个样本保留各自 task/grader 身份，[旧协议验证](replay-validation.md)和[用途对照](usefulness.md)中的噪声、失败与限制不改写。 diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md index 5c5bcda70..d369442ff 100644 --- a/docs/plans/issue-252/formal-integration.md +++ b/docs/plans/issue-252/formal-integration.md @@ -1,6 +1,6 @@ # 将共享任务接入正式 SkillEval 两臂比较 -这一步把 `rmsnorm-cpu` / `softmax-cpu` 接入现有 `tools.skill_eval_compare`：两边使用各自的旧/新 skill，交付真实 `implementation.py`，由共同的独立 verifier 检查，再进入现有配对报告。调用者不再需要手工拼接底层 `run_trial`。 +这一步把共享 CPU 任务接入现有 `tools.skill_eval_compare`：两边使用各自的旧/新 skill，交付真实 `implementation.py`，由共同的独立 verifier 检查，再进入现有配对报告。当前保留的内置 CPU 任务是 `softmax-cpu`；`rmsnorm-cpu` 已移除，下面的历史验证记录仍保留当时的两题结果。调用者不再需要手工拼接底层 `run_trial`。 **本次新增接入只判定产物正确性。** 它复用共享题目、变化输入与标量 oracle，并增加隔离执行的产物交接。性能仍由明确的 Workbench 实验回答；不能把 sandbox 启动时间或子进程声称的 latency 当作优化分数。 @@ -89,7 +89,7 @@ Windows、缺少 bubblewrap、不能创建 namespace、系统 NumPy 不满足运 以下是手工两臂交接示例。占位符须替换为实际已冻结的 skill 路径、Git commit、模型设置和预算；它不是默认实验门槛。无需编造一份自定义题目，直接生成当前 canonical descriptor： ```bash -python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('rmsnorm-cpu')])" +python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('softmax-cpu')])" ``` 配置示意： @@ -120,13 +120,13 @@ python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; fr ```bash python -m tools.skill_eval_compare prepare --config config.json --repo --output -python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id rmsnorm-cpu --condition baseline --repeat 1 --current-head +python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id softmax-cpu --condition baseline --repeat 1 --current-head ``` Agent 在该 workspace 内读取任务和分配的 skill，只修改 `implementation.py`。记录实际 stdout/stderr 和作者 `execution.json`，字段与边界沿用[本地交接文档](../../local-skill-evaluation.md)。完成后 capture： ```bash -python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id rmsnorm-cpu --condition baseline --repeat 1 --current-head +python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id softmax-cpu --condition baseline --repeat 1 --current-head ``` 对 candidate 使用新的 workspace 和独立日志，把 `--condition` 改为 `candidate`。两臂完成后： diff --git a/docs/plans/issue-252/npu-pilot.md b/docs/plans/issue-252/npu-pilot.md index 35b0675d2..f980c8eb4 100644 --- a/docs/plans/issue-252/npu-pilot.md +++ b/docs/plans/issue-252/npu-pilot.md @@ -85,4 +85,4 @@ RMSNorm 是推理中明确的归约与逐元素计算。基础运算组合和 NP 首轮运行已审核可信源码。现有正式 `engineering-benchmark-v1` 只支持 CPU 任务，Linux bubblewrap 控制没有验证 NPU 设备隔离；不得通过隐式宿主执行或 CPU fallback 绕过不支持状态。NPU 产物隔离需要后续单独验收。 -旧 CPU 任务继续服务回归和历史记录。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，按各自任务资格推进。当前跟踪已收敛为 #336 任务库和 #337 接入两项，见[总览](README.md)；本页的试点记录不因 Issue 调整而改变，NPU 演练完成不自动关闭整个 #252。 +CPU Softmax 继续服务软件回归；CPU RMSNorm 题包已移除，旧实验记录保留。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，按各自任务资格推进。当前跟踪已收敛为 #336 任务库和 #337 接入两项，见[总览](README.md)；本页的试点记录不因 Issue 调整而改变，NPU 演练完成不自动关闭整个 #252。 diff --git a/docs/plans/issue-252/task-expansion/01-operators.md b/docs/plans/issue-252/task-expansion/01-operators.md index cfcfcb33c..381e631f3 100644 --- a/docs/plans/issue-252/task-expansion/01-operators.md +++ b/docs/plans/issue-252/task-expansion/01-operators.md @@ -2,7 +2,7 @@ **当前题目：在 Ascend NPU 的给定输入、权重和容差下，比较 affine RMSNorm 的基础运算与融合实现，同时保留计算结果。** -状态：`rmsnorm-npu` 已在 Ascend 910B1 完成真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较，见[实测记录](../npu-validation.md)。资格限已记录 runtime、float32 与 workload，不能外推整类算子或其他硬件。Softmax/RMSNorm CPU 两题保留为软件控制；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 +状态：`rmsnorm-npu` 已在 Ascend 910B1 完成真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较，见[实测记录](../npu-validation.md)。资格限已记录 runtime、float32 与 workload，不能外推整类算子或其他硬件。`softmax-cpu` 保留为软件控制，`rmsnorm-cpu` 已移除；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 ## 为什么先在 NPU 做这个任务 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md index e6005213f..886ec031e 100644 --- a/docs/plans/issue-252/task-expansion/README.md +++ b/docs/plans/issue-252/task-expansion/README.md @@ -4,7 +4,7 @@ ## 本轮决定 -**先在真实 NPU 上跑通 affine RMSNorm 的基线、profiling、候选、独立检查与 Workbench 比较。** 原 CPU Softmax/RMSNorm 保留作软件控制和历史证据；Tokenizers、KV、通信与服务来源保留为后续扩展。每类先完成一个任务的执行资格，再扩充数量。 +**先在真实 NPU 上跑通 affine RMSNorm 的基线、profiling、候选、独立检查与 Workbench 比较。** CPU Softmax 保留作软件控制；CPU RMSNorm 题包已移除，旧实验只保留历史证据。Tokenizers、KV、通信与服务来源保留为后续扩展。每类先完成一个任务的执行资格，再扩充数量。 AgentKernelArena 适合学习 kernel 任务的提交与独立评测；整个 InferRSI 还需要仓库补丁、状态管理、资源配置和服务工作负载。不能把所有任务都变成 `compute(x)`，也不能把功能修复的通过率叫作性能提升。 diff --git a/docs/plans/issue-252/usefulness.md b/docs/plans/issue-252/usefulness.md index 81e687d7a..168b12b9b 100644 --- a/docs/plans/issue-252/usefulness.md +++ b/docs/plans/issue-252/usefulness.md @@ -1,6 +1,6 @@ # #252：实际工程用途对照 -本页是 **v1 历史对照**，对应源码 [c06b7255](https://github.com/Bounty-hunter/InferRSI/tree/c06b72555d4643df133c59fe7cead5a91e7b86f5)。下文 720 个样本、脚本摘要和计划均属于该版本。精确复现应在此版本运行；最新 fixture 使用 v2 五个计时 case，会产生不同任务/脚本身份和样本数。后续修复见 [v2 验证](replay-validation.md)，不修改本页历史结果。 +本页是 **v1 历史对照**，对应源码 [c06b7255](https://github.com/Bounty-hunter/InferRSI/tree/c06b72555d4643df133c59fe7cead5a91e7b86f5)。下文 720 个样本、脚本摘要和计划均属于该版本。当前已删除 CPU RMSNorm 题包及其专用 fixture；本页命令只能在上述历史 checkout 中执行，[当时的 fixture](https://github.com/Bounty-hunter/InferRSI/blob/c06b72555d4643df133c59fe7cead5a91e7b86f5/tests/fixtures/engineering_benchmark_usefulness.py)仍可追溯。后续修复见 [v2 验证](replay-validation.md)，不修改本页历史结果。 本补证回答一个具体问题：共享任务能否帮助使用者识别一个看似合理的错误改动，并让正确候选经过同一套工程检查和实际性能比较？它不以 CI 绿色或某次计时 PASS 作为“有用”的定义。 diff --git a/docs/plans/issue-252/walkthrough.md b/docs/plans/issue-252/walkthrough.md index a81360146..74b64290d 100644 --- a/docs/plans/issue-252/walkthrough.md +++ b/docs/plans/issue-252/walkthrough.md @@ -2,7 +2,7 @@ 这份导读先讲用途，再沿一个 RMSNorm 例子走到代码与结果。总览见 [README](README.md)，本轮审查结论见 [review](review.md)。 -本页保留早期 CPU 实现、四项分工与当时状态，供追溯使用；当前两项工作、NPU 范围和 PR 安排以[总览](README.md)为准。当前通用流程中 profiling 是可选工具，不按下文演示顺序强制 Agent 执行。 +本页保留早期 CPU 实现、四项分工与当时状态，供追溯使用；当前两项工作、NPU 范围和 PR 安排以[总览](README.md)为准。`rmsnorm-cpu` 已从当前题库删除；下文示例须在[删除前版本](https://github.com/Bounty-hunter/InferRSI/tree/125b06451a487e76a1c20c45abe4e22cd8631d68)中执行，历史实测仍以各节指定版本为准。当前通用流程中 profiling 是可选工具，不按下文演示顺序强制 Agent 执行。 **我们给 InferRSI 增加的是 Engineering Benchmarks（共享工程评测任务）：一份题目、一份候选实现，交给固定的独立评测，得到可追溯的正确性、诊断和计时结果。** 主 Agent 根据这些证据决定修改、保留或放弃实现。 @@ -12,7 +12,7 @@ 没有共享任务时，调用者仍然可以使用已有 Workbench，但需要自己提供这些命令和规则。不同实验若自行定义输入与答案，结果很难直接比较。 -这次把规则放进一个任务包。当前 `rmsnorm-cpu` 明确要求：CPU、NumPy、float32、沿 axis=1 归约，保留 epsilon，不修改输入。Agent 只交付 `implementation.py` 中的 `compute(x, *, eps)`。工具检查这个文件的实际行为，并记录是哪一版题目、裁判、源码和运行环境产生了结果。 +这次把规则放进一个任务包。当时的 `rmsnorm-cpu` 明确要求：CPU、NumPy、float32、沿 axis=1 归约，保留 epsilon，不修改输入。Agent 只交付 `implementation.py` 中的 `compute(x, *, eps)`。工具检查这个文件的实际行为，并记录是哪一版题目、裁判、源码和运行环境产生了结果。 它在 RSI 中的位置是**任务与评测层**。Workbench 继续负责运行实验；主 Agent 继续负责调查、读论文、分析 profile、写代码和决定下一轮。评估 skill 是否有效时，新增的[正式两臂产物 adapter](formal-integration.md)复用这层任务；实际技能收益仍需要受控 Agent 实验。 @@ -116,7 +116,7 @@ AB/BA 是调换执行顺序：一轮先原版再候选，下一轮先候选再 ## 5. 自己从哪里开始看和运行 -先读任务，再复制候选。安装/运行命令的详细参数见[使用文档](../../engineering-benchmarks.md)。下列演示在仓库根目录执行；输出目录必须未存在，重复试验应换新名字。 +先读任务，再复制候选。当前任务的安装/运行命令见[使用文档](../../engineering-benchmarks.md)。下列 CPU RMSNorm 历史演示仅在上述删除前版本的仓库根目录执行；输出目录必须未存在，重复试验应换新名字。 ```bash python -m pip install -e ".[dev,benchmarks]" @@ -182,7 +182,7 @@ AMD [AgentKernelArena 的固定版本任务契约](https://github.com/AMD-AGI/Ag | 顺序 | 入口 | 重点 | | --- | --- | --- | -| 1 | [RMSNorm task.json](../../../src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json)、[candidate.py](../../../src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py) | 题目规定了什么，Agent 改哪里 | +| 1 | [历史 RMSNorm task.json](https://github.com/Bounty-hunter/InferRSI/blob/125b06451a487e76a1c20c45abe4e22cd8631d68/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json)、[历史 candidate.py](https://github.com/Bounty-hunter/InferRSI/blob/125b06451a487e76a1c20c45abe4e22cd8631d68/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py) | 题目规定了什么，Agent 改哪里 | | 2 | [numpy_task.py](../../../src/inferrsi/harness/benchmarks/numpy_task.py) | 输入、独立 oracle、check、profile 与计时的真实边界 | | 3 | [contract.py](../../../src/inferrsi/harness/benchmarks/contract.py)、[worker.py](../../../src/inferrsi/harness/benchmarks/worker.py) | manifest 合法性、内容身份、执行前后检查、RunSpec | | 4 | [consumers.py](../../../src/inferrsi/harness/benchmarks/consumers.py)、[CLI](../../../src/inferrsi/harness/benchmarks/cli.py) | 如何复用执行器，如何呈现状态和结果 | diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index 9b07d9166..d1aa877b0 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -10,7 +10,7 @@ authorization and retained-result publication around these interfaces. The [offline improvement MVP](harness-improvement.md) reuses the same interfaces for bounded INV/DOM probes, successor rerun and a one-shot held-out gate. -Shared Softmax/RMSNorm CPU artifacts use the `engineering-benchmark-v1` adapter +Shared Softmax CPU artifacts use the `engineering-benchmark-v1` adapter in **update mode**, with the same prepare/execute/report and local stage/capture interfaces. The trusted parent grades output from a Linux bubblewrap NumPy worker; unsupported sandbox/runtime leaves INCONCLUSIVE without host fallback. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE deleted file mode 100644 index eaae6b11e..000000000 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -MIT License - -Copyright (c) 2023 Anne Ouyang, Simon Guo, Azalia Mirhoseini -Scaling Intelligence Lab, Stanford University - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md deleted file mode 100644 index 2f914e28d..000000000 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/README.md +++ /dev/null @@ -1,31 +0,0 @@ -# rmsnorm-cpu - -CPU adaptation of [KernelBench 36_RMSNorm_.py](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py). -Source license is retained in LICENSE. No upstream scoring result transfers here. - -Implement `compute(x, *, eps)` in a self-contained `implementation.py`. Return a -float32 NumPy array of the same shape. Preserve input values. Compute x / sqrt(mean(x*x, axis=1, keepdims=True) + eps), with no affine scale. -Use only NumPy, math and locally defined helpers. Source is reviewed/trusted; -the evaluator is not a security sandbox for model-produced arbitrary code. - -The task JSON owns six base correctness workloads, seeds, layouts, numerical -tolerances and the v2 timing policy. Five varying workloads have a frozen -per-call input sequence; invariant zero inputs are correctness-only. Baseline and example candidate are separate from the scalar -double-precision oracle. The oracle uses math.fsum and is not editable by a -candidate. The task package and grader are content-frozen per comparison. - -Each measured call actually runs compute and checks its output and input -preservation outside the timed region. Input generation, reference computation, -warmups, validation, process startup and profiling are excluded. Output allocation -inside compute is included. Samples use perf_counter_ns and report milliseconds. -Sum of per-case medians is only this task's fixed-suite objective, not a universal -score. Retain every per-case sample and input hash to inspect regressions. Inputs -are generated before timing and kept alive; each precheck, warmup and sampled -input has distinct bytes. Baseline and candidate consume the same sequence. -This prevents ordinary identity/content memo hits, not hostile-code execution. -Small CPU timings still need repeatability qualification; no cold-start serving -or device synchronization is claimed. - -All tests are public. This small subset cannot demonstrate production inference -speedup, memory savings, or GPU/NPU support. A fresh task version and device -qualification are needed for changed shapes, dtype, backend or timing. diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py deleted file mode 100644 index 7920c19b0..000000000 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/baseline.py +++ /dev/null @@ -1,8 +0,0 @@ -"""NumPy CPU adaptation of the attributed KernelBench task; see LICENSE.""" - -import numpy as np - - -def compute(x, *, eps): - rms = np.sqrt(np.mean(x * x, axis=1, keepdims=True) + np.float32(eps)) - return x / rms diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py deleted file mode 100644 index 89f3c6cf1..000000000 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Reviewed example using temporary buffers; not an agent-generated result.""" - -import numpy as np - - -def compute(x, *, eps): - squared = np.square(x) - rms = np.mean(squared, axis=1, keepdims=True) - rms += np.float32(eps) - np.sqrt(rms, out=rms) - return np.divide(x, rms) diff --git a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json b/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json deleted file mode 100644 index a17888f01..000000000 --- a/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json +++ /dev/null @@ -1,129 +0,0 @@ -{ - "schema_version": 1, - "id": "rmsnorm-cpu", - "adapter": "numpy-axis1-v2", - "description": "Optimize rmsnorm on finite float32 CPU arrays while preserving axis-1 semantics and input bytes.", - "searchspace": "docs/searchspace-matrix.md \u2014 Operator / Diagnose, Optimize, Verify; CPU adaptation, no matrix maturity promotion", - "source": { - "url": "https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py", - "revision": "423217d9fda91e0c2d67e4a43bf62f96f6d104f1", - "license": "MIT", - "adaptation": "Adapted mathematical task, not upstream scores. PyTorch/CUDA replaced by NumPy CPU; reduced shapes, public stress distributions and strided inputs added. Axis 1 retained. Baseline is a NumPy implementation; scalar math.fsum supplies independent reference." - }, - "environment": { - "backend": "cpu", - "runtime": "numpy", - "dtype": "float32" - }, - "candidate": { - "entrypoint": "compute", - "editable": [ - "implementation.py" - ] - }, - "oracle": { - "operation": "rmsnorm", - "axis": 1, - "eps": 1e-05, - "rtol": 1e-05, - "atol": 1e-07 - }, - "measurement": { - "key": "metrics.suite_latency_ms", - "unit": "ms", - "boundary": "synchronous-compute-call", - "aggregation": "sum-of-per-case-medians", - "warmups": 2, - "samples": 5, - "input_sequence": "seeded-distinct-per-call-v1", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ] - }, - "workloads": [ - { - "id": "normal", - "shape": [ - 4, - 17, - 9, - 7 - ], - "seed": 252, - "distribution": "normal", - "layout": "contiguous" - }, - { - "id": "strided", - "shape": [ - 4, - 17, - 9, - 7 - ], - "seed": 253, - "distribution": "normal", - "layout": "strided" - }, - { - "id": "zeros", - "shape": [ - 2, - 3, - 5, - 7 - ], - "seed": 254, - "distribution": "zeros", - "layout": "contiguous" - }, - { - "id": "large", - "shape": [ - 2, - 5, - 3, - 7 - ], - "seed": 255, - "distribution": "large", - "layout": "contiguous" - }, - { - "id": "tiny", - "shape": [ - 2, - 7, - 3, - 5 - ], - "seed": 256, - "distribution": "tiny", - "layout": "contiguous" - }, - { - "id": "singleton", - "shape": [ - 2, - 1, - 3, - 5 - ], - "seed": 257, - "distribution": "normal", - "layout": "contiguous" - } - ], - "coverage": [ - "Public development workloads, not held-out generalization", - "CPU float32 finite inputs only; no float16/bfloat16, NaN/Inf inputs or gradients", - "No CUDA/ROCm/Ascend execution or accelerator timing qualification", - "No service/E2E, communication, memory-capacity or skill/search benefit claim", - "Trusted self-contained reviewed source only; no hostile-agent isolation", - "Distinct frozen per-call inputs; zeros correctness-only; allocations inside compute included, setup/oracle/profile excluded from timing" - ] -} diff --git a/tests/check_installed.py b/tests/check_installed.py index 1c44bd195..914d57417 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -15,7 +15,7 @@ assert Path(inferrsi.__file__).resolve().parent.parent.name != "src", ( "This check requires an installed package" ) -assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu", "rmsnorm-npu"} +assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} for benchmark_id in builtin_tasks(): benchmark = load_task(benchmark_id) benchmark.assert_unchanged() diff --git a/tests/fixtures/engineering_benchmark_usefulness.py b/tests/fixtures/engineering_benchmark_usefulness.py deleted file mode 100644 index c19e5d6da..000000000 --- a/tests/fixtures/engineering_benchmark_usefulness.py +++ /dev/null @@ -1,341 +0,0 @@ -"""Frozen CPU engineering-use probe; trusted authored candidates, no agent calls. - -Prepare only writes a plan. Run consumes that exact plan once and keeps raw evidence. -Functionality, not a performance winner, determines this fixture's exit status. -""" - -import argparse -import hashlib -import json -import shutil -from pathlib import Path - -from inferrsi.harness.benchmarks import load_task -from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action -from inferrsi.harness.benchmarks.contract import digest -from inferrsi.harness.benchmarks.numpy_task import check_output, reference -from inferrsi.harness.runner import run_experiment -from inferrsi.harness.skill_evaluation import valid_checks -from inferrsi.infra.artifacts import atomic_write_json -from tools import engineering_benchmark_validate as integration - -EINSUM = """import numpy as np -def compute(x, *, eps): - rms = np.einsum('ij...,ij...->i...', x, x) - rms /= np.float32(x.shape[1]) - rms += np.float32(eps) - np.sqrt(rms, out=rms) - return x / np.expand_dims(rms, axis=1) -""" -NO_EPS = """import numpy as np -def compute(x, *, eps): - return x / np.sqrt(np.mean(x * x, axis=1, keepdims=True)) -""" -ROLES = ("unchanged", "einsum", "no-eps") - - -def read(path): - return json.loads(path.read_text(encoding="utf-8")) if path.is_file() else {} - - -def normalized(value): - return json.loads(json.dumps(value)) - - -def specs(task, root, plan): - return { - role: task.runspec( - root / (role + ".py"), - repo=Path(plan["repo"]), - output_dir=root / "results" / role / "workbench", - **plan["comparison"], - ) - for role in ROLES[:2] - } - - -def prepare(args): - root = args.output_dir.resolve() - root.mkdir(parents=True, exist_ok=False) - shutil.copytree( - load_task("rmsnorm-cpu").root, - root / "task", - ignore=shutil.ignore_patterns("__pycache__", "*.pyc"), - ) - task = load_task(root / "task") - shutil.copyfile(task.root / "baseline.py", root / "unchanged.py") - (root / "einsum.py").write_text(EINSUM, encoding="utf-8") - (root / "no-eps.py").write_text(NO_EPS, encoding="utf-8") - plan = { - "schema_version": 1, - "repo": str(args.repo.resolve()), - "script_sha256": digest(Path(__file__)), - "replay_helper_sha256": digest(Path(integration.__file__)), - "comparison": { - key: getattr(args, key) for key in ("repetitions", "max_minutes", "min_improvement_pct") - }, - "action_timeout_seconds": args.timeout_seconds, - "replay_bounds_seconds": {"client": 20, "trial": 40}, - "identities": {role: task.identity(root / (role + ".py")) for role in ROLES}, - "smoke": {"seed": 2026, "shape": [4, 17, 9, 7], "scale": 4, "dtype": "float32"}, - "hypothesis": "Source suggests einsum may avoid the full-sized square temporary; " - "CPU cProfile does not establish a memory bottleneck.", - "sequence": [ - "profiles", - "correctness-and-replay", - "ordinary-smoke", - "reject-wrong-measurement", - "unchanged-comparison", - "einsum-comparison", - ], - "success": "Both valid candidates accepted; no-eps rejected without a metric; " - "complete profiles/comparisons and identical consumer tasks/workloads. " - "Performance winner and ordinary-smoke outcome are not success gates.", - } - if not 0 < args.timeout_seconds <= 7 * 24 * 3600: - raise ValueError("Action timeout must be positive and at most seven days") - plan["runspecs"] = {role: spec.to_dict() for role, spec in specs(task, root, plan).items()} - atomic_write_json(root / "plan.json", plan) - print(json.dumps({"plan_sha256": digest(root / "plan.json"), "payload_executed": False})) - return 0 - - -def verify(root, expected): - if digest(root / "plan.json") != expected: - raise ValueError("Frozen plan changed") - plan = read(root / "plan.json") - if ( - digest(Path(__file__)) != plan["script_sha256"] - or digest(Path(integration.__file__)) != plan["replay_helper_sha256"] - ): - raise ValueError("Probe or replay helper changed") - task = load_task(root / "task") - if any(task.identity(root / (role + ".py")) != plan["identities"][role] for role in ROLES): - raise ValueError("Task, candidate, grader or runtime changed") - current = specs(task, root, plan) - if normalized({role: spec.to_dict() for role, spec in current.items()}) != plan["runspecs"]: - raise ValueError("RunSpec changed") - return plan, task, current - - -def smoke(root, plan, task): - import numpy as np - - config = plan["smoke"] - x = ( - np.random.default_rng(config["seed"]).standard_normal(config["shape"]) * config["scale"] - ).astype(np.float32) - expected = reference(x, task.manifest["oracle"], np) - records = {} - for role in ROLES: - namespace = {} - exec(compile((root / (role + ".py")).read_bytes(), "implementation.py", "exec"), namespace) - actual_input = x.copy() - output = namespace["compute"](actual_input, eps=task.manifest["oracle"]["eps"]) - assertions = check_output(output, actual_input, x, expected, task.manifest["oracle"], np) - records[role] = {"passed": all(assertions.values()), "assertions": assertions} - return {"input_sha256": hashlib.sha256(x.tobytes()).hexdigest(), "candidates": records} - - -def run(args): - root = args.output_dir.resolve() - plan, task, comparisons = verify(root, args.plan_sha256) - output = root / "results" - output.mkdir(exist_ok=False) # An interrupted attempt cannot silently resume or be replaced. - artifacts, profiles, candidates, measured = [], {}, {}, {} - - def checked(): - verify(root, args.plan_sha256) - - def retain(path, role): - document = read(path) - artifacts.append((role, document)) - return document - - for role in ROLES[:2]: - checked() - directory = output / role / "profile" - execution = run_action( - task, - root / (role + ".py"), - "profile", - directory, - timeout_seconds=plan["action_timeout_seconds"], - ) - result = retain(directory / "result.json", role) - profile = directory / "profile.txt" - profiles[role] = { - "status": execution["status"], - "diagnostic": result.get("diagnostic"), - "has_metric": "metrics" in result, - "artifact_sha256": digest(profile) if profile.is_file() else None, - } - for role in ROLES: - checked() - source, directory = root / (role + ".py"), output / role - grade = grade_artifact( - task, source, directory / "correctness", timeout_seconds=plan["action_timeout_seconds"] - ) - result = retain(directory / "correctness" / "result.json", role) - checked() - trial = integration.replay(task, source, directory / "skilleval") - retain(directory / "skilleval/trial/verification/artifact/result.json", role) - candidates[role] = { - "correctness": grade["verdict"], - "skilleval": trial["verdict"], - "failed_assertions": [ - key for key, value in result.get("assertions", {}).items() if not value - ], - } - checked() - smoke_result = smoke(root, plan, task) - atomic_write_json(output / "ordinary-smoke.json", smoke_result) - checked() - wrong = run_action( - task, - root / "no-eps.py", - "measure", - output / "no-eps/measure", - timeout_seconds=plan["action_timeout_seconds"], - ) - wrong_result = retain(output / "no-eps/measure/result.json", "no-eps") - for role in ROLES[:2]: - checked() - run_dir = run_experiment(comparisons[role], {"scope": "CPU engineering-use probe"}) - evaluation = read(run_dir / "evaluation.json") - trials = [] - for side in ("baseline", "candidate"): - for path in sorted((run_dir / side).glob("trial-*/result.json")): - result = retain(path, "unchanged" if side == "baseline" else role) - trials.append( - { - "side": side, - "trial": path.parent.name, - "cases": result.get("cases", []), - "metrics": result.get("metrics"), - } - ) - validation = retain(run_dir / "validation/result.json", role) - quality_checks = {key: validation.get(key) for key in ("passed", "assertions")} - measured[role] = { - key: evaluation.get(key) - for key in ( - "verdict", - "baseline", - "candidate", - "improvement_pct", - "statistical_significance", - ) - } - measured[role].update( - final_quality_passed=( - validation.get("status") == "completed" - and validation.get("action") == "check" - and validation.get("passed") is True - and valid_checks(quality_checks) - ), - source_integrity=evaluation.get("source_integrity", {}).get("verified"), - trials=trials, - measured_calls=sum( - len(case["samples_ms"]) for trial in trials for case in trial["cases"] - ), - ) - checked() - workloads = artifacts[0][1].get("workloads") - identities_match = all( - result.get("identity") == plan["identities"][role] for role, result in artifacts - ) - workloads_match = bool(workloads) and all( - result.get("workloads") == workloads for _, result in artifacts - ) - sequence = artifacts[0][1].get("input_sequence") - executed = artifacts[0][1].get("executed_calls") - checks = { - "valid_candidates_accepted": all( - candidates[r]["correctness"] == candidates[r]["skilleval"] == "PASS" for r in ROLES[:2] - ), - "wrong_candidate_rejected": candidates["no-eps"]["correctness"] - == candidates["no-eps"]["skilleval"] - == "FAIL", - "wrong_has_no_metric": wrong["status"] == "failed" and "metrics" not in wrong_result, - "profiles_complete": all( - p["status"] == "completed" - and p["artifact_sha256"] - and p["diagnostic"] - and p["diagnostic"].get("performance_verdict") is None - and not p["has_metric"] - for p in profiles.values() - ), - "consumer_identities_match": identities_match, - "actual_workloads_match": workloads_match, - "planned_input_sequences_match": bool(sequence) - and all(result.get("input_sequence") == sequence for _, result in artifacts), - "valid_executed_inputs_match": bool(executed) - and all( - result.get("executed_calls") == executed - for _, result in artifacts - if result.get("passed") - ), - "comparison_quality_passed": all(m["final_quality_passed"] for m in measured.values()), - "comparisons_complete": all( - m["source_integrity"] - and len(m["trials"]) == 2 * plan["comparison"]["repetitions"] - and m["measured_calls"] - == 2 - * plan["comparison"]["repetitions"] - * len(task.manifest["measurement"]["case_ids"]) - * task.manifest["measurement"]["samples"] - and all( - m[side] and m[side]["excluded_trials"] == 0 for side in ("baseline", "candidate") - ) - for m in measured.values() - ), - } - summary = { - "scope": "CPU engineering-use probe with trusted authored candidates", - "plan_sha256": args.plan_sha256, - "script_sha256": plan["script_sha256"], - "replay_helper_sha256": plan["replay_helper_sha256"], - "smoke_config": plan["smoke"], - "environment": artifacts[0][1].get("environment"), - "checks": checks, - "checks_succeeded": all(checks.values()), - "identities": plan["identities"], - "comparison": plan["comparison"], - "workloads": workloads, - "input_sequence": sequence, - "profiles": profiles, - "candidates": candidates, - "ordinary_smoke": smoke_result, - "comparisons": measured, - "unchanged_raw_pass": measured["unchanged"]["verdict"] == "PASS", - "interpretation": "Raw Workbench verdicts are descriptive; an unchanged PASS is " - "a false-positive warning, not optimization. No speedup claim.", - "campaign_acceptance": "INCONCLUSIVE", - "agent_or_skill_benefit": "not measured", - "accelerator_qualification": "not run", - } - atomic_write_json(output / "summary.json", summary) - print(json.dumps({"checks_succeeded": summary["checks_succeeded"], "checks": checks})) - return 0 if summary["checks_succeeded"] else 1 - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - commands = parser.add_subparsers(dest="action", required=True) - setup = commands.add_parser("prepare") - setup.add_argument("--repo", type=Path, required=True) - setup.add_argument("--repetitions", type=int, required=True) - setup.add_argument("--max-minutes", type=float, required=True) - setup.add_argument("--min-improvement-pct", type=float, required=True) - setup.add_argument("--timeout-seconds", type=float, required=True) - execute = commands.add_parser("run") - execute.add_argument("--plan-sha256", required=True) - for command in (setup, execute): - command.add_argument("--output-dir", type=Path, required=True) - args = parser.parse_args() - return prepare(args) if args.action == "prepare" else run(args) - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tests/test_engineering_benchmarks.py b/tests/test_engineering_benchmarks.py index 12f661ae8..97dc1f11b 100644 --- a/tests/test_engineering_benchmarks.py +++ b/tests/test_engineering_benchmarks.py @@ -12,6 +12,7 @@ from inferrsi.harness.benchmarks import builtin_tasks, load_task, skill_verifier from inferrsi.harness.benchmarks.consumers import grade_artifact, run_action +from inferrsi.harness.benchmarks.contract import load_builtin_task from inferrsi.harness.runner import run_experiment from inferrsi.harness.scenarios.external import execute_trial from inferrsi.harness.skill_evaluation import run_trial @@ -26,9 +27,8 @@ def read_result(execution): return json.loads(Path(execution["result_path"]).read_text(encoding="utf-8")) -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) -def test_real_baseline_candidate_measurement_and_profile(name, tmp_path, numpy_runtime): - task = load_task(name) +def test_real_baseline_candidate_measurement_and_profile(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") checked = [] measurement = task.manifest["measurement"] expected_calls = len(task.manifest["workloads"]) + len(measurement["case_ids"]) * ( @@ -71,16 +71,11 @@ def test_real_baseline_candidate_measurement_and_profile(name, tmp_path, numpy_r ] -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) @pytest.mark.parametrize("key", ["id(x)", "x.tobytes()"]) -def test_timed_calls_cannot_hit_precheck_or_warmup_memo(name, key, tmp_path, numpy_runtime): - task = load_task(name) +def test_timed_calls_cannot_hit_precheck_or_warmup_memo(key, tmp_path, numpy_runtime): + task = load_task("softmax-cpu") candidate = tmp_path / "implementation.py" - operation = ( - "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" - if name == "softmax-cpu" - else "y = x / np.sqrt(np.mean(x*x,axis=1,keepdims=True) + np.float32(eps))" - ) + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" candidate.write_text( "import numpy as np\nfrom pathlib import Path\ncache = {}\n" "def compute(x, *, eps):\n" @@ -100,15 +95,10 @@ def test_timed_calls_cannot_hit_precheck_or_warmup_memo(name, key, tmp_path, num assert not candidate.with_suffix(".hits").exists() -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) -def test_shape_cached_output_replay_is_rejected_after_precheck(name, tmp_path, numpy_runtime): - task = load_task(name) +def test_shape_cached_output_replay_is_rejected_after_precheck(tmp_path, numpy_runtime): + task = load_task("softmax-cpu") candidate = tmp_path / "implementation.py" - operation = ( - "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" - if name == "softmax-cpu" - else "y = x / np.sqrt(np.mean(x*x,axis=1,keepdims=True) + np.float32(eps))" - ) + operation = "y = np.exp(x-x.max(axis=1,keepdims=True)); y /= y.sum(axis=1,keepdims=True)" candidate.write_text( "import numpy as np\ncache = {}\ndef compute(x, *, eps):\n" " key = (x.shape, x.strides)\n" @@ -149,20 +139,6 @@ def test_bad_candidates_cannot_supply_performance(body, tmp_path, numpy_runtime) assert "metrics" not in read_result(execution) -def test_rmsnorm_last_axis_bug_detected(tmp_path, numpy_runtime): - candidate = tmp_path / "implementation.py" - candidate.write_text( - "import numpy as np\ndef compute(x, *, eps):\n" - " return x / np.sqrt(np.mean(x*x, axis=-1, keepdims=True) + eps)\n" - ) - assert ( - grade_artifact(load_task("rmsnorm-cpu"), candidate, tmp_path / "grade", timeout_seconds=20)[ - "verdict" - ] - == "FAIL" - ) - - def test_checks_repeated_timed_calls_not_only_initial_call(tmp_path, numpy_runtime): candidate = tmp_path / "implementation.py" candidate.write_text( @@ -221,7 +197,9 @@ def test_descriptor_does_not_mutate_task(): manifest = task.manifest manifest["oracle"]["axis"] = 0 assert task.manifest["oracle"]["axis"] == 1 - assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-cpu", "rmsnorm-npu"} + assert set(builtin_tasks()) == {"softmax-cpu", "rmsnorm-npu"} + with pytest.raises(ValueError, match="Unknown built-in task package"): + load_builtin_task("rmsnorm-cpu") @pytest.mark.parametrize( diff --git a/tests/test_skill_eval_benchmark.py b/tests/test_skill_eval_benchmark.py index 78c9a7b85..04b1018d7 100644 --- a/tests/test_skill_eval_benchmark.py +++ b/tests/test_skill_eval_benchmark.py @@ -19,7 +19,7 @@ READY = {"ready": True, "numpy": np.__version__, "python": "3.10.0", "machine": "x86_64"} -def answer(name="rmsnorm-cpu", source=None): +def answer(name="softmax-cpu", source=None): task = benchmark.task_descriptor(name) return task, {"artifact": benchmark.ARTIFACT, "summary": "Authored software control", "artifact_source": source or benchmark.starter(task).decode()} @@ -36,8 +36,8 @@ def oracle_transport(task): return {"outputs": outputs} -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) -def test_descriptor_freezes_builtin_task_and_shared_grader(name): +def test_descriptor_freezes_builtin_task_and_shared_grader(): + name = "softmax-cpu" task, _ = answer(name) benchmark.validate_task(task) package = load_task(name) @@ -92,7 +92,7 @@ def test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle(tmp_path, @pytest.mark.parametrize("change", ["adapter", "environment"]) def test_formal_artifact_path_rejects_new_backend_packages(change, monkeypatch): - package = load_builtin_task("rmsnorm-cpu") + package = load_builtin_task("softmax-cpu") manifest = package.manifest if change == "adapter": manifest["adapter"] = "torch-npu-rmsnorm-v1" @@ -111,7 +111,7 @@ def test_descriptor_drift_and_noncanonical_tasks_rejected(change): if change == "hash": task["benchmark"]["task_sha256"] = "0" * 64 elif change == "path": - task["benchmark"]["task_id"] = "../rmsnorm-cpu" + task["benchmark"]["task_id"] = "../softmax-cpu" elif change == "holdout": task["split"] = "holdout" elif change == "request": @@ -149,8 +149,8 @@ def test_missing_numpy_is_explicit_unavailable(): benchmark.verify(task, output) -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) -def test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call(name): +def test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call(): + name = "softmax-cpu" task, output = answer(name) transport = oracle_transport(task) calls = [] @@ -274,9 +274,8 @@ def run_real(task, output): pytest.skip(str(error)) -@pytest.mark.parametrize("name", ["softmax-cpu", "rmsnorm-cpu"]) -def test_real_sandbox_correct_and_wrong_authored_controls(name): - task, output = answer(name) +def test_real_sandbox_correct_and_wrong_authored_controls(): + task, output = answer() result = run_real(task, output) assert result["passed"] and len(result["assertions"]) == 164 wrong = {**output, "artifact_source": @@ -301,7 +300,8 @@ def test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay( def compute(x, *, eps): key = (x.shape, x.strides) if key not in cache: - cache[key] = x / np.sqrt(np.mean(x*x, axis=1, keepdims=True) + eps) + y = np.exp(x - x.max(axis=1, keepdims=True)) + cache[key] = y / y.sum(axis=1, keepdims=True) return cache[key] """ task, output = answer(source=source) diff --git a/tests/test_skill_eval_benchmark_compare.py b/tests/test_skill_eval_benchmark_compare.py index 46f38cbce..1165286b2 100644 --- a/tests/test_skill_eval_benchmark_compare.py +++ b/tests/test_skill_eval_benchmark_compare.py @@ -51,11 +51,13 @@ def __init__(self, modes=None): def command(self, workspace, final, schema, prompt): task = compare.read_json(workspace / "task.json") role = "baseline" if "-baseline-" in final.parent.name else "candidate" - mode = self.modes.get((task["id"], role), "correct") + repeat = int(final.parent.name.rsplit("-", 1)[1]) + mode = self.modes.get( + (task["id"], role, repeat), self.modes.get((task["id"], role), "correct")) source = ("import numpy as np\ndef compute(x, *, eps): return np.zeros_like(x)\n" if mode == "wrong-output" else benchmark.starter(task).decode("utf-8")) self.calls.append({ - "task": task, "role": role, "workspace": workspace, + "task": task, "role": role, "repeat": repeat, "workspace": workspace, "schema_sha256": sha256(schema), "source": source, "inputs": {p.relative_to(workspace).as_posix(): p.read_bytes() for p in workspace.rglob("*") if p.is_file()}, @@ -63,7 +65,7 @@ def command(self, workspace, final, schema, prompt): return (sys.executable, "-B", "-c", PRODUCER, str(workspace), str(final), source, mode) -def prepared(tmp_path, names=("rmsnorm-cpu",), repeats=1): +def prepared(tmp_path, names=("softmax-cpu",), repeats=1): compare.require_git_version(subprocess.check_output(("git", "--version"), text=True)) config_path, repo = configuration(tmp_path, ids=names, repeats=repeats) config = compare.read_json(config_path) @@ -87,13 +89,15 @@ def software_control_verifier(task, answer): def test_real_authored_processes_use_frozen_task_and_retain_both_sources(tmp_path, monkeypatch): - output, digest, _ = prepared(tmp_path, names=("rmsnorm-cpu", "softmax-cpu")) + # Two repetitions retain AB/BA isolation and both report outcomes with one task. + output, digest, _ = prepared(tmp_path, repeats=2) monkeypatch.setattr(benchmark, "verify", software_control_verifier) - client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "wrong-output"}) + client = AuthoredProducer({("softmax-cpu", "baseline", 1): "wrong-output"}) result = compare.execute(output, digest, client=client) assert result["adapter"] == benchmark.ADAPTER assert result["counts"]["fix"] == result["counts"]["unchanged_correct"] == 1 assert len(client.calls) == 4 + assert [call["repeat"] for call in client.calls] == [1, 1, 2, 2] assert [call["role"] for call in client.calls] == [ "baseline", "candidate", "candidate", "baseline"] for left, right in (client.calls[:2], client.calls[2:]): @@ -145,7 +149,7 @@ def test_task_grader_and_oracle_dependencies_are_frozen_before_producer(tmp_path names = ( "src/inferrsi/harness/skill_eval_benchmark.py", "src/inferrsi/harness/benchmarks/numpy_task.py", - "src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json", + "src/inferrsi/data/engineering_benchmarks/softmax-cpu/task.json", ) for name in names: assert plan["controllers"][name] == sha256(compare.ROOT / name) @@ -197,7 +201,7 @@ def test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow(tmp_path, def test_invalid_handoff_is_not_a_clean_comparison(tmp_path, monkeypatch, mode, error): output, digest, _ = prepared(tmp_path) monkeypatch.setattr(benchmark, "verify", software_control_verifier) - client = AuthoredProducer({("rmsnorm-cpu", "candidate"): mode}) + client = AuthoredProducer({("softmax-cpu", "candidate"): mode}) result = compare.execute(output, digest, client=client) trial = result["pairs"][0]["candidate"] assert trial["verdict"] == "INCONCLUSIVE" @@ -247,7 +251,7 @@ def test_retained_source_tamper_invalidates_report_without_rerunning_producer( def test_unknown_usage_stops_new_trials_without_inventing_cost(tmp_path, monkeypatch): output, digest, _ = prepared(tmp_path) monkeypatch.setattr(benchmark, "verify", software_control_verifier) - client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "unknown-usage"}) + client = AuthoredProducer({("softmax-cpu", "baseline"): "unknown-usage"}) result = compare.execute(output, digest, client=client) assert len(client.calls) == 1 assert result["status"]["stop_reason"] == "unknown_usage" @@ -269,10 +273,10 @@ def test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks(tmp_p (submission / name).write_text("authored control\n", encoding="utf-8") for role in compare.ROLES: workspace = tmp_path / role - local.stage(output, digest, "rmsnorm-cpu", role, 1, workspace, head) + local.stage(output, digest, "softmax-cpu", role, 1, workspace, head) if role == "baseline": (workspace / benchmark.ARTIFACT).write_bytes(b"# authored bad control\r\n") - result = local.capture(output, digest, "rmsnorm-cpu", role, 1, workspace, head, submission) + result = local.capture(output, digest, "softmax-cpu", role, 1, workspace, head, submission) assert result["counts"]["artifact_improved"] == 1 pair = result["pairs"][0] assert pair["comparability"] == "unverified_author_reported" @@ -292,18 +296,18 @@ def test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks(tmp_p {"grader": "custom-grader.py"}, ]) def test_adapter_does_not_accept_unfrozen_alternate_paths(tmp_path, change): - config_path, _ = configuration(tmp_path, ids=("rmsnorm-cpu",)) + config_path, _ = configuration(tmp_path, ids=("softmax-cpu",)) config = compare.read_json(config_path) config.update(adapter=benchmark.ADAPTER) config.update(change) with pytest.raises(ValueError): - compare.validate(config, [benchmark.task_descriptor("rmsnorm-cpu")]) + compare.validate(config, [benchmark.task_descriptor("softmax-cpu")]) def test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable(tmp_path): """Actual authored source replay; never represents an LLM or skill benefit.""" output, digest, _ = prepared(tmp_path) - client = AuthoredProducer({("rmsnorm-cpu", "baseline"): "wrong-output"}) + client = AuthoredProducer({("softmax-cpu", "baseline"): "wrong-output"}) result = compare.execute(output, digest, client=client) pair = result["pairs"][0] if any(pair[role]["evaluation_error"] == "artifact_sandbox_unavailable" From 6ed93bb41f7db9dafb420dcb6f8705f1864bc45c Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Wed, 4 Nov 2026 17:24:30 +0800 Subject: [PATCH 20/20] docs: keep benchmark usage separate from development archives Remove issue-specific plans and large historical dumps from the current tree, retain durable task and artifact handoff guidance, and drop obsolete path attributes and issue-specific utility wording. Historical records remain available at their original revision. Agent-assisted-by: Codex (documentation cleanup and independent review) Signed-off-by: dengyunyang <584797741@qq.com> --- .gitattributes | 7 - docs/engineering-benchmarks.md | 261 +- docs/engineering-task-artifacts.md | 115 + docs/plans/issue-252/01-research.md | 82 - docs/plans/issue-252/02-contract.md | 57 - docs/plans/issue-252/03-tasks.md | 69 - docs/plans/issue-252/04-integration.md | 60 - docs/plans/issue-252/README.md | 54 - docs/plans/issue-252/formal-integration.md | 180 - .../issue-252/formal-validation-summary.json | 325 - docs/plans/issue-252/npu-pilot.md | 88 - .../issue-252/npu-validation-summary.json | 12575 ---------------- docs/plans/issue-252/npu-validation.md | 62 - docs/plans/issue-252/replay-measurements.json | 1742 --- .../issue-252/replay-profiles/rmsnorm-cpu.txt | 19 - .../issue-252/replay-profiles/softmax-cpu.txt | 19 - .../issue-252/replay-validation-summary.json | 1255 -- docs/plans/issue-252/replay-validation.md | 52 - docs/plans/issue-252/review.md | 80 - docs/plans/issue-252/rsi-integration.md | 63 - .../issue-252/task-expansion/01-operators.md | 37 - .../task-expansion/02-host-runtime.md | 77 - .../issue-252/task-expansion/03-memory.md | 64 - .../task-expansion/04-communication.md | 69 - .../issue-252/task-expansion/05-serving.md | 78 - docs/plans/issue-252/task-expansion/README.md | 93 - .../task-expansion/gso-source-audit.json | 44 - .../issue-252/usefulness-profiles/einsum.txt | 25 - .../usefulness-profiles/unchanged.txt | 19 - docs/plans/issue-252/usefulness-summary.json | 2134 --- docs/plans/issue-252/usefulness.md | 74 - docs/plans/issue-252/validation-summary.json | 53 - docs/plans/issue-252/validation.md | 58 - docs/plans/issue-252/walkthrough.md | 195 - docs/skill-eval-comparison.md | 2 +- tools/engineering_benchmark_validate.py | 2 +- 36 files changed, 234 insertions(+), 19955 deletions(-) create mode 100644 docs/engineering-task-artifacts.md delete mode 100644 docs/plans/issue-252/01-research.md delete mode 100644 docs/plans/issue-252/02-contract.md delete mode 100644 docs/plans/issue-252/03-tasks.md delete mode 100644 docs/plans/issue-252/04-integration.md delete mode 100644 docs/plans/issue-252/README.md delete mode 100644 docs/plans/issue-252/formal-integration.md delete mode 100644 docs/plans/issue-252/formal-validation-summary.json delete mode 100644 docs/plans/issue-252/npu-pilot.md delete mode 100644 docs/plans/issue-252/npu-validation-summary.json delete mode 100644 docs/plans/issue-252/npu-validation.md delete mode 100644 docs/plans/issue-252/replay-measurements.json delete mode 100644 docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt delete mode 100644 docs/plans/issue-252/replay-profiles/softmax-cpu.txt delete mode 100644 docs/plans/issue-252/replay-validation-summary.json delete mode 100644 docs/plans/issue-252/replay-validation.md delete mode 100644 docs/plans/issue-252/review.md delete mode 100644 docs/plans/issue-252/rsi-integration.md delete mode 100644 docs/plans/issue-252/task-expansion/01-operators.md delete mode 100644 docs/plans/issue-252/task-expansion/02-host-runtime.md delete mode 100644 docs/plans/issue-252/task-expansion/03-memory.md delete mode 100644 docs/plans/issue-252/task-expansion/04-communication.md delete mode 100644 docs/plans/issue-252/task-expansion/05-serving.md delete mode 100644 docs/plans/issue-252/task-expansion/README.md delete mode 100644 docs/plans/issue-252/task-expansion/gso-source-audit.json delete mode 100644 docs/plans/issue-252/usefulness-profiles/einsum.txt delete mode 100644 docs/plans/issue-252/usefulness-profiles/unchanged.txt delete mode 100644 docs/plans/issue-252/usefulness-summary.json delete mode 100644 docs/plans/issue-252/usefulness.md delete mode 100644 docs/plans/issue-252/validation-summary.json delete mode 100644 docs/plans/issue-252/validation.md delete mode 100644 docs/plans/issue-252/walkthrough.md diff --git a/.gitattributes b/.gitattributes index 1dd2a9bab..16433b97f 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,12 +8,5 @@ /evidence/README.md text eol=lf /evidence/_template/** text eol=lf -# Preserve the recorded utility probe's JSON/profile bytes and their SHA-256s. -/docs/plans/issue-252/usefulness-summary.json -text whitespace=cr-at-eol -/docs/plans/issue-252/replay-validation-summary.json -text whitespace=cr-at-eol -/docs/plans/issue-252/usefulness-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof -/docs/plans/issue-252/replay-measurements.json -text -/docs/plans/issue-252/replay-profiles/*.txt -text whitespace=cr-at-eol,-blank-at-eof - # Retain this upstream license's original CRLF Git blob bytes. /src/inferrsi/data/skill_library/frameworks/torch-npu/torch-npu/references/licenses/op-plugin-LICENSE.txt -text diff --git a/docs/engineering-benchmarks.md b/docs/engineering-benchmarks.md index 1aabb6dd9..8c95f6553 100644 --- a/docs/engineering-benchmarks.md +++ b/docs/engineering-benchmarks.md @@ -1,174 +1,147 @@ # Shared engineering benchmarks -The `benchmarks` command exposes versioned task packages for Workbench and -offline artifact evaluation. The current device pilot is **Ascend affine -RMSNorm**, with a trusted NPU worker, independent scalar CPU oracle, synchronized -eager-call timing and separate device profiling. Existing CPU tasks remain -software controls. Packages own semantics, workload, oracle and metric; -consumers own comparison conditions. Search policy and agent execution remain -separate. - -| Adapter | Task and submission | Execution boundary | +The `benchmarks` commands check and measure an implementation against a versioned +task package. Packages define semantics, inputs, correctness rules and timing; +the existing Workbench runs paired comparisons and retains evidence. Agent +execution, knowledge selection and optimization search remain caller-owned. + +| Task | Candidate entry point | Execution boundary | | --- | --- | --- | -| `torch-npu-rmsnorm-v1` | `rmsnorm-npu`; `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | -| `numpy-axis1-v2` | `softmax-cpu`; `compute(x, *, eps)` | Trusted CPU worker; formal SkillEval also has a separate NumPy artifact sandbox | +| [Ascend affine RMSNorm](../src/inferrsi/data/engineering_benchmarks/rmsnorm-npu/README.md) | `compute(x, weight, *, eps)` | Reviewed source on one caller-allocated NPU; no formal device sandbox | +| [CPU Softmax](../src/inferrsi/data/engineering_benchmarks/softmax-cpu/README.md) | `compute(x, *, eps)` | Reviewed NumPy source; a separate SkillEval artifact sandbox also supports this task | + +The task READMEs retain source attribution, licenses and workload limits. To +optimize an implementation, copy the task's `baseline.py` to a separate +`implementation.py`. The packaged `candidate.py` is a reference control, not an +Agent-produced optimization. Keep task, oracle and baseline fixed while changing +the implementation. Review source before using the trusted worker: content +hashes detect drift, not malicious code. -## Run the NPU pilot +## Run an NPU implementation Discovery imports neither torch nor torch_npu. Execution requires the target's -compatible installed torch/torch_npu/CANN stack; this project does not install -or replace the device runtime. Enter an explicit single-NPU allocation before -executing any device command. The worker requires one visible device, uses its -logical index 0 and never falls back to CPU. +compatible installed torch/torch_npu/CANN stack; InferRSI does not install or +replace that runtime. Enter a caller-managed single-NPU allocation before any +device execution. The worker requires one `ASCEND_RT_VISIBLE_DEVICES` entry, +uses logical device 0 and never falls back to CPU. ```bash +inferrsi benchmarks list inferrsi benchmarks show rmsnorm-npu inferrsi benchmarks check rmsnorm-npu --candidate --output-dir --timeout-seconds -inferrsi benchmarks profile rmsnorm-npu --candidate --output-dir --timeout-seconds -inferrsi benchmarks compare rmsnorm-npu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct +inferrsi benchmarks compare rmsnorm-npu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct ``` -The task is contiguous 2D float32 affine RMSNorm with nonuniform weights. Inputs, -weights, scalar double/math.fsum reference results and transfers are prepared -outside timing. The measured clock covers the eager call and completion -synchronization after an untimed pre-synchronization. Dispatch and synchronization -costs are included: this is **not pure kernel latency**. Every output is checked -outside the timer, including shape/dtype/device and input/weight preservation. - -`profile` records real Ascend Hardware kernel events in `npu-trace.json`, separately -from latency sampling. Its local raw trace may contain private paths/identifiers; -review it before publication. The result includes a path-free kernel summary, -not a performance metric. Profiling includes verification copies, which must be -distinguished from compute kernels during diagnosis. +RMSNorm uses contiguous 2D float32 inputs and nonuniform weights. Input and +weight generation, scalar double/math.fsum oracle computation and transfers +occur outside timing. The clock covers eager dispatch and completion +synchronization after an untimed pre-synchronization. It measures synchronized +eager-call latency, not pure kernel time. Shape, dtype, device, numerical values +and input/weight preservation are checked outside the timer for every call. -To qualify the path with correct, wrong, replay, same-code and fused-reference -controls through the existing Workbench: +Profiling is optional and separate from measurement: ```bash -python -m tools.npu_benchmark_validate --repo --output-dir --repetitions --min-improvement-pct --max-aa-difference-pct --max-minutes --timeout-seconds +inferrsi benchmarks profile rmsnorm-npu --candidate --output-dir --timeout-seconds ``` -The utility freezes policy and implementation identities before execution, -preserves every sample and checks actual device/CANN/runtime consistency across -actions, trials and final validation. The general runtime fingerprint binds -package versions and selected device IDs; it alone does not qualify device/CANN -equivalence or host contention. Qualification is an authored-control exercise, -not Agent/skill efficacy or service acceptance. See the [NPU plan and results](plans/issue-252/npu-pilot.md). +This retains real device events in `npu-trace.json` and a path-free kernel +summary, with no performance verdict. Distinguish verification copies from +compute kernels. Review raw traces for private paths/identifiers before sharing. -## Use the CPU regression tasks +## Use the CPU software control -Install with `pip install -e ".[dev,benchmarks]"`. Discovery and planning do not -import NumPy; execution requires it and never silently switches backend. +Install the development checkout with `pip install -e ".[dev,benchmarks]"`. +Discovery and planning do not import NumPy; CPU execution requires it. ```bash -inferrsi benchmarks list inferrsi benchmarks show softmax-cpu -inferrsi benchmarks check softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/check --timeout-seconds 30 -inferrsi benchmarks profile softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/profile --timeout-seconds 30 -inferrsi benchmarks measure softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/measurement --timeout-seconds 30 -inferrsi benchmarks grade softmax-cpu --candidate /path/to/implementation.py --output-dir /path/to/grade --timeout-seconds 30 -inferrsi benchmarks plan softmax-cpu --candidate /path/to/implementation.py --repo /path/to/git-checkout --output-dir /path/to/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 +inferrsi benchmarks check softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks measure softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks grade softmax-cpu --candidate --output-dir --timeout-seconds +inferrsi benchmarks plan softmax-cpu --candidate --repo --output-dir --repetitions --max-minutes --min-improvement-pct ``` -Use `compare` instead of `plan` to execute the emitted RunSpec using existing -Workbench. Single-action output directories must be new. These are example -budgets, not campaign acceptance gates. The supplied Git checkout identifies -the source context; task and candidate files are additionally bound by hashes. - -`compare` returns the retained `status.json` alongside `run_dir`. Its exit code -matches Workbench: PASS is 0, FAIL is 1, and INCONCLUSIVE is 2. A completed -comparison missing its gain gate is FAIL; an incorrect candidate that prevents -measurement leaves the comparison INCONCLUSIVE, even though artifact correctness -grading can independently report FAIL. Inspect the retained state and errors. - -The candidate is a reviewed self-contained Python file with `compute(x, *, eps)`. -Use NumPy, math and local helpers. `load_task("softmax-cpu").root` locates its -baseline and example candidate. Copy `candidate.py` to `implementation.py` before -editing. The package and grader remain protected comparison inputs. This adapter -executes trusted source and is **not an untrusted-code sandbox**. Hash checks -detect drift, not malicious same-process code. Do not connect raw model output. - -## CPU measurement and shared evidence - -`TaskPackage` validates the schema/adapter/environment and freezes manifest, -package and grader identities. Commands bind candidate and runtime fingerprints. -The worker checks bindings before and after execution. Runtime captures Python, -NumPy version, OS and machine architecture; it does not prove an uncontended host -or a fully locked native dependency installation. - -Finite float32 inputs include strided layouts. RMSNorm reduces axis 1, including -for four-dimensional arrays. Scalar double precision with math.fsum supplies the -independent oracle. Numerical tolerances, shapes, seeds, warmups and sample counts -are task-specific and fixed across both consumers. - -`check` evaluates all six base cases plus the same varying-input sequence used -by measurement and profiling. `measure` checks first, times synchronous compute -calls, and verifies every invocation outside the timer. Inputs and references -are prepared in advance; all input objects stay alive and every timed input has -distinct bytes from precheck/warmup/other sampled inputs. This prevents an earlier -call from priming an identity/content memo for a later timed call. It does not -detect hostile code or public-suite overfitting. - -The task's `measurement.case_ids` declares five varying workloads. The invariant -zero-input case remains a correctness case and contributes no timing score. Every -result retains the planned `input_sequence` and actual `executed_calls`; each -sample records its input hash. Invalid candidates may stop before the full -sequence, so planned inputs must not be described as completed execution. - -Input generation, oracle, validation, startup and warmups are excluded; allocations -inside compute are included. The objective sums the five per-case medians for this -fixed suite, not a universal score. Inspect per-case regressions before adopting a -candidate. This protocol does not establish cold-start serving or statistical -resolution. The earlier `numpy-axis1-v1` repeated-input evidence remains historical; -its task/grader identities and six-case scores cannot be pooled with v2. - -`profile` retains a separate CPU `profile.txt` with no performance metric/verdict. -It is not a device timeline. Hardware adapters need their own synchronization, -timing and qualification. Missing or failed execution never becomes qualification. - -`grade` produces artifact-correctness PASS/FAIL/INCONCLUSIVE and SkillEval's -existing `passed`/`assertions` structure. `skill_verifier.task_descriptor` binds a -controller-reviewed retained artifact; `skill_verifier.py` plugs into the existing -`skill_evaluation.run_trial`. The answer supplies only `candidate_sha256`. -The shared oracle independently grades actual implementation behavior. This -offline handoff does not modify production engineering sandbox or paid execution. -Live artifact production and skill/search benefit need separate controlled runs. - -For formal paired artifact evaluation, `tools.skill_eval_compare` supports -`engineering-benchmark-v1` for the CPU NumPy tasks only: freeze the common public task and two skill bundles, -retain each arm's actual `implementation.py`, and independently check its output -outside the artifact's Linux bubblewrap sandbox. Local stage/capture uses the -same controller. This adapter evaluates correctness only; it does not accept -child-reported timing, launch an unqualified default agent, or fall back to host -execution when the sandbox/NumPy runtime is unavailable. It is separate from the -trusted-source worker above. Formal loading uses canonical packaged resources, -independent of same-named cwd task folders; the trusted Workbench retains explicit -path loading. NPU tasks are explicitly rejected by the formal adapter until -device isolation is separately qualified. See [setup and handoff](plans/issue-252/formal-integration.md). - -The offline harness-improvement controller and provider generation mode do not -yet support these tasks. See [the RSI integration audit](plans/issue-252/rsi-integration.md) -for decisions and remaining acceptance work. - -Exceptions, missing dependencies, drift and timeouts leave incomplete evidence. -Wrong results emit no usable performance metric. Workbench statistics remain -descriptive; a local verdict is not service gain or campaign acceptance. - -## Reproduce the historical CPU integration +The Softmax contract reduces axis 1 and includes strided and boundary inputs. +The independent oracle uses scalar double precision and `math.fsum`. `check` +evaluates the base cases and the same varying-input sequence used by measurement. +Every timed input has distinct retained objects and bytes, so an earlier call +cannot prime an identity/content memo for a later sample. This public suite does +not prevent hostile implementations or establish held-out generalization. + +Input generation, oracle evaluation, validation, startup and warmups are outside +timing; allocations inside `compute` are included. The score sums per-case +medians for the task's selected workloads. Inspect individual cases as well as +the aggregate. The zero-input case checks correctness but contributes no timing +score. CPU `profile` retains `profile.txt`, not a device timeline or speed score. + +## Comparison and retained results + +Replace `plan` with `compare` to execute through Workbench. `plan` prints a +RunSpec; `compare` constructs its own RunSpec rather than consuming a previously +printed one. Freeze the actual inputs and gates before candidate search using +the [acceptance procedure](contributing.md#acceptance-checklist). All placeholders +above come from that experiment, not repository defaults. Single-action output +directories must be new. + +`TaskPackage` freezes task and grader identities. Commands also bind candidate +bytes and runtime fingerprints, checked before and after worker execution. The +Git checkout supplied by `--repo` identifies source context; task and candidate +files have additional hashes. Runtime fingerprints alone do not establish an +uncontended host, equivalent device/CANN installations or native dependencies. + +Results retain the planned `input_sequence`, actual `executed_calls` and sample +input hashes. Failed candidates may execute only a prefix. Wrong outputs produce +no usable performance metric. Preserve unsuccessful trials and inspect +`execution.json`, per-trial `result.json`, `evaluation.json`, `status.json` and +source-integrity records. + +`compare` returns `run_dir` and the retained status. Its exit codes are PASS = 0, +FAIL = 1 and INCONCLUSIVE = 2. A complete comparison missing its gain gate is +FAIL. A wrong candidate preventing measurement leaves the comparison +INCONCLUSIVE, while independent artifact correctness can be FAIL. Exceptions, +missing dependencies, drift and timeouts must not become successful comparisons. +Workbench statistics are descriptive; these results do not establish service +gains, statistical significance or Agent/knowledge efficacy. + +`grade` returns artifact-correctness PASS/FAIL/INCONCLUSIVE with SkillEval's +`passed`/`assertions` structure. The lower-level +`benchmarks.skill_verifier.task_descriptor` binds an already reviewed artifact +for `skill_evaluation.run_trial`; its answer supplies `candidate_sha256`. +For actual CPU file handoff through `tools.skill_eval_compare`, see +[engineering task artifacts](engineering-task-artifacts.md). That formal adapter +checks correctness only and rejects NPU tasks. It does not automatically send +generated NPU code to this trusted worker. Provider generation mode and the +offline harness-improvement controller do not yet support this task adapter. + +## Qualify the tools with authored controls + +The NPU utility exercises correct, wrong, replay, same-code and reference +implementations through the worker and Workbench: ```bash -python tools/engineering_benchmark_validate.py --repo /path/to/checkout --output-dir /path/to/new-validation --repetitions 3 +python -m tools.npu_benchmark_validate --repo --output-dir --repetitions --min-improvement-pct --max-aa-difference-pct --max-minutes --timeout-seconds ``` -The tool freezes copied tasks and candidate identities, executes the retained CPU task, -retains profiles and AB/BA samples, and runs the actual SkillEval trial path with -correct and deliberately wrong **authored controls**. No model is called. -`summary.json` separates integration checks from campaign acceptance. Local raw -bundles contain resolved paths; review/redact a copy before sharing. +It freezes policy and implementation identities before execution, retains all +samples and checks actual device/CANN/runtime consistency. The utility includes +profiling as a tool control; this does not make profiling mandatory for ordinary +optimization. `--timeout-seconds` caps its standalone actions, while paired +Workbench trials use the comparison deadline. It does not enforce a separate +whole-utility budget or the standalone cap on each paired trial. Establish the +required outer allocation/process deadlines and verify cleanup before relying +on those budget boundaries. + +The CPU integration utility dynamically selects the packaged NumPy task: -See [#252 research and subtask plans](plans/issue-252/README.md). Formal Issue -acceptance, independent review, communication and memory-capacity coverage remain -distinct from each task's scoped qualification. +```bash +python -m tools.engineering_benchmark_validate --repo --output-dir --repetitions +``` -Start with the current [Chinese NPU pilot guide](plans/issue-252/npu-pilot.md). -The earlier [CPU walkthrough](plans/issue-252/walkthrough.md) retains its historical scope. +It retains profiles and AB/BA samples and runs the existing SkillEval trial path +with correct and wrong authored controls. Neither utility generates Agent +candidates. Their reports separate software controls from campaign acceptance. +Preserve experiment plans and original results with their tested revisions using +the [evidence conventions](contributing.md#evidence-and-claims); review/redact +copies of raw logs before publication. A successful control run does not finish +the Agent-to-NPU handoff or prove an optimization benefit. diff --git a/docs/engineering-task-artifacts.md b/docs/engineering-task-artifacts.md new file mode 100644 index 000000000..75f66063c --- /dev/null +++ b/docs/engineering-task-artifacts.md @@ -0,0 +1,115 @@ +# Engineering task artifacts in SkillEval + +`engineering-benchmark-v1` connects the packaged `softmax-cpu` task to +`tools.skill_eval_compare` in **update mode**. It freezes two skill versions, +retains each producer's actual `implementation.py`, and grades correctness with +the shared task oracle. It does not measure speed. Reviewed implementations can +later enter a separate [Workbench comparison](engineering-benchmarks.md). + +## Prepare the task and comparison + +Run from a development checkout. Generate a canonical case rather than editing +its contract or relying on a same-named directory in the working tree: + +```bash +python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('softmax-cpu')])" +``` + +Create `config.json` beside `cases.json`: + +```json +{ + "adapter": "engineering-benchmark-v1", + "mode": "update", + "skill": "", + "skill_root": "", + "cases": "cases.json", + "snapshots": { + "baseline": {"git_revision": ""}, + "candidate": {"git_revision": ""} + }, + "model": {"name": "", "effort": ""}, + "limits": { + "repeats": 1, + "max_task_calls": 2, + "timeout_seconds": 60, + "campaign_seconds": 180, + "token_stop_new_trials": 10000 + } +} +``` + +The example limits are illustrative; choose and freeze the actual bounds before +execution. Use full commits available in the local repository. Do not add a +provider client, custom schema or grader. See [comparison configuration](skill-eval-comparison.md) +for Git prerequisites, snapshot rules and budget semantics. + +```bash +python -m tools.skill_eval_compare prepare --config config.json --repo --output +``` + +Preparation freezes task, starter, schema, verifier/controllers and skill +bundles. Formal loading always uses canonical packaged resources. Retained task +copies must match that descriptor. The supplied CPU starter is already correct; +leaving it unchanged can pass, so correctness alone cannot identify optimization. + +## Hand off and retain actual files + +Stage a fresh workspace outside the evidence tree for one arm: + +```bash +python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Stage does not launch an Agent. Run the producer under the caller's established +permissions and budget. Only `implementation.py` may change; keep task, +instructions and assigned knowledge frozen. The file implements +`compute(x, *, eps)` with self-contained NumPy/math helpers. Its final answer is: + +```json +{"artifact": "implementation.py", "summary": "Describe the actual change"} +``` + +Stop the producer before capture. Retain `stdout.log`, `stderr.log` and +`execution.json` in a separate submission directory, following the exact fields +in [author-run handoff](local-skill-evaluation.md#author-run-handoff). +Model, access, resources and usage in these files are author-reported, not +authenticated observations. Missing usage stays unknown. + +```bash +python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id softmax-cpu --condition baseline --repeat 1 --current-head +``` + +Repeat stage/produce/capture for `--condition candidate` with distinct workspace +and log directories, then run: + +```bash +python -m tools.skill_eval_compare report --output +``` + +The plan hash binds this attempt; it does not authorize model spending. Changed +inputs require a new plan. Capture preserves actual source bytes and hashes; +report rechecks retained artifacts without rerunning the producer. Failed or +incomplete attempts remain recorded and cannot be overwritten as clean results. + +## Execution boundary and interpretation + +The trusted parent owns inputs, the scalar oracle and assertions. Submitted +source executes in a Linux bubblewrap child with system Python, matching NumPy +and a bounded runner; the repository, grader and other arm are not mounted. +Every returned array is checked by the parent. The public suite does not prove +hidden-test generalization, hostile-protocol resistance or independent tracing +of every candidate operation. Producer permissions are a separate boundary. + +Windows, missing bubblewrap/namespace support or an incompatible NumPy runtime +produce `artifact_sandbox_unavailable` / INCONCLUSIVE without host fallback. +NPU tasks are explicitly rejected. CPU replay qualification does not establish +NPU isolation, and the default unrestricted Agent client is refused for automatic +engineering runs. Provider generation and the offline improvement controller do +not support this adapter. + +A PASS means correctness on the public input sequence, not faster code. A local +handoff report showing `artifact_improved` establishes a difference in retained +artifact checks, not a causal skill benefit or verified total cost. Performance, +Agent efficacy and service claims need their own controlled evidence. Profiling +is optional and is not invoked by this adapter. diff --git a/docs/plans/issue-252/01-research.md b/docs/plans/issue-252/01-research.md deleted file mode 100644 index 2c41e70fe..000000000 --- a/docs/plans/issue-252/01-research.md +++ /dev/null @@ -1,82 +0,0 @@ -# 研究支持文档：选定任务和复用方案 - -原独立 Issue #334 的范围已并入 [#336 任务库建设](03-tasks.md)和 [#337 流程接入](04-integration.md)。本文保留论文、固定源码与历史采用理由；研究仍随每批任务和接口工作进行，不再作为独立交付 Issue。旧 Issue 关闭不代表所有研究候选已实现或获验收。 - -2026-11-04 范围更新：用户已指定 NPU 试点，当前优先交付 [RMSNorm NPU 方案](npu-pilot.md)。[五类任务研究](task-expansion/README.md)继续保留，但 Tokenizers.encode 不再作为当前试点。下面的初始 CPU 选型作为历史设计依据，不代表本轮设备验收或整个 SearchSpace 的最终覆盖。 - -**这项工作的交付是选型结论：用真实 NPU affine RMSNorm 跑通任务、独立检查、profiling 和比较，复用已有任务契约与 Workbench，而不是再建一个执行平台。** - -| 要说清楚的问题 | 本子项的答案 | -| --- | --- | -| 具体做什么 | 阅读论文与固定版本源码，比较任务、正确性参考、测量和执行边界，确定首批范围 | -| 交付什么 | 下文的一手来源表、方案比较、采用/不采用理由和任务选择 | -| 谁使用 | 子项 02 的接口开发者、子项 03 的任务开发者，以及范围评审者 | -| 实际作用 | 明确 NPU 试点问题、基线与参考候选，复用已有 Workbench；CPU SkillEval 资格不自动延伸到设备 | -| 当前状态 | 既有研究保留，NPU 选择与资格计划明确；设备结果和维护者验收分别记录 | - -它是设计依据，运行时不会执行“调研模块”。具体成果落实在任务库和流程接入两项工作中。 - -## 理解 - -研究目标是选出适合现有 InferRSI 的最小可用组合。不能把一个外部 benchmark 的评分、runner、agent 和 CUDA 假设整体搬进来。 - -## 计划 - -1. 重读 #252 正文与两条评论，区分已要求的任务契约和独立 policy RFC。 -2. 核对 KernelBench、AgentKernelArena、BackendBench、SkillsBench 的论文和源码边界。 -3. 用 AlphaEvolve、AIDE 检查任务评估和搜索策略的职责划分。 -4. 核对现有 RunSpec、Workbench 和 SkillEval，选择最薄的接入方式。 -5. 记录固定来源、许可证、适配变化以及未采用原因；让实现和反例验证检验决策。 - -## 一手来源与采用边界 - -2026-11-03 重新读取下列来源。论文解释方法，固定版本代码确定可复用接口。作者的测量结果不作为本地结果。 - -| 工作 | 证据与具体设计 | 本次决定 | -| --- | --- | --- | -| KernelBench | [论文 §3、§4.2](https://arxiv.org/html/2502.10517v1) 将正确性和速度分开；[Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py) 和 [RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) 明确 axis=1；[MIT 许可](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/LICENSE) 已读取 | 适配数学题意并保留许可。CPU/NumPy、小形状、边界分布是明确变更。原 Softmax 的 4096×393216 FP32 输入约 6 GiB，不直接搬用 | -| AgentKernelArena | [论文 §3.1](https://arxiv.org/html/2605.16819v1) 分离 workspace、baseline、agent、central evaluation；[task v2 contract](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) 把 semantics/candidate/baseline/reference/workload/evaluation 放在任务包 | 采用责任划分、baseline/oracle 分离、统一最终评估。没有复制 GPU 平台或评分器，没有声称满足其 GPU 资格标准 | -| BackendBench | [README](https://github.com/meta-pytorch/BackendBench/blob/2a8d7e19e2c13c789ff2dceb44883ea8ea04dab4/README.md) 明确用 OpInfo/FACTO 做边界检查，用真实模型 shapes 做性能 | 借鉴边界和布局验证。首批不引入整个 PyTorch backend override、FACTO 和 TorchBench，也不声称覆盖它们 | -| SkillsBench | [固定版本 README](https://github.com/benchflow-ai/skillsbench/blob/f860ecf02ef53412a030f8661c83d644b739ff01/README.md) 分离 task.md、environment、oracle、verifier；[论文](https://arxiv.org/abs/2602.12670) 是技能受控比较参考 | 保持题目与技能条件独立，沿用现有 SkillEval verifier，不接入另一套云执行平台 | -| AlphaEvolve | [§2.1、§2.4–2.6](https://arxiv.org/html/2506.13131v1) 分离 task/evaluate、evaluation cascade、程序库和演化控制 | 采用正确性先于测量。候选生成、选择和程序库不放进 #252 | -| AIDE | [§3.1–3.2](https://arxiv.org/html/2502.13138v1) 显式区分 evaluator h、search policy π、coding operator f、solution tree T | search policy 是独立模块。本次提供任务评估，不实现搜索树或停止规则 | -| InferRSI 当前源码 | RunSpec 要求 baseline/candidate；runner 已有 AB/BA、超时和快照；SkillEval run_trial 已有独立 verifier；生产 engineering verifier 有 Linux bubblewrap 边界 | 生成真实 RunSpec，诊断用 execute_trial；保留产物接入现有 verifier。生产 sandbox 和执行权限不变 | - -## 当前 NPU 选择 - -官方算子与测试固定在 op-plugin `9f23f00b1ebe54cca612b0537bc582dba409a0af`，NPU profiler 使用方式参考 Ascend/pytorch `9065e614f02639f5ddde0cf370bb9d38de7bd8a9`；[NPU 来源表](npu-pilot.md#为什么选择-rmsnorm)给出具体文件与许可。它们提供 API 和任务设计依据，不替代安装环境验证。 - -1. 首个试点选 **affine RMSNorm**：二维 contiguous float32 输入、非全一权重，基线使用 NPU 上的基础运算组合，参考候选使用 `torch_npu.npu_rms_norm`。独立 CPU scalar double / `math.fsum` 路径计算参考结果。官方 NPU 算子的语义与运行时支持需单独核对，不直接继承旧 CPU 四维无 affine 任务。 -2. 保留 `check / profile / measure` 与现有 RunSpec / Workbench 接口，由 NPU adapter 明确依赖、设备身份、同步、输入准备和计时边界。实际设备不支持时拒绝执行，不静默退回 CPU。 -3. 先用正确/错误候选、A/A 和基础算子/融合候选对照验证任务能否产生可信决策，再扩充题数或评估 Agent。融合候选是参考控制，不预设其必然加速。 -4. 真实 CPU/NPU profiling 单独运行；记录观察如何影响改法或最终决定。仅生成 profile 文件不能证明 Agent 使用了诊断。 -5. 正式 SkillEval 的现有 Linux sandbox 是 CPU 路径。NPU Workbench 先执行已审核可信源码；NPU 产物隔离与真实 skill 收益单独跟踪。 - -具体契约、验收矩阵和未完成项见 [NPU 试点](npu-pilot.md)。后续 Tokenizers、KV、通信与服务题仍按各自来源、环境和裁判重新资格，不因已有 CPU 或 NPU 任务完成而宣布全部覆盖。 - -## 历史 CPU 方案比较与采用理由 - -以下记录最初硬件未定时的 CPU 决策，不作为当前优先级。原设计和失败证据保留便于审查，本轮已由上面的 NPU 决定更新交付目标。 - -| 方案 | 优点 | 当前问题 | 决策 | -| --- | --- | --- | --- | -| 完整引入外部 benchmark runner | 复用成熟 GPU 生态 | 硬件未定，重复 Workbench，带入 agent/评分假设 | 不选 | -| 只写统一 schema | 成本低、无需设备 | 无法验证真实任务或消费者 | 不选 | -| 首版实现所有硬件与三大类别 | 覆盖广 | 无资格环境，容易只剩空协议 | 不选 | -| 小任务包 + adapter + 既有执行器 + 两个消费者 | 语义和证据共享，可立即跑通和检验反例 | 首批只证明 CPU 子集 | **采用** | - -1. **首批选 Softmax 和 RMSNorm**。都是推理相关归约算子。Softmax 暴露 exp 溢出，RMSNorm 四维输入暴露错误轴，使第二道题有不同验证价值。通信/显存不放空实现。 -2. **消费者接口通用，adapter 能力具体**。`command(check|measure|profile)` 和 `runspec` 是共享边界，当前 `numpy-axis1-v2` 以变化输入修复最初 v1 的重放问题，见 [协议验证](replay-validation.md)。未知 backend/adapter 被拒绝。未来适配器沿用消费者入口，但必须定义自身同步、计时、manifest 校验和环境资格。 -3. **独立 oracle**。baseline/candidate 用 NumPy 归约，oracle 用标量 double 与 math.fsum，减少共享实现错误。容差属于具体任务数值规范，非全仓库性能门槛；每次比较冻结。 -4. **检查真实计时路径**。计时后的实际输出再验证，能发现预检后变错的状态行为。公开输入、同进程评估不能抵御恶意作弊，入口限定已审核可信文件，不能直接接任意模型源码。 -5. **profiling 独立**。CPU cProfile 解释 CPU 调用，不进入速度统计、不输出 A/B verdict。设备 trace 待硬件 adapter。 -6. **SkillEval 采用保留产物 replay**。controller 固定审核后的候选及其哈希，已有 run_trial 调用独立 verifier。本次 authored control 验证真实通路，没有模型调用和技能收益结论。 -7. **共享身份与原始样本**。task、grader、candidate、runtime 分别有哈希，记录实际输入摘要及每次样本。漂移、缺依赖、超时保留不完整证据。没有新建 AB/BA 调度器或统计规则。 - -## 对评论中三个问题的处理 - -- 没有引入 `upper_bound < residual_target`，增量贡献仍由独立 policy 处理。 -- correctness/profile 有独立执行记录，不需要伪造 baseline；只有 measurement 构成 A/B。 -- 没有把某 cell 一条失败记录解释成搜索耗尽，也没有增加停止控制器。 - -以上 CPU 选择针对当时硬件未定的约束；当前设备方向已明确为 NPU。无论哪一阶段，来源调研和贡献者方案都不代替维护者对 #252 的最终验收。 diff --git a/docs/plans/issue-252/02-contract.md b/docs/plans/issue-252/02-contract.md deleted file mode 100644 index 6e8e47f91..000000000 --- a/docs/plans/issue-252/02-contract.md +++ /dev/null @@ -1,57 +0,0 @@ -# 接口支持文档：规定任务怎样提交、运行和判定 - -原独立 Issue #335 的范围已并入 [#337 流程接入](04-integration.md)；任务作者需要的契约也由 [#336](03-tasks.md)引用。本文继续维护共享接口与已支持范围，不因旧 Issue 关闭而把待补的 NPU 产物执行边界或新 adapter 记为完成。 - -**这项工作的交付是一套共同规则：同一道优化任务交给 Workbench 或评估工具，都使用同一份输入、答案、计时边界和版本。** - -| 要说清楚的问题 | 本子项的答案 | -| --- | --- | -| 具体做什么 | 定义任务包字段、候选函数入口、检查/诊断/测量操作和结果身份 | -| 交付什么 | `TaskPackage`、`check/profile/measure` 操作及生成既有 RunSpec 的接口 | -| 谁使用 | Workbench 适配代码、离线 verifier 和后续任务作者 | -| 实际作用 | 一次比较中的题目和裁判固定；源码或规则改变时会被识别，两个调用者不用分别维护答案 | -| 当前状态 | CPU 接口与软件验证保留；本轮增加 NPU 设备语义、同步计时与环境身份，实际资格单独记录 | - -NPU RMSNorm 的入口为 `compute(x, weight, *, eps)`，采用二维 contiguous float32 与非全一权重；旧 CPU 题的 `compute(x, *, eps)`、四维 axis=1、无 affine 语义保持原身份。不能因同名算子就混用题目或裁判。接口不决定候选怎么生成。 - -## 本轮 NPU 契约 - -[NPU 试点](npu-pilot.md)列出当前目标：独立 adapter 在实际设备上验证输入/输出设备、dtype、数值与输入保持，使用 CPU scalar double / `math.fsum` oracle；数据准备与传输、oracle、校验在性能计时外完成。同步边界必须覆盖实际完成的 NPU 计算。独立 CPU/NPU profile 不进入性能分数。 - -运行时身份需绑定实际设备与 torch / torch_npu / CANN 版本，不沿用只包含 Python/NumPy 的 CPU 身份。当前命令指纹包含 torch/torch_npu 版本与分配的可见设备；worker 另记录实际型号和 CANN，资格工具检查所有 action/trial/final validation 的实际环境一致。不能把命令指纹单独称为完整硬件资格。任务、候选、oracle、输入和计时规则继续使用共享内容身份；缺依赖、无设备或不支持语义不能回退 CPU。已支持范围及执行证据见[本轮记录](npu-validation.md)。 - -Workbench 接入先运行已审核可信源码。正式 `engineering-benchmark-v1` 目前只接受已声明的 CPU 任务；NPU 题出现在 catalog 中，不代表正式 SkillEval 具备设备隔离或产物重验能力。adapter 的能力检查必须显式拒绝不支持的组合。 - -## 理解 - -任务包负责题目事实。消费者负责执行组织或 agent 对照。共享操作接口保持可复用，具体 adapter 必须声明硬件语义与支持范围，不承担搜索调度。 - -## 计划 - -- 用版本化 JSON manifest 记录任务、SearchSpace 关联、来源、环境、输入、可编辑范围、oracle、单位、计时边界与局限。 -- 用内容哈希冻结包、grader 和候选；任务改变后不得继续冒充原比较。 -- 暴露 correctness、measurement、diagnostic 三个操作。实际命令仍由现有执行器运行。 -- 将真正的 A/B 比较转成现有 RunSpec；不给诊断填假 baseline。 -- 为路径、格式、未支持环境、非有限指标和漂移编写有实际失败意义的检查。 - -## 验收 - -同一包身份被支持它的消费者使用；修改冻结输入后被拒绝；缺失环境不记成功。本轮 NPU 需证明真实设备检查、同步计时和 Workbench 证据，不能由 CPU 消费者测试替代。已有 CPU 变化输入协议见 [v2 验证](replay-validation.md)；早期安装包检查和 v1 运行保留在[历史验证记录](validation.md)，不作为当前 NPU 的执行结果。 - -## 已有软件接口 - -代码在 `src/inferrsi/harness/benchmarks/contract.py`，CLI 入口为 `inferrsi benchmarks`。 - -| 对象 | 实际字段或操作 | 用途 | -| --- | --- | --- | -| TaskPackage | schema_version、adapter、source、environment、candidate、oracle、measurement、workloads、coverage | 消费者读取同一份题目事实 | -| 冻结身份 | task_sha256、grader_sha256、candidate_sha256、runtime_sha256 | 记录实际包、实现与运行时；NPU 需扩充实际设备环境身份 | -| 单次操作 | check、measure、profile | 都生成结构化结果，只有 measure 提供性能指标 | -| Workbench 绑定 | runspec(candidate, repo, output_dir, repetitions, max_minutes, min_improvement_pct) | 使用已有 AB/BA、快照、截止时间和报告 | -| 工程评估绑定 | grade_artifact；skill_verifier.task_descriptor | 用实际实现与独立 oracle 生成 SkillEval checks | - -选择 JSON 是为了保持核心包零第三方依赖并便于命令快照。任务包描述题目，不保存密钥或用户运行配置。执行预算和增益门槛仍由 CLI/Python 调用方提供。 - -已有 `numpy-axis1-v2` 明确限定 CPU/NumPy/float32；本轮 NPU 使用独立 adapter，不改变上述操作分工。未知后端直接拒绝。没有承诺任意任务 manifest 都能执行，也没有提供空的通信/显存适配。 - -真实 payload 仍由已有 `execute_trial/execute_command` 运行，独立 worker 仅执行一道题。`profile` 不需要 RunSpec，`compare` 才使用 RunSpec。保留缺依赖、超时、候选/任务漂移等失败路径。哈希冻结不是恶意代码隔离。 diff --git a/docs/plans/issue-252/03-tasks.md b/docs/plans/issue-252/03-tasks.md deleted file mode 100644 index ece3e86b1..000000000 --- a/docs/plans/issue-252/03-tasks.md +++ /dev/null @@ -1,69 +0,0 @@ -# #336：持续建设优化任务库 - -**本项长期维护可复用的优化任务，每批独立研究、实现和验收。** 首题是下面的 `rmsnorm-npu`；后续任务沿现有接口增加，不因为题库存在就限制用户只能做内置题。真实 Agent 与外部项目的接入归 [#337](04-integration.md)。 - -原 #334 的跨任务选题、来源核查与复用决定并入本项；原 #335 中任务作者需要的共享接口说明继续使用 [02-contract.md](02-contract.md)。两项旧 Issue 的关闭仅表示范围迁移，不表示所有来源或接口已实现。 - -## 每批任务交付什么 - -- 明确任务目标、允许的修改范围及对应工程问题，不按 skill 或 Agent 建专用题库。 -- 调研论文和固定版本开源源码，记录许可证、采用/不采用理由与适配变化；研究条目不计为执行覆盖。 -- 提供任务输入、基线、独立正确性检查、计时方法、设备/运行时条件和版本身份；profiling 按需提供，不作为所有任务的默认门槛。 -- 用正确、错误和同码等适用控制验证裁判与测量，再记录实际支持范围、失败和复现方法。 -- 复用既有 Workbench 与 #337 的接入。需要新任务形式时先明确 adapter 缺口，不增加空实现来充覆盖。 - -## 持续跟踪 - -- [x] 首个 NPU RMSNorm 题包及工具控制实验已在 Draft #338 交付，尚未合并或完成项目验收。 -- [x] 保留 [已有选型研究](01-research.md)与[五类来源对照](task-expansion/README.md)，供后续按实际需求选择。 -- [ ] 修复并复审 #338 的单次/整体超时执行缺口；原始样本不改标为修复后的运行。 -- [ ] 由 #337 验证真实 Agent 消费首题及外部项目任务；题包可运行不等于这一步完成。 -- [ ] 后续每批任务分别记录来源、实现 PR、实际执行环境、结果和剩余限制。 - -本 Issue 不因首题通过而关闭，也不要求一次覆盖所有类别。后续加题按任务或紧密相关的一批任务提交 PR；当前首轮基础 PR 为 #338，后续端到端接入 PR 尚未创建。 - -## 首题：NPU RMSNorm - -**这项工作的当前交付是 `rmsnorm-npu`：给出可以在真实 Ascend NPU 上运行的基线、候选、输入、独立正确性检查、profiling 和测量规则。** 用户已经指定 NPU，不能以 CPU 软件检查作为试点验收。 - -| 要说清楚的问题 | 本子项的答案 | -| --- | --- | -| 具体做什么 | 将 affine RMSNorm 的数值语义接到 PyTorch / torch_npu，独立定义设备同步与计时 | -| 交付什么 | 任务包、基础算子 baseline、融合候选、数值 oracle、变化输入、错误控制与原始结果 | -| 谁使用 | 修改实现的 Agent/开发者，以及 Workbench 的任务操作 | -| 实际作用 | 验证候选确实在 NPU 计算且算对，并比较真实完成一次计算的成本 | -| 当前状态 | 已在 Ascend 910B1 完成 [NPU 资格实验](npu-validation.md)，保留全部 800 样本与错误控制；CPU 旧结果不替代它 | - -## 任务定义 - -首版使用二维 contiguous float32 输入 `[rows, features]` 与一维 float32 `weight`，沿 feature 轴计算 `x / sqrt(mean(x²) + eps) * weight`。这一版本明确采用真实 affine 语义，权重不是全一。二维的 axis=1 与末轴一致；不能直接将旧 CPU 四维 axis=1、无 affine 任务送给按末维归一化的融合算子。 - -基线由在 NPU 上执行的 PyTorch 基础运算组成；候选验证 `torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0]`。接口为 `compute(x, weight, *, eps)`。具体形状、epsilon、容差与变化输入协议在最终 manifest 中冻结；不从 CPU 结果推导设备支持范围。融合算子是可检验的参考候选，不要求所有合法实现都调用它。 - -独立参考计算在 CPU 使用 scalar double / `math.fsum` 做归约，再乘权重并转换为 float32。输入与权重准备、传到设备、oracle、复制回 host 检查均在性能计时区间之外。正确性包括输出形状、dtype、设备、有限性、数值误差和输入/权重内容不被改变。普通输入之外需覆盖全零、小值及不同形状，防止省略 epsilon、忽略权重或硬编码结果通过。 - -## 测量与诊断 - -- 输入在调用前准备并绑定摘要；变化输入避免前次正确输出被重放成“优化”。不得将缓存后的答案当作实际计算时间。 -- NPU 是异步设备；计时边界必须明确等待调用前后的设备工作完成。仅统计 Python 发起调用的时间不能作为 NPU 完成延迟。 -- 预热、首次编译或初始化、输入准备、oracle 与输出校验不计入正式样本；具体次数及策略来自冻结任务与实验计划。 -- 调用 profiling 时独立于正式计时，保留 CPU/NPU 活动与 trace 身份。若设备 trace 不可用，不能声称已有设备诊断；未调用 profile 不默认判任务失败。 -- 运行同码 A/A、错误但快的候选、基础算子与融合候选；保留全部有效和失败记录，避免只报告最快样本。 - -已执行的工具资格试点按冻结计划包含 profiling、A/A 与错误控制，完整流程和证据清单见 [NPU 试点](npu-pilot.md)。这些工具控制不要求每次真实 Agent 优化都重复执行。 - -## 验收 - -任务能在明确设备与运行时上重现；基线和合法候选计算正确；错误候选被拒绝且没有可用性能分数；真实 NPU profile 与同步计时进入 Workbench 证据。是否有可分辨加速另行判断，不能预先承诺融合必快。 - -这只是算子级 NPU 验收，不证明模型质量、服务吞吐、其他 dtype/形状、通信或显存容量。正式 SkillEval NPU 候选隔离仍需另外设计和验证，不能把可信 Workbench 源码执行包装为该能力。 - -## 保留的 CPU 工作 - -`softmax-cpu` 保留作软件控制，保留原 KernelBench 来源和 MIT 许可。其 NumPy 基线、标量 double / math.fsum oracle、变化输入和错误反例可以检验共享软件接口；不向 NPU 迁移其计时结果或环境资格。 - -按用户要求移除 `rmsnorm-cpu` 题包及其专用用途对照 fixture；当前内置题只有 `softmax-cpu` 和 `rmsnorm-npu`。通用接口与正式产物测试使用 Softmax，旧 CPU RMSNorm 的实测记录仍按原版本保留，复现须使用对应历史 checkout。 - -旧 `numpy-axis1-v2` 每题一次完整 trial 为 41 次调用、164 项断言、25 个计时样本；CPU profile 仅解释 CPU 调用。v1 的 360 个样本与 v2 的 300 个样本保留各自 task/grader 身份，[旧协议验证](replay-validation.md)和[用途对照](usefulness.md)中的噪声、失败与限制不改写。 - -[五类扩展方案](task-expansion/README.md)仍保留后续题源。NPU RMSNorm 工具试点已有记录，真实 Agent 接入归 #337；Tokenizers、KV 复用、通信和服务任务分别按实际需求独立评估，不因本任务完成而标记整类已覆盖。 diff --git a/docs/plans/issue-252/04-integration.md b/docs/plans/issue-252/04-integration.md deleted file mode 100644 index 428708939..000000000 --- a/docs/plans/issue-252/04-integration.md +++ /dev/null @@ -1,60 +0,0 @@ -# #337：将工程优化任务接入现有执行与评测流程 - -**交付目标：让已有 Agent + RSI 工作流接收题库或外部项目任务，留存 Agent 实际修改的代码，独立验收，并关联最终效果和整次执行的消耗。** 首轮使用一种真实 Agent，在 NPU 上验收一个内置任务和一个外部目标。 - -这是一项尚未完成的接入工作。现有 Workbench、SkillEval、Agent 执行与日志能力继续复用；Draft #338 提供任务与工具基础，预写参考候选的 NPU 演练不代替真实 Agent 验收。原 #334 的接入调研和 #335 的接口工作归本项跟踪，不另建平台。 - -## 已有什么，还需补什么 - -| 环节 | 仓库或 Draft PR 中已有 | 本项待补 | -| --- | --- | --- | -| 任务与条件 | 既有 Workbench 可接外部 benchmark/validation 命令；#338 提供共享题包和版本身份 | 让两种来源接入同一轮任务记录，事先确定基线、输入、检查、环境和预算 | -| Agent 执行 | 已有客户端、日志和部分受控编辑机制 | 选一个适用入口，接实际任务、工具操作和真实代码交付；不能把现有专用 CPU actor 当通用 NPU 执行器 | -| 独立评测 | #338 已有 NPU 正确性、可选 profile、同步计时与 Workbench 比较 | 对 Agent 实际产物建立不可混淆的版本绑定和最终重验，不接收自报分数作为结论 | -| 产物执行边界 | CPU 正式产物路径已有隔离；NPU worker 当前执行已审核可信源码 | 确定并验证适合 NPU 产物的受限执行或审核交接方式；不因能访问设备就视为已经隔离 | -| 完成判定 | 已有单次执行和比较 verdict | 检查事先声明的实际产物、质量、性能和必要记录，漏检、超时或中断不记成完整成功 | -| 整轮消耗 | 既有 Agent 耗时、部分 token/调用记录及 Workbench 执行记录 | 按同一任务身份关联全过程时间、可获取的用量、调用和设备实验消耗；不能获取的值保持未知 | -| 实际验收 | #338 已运行参考、错误和同码控制 | 真实 Agent 运行并交付，覆盖两种任务来源；实际重复执行，验证记录可以供调用方比较 | - -## 流程也是交付内容 - -1. **准备任务。** 接收内置题包，或外部项目的仓库、基线、允许修改范围、工作负载、测试和 benchmark。缺少可靠验收方法时先补齐，不能凭目标文字生成成功判定。 -2. **记录运行条件并启动。** 复用已有入口保存任务、Agent/RSI 配置、版本、环境和预算；保留实际执行记录。 -3. **Agent 自主优化。** 在允许的工具和预算内选择方法、修改和测试，可以尝试多个候选。 -4. **留存实际产物。** 保存最终代码及其身份、过程和失败记录，不能替换成预写参考候选。 -5. **独立验收。** 工具按执行前确定的规则检查实际产物，并在固定条件下重测基线与候选。结论不采用 Agent 自报的加速数字。 -6. **汇总效果与消耗。** 将正确性、性能、实际执行状态、时间和可获取用量关联，生成可追溯记录。 - -Profiling 是可选诊断，不是通用必做步骤，不因没有 profile 默认判失败，也不要求另行说明或申请跳过。只有具体任务在执行前明确要求诊断产物时，才检查该项。已经完成的 NPU 工具资格试点仍保留原冻结的 profiling 检查和证据。 - -评测层不识别 indexer、知识库开关等组件语义。调用方负责切换条件、控制变量、重复执行和解释差异；本项保存配置身份、实际产物与统一结果。普通用户优化一次任务不需要额外运行对照 Agent。 - -## 外部项目怎样独立验收 - -以 vLLM 项目的一次优化为例，调用方先明确目标、模型、输入、硬件和工作负载，优先复用项目已有测试与 benchmark，必要时增加专项检查。改动语义、模型质量和目标性能按任务要求分别检查；算子级结果不能代替模型或服务结果。验收规则、容差和输入不随候选一起降低。 - -本项提供产物交接、执行、身份和结果关联，不提供适用于所有项目的万能 oracle，也不承诺把任意自然语言目标自动转为可靠验收方案。首轮外部目标应有明确、可复现的质量检查和 benchmark，再按实际选择完善适配。 - -## 首轮完成条件 - -- [ ] 一种真实 Agent 从受支持入口完成内置 `rmsnorm-npu` 任务，实际代码被留存并由独立工具重验；不得回放预写 reference 充当 Agent 产物。 -- [ ] 一个选定的外部项目 NPU 目标复用自己的测试与 benchmark，进入同一结果与消耗记录链路；项目、模型与 workload 在执行前确定。 -- [ ] NPU 产物执行边界、任务/候选版本绑定、单次与整轮预算、中断清理有相应检查；缺环境与失败保留原因。 -- [ ] 缺失产物、错误结果、超时或不完整测量不能被记录成优化成功；profiling 未调用不是默认失败原因。 -- [ ] 实际重复执行并保留全部记录，验证效果与代价可对照；缺失 usage/费用保持未知，不伪造精确总成本。 -- [ ] 文档说明已有入口如何调用、输出如何解释以及如何接外部任务；实现和验收证据随同后续 Draft PR 交付。 - -重复次数、模型、预算与性能门槛属于具体实验计划，执行前固定，不在本页添加仓库默认值。一个任务没有取得改善可以是完整的负结果；工具可靠性与 RSI 收益是不同结论。 - -## 与其他工作和 PR 的关系 - -- #336 持续提供可复用任务；原 #334/#335 的研究和接口随具体任务或本接入工作完成。 -- [#338](https://github.com/InferMatrixLab/InferRSI/pull/338) 是基础交付，保持 Draft；后续计划一个 Draft PR 完成本页的实际接入与验收，尚未创建。更多题目按批次另交 PR。 -- [#248](https://github.com/InferMatrixLab/InferRSI/issues/248) 负责整体 Agent 收益实验；本项给它或其他调用方提供可用的 NPU 任务和记录，不接管其全部实验或重启旧运行。 -- #249 / Optimization Search Policy 的并行搜索、候选选择与停止策略不作为本项交付。也不将所有 provider、所有 Agent、自动知识晋升或 `skill_eval_improve` 支持列为首轮前置条件。 - -## 现有证据与基础缺口 - -[已执行 NPU 工具试点](npu-validation.md)与[800 个样本及身份](npu-validation-summary.json)保留正确/错误控制、真实 profile、A/A 和参考 AB/BA；候选是已有官方融合算子，未调用真实 Agent 生成实现。正式 CPU 产物接入见 [formal-integration.md](formal-integration.md)，不能外推 NPU 支持。 - -[Canonical loader finding 已由原 reviewer 清除](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)。[NPU action/total-timeout finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决，归 #338 修复，不因 Issue 合并而消失。GitHub 的整份 review 状态和项目验收仍分别记录。 diff --git a/docs/plans/issue-252/README.md b/docs/plans/issue-252/README.md deleted file mode 100644 index 5fd4079eb..000000000 --- a/docs/plans/issue-252/README.md +++ /dev/null @@ -1,54 +0,0 @@ -# #252：共享优化任务与现有执行、评测流程的接入 - -**目标是让真实 Agent 使用已有 RSI 流程完成优化，交付实际代码，由事先确定的测试和 benchmark 验收，并将优化效果与整次执行的消耗关联起来。** 任务可以来自题库，也可以来自外部项目；首轮设备验收使用 NPU。 - -当前已实现的是 Draft PR #338 的共享任务与评测基础，以及预写参考候选的 Ascend RMSNorm 试点。真实 Agent 产物到 NPU 验收、外部任务贯通与整轮消耗汇总仍未完成。不能把本页的目标能力当成已交付能力。 - -## 两项工作、首轮两个 PR - -用户明确委托将原四项工作收敛为以下两项。#252 保留总索引，不形成第三条实施工作流。 - -| 工作 Issue | 交付与边界 | 说明 | -| --- | --- | --- | -| [#337：接入现有执行与评测流程](https://github.com/InferMatrixLab/InferRSI/issues/337) | 题库/外部任务 → 真实 Agent → 实际产物 → 独立验收 → 效果与执行消耗记录 | [已有能力、待补工作与验收](04-integration.md) | -| [#336：持续建设优化任务库](https://github.com/InferMatrixLab/InferRSI/issues/336) | 按实际需求研究、实现和验证任务；首题为 `rmsnorm-npu`，后续逐批交付 | [任务建设与首题](03-tasks.md) | - -#334 的研究与来源选择、#335 的共享接口分别并入这两项，不再独立跟踪。关闭旧 Issue 表示范围迁移，不能解释为任务已全部实现或验收。原[调研](01-research.md)和[接口说明](02-contract.md)继续作为支持文档。 - -| 首轮 PR | 范围 | 当前状态 | -| --- | --- | --- | -| [#338](https://github.com/InferMatrixLab/InferRSI/pull/338) | 共享题包、独立检查/测量、可选诊断、NPU RMSNorm 首题；保留 CPU 软件控制与既有产物适配 | 已有 Draft，尚未合并，仍有超时 review 待解决 | -| 后续一个 Draft PR | 在现有入口接真实 Agent，完成产物交接、NPU 执行边界、外部任务适配、消耗关联与实际验收 | 计划项，尚未创建；验收随实现交付 | - -后续加题按任务或紧密相关的一批任务提交 PR，归 #336 跟踪，不预设总数量。 - -## 完成后新增什么能力 - -仓库已经有 Workbench、SkillEval、Agent 日志与部分用量记录。本次补齐的是这些能力在真实工程优化任务上的接入。 - -| 当前缺口 | 完成后的目标能力 | -| --- | --- | -| NPU 工具试点运行的是预写参考候选 | 接收真实 Agent 产生的代码，独立检查与测量 | -| 外部目标已有通用命令入口，但产物和评测记录需手工拼接 | 有明确测试和 benchmark 的外部项目任务也能进入同一条链路 | -| Agent 消耗与 NPU 代码性能尚未完整关联 | 每次任务记录实际产物、正确性、性能、整体耗时及可获取的 token/调用/设备消耗；未知值保持未知 | -| 现有对照评测尚未覆盖新增 NPU 任务 | 调用方控制变量并重复执行，利用同一格式的记录分析效果与代价 | - -评测层不需要理解调用方切换的是 indexer、Agent 还是知识库；它保存配置身份和真实结果。整体实验的设计、重复安排与归因沿 [#248](https://github.com/InferMatrixLab/InferRSI/issues/248) 或调用方的职责进行，不在这里另建组件实验平台。并行搜索与优化策略仍有独立范围。 - -## 流程与独立验收 - -目标链路是：**提供任务和运行条件 → Agent 自主优化 → 留存实际产物 → 工具独立验收 → 效果与消耗报告。** Agent 在允许的工具和预算内自行选择方法；profiling 保留为可选工具，不是必做步骤，也不因没有 profile 默认判失败。 - -独立验收来自具体任务的测试与 benchmark，不是万能裁判。题库任务预先提供这些内容；外部项目优先复用项目自身测试，必要时补充专项检查，并在优化前固定输入、质量规则、测量条件和允许修改范围。比如 vLLM 优化，要按实际目标检查改动语义、模型质量及固定负载性能；单个算子变快不能代替模型或服务验收。当前工具不会自动从任意一句目标生成可靠裁判。 - -## 已有证据及尚未解决的问题 - -已在 Ascend 910B1 执行 [RMSNorm 工具资格试点](npu-validation.md)，[公开摘要](npu-validation-summary.json)保留 800 个计时样本、独立检查、错误控制、A/A 和参考比较。该试点使用已有融合算子作为参考候选，证明的是限定任务下的工具行为，不是 Agent 自主发现、技能收益或服务收益。 - -已执行试点的 profiling、冻结规则和原始 verdict 保持原样。当前 [NPU 单次/整体超时 review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决：不能把脚本记录的门槛等同于所有配对 trial 和整体预算均有执行保障。历史样本不因文档整理改标为新运行。 - -[Canonical task loader 的具体 finding 已获原 reviewer 清除](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)，这不等于整个 PR 批准或 #252 验收。正式 SkillEval 当前仍只支持 CPU NumPy 产物，新 NPU worker 运行已审核源码；NPU 产物执行边界仍需在 #337 接入中解决。 - -已有 CPU 任务和失败记录保留原身份：[v1 集成](validation.md)、[v1 用途](usefulness.md)、[v2 协议](replay-validation.md)、[正式 CPU 接入](formal-integration.md)。[五类来源研究](task-expansion/README.md)供后续选题使用，不等于已实现五类设备任务。 - -用户已授权本次整理和关闭被合并的旧 Issue；这不代替维护者对剩余交付的项目验收。#336、#337 与父 #252 保持打开，#338 保持 Draft。原生父子关系此前受写权限限制，继续通过引用和父索引跟踪。 diff --git a/docs/plans/issue-252/formal-integration.md b/docs/plans/issue-252/formal-integration.md deleted file mode 100644 index d369442ff..000000000 --- a/docs/plans/issue-252/formal-integration.md +++ /dev/null @@ -1,180 +0,0 @@ -# 将共享任务接入正式 SkillEval 两臂比较 - -这一步把共享 CPU 任务接入现有 `tools.skill_eval_compare`：两边使用各自的旧/新 skill，交付真实 `implementation.py`，由共同的独立 verifier 检查，再进入现有配对报告。当前保留的内置 CPU 任务是 `softmax-cpu`；`rmsnorm-cpu` 已移除，下面的历史验证记录仍保留当时的两题结果。调用者不再需要手工拼接底层 `run_trial`。 - -**本次新增接入只判定产物正确性。** 它复用共享题目、变化输入与标量 oracle，并增加隔离执行的产物交接。性能仍由明确的 Workbench 实验回答；不能把 sandbox 启动时间或子进程声称的 latency 当作优化分数。 - -## 接入前后，分别能做什么 - -| 入口 | 原有能力 | 本次新增能力 | -| --- | --- | --- | -| `benchmarks compare` | 对已审核可信实现执行 Workbench 比较 | 仍承担已审核源码的正确性与测量，不改它的计时协议 | -| `benchmarks.skill_verifier` | 单个 controller 指定的已审核文件经过底层 verifier | 历史离线接口保留，不把它直接开放给任意模型源码 | -| `tools.skill_eval_compare` | audit、diagnostic，以及固定 parser 工程任务 | 新增 `engineering-benchmark-v1`，可冻结共享任务、两臂 skill、模型/预算、schema、starter 和 grader，并接收两边实际文件 | -| `tools.skill_eval_local` | parser 的 stage/capture 本地交接 | 共享 benchmark 也能导出每臂 workspace、留存实现和作者日志，再独立检查 | -| `tools.skill_eval_improve` | 离线 audit 的选择、重跑和 holdout 控制 | 本次仍不支持共享 benchmark；公开 development 题不能冒充独立 holdout | - -两臂改变的是提供给 Agent 的 skill。题目、正确性标准、候选入口和输入序列保持相同。固定作者脚本可以检查接线行为，但不是 skill 收益实验。 - -starter 本身是正确实现，因此原样保留也能通过正确性检查。这层入口能检查优化过程有没有破坏语义、拒绝错误产物，并保留两边真实实现；它不能区分“正确但未优化”和“正确且更快”。性能选择必须另有经过资格验证的测量，不能把正确性 PASS 当作优化成功。 - -## 规范题包来源与设备边界 - -[同名工作目录替代问题](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4462417808)暴露了“冻结内容”和“选对内容”的区别：旧版正式路径按名字调用可信 Workbench 的路径优先 loader，可能把工作目录中放宽容差的题包当作内置题，再一致地冻结和重验错误规则。 - -正式路径现在显式调用 `load_builtin_task`，只从已安装的 `inferrsi` 包资源加载题包。descriptor、starter、verifier、prepare 的题包复制及依赖收集使用同一入口；留存副本则与规范 descriptor 的哈希比较。可信 Workbench 的 `load_task(path)` 保留，不能用于替代正式内置题的来源。 - -回归覆盖同名 `softmax-cpu/` 将 `atol` 改成 1 并替换 starter 的情况，检查规范 task hash / starter、prepare 的依赖和冻结副本，以及父进程拒绝全零数组。模拟返回数组的测试不声称执行了候选；单独的真实 Linux sandbox 控制负责实际源码执行，不支持的本地平台明确跳过。 - -此正式产物 adapter 仍只支持 `numpy-axis1-v2` 的 CPU / NumPy / float32 契约。新增 NPU 内置题不自动进入该 NumPy sandbox；不支持的 adapter 或环境被拒绝。真实 NPU 的可信 Workbench 运行和后续 NPU 产物隔离需要各自的资格，不能借本页的 CPU 隔离结果声明已经覆盖。 - -本次 loader 修复改变 grader 身份；历史软件控制与计时样本保留原被测版本，不能重新标为修复后的执行。 - -本次设计沿用[既有调研](01-research.md)中的任务、候选、独立评测分工，并结合现有 RSI 接口作了以下选择： - -| 接入方式 | 当前决定与理由 | -| --- | --- | -| 将模型生成源码直接交给可信 CPU worker | 不采用；该 worker 的前提是已审核可信源码，不能充当正式生成产物的隔离边界 | -| 新建一套两臂控制器 | 不采用；现有 compare 已有 skill 快照、预算、日志、配对和留存复核 | -| 把整个 grader 与候选一同放进子进程并直接接受 verdict | 不采用；应由受信父进程持有答案并计算断言，候选不能提交自己的通过结论 | -| 沿现有 artifact 交接扩展 adapter，隔离执行候选，父进程检查数组 | 采用；共享数值规则并复用既有控制器，同时明确保护宿主与判定正确性的职责 | - -先交付正确性接入，是因为当前计时仍未完成可靠性能排序资格。引入隔离子进程不会自动解决测量噪声；性能比较需要独立定义和验证计时边界。 - -## 一次比较怎样运行 - -```mermaid -flowchart TD - T[同一共享题目及独立 oracle] --> F[prepare 冻结计划与全部输入] - S[旧 skill / 新 skill 的 Git 快照] --> F - F --> B[baseline workspace：旧 skill] - F --> C[candidate workspace：新 skill] - B --> BI[Agent 或已授权本地执行者交付 implementation.py] - C --> CI[Agent 或已授权本地执行者交付 implementation.py] - BI --> V[独立产物 verifier] - CI --> V - V --> R[配对报告：正确性差异、错误、版本、预算与证据] -``` - -| 环节 | 实际输入和操作 | 保留什么 | -| --- | --- | --- | -| prepare | canonical task descriptor、旧/新 skill revision、模型与共同预算；冻结题包、schema、starter、grader/controller | `plan/freeze.json`、`plan/tasks.json`、各臂 bundle、每题 starter 和题包副本 | -| stage / 调用客户端 | 每次独立 workspace，只给同一任务、starter 和该臂 skill；候选只能改 `implementation.py` | 冻结输入哈希与执行日志；默认不启动不受限 Agent | -| capture | 在 producer 停止后有界读取真实文件，拒绝危险路径/特殊节点，保留源码字节与 SHA-256 | `implementation.py`、原始答案、扩展后的留存答案、作者日志或 controller trace | -| verifier | 受信父进程生成输入与 oracle；Linux bubblewrap 子进程只执行候选并返回数组；父进程独立比较 | 164 个正确性断言及输入/任务身份；没有性能分数 | -| report | 检查留存文件与 receipt，使用已冻结、受信的 verifier 复核实际产物 | 两臂结果、错误原因、版本、资源记录和相对证据链接；不再次调用 Agent | - -当前内置题每次完整检查为 6 个基础输入 + 35 个变化输入，runner 按该顺序调用 41 次并由父进程检查 164 个断言。该顺序复用 `numpy-axis1-v2` 的规则。输入对象在单个子进程中保留，runner 只加载一次 `compute`，避免每次起进程而掩盖普通状态错误。 - -## 产物执行的边界 - -正式 adapter 为 `inferrsi.harness.skill_eval_benchmark`，候选文件固定为 `implementation.py`，入口仍是 `compute(x, *, eps)`。本地 producer 的答案仅为： - -```json -{"artifact": "implementation.py", "summary": "描述实际改动"} -``` - -controller 从文件读取源码，再添加留存用的 `artifact_source`；Agent 不能用一句“测试通过”替代实际产物。 - -受信父进程持有题包、输入生成器、独立 oracle 和断言。候选只在既有 Linux bubblewrap namespace/read-only mount 机制内运行，挂载所需系统 Python、匹配版本的 NumPy、runner 和实现；不挂载仓库、另一臂、答案或 grader。输出规模、进程资源和时间有界，父进程只解释数组数据，不执行候选提供的校验代码。 - -留存的 `returned_calls` 表示父进程收到并检查的格式合法回复，不是独立设备 trace，也不能证明任意候选都按约定执行了 `compute`。OS 隔离保护宿主资源；公开题集及同一子进程里的 Python runner 不提供隐藏测试、恶意协议伪造防护或独立的输入修改证明。结果的适用范围必须保留这些限制。 - -这仍不是整台机器上所有 Agent 活动的隔离保证。**产物 replay 的隔离与产物生产者的权限是两个边界。** 注入客户端需由调用者负责实际执行权限与预算；手工提交的作者日志无法认证模型身份、访问权限或全部尝试。没有合格本地客户端时，正式入口拒绝启动默认 danger-full-access Agent。 - -Windows、缺少 bubblewrap、不能创建 namespace、系统 NumPy 不满足运行条件时，都保留 `artifact_sandbox_unavailable` / INCONCLUSIVE，不回退到宿主执行。源码仍可留存，之后在支持的 reviewer 环境中复核；原始 unavailable 记录不能改写成当时已通过。 - -## 如何从现有入口开始 - -以下是手工两臂交接示例。占位符须替换为实际已冻结的 skill 路径、Git commit、模型设置和预算；它不是默认实验门槛。无需编造一份自定义题目，直接生成当前 canonical descriptor： - -```bash -python -c "from inferrsi.harness.skill_eval_benchmark import task_descriptor; from inferrsi.infra.artifacts import write_json; from pathlib import Path; write_json(Path('cases.json'), [task_descriptor('softmax-cpu')])" -``` - -配置示意： - -```json -{ - "adapter": "engineering-benchmark-v1", - "mode": "update", - "skill": "", - "skill_root": "", - "cases": "cases.json", - "snapshots": { - "baseline": {"git_revision": ""}, - "candidate": {"git_revision": ""} - }, - "model": {"name": "", "effort": ""}, - "limits": { - "repeats": 1, - "max_task_calls": 2, - "timeout_seconds": 60, - "campaign_seconds": 180, - "token_stop_new_trials": 10000 - } -} -``` - -冻结、导出 baseline workspace： - -```bash -python -m tools.skill_eval_compare prepare --config config.json --repo --output -python -m tools.skill_eval_compare execute --output --authorize --stage-local --task-id softmax-cpu --condition baseline --repeat 1 --current-head -``` - -Agent 在该 workspace 内读取任务和分配的 skill，只修改 `implementation.py`。记录实际 stdout/stderr 和作者 `execution.json`，字段与边界沿用[本地交接文档](../../local-skill-evaluation.md)。完成后 capture： - -```bash -python -m tools.skill_eval_compare execute --output --authorize --capture-local --submission --task-id softmax-cpu --condition baseline --repeat 1 --current-head -``` - -对 candidate 使用新的 workspace 和独立日志，把 `--condition` 改为 `candidate`。两臂完成后： - -```bash -python -m tools.skill_eval_compare report --output -``` - -`--authorize` 绑定 prepare 输出的计划哈希；配置、任务或代码变动后必须重新 prepare。它不是重新授权远端模型费用的替代机制。本次没有新增 provider、generation-only 或 `skill_eval_improve` 支持。 - -## 怎样判断结果有实际用途 - -| 观察 | 正确解释 | -| --- | --- | -| 正确实现 PASS，错误实现 FAIL | 共用裁判检查了实际源码行为 | -| baseline FAIL、candidate PASS | 产物正确性出现差异；固定脚本对照仅证明流程作用 | -| 手工两臂报告 artifact_improved | 作者提交的产物检查改善；身份、环境和成本仍是作者报告，不能直接证明 skill 因果收益 | -| sandbox unavailable、预算不完整、输入漂移 | INCONCLUSIVE 或拒绝执行；不能计作干净胜负 | -| 两边均 PASS | 在当前公开输入上都正确；并不表示候选更快 | - -接入验证分两层：跨平台 controller 测试使用真实作者脚本进程写候选，模拟 verifier 响应，验证冻结、交接、报告、预算和错误传播；Linux 软件对照执行真实 sandbox 与独立 oracle，验证正确/错误实现确实产生不同结论。前者不能代替后者。 - -复现测试入口： - -```bash -python -m pytest tests/test_skill_eval_benchmark_compare.py tests/test_skill_eval_benchmark.py -``` - -严格 Linux 软件对照设置 `INFERRSI_REQUIRE_BENCHMARK_SANDBOX=1`：缺少隔离环境时测试失败，不能用 skip 形成绿色资格结论。实际执行结果由本轮验证记录/CI 链接另行记录；本文的命令和测试设计本身不代表执行已经通过。 - -## 已执行的软件验证 - -在 `dfcc18557fd065dbe499f5cbaf77ad20cf6c5870`，现有 ready/review CI 内新增的 [benchmark-artifact-controls](https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558/job/126297873523) 已实际通过：**49 tests、0 failures、0 errors、0 skipped**。这是 Ubuntu 22.04 系统 Python + 匹配 NumPy + bubblewrap 的软件对照，严格环境变量已开启，未调用模型。下载的原始 JUnit 与[可公开汇总](formal-validation-summary.json)记录所有测试名称、原件 SHA-256 和来源。 - -job 日志确认实际 checkout 为 PR 合并快照 `e74eb3d0a693f88599805faa8bc1afaf05544dbc`，与上述 head 比较为零文件差异；隔离运行时预检输出 NumPy 2.2.6。不会把 CI 的合并快照误称为直接 checkout 了 head。 - -其中 5 个 case 明确执行真实 sandbox：Softmax 与 RMSNorm 各自的正确/错误实现；同进程状态缓存重放拒绝；宿主 sentinel 与 oracle 文件不可见；正式 prepare/execute/report 的两臂文件交接。其余测试覆盖控制器、协议和拒绝路径，并不全部属于真实 sandbox 执行。 - -本地 Windows 新测试为 44 passed、5 个真实 Linux 控制 skipped。旧路径的两批回归分别为 66 passed/14 failed、133 passed/1 failed；全部 15 个失败报告在未修改基线复现，来自 symlink 权限、CRLF 字面量或 Windows 路径行为，该本地批次仍不绿。新 hosted 全量检查与上述专门软件控制分开记录。 - -同一 head 的 [ready/review 全流程](https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558) 已完成：Python 3.10 / 3.11 full-check、两套 installed-package 检查和 PR references 均通过；Draft 的既有 KV qualification 按原规则跳过。全量日志中的精确计数保存在上述汇总，真实 NumPy sandbox 的无跳过结果由专门 job 单独证明。之后仅文档/证据整理的提交不冒充这次原始执行版本。 - -首个 Linux CI 准备阶段曾因系统 pip 的 editable-build hook 失败，未进入 sandbox 测试。随后改为显式安装系统 NumPy 与测试依赖、直接从 checkout 运行，才得到上述成功记录；没有改变宿主 namespace 安全策略或加入 host fallback。该成功只验证当前软件环境中的接口、数值和隔离控制，不证明性能收益、真实技能收益或全硬件资格。 - -## 本轮交付结束后，仍然分开验收的内容 - -共享任务已经有正式两臂产物入口的设计与实现位置；是否在某个具体平台完成真实 sandbox 软件对照，以对应执行记录为准。性能排序、真实旧/新 skill 的 Agent 实验、生产者隔离资格、成本效果、改进控制器及目标硬件仍需各自证据。 - -Profiling 仍由主 Agent 根据实际瓶颈调用：对已审核实现可使用 `benchmarks profile`，真实设备需对应 profiler。当前 formal adapter 不会自动做 profile、选择下一次候选或晋升 skill。它提供“同题、两臂、真实产物、独立检查”的接入层，供这些现有流程使用。 - -在当前 RSI 中继续推进一次实现优化时，主 Agent 从报告的 `implementation.py` 证据链接取得候选，审核其源码与适用范围后，再用现有 Workbench 做配对测量，需要诊断时可单独调用 profile。Profiling 不作为通用完成条件。formal adapter 的通过结果不会自动把未经审核的代码送入可信 worker，也不会自动批准该实现。 diff --git a/docs/plans/issue-252/formal-validation-summary.json b/docs/plans/issue-252/formal-validation-summary.json deleted file mode 100644 index f3f48a8f3..000000000 --- a/docs/plans/issue-252/formal-validation-summary.json +++ /dev/null @@ -1,325 +0,0 @@ -{ - "scope": "Software controls for the formal shared-benchmark artifact adapter; no model or skill-efficacy experiment", - "tested_head": "dfcc18557fd065dbe499f5cbaf77ad20cf6c5870", - "adapter": "engineering-benchmark-v1", - "ci": { - "run_id": 41745549558, - "job_id": 126297873523, - "url": "https://github.com/InferMatrixLab/InferRSI/actions/runs/41745549558/job/126297873523", - "job": "benchmark-artifact-controls", - "conclusion": "success", - "runner": "ubuntu-22.04", - "strict_environment": { - "INFERRSI_REQUIRE_BENCHMARK_SANDBOX": "1" - }, - "artifact": "benchmark-artifact-controls-41745549558-1", - "original_junit_sha256": "ac2d216425f07adfde4d599d14b0e1203bd5bbe258e23640857a0d56deb339d0", - "tests": 49, - "failures": 0, - "errors": 0, - "skipped": 0, - "checkout_merge_sha": "e74eb3d0a693f88599805faa8bc1afaf05544dbc", - "merge_tree_matches_tested_head": true, - "numpy_version": "2.2.6", - "artifact_zip_sha256": "37ced936f75faa9caad0b95051de4a1a5b2bca30ff01b2d7e09541298f46518c" - }, - "transformation": "Read-only extraction of suite counts and all testcase identities/outcomes from the original CI JUnit artifact. Hostname, timestamp, absolute paths and test durations omitted. Original XML stays in the linked CI artifact and local validation archive.", - "real_sandbox_cases": [ - "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", - "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", - "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", - "test_real_sandbox_cannot_read_host_sentinel_or_oracle", - "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable" - ], - "cases": [ - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_freezes_builtin_task_and_shared_grader[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_freezes_builtin_task_and_shared_grader[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[hash]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[holdout]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[path]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[request]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[extra]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[contract]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change0]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change1]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change2]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change3]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_no_sandbox_means_explicit_unavailable_never_host_execution", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_missing_numpy_is_explicit_unavailable", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[dtype]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[values]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[mutation]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[truncated]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[malformed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_workspace_freeze_excludes_only_implementation", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_runtime_version_mismatch_is_unavailable", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_worker_startup_is_distinct_from_artifact_failure[False]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_worker_startup_is_distinct_from_artifact_failure[True]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_cannot_read_host_sentinel_or_oracle", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_real_authored_processes_use_frozen_task_and_retain_both_sources", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[task]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[schema]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[starter]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_task_grader_and_oracle_dependencies_are_frozen_before_producer", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[changed-input-inputs_changed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[extra-file-inputs_changed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[missing-artifact-missing_or_invalid_artifact]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[oversized-missing_or_invalid_artifact]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[invalid-answer-invalid_artifact_answer]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_no_unsafe_default_launch_or_silent_sandbox_fallback", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_retained_source_tamper_invalidates_report_without_rerunning_producer", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_unknown_usage_stops_new_trials_without_inventing_cost", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change0]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change1]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change2]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change3]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable", - "outcome": "passed" - } - ], - "local_windows": { - "new_tests_passed": 44, - "real_sandbox_tests_skipped": 5, - "legacy_batches": [ - { - "passed": 66, - "failure_reports": 14, - "baseline_reproduced": 14 - }, - { - "passed": 133, - "failure_reports": 1, - "baseline_reproduced": 1 - } - ], - "legacy_baseline": "663f3b327998526a13aa31e72ad729025c18983c", - "legacy_batches_green": false - }, - "limits": [ - "Correctness-only; starter is already correct and retaining it can pass.", - "Authored producers are software controls, not real Agent or old/new skill efficacy evidence.", - "No timing, hardware or service-performance qualification.", - "Artifact sandbox protects the host boundary; public inputs and returned-call records are not tamper-proof invocation evidence.", - "Provider generation and skill_eval_improve are outside this integration." - ], - "hosted_full_checks": [ - { - "python": "3.10", - "job_id": 126297873658, - "conclusion": "success", - "summary": "3631 passed, 18 skipped, 1 warning, 284 subtests passed in 308.28s (0:05:08)" - }, - { - "python": "3.11", - "job_id": 126297873629, - "conclusion": "success", - "summary": "3631 passed, 18 skipped, 1 warning, 284 subtests passed in 281.39s (0:04:41)" - } - ], - "hosted_installed_package_checks": { - "3.10": "success", - "3.11": "success" - }, - "ci_run_conclusion": "success" -} diff --git a/docs/plans/issue-252/npu-pilot.md b/docs/plans/issue-252/npu-pilot.md deleted file mode 100644 index f980c8eb4..000000000 --- a/docs/plans/issue-252/npu-pilot.md +++ /dev/null @@ -1,88 +0,0 @@ -# NPU 试点：用 RMSNorm 跑通一次真实优化比较 - -2026-11-04，用户将试点明确指定为 NPU。**本轮已在 Ascend 910B1 完成实际运行，七项软件资格检查通过**，见[实测解释与保留决定](npu-validation.md)和[全部样本及身份](npu-validation-summary.json)。本页保留任务接口与执行方法。已有 CPU 测试和样本仅作历史软件证据；项目接受、NPU 产物隔离与真实 Agent 收益另行验收。 - -以下图示、步骤和验收矩阵描述本轮已执行的参考控制与工具资格试点，其 profiling 检查继续按原冻结计划保留。通用 Agent 优化流程中，profiling 是按需使用的工具，不是必做步骤，也不因缺少 profile 默认判失败；见[当前流程说明](04-integration.md#流程也是交付内容)。 - -## 这次究竟要跑通什么 - -给 Agent 一个具体问题：**在 Ascend NPU 上保持 affine RMSNorm 计算结果，检查基础算子组合能否由更合适的实现替代。** Agent 交付真实源码；InferRSI 负责独立检查、诊断和测量，再用证据决定是否保留。首个参考候选采用 `torch_npu.npu_rms_norm`，是否更快由实际实验判断。 - -```mermaid -flowchart LR - A["固定题目、环境和验收"] --> B["运行 NPU 基线并 profiling"] - B --> C["记录改法依据并交付候选"] - C --> D["独立正确性和错误控制"] - D --> E["同码对照与 NPU 同步计时"] - E --> F["Workbench 报告与保留决定"] -``` - -这些步骤由主 Agent 调用已有接口完成；本次没有新增跨轮次搜索控制器，也没有通过图示保证任何 Agent 都自动遵守顺序。留存的诊断、代码与决定才是演练证据。 - -## 为什么选择 RMSNorm - -RMSNorm 是推理中明确的归约与逐元素计算。基础运算组合和 NPU 融合算子提供两条可实际比较的实现路径，且遗漏 epsilon、忽略权重、错误 dtype 或旧结果重放都有可构造的反例。相比先接入整个服务，它能在一个受限问题上验证设备执行、数值裁判、同步计时和 profiler 是否真正接通。 - -已有 [KernelBench 语义与评测研究](01-research.md)提供任务/基线/裁判分离的来源；[AgentKernelArena task contract](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md)提供任务责任划分。它们的 GPU/CPU 结果不能证明 Ascend 资格；本轮任务采用独立的 NPU affine 语义，torch_npu API、运行时兼容性和设备行为必须以源码核对与真实执行补证。 - -| NPU 一手来源 | 采用与验证边界 | -| --- | --- | -| [op-plugin RmsNormKernelOpApi.cpp](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) 与 [对应测试](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py) | 固定 `9f23f00`，核对 affine 权重、epsilon 和输出接口；[BSD-3-Clause 许可](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/LICENSE)。实际安装版本的支持范围仍需本轮执行验证 | -| [Ascend PyTorch profiler tests](https://github.com/Ascend/pytorch/blob/9065e614f02639f5ddde0cf370bb9d38de7bd8a9/test/profiler/test_npu_profiler.py) | 固定 `9065e61`，核对 NPU profiler 使用方式；trace 成功和实际设备活动须检查最终产物，不能只凭 API 存在 | - -本次使用原生算子 API 和自编任务适配，不搬入上游整套执行器。源码参考、许可、版本与实测分开记录。 - -## 任务接口与冻结内容 - -| 项目 | 本轮约定 | -| --- | --- | -| 任务 | `rmsnorm-npu`；单个实现文件 `implementation.py` | -| 候选入口 | `compute(x, weight, *, eps)`，输入输出是目标 NPU 上的 tensor | -| 数值语义 | 二维 contiguous float32 `[rows, features]`；`x / sqrt(mean(x²) + eps) * weight`，非全一的一维 float32 权重 | -| baseline | 在 NPU 上执行的 PyTorch 基础运算组合 | -| 参考候选 | `torch_npu.npu_rms_norm(x, weight, epsilon=eps)[0]`；取规范化输出而非辅助结果 | -| oracle | CPU scalar double / `math.fsum` 归约后乘权重并转 float32；不复用设备归约或将 baseline 当唯一真值 | -| 环境 | 冻结实际设备型号、torch / torch_npu / CANN 版本、设备选择与源码身份；缺少 NPU 时明确失败，不回退 CPU | -| 输入范围 | 形状、分布、epsilon、容差与变化输入序列写入 manifest；仅宣称实际检查范围 | -| 修改权限 | 候选源码可改；输入、oracle、容差、计时规则和报告不可由候选改写 | -| 性能边界 | 输入/权重和 oracle 在计时外准备；使用声明的设备同步边界统计完成调用；预热/初始化与正式样本分开 | -| profiler | 单独运行并保存 CPU/NPU trace 或表格、配置与文件摘要，不把 profiler 开销混入性能分数 | - -共享接口保持 `check / profile / measure / plan / compare` 的分工；NPU adapter 负责自身语义、依赖、设备检查、同步与运行时身份。不能只把 CPU manifest 的 backend 改名就称为设备适配。 - -普通 `plan` 只是命令/RunSpec 预览。正式实验仍要在搜索前保存任务、基线、候选、oracle、环境与验收条件；重复次数、时间预算及判断门槛放在本次实验计划与 acceptance issue，不能写成仓库通用常量。 - -## 实际执行顺序 - -1. **确认环境。** 在已有 NPU 容器内核对个人身份、设备映射和共享调度器。所有 torch_npu 导入、设备诊断、测试与基准在有效设备预约内执行；不改共享安装，不切换正在被其他任务使用的源码。 -2. **固定本轮输入与实现。** 保留源码、task/grader、运行时及 manifest 身份；记录工作目录和实际导入位置，避免只改变目录却导入其他 checkout。 -3. **运行基线与独立 profiling。** 先确认数值正确；保留真实 NPU 活动，记录观察到的算子、调用或同步行为。没有 device trace 时不得声称已诊断设备瓶颈。 -4. **记录假设并运行候选。** 将观察与候选改动关联；如果融合候选来自预先选定的参考实现，明确是 authored control，不能声称 Agent 从 profile 自主发现它。 -5. **执行正确性和错误控制。** 检查合法候选、全零错误、遗漏 epsilon、忽略权重等语义错误；失败不得获得可用性能分数。实际被计时的输出也要核对。 -6. **执行 A/A 和候选比较。** 相同设备与固定条件下保留逐次样本；使用现有 Workbench 组织 AB/BA。诊断、H2D/D2H、oracle 和初始化开销单独记录。 -7. **记录决定。** 引用数值、trace 和原始样本，说明保留、放弃或证据不足。信号无法区分、运行不完整或环境漂移时，不得写成优化成功。 - -## 验收矩阵与实际证据 - -| 要回答的问题 | 必须执行的控制/记录 | 本轮状态 | -| --- | --- | --- | -| 是否确实在 NPU 运算？ | 输入/输出设备、实际运行时、设备预约、真实设备活动、退出码 | Ascend 910B1 已运行，退出 0；实际 kernel trace 已检查 | -| 是否算对？ | baseline 与参考候选对独立 oracle；声明范围内的边界输入、输入与权重保持 | 两者各 75 次调用、375 项断言通过 | -| 错误但快是否会被拒绝？ | 全零、遗漏 epsilon、忽略权重等候选；无可用性能指标 | 六类错误控制均拒绝；另有预检后重放控制 | -| 计时是否覆盖完成的设备计算？ | 同步边界、变化输入、预热与正式样本、实际输出校验 | 同步 eager 计时，保留全部 800 样本 | -| 同码是否被误判成进步？ | A/A 或 no-op 对照，保留波动与原始 verdict | 两个 shape 差异小于 0.4%；未达到收益门槛 | -| 融合候选是否值得保留？ | composed baseline 对 fused 候选的 AB/BA、分工作负载样本和结论 | 本轮两个 workload 达到预定门槛，保留参考实现 | -| profiling 是否发挥作用？ | 真实 profile、据此作出的具体判断及后续动作；预定控制与自主发现分开 | 核对每输入 6 个基础 kernel → 1 个 RmsNorm；非自主发现证明 | -| 结果是否接入 RSI？ | Workbench 的 RunSpec、实际实现身份、检查/比较报告与最终决定 | 实际 A/A 与参考 AB/BA、最终 validation 和身份重验已完成 | -| 正式 SkillEval 是否能隔离 NPU 产物？ | 设备访问、父子边界、可信 oracle、重验与攻击控制 | 本轮尚未建立，不能用 CPU sandbox 替代 | -| 是否提高了 Agent/skill 或服务能力？ | 受控 Agent 两臂或目标模型/服务实验 | 未测，不由本试点推导 | - -“有用”首先指可以识别错误、记录真实设备行为，并支持有根据的保留决定。算子完成延迟有改善，只能支持对应 NPU、dtype 和 workload 的结论；没有模型/服务验收就不声称端到端收益。 - -## 产物与边界 - -保留任务/源码/环境摘要、独立检查结果、全部计时样本、profile 文件及其摘要、Workbench 报告、日志和退出码。公开前移除私有主机、账号与个人路径，完整原始日志留在受控执行归档中；任何删减或转换说明来源和范围。 - -首轮运行已审核可信源码。现有正式 `engineering-benchmark-v1` 只支持 CPU 任务，Linux bubblewrap 控制没有验证 NPU 设备隔离；不得通过隐式宿主执行或 CPU fallback 绕过不支持状态。NPU 产物隔离需要后续单独验收。 - -CPU Softmax 继续服务软件回归；CPU RMSNorm 题包已移除，旧实验记录保留。Tokenizers、KV、通信与服务的研究保留在[扩展方案](task-expansion/README.md)，按各自任务资格推进。当前跟踪已收敛为 #336 任务库和 #337 接入两项，见[总览](README.md)；本页的试点记录不因 Issue 调整而改变，NPU 演练完成不自动关闭整个 #252。 diff --git a/docs/plans/issue-252/npu-validation-summary.json b/docs/plans/issue-252/npu-validation-summary.json deleted file mode 100644 index 1f1ea6e8c..000000000 --- a/docs/plans/issue-252/npu-validation-summary.json +++ /dev/null @@ -1,12575 +0,0 @@ -{ - "scope": "NPU software controls; no model/skill benefit or service claim", - "tested_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", - "acceptance_plan": "https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6621316099", - "software_qualification": "PASS", - "campaign_acceptance": "not established", - "checks": { - "correctness": true, - "device_profiles": true, - "reject_wrong_outputs": true, - "fixed_environment": true, - "frozen_identities": true, - "aa_qualified": true, - "reference_qualified": true - }, - "agent_benefit": "not measured", - "execution": { - "qualification_exit_code": 0, - "postcheck_replay_exit_code": 0, - "scheduled_devices": 1, - "user": "non-root personal UID with existing device-group membership", - "release": "scheduler completed and allocated device was available afterward" - }, - "source": { - "summary_sha256": "c2baefb9bb9624a9542e4c3ec03d42db09025511cbb8d526604a3f01b37e38e2", - "archive_sha256": "c39bb59536b59cb6a9c743d29290f7e0a4d9f16d157b26a3ad2ea9b849b35d91", - "full_raw_archive": "retained locally, not embedded; contains private paths" - }, - "transformation": "Read-only extraction. All 800 timing samples, original evaluations, frozen identities and profile kernel summaries retained. One identical planned input sequence/environment stored once; repeated assertions summarized by counts and every failed key. Every executed call retains id/index/phase mapping to that sequence. Raw result and trace SHA-256 identify local originals; raw traces/logs are not included and hashes do not substitute for those files.", - "frozen": { - "scope": "NPU software controls; no model/skill benefit or service claim", - "source_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", - "policy": { - "repetitions": 4, - "min_improvement_pct": 10.0, - "max_aa_difference_pct": 10.0, - "max_minutes": 8.0, - "timeout_seconds": 180.0 - }, - "implementations": { - "baseline": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "reference": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "zeros": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "1615e05c422cfc824b5d85b589d751289a14fd92268b96ba92895adc4887564a", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "missing-epsilon": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "025fc929dd888697240d78b8d58030c7a19df0e32b7bffdc86ed4297be55d73b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "missing-weight": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "a7c77754e4e305fc6c88b5ebaec06538489d024cead945ef67ad64c87beae7af", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "wrong-axis": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "f522d375639326991e4f4f8bdb8d2a6e761e51f2470c116162f359916837eebf", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "input-shape": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "9ed9d0539bb07913db3d117311e12b971eaf324746b600c3b9a9f6cee7157efd", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "replay": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "d2307fb34cac2e6e127c288578322679ecb04459c92ae30c8214a2831afded62", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - } - } - }, - "environment": { - "backend": "ascend", - "device": "Ascend910B1", - "logical_device": 0, - "python": "3.12.13", - "torch": "2.10.0+cpu", - "torch_npu": "2.10.0.post4", - "numpy": "2.5.3", - "cann": "9.1.0", - "os": "Linux", - "machine": "aarch64", - "allocation": "caller-owned; one visible device" - }, - "input_sequence": [ - { - "id": "decode", - "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "input_sha256": "67e5405e42f45a9022c5d100d708f939a21a1db991f7d88a09cd2badfbc73faf", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "bbcce025be7b34405e19e3e2c51ae1a9ee54556c8a2c50f8310709b5645b19e4", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "1b9d2b1b11300da022b73370314a92f7338fb4f519e106b2c7b440b7fff30992", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "5c47fe8288727110b526fb241677814b361c013e48feb52ebe7d062773ef6e81", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "adf128f85e21b7a1b635450bcb9bb2f478e813a352325f15f98811d678002f03", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "94e4f4a69f42e13540f6d0d85492ce1a7c12f66e8d71a0f863e8b957590691c6", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "02e8ebb8c6143b3d1121af61856ea948e4eae5cabe8dd2f37ecf3e96c7d72c73", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "ec0adadf85c19c1c0c70947b33ce97c4fa4e045d2db1d8e7ed91d3fb6754a7a3", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "c06ae630eb37f04c3469f46b288eb6718f2c11253edf73fdd2f1796d0254ca4b", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "35579c2e878a529f9858865aacc2b6c29a18fe2b14adf3a99fe613b2bee48316", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "f02bc6ff8c5b21941045b08b339451b0155fbb02298454097235356807b7d7b5", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "31cefd7002d981c70ff2a9f029843fba0a844863022103fb664383d33ec89136", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "e14628f15ec7e8a972ed4e537affe9113682f6ec7eb2c9857a62b42f50f8f06f", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "cc92864419bb9e875efb43f8532e6aa7f6813caa7c294be27ca15612b2ae2689", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "77325d78aad1312351252a5ba9449831ae7c2c0a69b707ed139fa2290b6a9edf", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "0979c82f4f16d80bc3c4dccb42048fdf0a1ca091406c4686f2157ad497dd9338", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "32ce39afe09617927c5690adfbf682f2c3b7bdbf2a1305df2c85466673be74b9", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "a97206ff3175de40cb7f18fc72f13704925019ef94a39f2981f76df097780ea2", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "bd55b85cf2b3feebb78c06fdfdcc02edfc005211414f6a9d88772fb5716498a2", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "6365fe4e41f62c29341869a54d536730e55ed2ff071580ba3ce9bda2730584ab", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "input_sha256": "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "7bf20a4d149fffd1227663f4ea610b29150eee58eeac96a6535d3ea49c0bd7da", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "fd2e53466459343f302f3ca60ccc5d725af8d8bdd1a328a9e69e9e4b6da6a84a", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "6f84d8ecfafe81a7e6fa6b1adff784e695e255a8f9c8cdd25dded5b0c566cf27", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "ef24320331639ec55deb6318ebad0ae20d33d9d9d78ca55a04f72b944ba6576e", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "260b7bf862444f39605bb1d8b866e3fab3f80d50ae4d200cca17dfbc4d52cac3", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "734e4b02cb2ce77cf33d5541ac91a70ba4a3d337110575c06fd8c841710e01f7", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "852852f78bf3e5dd994dd0a0ee6c7d9d8884d9a179b12a110c500e8ebf7bb29e", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "1db290780ae85fd2f7e956a75c2e1112cf3ffadeca8765d512072dbad7092606", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "fbc2e40841970fc9f0032765a8846c1f561ecf81d7968ff224eb2877d339e2be", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "d87af900bb9847a6d803262c1424f46ceed2a4e9114e3f3f3b474bfb79f5912e", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "6aa37f0d4d395fc7b1a6e1ac363655c7522e016f9af9ed6eea2ece4861546a9d", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "8b5d6e7fda0a0cd7ef01fa541c9106001ef33003986da4702fb8d930ec0ee1fb", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "30ae9e7b8c82f09b8090370c9b26635e0c9938df8afdc25ea1cc91f9e49acc7a", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "716c54f925401dcae62b378adfc416ad6ec2bcfd5baa1450550564f13c160d82", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "052d8abdfa0f612878b5942053f335668867391a8d5a1d526f949d6015701bde", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "f54464abd4d7e88456f7af0f383a8f87a272ff14f056a5871ff7287b0897b7e4", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "cdb0a669d90aab982f348db84aa3352a9712a92e8e671a35115e492c8e546bc0", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "5f5195ff2db02fa76d087bda9a4088ae84b05513e380979dabab6e1a0debf175", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "d3970bb0c428d50034061239e065b20729b0a3ab4aa448c3b3042493eafad9ed", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "ba5097c249135a071fe4d4951d7282a55f3a08fad85ccb445140993b26446d08", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "ece71d95686d8b6e860d1db955705edad6f7f82fa994f7509ebf5a9d0e9468f1", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "a522dfb1ff7a38d77b3616e757e76a745075d3bbec3a7cdfc6d55d8ce3371ac0", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "69a8f334773a595947f42776377f2e602c43dcd55108d649b8bd6f1135744141", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "b72bfd4faaa5decc5ec4a1933801ad95134cbf97afd777e15b7e51e80baef7aa", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "input_sha256": "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "e19ebeab7be1a93cf416bb8cbff53efcb13e9b6b466ed283b96efeb7511fdc9f", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "308e41ee69b40074c40bd3b2eee74773baba275bc1b47284f0c200d1fff310b4", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "d024b54d7dac98b718d67fb80a4cafcbba7df2e001eeb45e9811c1d9f8eb74db", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "input_sha256": "9ed37ae2d2d07b85395e3d8eae98af70654c2fc76c9213cfba7bf213639b560c", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "5e14f8a1d2c798eebc415fa78392b013ff15549dbea9a6c98ffbd38b37129418", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "6c877742ace1d75b6858f58f5627ffd6b0b51e1c8420e16eed426ac7ce1d7723", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "1568551c28137ddbc29e25b2340e63b7d579c13ad1887c82ac3a6523d90fa88b", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "8dcf06509296931f3f807d51c8f4e35a527da84af9d0b48e7367aae90aff4229", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "f9c76c905fc246b5535009c9bb16301721abbde6ac0045a9eb7b84f17a85dbef", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "c5595da004e11308e95cf7ce9ac80368ed0bcc5b7ea433f3ae22a836ef29b4dd", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "4dd078d9682e3baedaac103f0a49f720d7333ed35a2c0dfa989a9d8bc1e0114b", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "e3bb7889247dc9ea493a0da78053a5da4b34ea7a4504728d7e5af50d3552c0e4", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "2851e139d0494ab60090270665e522d47a12b59e9964df9c2088dcb834d2cae3", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "b393f65f71eaad27bf958332fd70fdaa5c2271a22124bcf7544836a8155eadad", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "e0914bd356fbbbdfa8d132cab980fdadf443c8f710ef1e338c878008644f7cf1", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "3664ee501755106b5bc0e9a904c65ad1ff609226ba55245d336d8ac5b8a83d33", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "336ddcb67c73f5c1434edcf5bf89e6e21e63613b6ba2e956e7cdb15e2911e416", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "829455e1b93dad13a73ff070e426bb0e2069c0e67770a135c3c06eadc1fa5c96", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "54074b89e4b968ce56413d46520140563de8516c0e7fa6ab62c34bcd5fe93b47", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "56401b4db5df9b3f6677d071a2d17e9b75c583a0ab68544a58c9e2c698a8006d", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "e6ea20b07896b49ca3a21e1b285c020f9ab2cb0d9923213a777c72a1bf8a924f", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "7d037f31a44526b4be24c48f34ca2666770808c50b58c5a27e053b72a7f01460", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "c27e5de79f782c022a34d2127e08d07fb4b65ca25879170524d44f51b6f5ca80", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "input_sha256": "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "b2ed3add5da5efe3af5628096d5ff8b1fbe33f28e6a8c985e72ff5510ad8e500", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "6dfdb3ae75e8821c2a5b3f6526755f40b593a77d732855e01dfd7ee6a8f9d346", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "9ae2936b3322d8bf03ce38f0ee4fa91df43abc6e923b5156155f85a8ce4eae44", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "cc1f2a67b8e4664c723cc3eb0fed4d3e01c073bbab7a9ffd1f3d4bfeaaf0ba16", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "4c22aceffc31159f51e1ccd41fed6d9b8e752d2dc00812ed9ff99edbd4a8b058", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "0446209c7a1cf2cf58d516ae074e13c4112769460581d5d4422dbc78adffcb7a", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "2ca61fd77bdaf215122a3c01b02734ce0d874d66a8cdeb35201818386dceaf89", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "2865af940cde618eed38bbd4732ebbbc9d7f7917677178203ae8045ac51ea2d3", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "2c1d1b19e0d5d3168369723c30348ce95aa77a16be0e2236d8580181386175bc", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "8f21a4e0eab656713ddfad4bd876bf2325062a5ffc8da57ef29c64a67430697f", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "4444006264a044c02eef87b9bcf7edd648c4ea6f2cfb8add3f260b2327d853c6", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "12c29f46c17552f1d848c345d62c6895e4938f95d4c2b3ffc8ac0e6d203122da", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "882b2bd2626047f2bfa81bd8378e1ff762f4883540cad696087623d4f8c0bb4e", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "61f8c98043d2576ed42ceafb1c173e652c2f957eb385dc62a3e8919ecd433e3c", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "f3fb6d9580316932f297013ba11d18584dbde77092015d22956f6c5615d7d50a", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "3253836502182f819a81c83e8a205fe515d791b69a0266109c2010e69935f2c5", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "f4a28650d4372458fa5f4ca2cdcdb38cd90b0a9e75d38a098e3e608e8e023b29", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "e5a8309d80d6f7d699f9ff3c74c258fbb06f4e777b955c56603319c069c8b3fc", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "62b0a4928daf34cb9c2d9d5cf2f3666e91e9a4b28848b9b15f5dc0c59279e96b", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "df29a68e0b10e1987dec3fdfa32d0bb89f97838ae67b1330df0cbc548cada712", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "247a9bfbe32f0fa31b8fcf46c713e8bed2a6d7aabe22dbae268270ad4a64cd8e", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "4dbc040b0adce5d633006cf20f1d6f09f3c56a641d8d8696896ae7d4315b784c", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "7c5d608b6615654e860046f3df610938c7575db80def275a56c138a1f797068e", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "ab674012075f97a469bf85f53e85c80bee1ae4fafa62823d1544fe38ff1b8a6e", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "input_sha256": "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf", - "shape": [ - 64, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "de454aebaf27a9b49554e330f720ed86a51fed964a201aec50a4e998c4cd7259", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "zeros", - "input_sha256": "c35020473aed1b4642cd726cad727b63fff2824ad68cedd7ffb73c7cbd890479", - "shape": [ - 2, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "7e7c7ad1a45dd9288204324e4f98ea57aca3cbce4b06ed2c7700f09fcba6a40f", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "input_sha256": "a8e8f83cff88a89a0c7019c1246f7c087896fd77c6f5c44f22a2c3bd03f04312", - "shape": [ - 4, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "0629c729a9360829ccac972a3021cea3f534ced7b0d2d81e9f6894d394093bdc", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "input_sha256": "73a7d73ac83fe1d628b5c39d1469bc2393d3715f9702ace8e6d8e69187c7e215", - "shape": [ - 4, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "22366ccbb58dd678cff742c25370137369c5fc27a6f345ac80e9a34688977e45", - "call_index": 0, - "phase": "precheck" - } - ], - "shape_comparisons": { - "aa": { - "decode": { - "baseline": 0.280772, - "candidate": 0.2797225, - "improvement_pct": 0.3737908338438456 - }, - "prefill": { - "baseline": 0.311018, - "candidate": 0.310098, - "improvement_pct": 0.29580281527115115 - } - }, - "reference": { - "decode": { - "baseline": 0.278377, - "candidate": 0.18149500000000002, - "improvement_pct": 34.80244416744198 - }, - "prefill": { - "baseline": 0.31132800000000005, - "candidate": 0.216051, - "improvement_pct": 30.603415047795266 - } - } - }, - "timed_samples": 800, - "actions": { - "baseline-check": { - "implementation": "baseline", - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "check", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "4cd62c65c5c13a7c65729bd77473e6563ceaf3a89d51c218d7ffa208efe578f9", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ] - } - }, - "baseline-profile": { - "implementation": "baseline", - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "profile", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "67fbbee82b21ab6c364c0dc68d61db1c9fd88851fddd414ed681438853e515ea", - "assertions": { - "total": 135, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - } - ], - "diagnostic": { - "kind": "ascend-torch-profiler", - "artifact": "npu-trace.json", - "sha256": "e4dfa493b9066a8bcb4194d77928de9780e5f946dce3cbbadd3f4402f2607950", - "performance_verdict": null, - "kernels": [ - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 1.28 - }, - { - "name": "aclnnMean_ReduceMeanAiCore_ReduceMean", - "duration_us": 2.9 - }, - { - "name": "aclnnAdds_AddAiCore_Add", - "duration_us": 1.12 - }, - { - "name": "aclnnRsqrt_RsqrtAiCore_Rsqrt", - "duration_us": 0.94 - }, - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 1.4 - }, - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 1.38 - }, - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 3.1 - }, - { - "name": "aclnnMean_ReduceMeanAiCore_ReduceMean", - "duration_us": 8.5 - }, - { - "name": "aclnnAdds_AddAiCore_Add", - "duration_us": 1.1 - }, - { - "name": "aclnnRsqrt_RsqrtAiCore_Rsqrt", - "duration_us": 0.88 - }, - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 5.26 - }, - { - "name": "aclnnMul_MulAiCore_Mul", - "duration_us": 5.66 - } - ], - "limitation": "Instrumented diagnostic; includes verification copies; not a score" - } - } - }, - "reference-check": { - "implementation": "reference", - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "check", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "1455ed6031f651524506adb1e834d4d069c48c6742c0f8a1dd27641dec35ea8f", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ] - } - }, - "reference-profile": { - "implementation": "reference", - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "profile", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "4926cb61be9031760239f7b269cef7b519f21f719e600a522cf2e11d01bc4ebd", - "assertions": { - "total": 135, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - } - ], - "diagnostic": { - "kind": "ascend-torch-profiler", - "artifact": "npu-trace.json", - "sha256": "3a3303354707d65cc2dccf438cdac1c0a9b39920bfcbad64f7b8cd714cd870be", - "performance_verdict": null, - "kernels": [ - { - "name": "RmsNorm", - "duration_us": 2.02 - }, - { - "name": "RmsNorm", - "duration_us": 6.44 - } - ], - "limitation": "Instrumented diagnostic; includes verification copies; not a score" - } - } - }, - "zeros": { - "implementation": "zeros", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "1615e05c422cfc824b5d85b589d751289a14fd92268b96ba92895adc4887564a", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "2387d984bd57af12b739482198340c79ec9578a0bb98b550d2c6f8c74b1853f7", - "assertions": { - "total": 5, - "failed": [ - "decode.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "decode", - "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - }, - "missing-epsilon": { - "implementation": "missing-epsilon", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "025fc929dd888697240d78b8d58030c7a19df0e32b7bffdc86ed4297be55d73b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "5c2e2a13a0e81a2420fec1b769693712b73cc7b7e50291d83a84c3cf5bfa1ce3", - "assertions": { - "total": 15, - "failed": [ - "zeros.call-0.finite", - "zeros.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "zeros", - "input_sha256": "c35020473aed1b4642cd726cad727b63fff2824ad68cedd7ffb73c7cbd890479", - "shape": [ - 2, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "7e7c7ad1a45dd9288204324e4f98ea57aca3cbce4b06ed2c7700f09fcba6a40f", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - }, - "missing-weight": { - "implementation": "missing-weight", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "a7c77754e4e305fc6c88b5ebaec06538489d024cead945ef67ad64c87beae7af", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "079737c082a1b4a768755c81f636dd43fbc2478a2c1db1fe2da494675e371a77", - "assertions": { - "total": 5, - "failed": [ - "decode.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "decode", - "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - }, - "wrong-axis": { - "implementation": "wrong-axis", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "f522d375639326991e4f4f8bdb8d2a6e761e51f2470c116162f359916837eebf", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "c02bc2a197e9ad4c287f2f7e9a645f6bdc1efade6ec03d601486f6eb165b5efe", - "assertions": { - "total": 5, - "failed": [ - "decode.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "decode", - "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - }, - "input-shape": { - "implementation": "input-shape", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "9ed9d0539bb07913db3d117311e12b971eaf324746b600c3b9a9f6cee7157efd", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "b260a08297f3fd9a0cfe9806cc9653c8e7b9275586941d399e632b01f05413e9", - "assertions": { - "total": 5, - "failed": [ - "decode.call-0.shape_dtype_device", - "decode.call-0.finite", - "decode.call-0.input_preserved", - "decode.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "decode", - "input_sha256": "d6665d6edaebf785b04f9ebe2e7b58c5dd20711db027364be50ae68502f5d273", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "724406de5202d27a210615975784c4f87974babb5068d3503c05547441ed905b", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - }, - "replay": { - "implementation": "replay", - "execution_status": "failed", - "returncode": 1, - "result": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "d2307fb34cac2e6e127c288578322679ecb04459c92ae30c8214a2831afded62", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "cf91847bdedc0b7377a609e9e474b92a8e0f829081abd2a813b716df311c08ac", - "assertions": { - "total": 25, - "failed": [ - "tiny.call-0.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - } - ], - "failed_call": { - "id": "tiny", - "input_sha256": "73a7d73ac83fe1d628b5c39d1469bc2393d3715f9702ace8e6d8e69187c7e215", - "shape": [ - 4, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "22366ccbb58dd678cff742c25370137369c5fc27a6f345ac80e9a34688977e45", - "call_index": 0, - "phase": "precheck", - "usable_measurement": false - } - } - } - }, - "comparisons": { - "aa": { - "run": "results/aa/20261104T030249Z-4f3c7e153a", - "evaluation": { - "verdict": "FAIL", - "reasons": [ - "measured gain 0.493466% must be positive and meet 10% threshold" - ], - "metric_key": "metrics.suite_latency_ms", - "metric_unit": "ms", - "comparison_scope": "rmsnorm-npu:device-synchronized-eager-call", - "direction": "minimize", - "min_improvement_pct": 10.0, - "baseline": { - "values": [ - 0.562494, - 0.596805, - 0.594275, - 0.591216 - ], - "median": 0.5927454999999999, - "min": 0.562494, - "max": 0.596805, - "range_pct_of_median": 5.78848763929882, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.5784050000000001, - 0.6017159999999999, - 0.586865, - 0.592776 - ], - "median": 0.5898205, - "min": 0.5784050000000001, - "max": 0.6017159999999999, - "range_pct_of_median": 3.952219361653225, - "excluded_trials": 0 - }, - "improvement_pct": 0.4934664202427443, - "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", - "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", - "source_integrity": { - "verified": true, - "reason": "Git source snapshots unchanged", - "command_checks": 9 - }, - "statistical_significance": "not established" - }, - "validation": { - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "check", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "4cd62c65c5c13a7c65729bd77473e6563ceaf3a89d51c218d7ffa208efe578f9", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ] - } - }, - "trials": { - "baseline": [ - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "e6c4f930c7013e4fdfadde5df57f90cc79a3ec5a1359b50cd0bc898d559b479a", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.270847, - 0.266096, - 0.270057, - 0.271407, - 0.271567, - 0.264917, - 0.268437, - 0.265727, - 0.264737, - 0.270917, - 0.263607, - 0.269307, - 0.265517, - 0.268677, - 0.276737, - 0.263507, - 0.280237, - 0.276237, - 0.278567, - 0.274337, - 0.266127, - 0.277277, - 0.273037, - 0.288907, - 0.270107 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.270107 - }, - { - "id": "prefill", - "samples_ms": [ - 0.284087, - 0.292387, - 0.308548, - 0.306467, - 0.311088, - 0.281727, - 0.299718, - 0.323208, - 0.291927, - 0.310288, - 0.283168, - 0.290687, - 0.311578, - 0.359399, - 0.307678, - 0.287588, - 0.290848, - 0.281768, - 0.283218, - 0.361129, - 0.285997, - 0.283188, - 0.292227, - 0.303428, - 0.302648 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.292387 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.562494 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "daa8ed909c7317ec7945e8782ca553fb5ec64a81b9d4cb3486f2b6148e28e0a6", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.281927, - 0.279458, - 0.276987, - 0.284948, - 0.278717, - 0.279757, - 0.276777, - 0.278908, - 0.279247, - 0.281868, - 0.275567, - 0.280217, - 0.276387, - 0.305428, - 0.292888, - 0.279097, - 0.271337, - 0.292507, - 0.289458, - 0.292607, - 0.276958, - 0.290837, - 0.292118, - 0.293278, - 0.289917 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.280217 - }, - { - "id": "prefill", - "samples_ms": [ - 0.299068, - 0.325109, - 0.306588, - 0.307958, - 0.315788, - 0.295298, - 0.321088, - 0.337188, - 0.363559, - 0.348829, - 0.308698, - 0.318308, - 0.325238, - 0.318338, - 0.317199, - 0.304188, - 0.304588, - 0.329579, - 0.307348, - 0.340608, - 0.299978, - 0.311048, - 0.317938, - 0.307848, - 0.316588 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.316588 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.596805 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "e6ecc746fcee67b464c4d56ffd40e07f998aac1022e813f1b484f4c93196411a", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.282347, - 0.280247, - 0.276077, - 0.275798, - 0.277887, - 0.279207, - 0.284948, - 0.292547, - 0.275967, - 0.281327, - 0.280788, - 0.284397, - 0.294678, - 0.280827, - 0.275487, - 0.295578, - 0.296628, - 0.295377, - 0.293608, - 0.273857, - 0.293438, - 0.294307, - 0.278057, - 0.295888, - 0.268257 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.281327 - }, - { - "id": "prefill", - "samples_ms": [ - 0.348389, - 0.310048, - 0.37866, - 0.310458, - 0.302858, - 0.313628, - 0.316648, - 0.311838, - 0.310128, - 0.342249, - 0.319818, - 0.305708, - 0.317698, - 0.312948, - 0.308288, - 0.345349, - 0.306908, - 0.336739, - 0.298448, - 0.291778, - 0.329348, - 0.326248, - 0.308348, - 0.293687, - 0.340649 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.312948 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.594275 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "44552c11f81550c1c1150499927f0409a71fd1033b999bc6ed9f4809702072aa", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.279918, - 0.277137, - 0.276218, - 0.283837, - 0.278237, - 0.285907, - 0.279707, - 0.277067, - 0.301048, - 0.278927, - 0.282128, - 0.286027, - 0.277818, - 0.295238, - 0.282007, - 0.291728, - 0.276038, - 0.286067, - 0.289208, - 0.292457, - 0.290978, - 0.270327, - 0.289688, - 0.286017, - 0.281787 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.282128 - }, - { - "id": "prefill", - "samples_ms": [ - 0.300278, - 0.290167, - 0.324408, - 0.305528, - 0.302278, - 0.309088, - 0.332289, - 0.322138, - 0.316509, - 0.307378, - 0.289108, - 0.312208, - 0.327008, - 0.308968, - 0.304528, - 0.286548, - 0.310698, - 0.301538, - 0.313818, - 0.313968, - 0.293087, - 0.318148, - 0.310828, - 0.345019, - 0.300628 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.309088 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.591216 - } - } - ], - "candidate": [ - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "43d40b486033db5eb73393ce1c96073cbbdd94701c4d6f0ace5738c672095ab7", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.268957, - 0.274997, - 0.269927, - 0.270507, - 0.274437, - 0.269557, - 0.270717, - 0.274387, - 0.276287, - 0.274607, - 0.270547, - 0.285797, - 0.270807, - 0.275297, - 0.275727, - 0.265337, - 0.296247, - 0.288358, - 0.284787, - 0.280047, - 0.266397, - 0.288228, - 0.281347, - 0.270727, - 0.275027 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.274607 - }, - { - "id": "prefill", - "samples_ms": [ - 0.283128, - 0.328019, - 0.316228, - 0.310008, - 0.303258, - 0.368039, - 0.311428, - 0.299017, - 0.305118, - 0.299138, - 0.282677, - 0.310678, - 0.292498, - 0.296418, - 0.293128, - 0.286247, - 0.313318, - 0.303798, - 0.304588, - 0.276517, - 0.309998, - 0.313139, - 0.314248, - 0.299268, - 0.283177 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.303798 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.5784050000000001 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "c69faca2121e76d949e80543a81da9807a2a67515270985f5ea08db136b12f80", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.289408, - 0.291968, - 0.287238, - 0.287267, - 0.281397, - 0.280077, - 0.282908, - 0.293968, - 0.281697, - 0.284557, - 0.283707, - 0.287858, - 0.282117, - 0.278748, - 0.302808, - 0.279927, - 0.296038, - 0.297638, - 0.291787, - 0.294068, - 0.279767, - 0.287098, - 0.289248, - 0.288517, - 0.283037 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.287238 - }, - { - "id": "prefill", - "samples_ms": [ - 0.311598, - 0.318548, - 0.314478, - 0.319678, - 0.308078, - 0.340629, - 0.316158, - 0.316878, - 0.327178, - 0.298378, - 0.332238, - 0.39341, - 0.329178, - 0.298788, - 0.289857, - 0.331209, - 0.305818, - 0.318578, - 0.301828, - 0.307138, - 0.310748, - 0.353699, - 0.313078, - 0.306628, - 0.285477 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.314478 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.6017159999999999 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "707d18426bc85667b32bc24cb95094f4370f9a1508af27fdedefa0544a3bcf0b", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.276737, - 0.275457, - 0.277717, - 0.273337, - 0.269667, - 0.276157, - 0.279028, - 0.279767, - 0.274217, - 0.275427, - 0.274507, - 0.279858, - 0.272408, - 0.270557, - 0.274467, - 0.294408, - 0.289007, - 0.288408, - 0.275858, - 0.281447, - 0.276027, - 0.283317, - 0.276807, - 0.268507, - 0.281448 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.276157 - }, - { - "id": "prefill", - "samples_ms": [ - 0.310398, - 0.318938, - 0.308028, - 0.296927, - 0.310728, - 0.324799, - 0.298108, - 0.312448, - 0.293347, - 0.316298, - 0.328679, - 0.310708, - 0.328508, - 0.305748, - 0.304708, - 0.35932, - 0.293017, - 0.327178, - 0.287658, - 0.334258, - 0.310108, - 0.315598, - 0.313748, - 0.290037, - 0.308377 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.310708 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.586865 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "2f2735d06ad4d7e6e091dbc618a6bec04adc35d3611ed4fc577e298c7041e350", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.277157, - 0.286788, - 0.286737, - 0.279377, - 0.283347, - 0.279617, - 0.280617, - 0.276967, - 0.279507, - 0.284868, - 0.280347, - 0.288298, - 0.280207, - 0.277538, - 0.287137, - 0.277607, - 0.289497, - 0.287818, - 0.292537, - 0.278067, - 0.312748, - 0.292008, - 0.283288, - 0.288028, - 0.278667 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.283288 - }, - { - "id": "prefill", - "samples_ms": [ - 0.305378, - 0.318388, - 0.310698, - 0.306658, - 0.301188, - 0.329069, - 0.322308, - 0.324909, - 0.304208, - 0.295508, - 0.311948, - 0.309128, - 0.311028, - 0.35008, - 0.297178, - 0.309488, - 0.326808, - 0.309538, - 0.352869, - 0.296298, - 0.300757, - 0.310768, - 0.299508, - 0.299807, - 0.290638 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.309488 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.592776 - } - } - ] - } - }, - "reference": { - "run": "results/reference/20261104T030558Z-3dea96ead1", - "evaluation": { - "verdict": "PASS", - "reasons": [], - "metric_key": "metrics.suite_latency_ms", - "metric_unit": "ms", - "comparison_scope": "rmsnorm-npu:device-synchronized-eager-call", - "direction": "minimize", - "min_improvement_pct": 10.0, - "baseline": { - "values": [ - 0.576635, - 0.600085, - 0.576815, - 0.601486 - ], - "median": 0.5884499999999999, - "min": 0.576635, - "max": 0.601486, - "range_pct_of_median": 4.223128558076295, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.38543000000000005, - 0.391621, - 0.418691, - 0.403471 - ], - "median": 0.397546, - "min": 0.38543000000000005, - "max": 0.418691, - "range_pct_of_median": 8.366578961931431, - "excluded_trials": 0 - }, - "improvement_pct": 32.44183872886396, - "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", - "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", - "source_integrity": { - "verified": true, - "reason": "Git source snapshots unchanged", - "command_checks": 9 - }, - "statistical_significance": "not established" - }, - "validation": { - "execution_status": "completed", - "returncode": 0, - "result": { - "status": "completed", - "action": "check", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "1455ed6031f651524506adb1e834d4d069c48c6742c0f8a1dd27641dec35ea8f", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ] - } - }, - "trials": { - "baseline": [ - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "0d08d29a995bcf420bb0d6aef000c66b9974ba9cf4a4b608ce68cdaa46fe332b", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.267757, - 0.265677, - 0.256887, - 0.268697, - 0.268847, - 0.257436, - 0.267587, - 0.268607, - 0.262027, - 0.268957, - 0.279077, - 0.271568, - 0.263547, - 0.274217, - 0.273097, - 0.258777, - 0.265856, - 0.273887, - 0.277548, - 0.264257, - 0.262926, - 0.278037, - 0.266266, - 0.278607, - 0.270247 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.268607 - }, - { - "id": "prefill", - "samples_ms": [ - 0.299667, - 0.346879, - 0.313618, - 0.309108, - 0.303218, - 0.292688, - 0.312638, - 0.313888, - 0.309178, - 0.308868, - 0.290317, - 0.308028, - 0.306678, - 0.305498, - 0.396641, - 0.290727, - 0.324258, - 0.299758, - 0.327229, - 0.296688, - 0.287847, - 0.310688, - 0.300957, - 0.324979, - 0.296638 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.308028 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.576635 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "f693b899f85c79dc16d47fb4904ed471e4ffb565b0c50188d0072e9051e39830", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.284207, - 0.286007, - 0.282617, - 0.282627, - 0.282278, - 0.280157, - 0.278847, - 0.278977, - 0.283977, - 0.283128, - 0.281907, - 0.294708, - 0.282538, - 0.282777, - 0.289068, - 0.296658, - 0.296707, - 0.284458, - 0.296647, - 0.294347, - 0.293258, - 0.278618, - 0.295507, - 0.293528, - 0.288477 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.284207 - }, - { - "id": "prefill", - "samples_ms": [ - 0.296308, - 0.310748, - 0.315878, - 0.311748, - 0.318928, - 0.303837, - 0.321958, - 0.318509, - 0.345599, - 0.314638, - 0.318058, - 0.322178, - 0.323429, - 0.316218, - 0.303997, - 0.294498, - 0.315908, - 0.309688, - 0.327929, - 0.308798, - 0.308478, - 0.319508, - 0.325189, - 0.308988, - 0.298388 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.315878 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.600085 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "a72aa38bca6d047312aeb27cd5a4cdb4b1aa1942580475059b39bd3c728f3f8d", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.285008, - 0.272907, - 0.272557, - 0.269076, - 0.272347, - 0.289628, - 0.270177, - 0.271747, - 0.270787, - 0.266397, - 0.286838, - 0.272547, - 0.268597, - 0.268837, - 0.265257, - 0.302148, - 0.280647, - 0.276927, - 0.274677, - 0.262726, - 0.295267, - 0.278137, - 0.269807, - 0.275467, - 0.263237 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.272547 - }, - { - "id": "prefill", - "samples_ms": [ - 0.310008, - 0.303008, - 0.322589, - 0.304388, - 0.300998, - 0.304848, - 0.323119, - 0.293837, - 0.346969, - 0.295828, - 0.311328, - 0.290568, - 0.304638, - 0.294747, - 0.294827, - 0.334908, - 0.305898, - 0.321639, - 0.295658, - 0.282977, - 0.312888, - 0.301587, - 0.304268, - 0.292718, - 0.301867 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.304268 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.576815 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "cbd0bbda5e5dc5b9cb9e8b697a418687f54d356f5d9285a06c415f2d1f99531b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "2535a0b44525b0dfa72952f9d16d9a779fee5f30337ad2f758f8a502e07346cb", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.305708, - 0.283368, - 0.283817, - 0.287737, - 0.284228, - 0.288518, - 0.279467, - 0.287807, - 0.280578, - 0.282327, - 0.285267, - 0.287397, - 0.283177, - 0.284328, - 0.283157, - 0.286858, - 0.285507, - 0.299728, - 0.294537, - 0.320048, - 0.293948, - 0.284608, - 0.294488, - 0.300218, - 0.297227 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.286858 - }, - { - "id": "prefill", - "samples_ms": [ - 0.304908, - 0.309648, - 0.323928, - 0.314628, - 0.318128, - 0.313288, - 0.38072, - 0.320378, - 0.314718, - 0.322139, - 0.317569, - 0.291758, - 0.325328, - 0.326618, - 0.312038, - 0.319998, - 0.312948, - 0.315088, - 0.300598, - 0.304678, - 0.297107, - 0.294867, - 0.313938, - 0.308348, - 0.317828 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.314628 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.601486 - } - } - ], - "candidate": [ - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "f10e62ed2cea8643f3117c0eb973ead40196e5c67b753c5bbdc8d4fdc2a3955e", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.181515, - 0.178065, - 0.179005, - 0.182205, - 0.176414, - 0.175604, - 0.181525, - 0.174984, - 0.176575, - 0.179895, - 0.182755, - 0.177965, - 0.175754, - 0.181194, - 0.176764, - 0.176855, - 0.179594, - 0.175994, - 0.175415, - 0.178925, - 0.174664, - 0.176605, - 0.177355, - 0.177714, - 0.176105 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.177355 - }, - { - "id": "prefill", - "samples_ms": [ - 0.221756, - 0.217806, - 0.232246, - 0.226976, - 0.211076, - 0.216186, - 0.230906, - 0.205366, - 0.206545, - 0.208616, - 0.204075, - 0.207695, - 0.204405, - 0.204425, - 0.205185, - 0.216805, - 0.208755, - 0.206635, - 0.203855, - 0.208075, - 0.199936, - 0.204866, - 0.210575, - 0.210916, - 0.204786 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.208075 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.38543000000000005 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "eca6c01839b942f61805159230039382e439563d64cf837c5c27ccd7872e36dd", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.179844, - 0.182995, - 0.179665, - 0.176995, - 0.179974, - 0.179614, - 0.183784, - 0.175794, - 0.176565, - 0.192985, - 0.182935, - 0.182015, - 0.178125, - 0.176615, - 0.183165, - 0.176175, - 0.176875, - 0.176795, - 0.175054, - 0.174945, - 0.197355, - 0.181685, - 0.174924, - 0.178985, - 0.173115 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.178985 - }, - { - "id": "prefill", - "samples_ms": [ - 0.217126, - 0.217286, - 0.219296, - 0.225376, - 0.212636, - 0.211676, - 0.212565, - 0.210786, - 0.212675, - 0.225246, - 0.224156, - 0.217836, - 0.216165, - 0.209086, - 0.210585, - 0.213206, - 0.214775, - 0.209486, - 0.208496, - 0.208235, - 0.211406, - 0.213915, - 0.211525, - 0.211666, - 0.208085 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.212636 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.391621 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "f5410d252698f682a3f329db06602aeebd84a9bce2f80801dddd4e5761b224c4", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.190515, - 0.179955, - 0.192655, - 0.183875, - 0.183085, - 0.183675, - 0.183365, - 0.184325, - 0.184784, - 0.187925, - 0.186285, - 0.182764, - 0.183875, - 0.187485, - 0.185304, - 0.184644, - 0.188105, - 0.180804, - 0.184775, - 0.181704, - 0.194745, - 0.183125, - 0.179784, - 0.188125, - 0.182044 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.184325 - }, - { - "id": "prefill", - "samples_ms": [ - 0.233746, - 0.235126, - 0.239556, - 0.235576, - 0.231426, - 0.234366, - 0.229576, - 0.235086, - 0.229916, - 0.232946, - 0.233096, - 0.238186, - 0.236866, - 0.235816, - 0.239157, - 0.235706, - 0.231616, - 0.237136, - 0.238596, - 0.243846, - 0.232456, - 0.232976, - 0.231946, - 0.233166, - 0.231306 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.234366 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.418691 - } - }, - { - "status": "completed", - "action": "measure", - "passed": true, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "4c0611670fd8f347d0fbc4e9e995e0c7fe9a45578931ae1f713d85a807d0d4de", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "a71402152841bb12eb1d6d9d7bd4416b94fcf716835ca2fc13e26c9b4fd19936", - "assertions": { - "total": 375, - "failed": [] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "decode", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "decode", - "call_index": 35, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 1, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 2, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 3, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 4, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 5, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 6, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 7, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 8, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 9, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 10, - "phase": "warmup" - }, - { - "id": "prefill", - "call_index": 11, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 12, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 13, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 14, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 15, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 16, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 17, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 18, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 19, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 20, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 21, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 22, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 23, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 24, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 25, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 26, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 27, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 28, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 29, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 30, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 31, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 32, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 33, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 34, - "phase": "measurement" - }, - { - "id": "prefill", - "call_index": 35, - "phase": "measurement" - } - ], - "cases": [ - { - "id": "decode", - "samples_ms": [ - 0.187135, - 0.190325, - 0.182625, - 0.189474, - 0.192355, - 0.181384, - 0.183195, - 0.181024, - 0.189715, - 0.182675, - 0.181085, - 0.190195, - 0.181104, - 0.184575, - 0.181755, - 0.186085, - 0.185894, - 0.186874, - 0.178724, - 0.199585, - 0.186425, - 0.184005, - 0.183164, - 0.181894, - 0.180184 - ], - "sample_input_sha256": [ - "ac0759b68f293f2218d316a480b29d925658c19a8a3b3540e0f28af1a560f833", - "bcdeee67f0f1d9fbc81fa4a33bc87206484ba248258c53c70976d17c2aeb0f60", - "1a3ba43551c6e7f0a4dada746dce7f47aa28ee2987b80fc0023ee61a98b9e19b", - "f4915ad0ea3320849f0df04b797ef3a01ef7337dc29c0a8da4bdf4c19d73ea61", - "f7d4aa5385de70f5f40335aa0f7f1e0557b1e4e64c515ad335d28d6415435981", - "5a18e910f3ec48875d12667aabf04032769c6779112db411926c3e8f65305713", - "ca3305dd1d246d3b724c1f437dd81603c4f560932ebee77c45763bfef7add379", - "db78e9392f255ce4239106b3ba024d2ae3841f96f3c9bee1e3cec3448c908473", - "8d214b2f9ac39748db638cebd5838affe99edd9c81e0fcff53f7c015c9d0a58d", - "75f605a706e640ba731724c4f19c27f878dde7d354a28b2e6b00158968f90774", - "7c4c2d9223fd4b04dff66c4adc2c4d16252cf72f1de815aa7436f6f823bb5607", - "c6a04e43854fcc3efdb4fe1234e541f9dfcc39ac1c29abbc75ed5ef6f48d1a89", - "5e67f5baefe8951d451411f63966faef35747627b2046c2b33559f144d68f548", - "9a85e8a17f4c900f2948b886705ec19a83e307ffaa8d7d2b248bb627f3362350", - "63db27c0aeaa67836b5c42c4c58a21660cf08bf11460e972b3600d25a561017b", - "d29f23b0d9fe5c0aee1e0808dbc965aa52cd74b4c1cc4301149ad6da0b4872a1", - "e597dd7b49aa98b822df10f0fab44b90f1c0142e74b95955c76548b2f4a7d698", - "4e1a5bb895af2de2a2eee1ec3f085100d7e209e00fec35827f637c4de62ecd08", - "3c1c7dd12d19b3f07b4569a7bb75617ade9bb2816157fbe178b276b5b4d41c77", - "d569ffc4c2f361f79615ddb8dd9e323d9ed2dd862fe8cae298e425c5719d6758", - "483a12934d9daaa69765261c2e315c7282085dec643a7bdd7a02cd98dbb101ee", - "c9592cb9c161c22ace238d3e455eed57d4b92546b4e6b93154fa61a76fb792dd", - "c03f3e863ec6fbdb7b3f5bff80d2d3de08bfc7a5b6c76c7eb9fc569c097ee938", - "28022c53ec7e2436756a0f7e5e4121c7ac708d0054ab59f33346fb205ebb7181", - "2849b120effda0d5de9d7d7e838236da38ebfa0f336214ca31b5d1c9043382ce" - ], - "median_ms": 0.184005 - }, - { - "id": "prefill", - "samples_ms": [ - 0.220516, - 0.222816, - 0.226316, - 0.215386, - 0.221336, - 0.219466, - 0.218496, - 0.217926, - 0.220756, - 0.225896, - 0.212485, - 0.215786, - 0.220526, - 0.232436, - 0.212225, - 0.213636, - 0.223476, - 0.220956, - 0.248296, - 0.218096, - 0.214235, - 0.216275, - 0.218026, - 0.222556, - 0.213775 - ], - "sample_input_sha256": [ - "729beb7ed2a0cf3ebb4a9567815c8ac2ccc7b89c9b528d5f6dc73b7d74c250d2", - "a7977b24dfacc10620ae9472c37a76ac7b6f1258b413316deaac857758d4034a", - "e826506626c3d2fca070253ab2b9658b91d16a5003a3599303d4b79e004304ca", - "c310d0905163b40c55291f33d6a04b652df873724e6591e8a00cade2bd39f1ce", - "51a61b6f2aa62f260dae68459d4723275e560d151fc33e7d879b2f73d3d68594", - "99154174a006a0c4ae233ffcb7f078165fbe209759ec61c363c3c59d27546078", - "d0c9b16582d9bee903d0d7ca3396ca1e1745164d52f72fc338085105ec610a85", - "e28f7123e5f9730001ef725d09ef497c16c472b78699e5cbd5fd08c35537146e", - "dde6e74eb639f7d337e1fdd9bd4472e998f426a8b7e4970aaac36d96daad79c2", - "75fa6ec9512b5d755d6ac63c5fcc99d68dcf2ffc7aa5c77d1e4978ed2107cc65", - "caccc090e10382ad5234b675d697fb3ec5e241fc08bfe8a224d228f6e6863a52", - "327c5c1d91d34e0ab0f2d3039d14e6c14e98a9de4291aeb0fb6d6f92f8f10e5e", - "f2cf3325923f3a934ed7992d760c9963a9ae38bc6fe653d641e945b1bd35b6ac", - "1461381778779b24f5ad9935c739a6561328143b9b6327bdc0cb20c0740627ea", - "7bc1d3219951dcfedf657881f6503bed8b4f1bf552ea1b32d4e72e0fb0c71152", - "75cfea49bb8cd3582254bb4d814dcea737f5155fe82502c8d5217e5e7a47d8f2", - "0e8f9dbe600f525c36da50aea90014a7afd1a35af5582605988dfd9db5c6caaa", - "845c70077e4484d7fe5fb56679f85feb44436d4021870fba23e7838876ca8b96", - "d5a6a26fe8b3e2dc42fd5a63bec50b2f178e4328c8c3e3969d0a41dfc21ff992", - "2bfb6707094254bce9f1cd74cc25253c7d2ad9182e8257e455a6343734fbd68e", - "cacc8388529fd07d0527f57e60108e017b828e52c68e3ec8399748799f00e3ff", - "b81ce1aa912f624d6f15d28ecdd12edb58ca396d4bed4b6de84c873278d07763", - "750b46834feb1d118788b5bae96a5db81b44dcd84f9f1c4cff1eddbb597c7927", - "7f1d277acf7bd79f2ea44dcdbd26b65f71768f4155557afa93609ab66ff527c1", - "023a65951412d202b3b9809bc7019b696c29dcf738f9cf278a2cf76f81818dcf" - ], - "median_ms": 0.219466 - } - ], - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "device-synchronized-eager-call", - "case_ids": [ - "decode", - "prefill" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 25, - "unit": "ms", - "warmups": 10 - }, - "metrics": { - "suite_latency_ms": 0.403471 - } - } - ] - } - } - }, - "postcheck_replay": { - "status": "completed", - "action": "measure", - "passed": false, - "identity": { - "task_id": "rmsnorm-npu", - "task_sha256": "d01163f9e38d36355fe4d2cb8db44da01d2f6e0d3da650ec290a3f9e5a4b5945", - "grader_sha256": "04e638f5510d6bf875efed75469c374e9741b02d7d7cf60f63b791643506a25a", - "candidate_sha256": "6e8d6bc8603515a3a7f3ecf697775bc2fa530d894a0fdb9d5b332d4a4936601b", - "runtime_sha256": "b6b1da63eac7f4a710378b9e7f7cf3fb5e3206b7b0e900f6e390c4195c2c5195" - }, - "raw_result_sha256": "8040f42798852733230ed28ad8d945fbfc8421fd0139c75a98a095128b53afaf", - "assertions": { - "total": 30, - "failed": [ - "decode.call-1.values" - ] - }, - "executed_calls": [ - { - "id": "decode", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "prefill", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "zeros", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "large", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "tiny", - "call_index": 0, - "phase": "precheck" - }, - { - "id": "decode", - "call_index": 1, - "phase": "warmup" - } - ], - "failed_call": { - "id": "decode", - "input_sha256": "67e5405e42f45a9022c5d100d708f939a21a1db991f7d88a09cd2badfbc73faf", - "shape": [ - 1, - 4096 - ], - "dtype": "float32", - "strides": [ - 16384, - 4 - ], - "weight_sha256": "bbcce025be7b34405e19e3e2c51ae1a9ee54556c8a2c50f8310709b5645b19e4", - "call_index": 1, - "phase": "warmup", - "usable_measurement": false - }, - "candidate_source": "import torch\nn = 0\ncache = {}\ndef compute(x, weight, *, eps):\n global n\n n += 1\n key = tuple(x.shape)\n if n <= 5:\n cache[key] = x * torch.rsqrt((x*x).mean(-1,keepdim=True) + eps) * weight\n return cache[key]\n" - }, - "software_ci": { - "url": "https://github.com/InferMatrixLab/InferRSI/actions/runs/41764076901", - "tested_head": "6308268fc6d1fbf90e9ab97b5da8af79152f39ea", - "checkout_merge_sha": "6f1506bcb366c10836991701f8beaf7ac89e9a11", - "head_and_merge_tree": "5c22811e28f1c8c0182852916f43804c18b35edf", - "full_suites": { - "3.10": { - "passed": 3649, - "skipped": 20, - "subtests_passed": 284 - }, - "3.11": { - "passed": 3649, - "skipped": 20, - "subtests_passed": 284 - } - }, - "installed_packages": "passed on 3.10 and 3.11", - "cpu_artifact_controls": { - "passed": 54, - "skipped": 0, - "junit_sha256": "e724dce4b3f7ddfe753832bea4150b2ab6aa490d4e410847b34679cb8fe13dd2", - "cases": [ - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_freezes_builtin_task_and_shared_grader[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_freezes_builtin_task_and_shared_grader[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_cwd_shadow_cannot_replace_descriptor_starter_or_parent_oracle", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_formal_artifact_path_rejects_new_backend_packages[adapter]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_formal_artifact_path_rejects_new_backend_packages[environment]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[hash]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[holdout]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[path]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[request]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[extra]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_descriptor_drift_and_noncanonical_tasks_rejected[contract]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change0]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change1]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change2]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_invalid_artifact_does_not_start_source[change3]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_no_sandbox_means_explicit_unavailable_never_host_execution", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_missing_numpy_is_explicit_unavailable", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_oracle_uses_identical_shared_sequence_and_checks_every_call[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[dtype]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[values]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[mutation]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[truncated]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_parent_rejects_wrong_or_malformed_returned_arrays[malformed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_workspace_freeze_excludes_only_implementation", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_numpy_mounts_extend_existing_boundary_without_host_venv_or_repository", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_runtime_version_mismatch_is_unavailable", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_worker_startup_is_distinct_from_artifact_failure[False]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_worker_startup_is_distinct_from_artifact_failure[True]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_correct_and_wrong_authored_controls[softmax-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_correct_and_wrong_authored_controls[rmsnorm-cpu]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_rejects_zero_output_despite_cwd_shadow", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_preserves_sequence_state_and_rejects_cached_output_replay", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark", - "name": "test_real_sandbox_cannot_read_host_sentinel_or_oracle", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_real_authored_processes_use_frozen_task_and_retain_both_sources", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[task]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[schema]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_changed_frozen_input_prevents_any_producer_call[starter]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_task_grader_and_oracle_dependencies_are_frozen_before_producer", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_prepare_and_revalidation_use_packaged_task_despite_cwd_shadow", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[changed-input-inputs_changed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[extra-file-inputs_changed]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[missing-artifact-missing_or_invalid_artifact]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[oversized-missing_or_invalid_artifact]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_invalid_handoff_is_not_a_clean_comparison[invalid-answer-invalid_artifact_answer]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_no_unsafe_default_launch_or_silent_sandbox_fallback", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_retained_source_tamper_invalidates_report_without_rerunning_producer", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_unknown_usage_stops_new_trials_without_inventing_cost", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_manual_handoff_keeps_author_reports_distinct_from_artifact_checks", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change0]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change1]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change2]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_adapter_does_not_accept_unfrozen_alternate_paths[change3]", - "outcome": "passed" - }, - { - "class": "tests.test_skill_eval_benchmark_compare", - "name": "test_real_benchmark_sandbox_two_arm_handoff_or_explicit_unavailable", - "outcome": "passed" - } - ] - }, - "npu_in_hosted_ci": "not executed; hosted device test skipped, remote evidence above is separate" - } -} diff --git a/docs/plans/issue-252/npu-validation.md b/docs/plans/issue-252/npu-validation.md deleted file mode 100644 index 35efb4345..000000000 --- a/docs/plans/issue-252/npu-validation.md +++ /dev/null @@ -1,62 +0,0 @@ -# NPU 试点实测：RMSNorm 已接通检查、profiling 和 Workbench - -2026-11-04。**本轮已在一张 Ascend 910B1 上跑通完整的软件控制试点。** 正确实现通过，错误实现被拒绝，实际设备 profile 可读，同码对照没有被选为优化，融合参考实现在两个规定 workload 上达到执行前固定的改善门槛。七项资格检查均通过；项目验收、真实 Agent/skill 收益和服务收益仍未建立。 - -后续审阅与范围更新：原 reviewer 已[清除 canonical-loader 的具体 finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463161383)，另一个 [NPU 单次/整体超时 finding](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4463081791)仍未解决。下文的脚本检查结果和性能样本保持原身份，不能据此声称所有声明的单次/总时限均已强制执行。Issue 已收敛为 #336 与 #337 两项，见[当前总览](README.md)。 - -执行前的[范围、源码和判断门槛](https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6621316099)先于本次资格实验发表。被测源码为 `6308268fc6d1fbf90e9ab97b5da8af79152f39ea`；后续本记录仅整理证据，不将结果改标为新实现的运行。 - -[可公开证据与全部 800 个计时样本](npu-validation-summary.json)保留原 verdict、冻结身份、环境、逐项错误和 profile kernel 摘要。完整 raw archive 留在执行归档；公开转换及每份原始结果、trace、JUnit 的摘要也在该文件中。 - -## 实际做了什么，它有什么用 - -| 步骤 | 本轮实际结果 | 在 RSI 中发挥的作用 | -| --- | --- | --- | -| 固定任务 | `rmsnorm-npu`，二维 contiguous float32，`compute(x, weight, *, eps)`；5 个基础 case，非全一权重 | baseline 和候选做同样的工作，不能减少工作量换分 | -| 独立检查 | baseline、fused reference 各 75 次调用、375 项断言通过；oracle 使用 scalar double / math.fsum | 检查数值、shape/dtype/device、输入与权重保持 | -| profiling | 两个受测 shape 上，baseline 共 12 个计算 kernel，fused 共 2 个 `RmsNorm` kernel | 验证这份候选确实走融合设备路径；辅助解释调用延迟差异 | -| 错误控制 | 全零、漏 epsilon、漏权重、错误轴、原地改 shape、缓存旧输出均失败且无可用 metric | 阻止错误但看似很快的实现进入性能选择 | -| 同码 A/A | 两侧都运行 baseline；各 shape 中位数差小于 0.4%；Workbench 原始 verdict 为 FAIL（没有达到收益门槛） | 没有将 no-op 当作优化；这里的 FAIL 是预期控制结果 | -| 参考优化比较 | composed baseline → 既有 fused 实现；两个 shape 分别降低约 34.80%、30.60% 的同步调用延迟 | 在本轮范围内支持保留融合参考实现 | -| 留存与重验 | 16 个 trial / 800 个计时样本、最终 validation、完整环境和冻结 identity 均核对通过 | 决定可以回溯到真实源码、实际执行和原始数据 | - -另外执行了**预检后才重放**的控制：前 5 个基础 case 的 25 项断言全通过，首个变化输入 warmup 被拒绝，无性能 metric。它补充证明检查没有停在最初的正确输出。该额外控制的源码、身份、失败 phase 和结果摘要也保留在公开证据中。 - -## NPU 与计时条件 - -实际环境为 Ascend 910B1、Linux aarch64、Python 3.12.13、torch 2.10.0+cpu、torch_npu 2.10.0.post4、NumPy 2.5.3、CANN 9.1.0。PyTorch 的 `+cpu` 包版本后缀不表示在 CPU 计算：设备 tensor、实际 `Ascend Hardware` kernel 事件和 torch_npu 执行分别核对。使用个人非 root UID、已有设备组和共享调度器单卡预约，主资格与额外重放控制退出码均为 0，完成后设备已释放。 - -计时为 **同步 eager 完成调用延迟**：预同步后开始计时，调用实现，再等待 NPU 完成才停止。包含 Python dispatch 和完成同步成本；输入/权重准备、H2D、独立 oracle、D2H 校验、profiling 均不进入该计时。因此下表不能称作纯 kernel 时间，更不能直接转换成模型或服务收益。 - -每个 shape 先做 10 次变化输入预热，再保留 25 个样本；每个比较使用 4 对 AB/BA。下表为各臂 4 个 trial 的对应 shape 中位数再取中位数。所有慢样本和原始 Workbench 聚合均保留，没有按结果筛选。 - -| 对照 | shape | baseline（ms） | 对侧（ms） | 同步调用延迟降低 | -| --- | --- | ---: | ---: | ---: | -| A/A，同实现 | `[1,4096]` | 0.280772 | 0.2797225 | 0.374% | -| A/A，同实现 | `[64,4096]` | 0.311018 | 0.310098 | 0.296% | -| baseline → fused | `[1,4096]` | 0.278377 | 0.181495 | 34.802% | -| baseline → fused | `[64,4096]` | 0.311328 | 0.216051 | 30.603% | - -这证明了本轮固定环境和 workload 下的区分能力；仍是有限次描述性比较，没有建立跨机器稳定性、置信区间或服务增益。门槛来自执行前的 Issue 计划，不是新增仓库默认性能门槛。 - -## Profiling 怎样影响判断 - -baseline 在每个选定输入上出现 3 次 Mul、1 次 ReduceMean、1 次 Adds、1 次 Rsqrt；融合路径每个输入只有 1 次 `RmsNorm`。因此这份实现确实减少了设备算子调用，性能差异不是把结果移到 CPU 或返回另一个形状造成的。 - -本轮候选来自预先选择的官方 fused reference。主 Agent 根据 profile 核对机制，再根据独立检查、A/A 与固定条件对照决定保留它；没有声称 Agent 从 trace 自主发明或搜索出新 kernel。profile 属于诊断证据，不能用其中的 instrumented 时间替换上表。 - -## 软件回归与 review 修复 - -同一 source head 的 [Linux ready/review CI](https://github.com/InferMatrixLab/InferRSI/actions/runs/41764076901)通过：Python 3.10 和 3.11 各 **3649 passed / 20 skipped / 284 subtests passed**，两个安装包检查通过。托管 CI 中没有 NPU；设备验证来自上面的独立远端运行。 - -正式 CPU artifact job **54 passed / 0 skipped**，其中新增的 `test_real_sandbox_rejects_zero_output_despite_cwd_shadow` 真正执行，错误零输出在同名 cwd 包放宽容差后仍被规范 oracle 拒绝。另有 descriptor/starter/prepare/dependency 的来源回归。CI checkout merge `6f1506bcb366c10836991701f8beaf7ac89e9a11` 与被测 head 的 tree 均为 `5c22811e28f1c8c0182852916f43804c18b35edf`。这对应已实现的 P2 修复；原 reviewer 后续仅清除了该具体 finding，未批准整个 PR 或项目验收。 - -新增 NPU 无设备回归本地最终 **14 passed / 1 device test skipped**。开发首轮的 6 项失败分别来自新测试的文本编码和输出目录使用错误，修复后重跑；没有归因于既有 baseline。独立源码审查还发现并修复了输入 shape 原地修改、A/A 最终 validation 失败混同无收益、冻结控制身份未重验三处问题。 - -早期探测的容器用户名初始化失败，以及一次 profile 后调用不存在的 `key_averages()` 导致退出 1，保留在原始日志中。正式 adapter 不使用该方法，直接核对硬件 trace；失败探测不参与 800 个性能样本。 - -## 如何复现和继续 - -使用 [NPU 试点的操作说明](npu-pilot.md)和 [共享接口](../../engineering-benchmarks.md)，在已预约的兼容 NPU 环境中运行 `python -m tools.npu_benchmark_validate`，显式传入本轮 Issue 的重复次数、预算与门槛。工具先写入 `plan/frozen.json`、`validator/task.json` 与 `patch/`，随后通过既有 Workbench 保留 `results/`、报告及 `summary.json`。 - -当前可保留的是这个 float32 NPU 参考候选及共享评测通路。BF16/FP16、更多形状、真实模型/服务、NPU 产物隔离和真实 Agent 两臂仍需分别验收。#336、#337 和父 #252 继续跟踪未完成工作，PR #338 保持 Draft；#334/#335 的范围迁移不构成这些能力的验收。 diff --git a/docs/plans/issue-252/replay-measurements.json b/docs/plans/issue-252/replay-measurements.json deleted file mode 100644 index f8ef51647..000000000 --- a/docs/plans/issue-252/replay-measurements.json +++ /dev/null @@ -1,1742 +0,0 @@ -{ - "scope": "Read-only export of the original 2026-11-03 UTC v2 integration; no new run", - "protocol": "numpy-axis1-v2", - "source_summary": "replay-validation-summary.json", - "source_summary_sha256": "a9e9d7d17f3de30344229332056071ab6450b283fbe3272352f48b68855f4842", - "provenance": "Measurements predate ce84506e; task/grader bytes match the frozen identities published there. No evidence is rebound to a later source revision.", - "transformation": "Retain all 12 trials and 300 samples unchanged; project result fields to omit repeated assertions/input records and local paths. Evaluations and stripped-path profiles are retained unchanged. Original full-result hashes identify local source files; those full files are not published.", - "campaign_acceptance": "INCONCLUSIVE", - "timing_qualification": "not established", - "agent_or_skill_benefit": "not measured", - "measured_calls": 300, - "tasks": [ - { - "task": "rmsnorm-cpu", - "trials": [ - { - "side": "baseline", - "trial": "trial-001", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-001/result.json", - "source_sha256": "7d7f50721b3fdcf42621799a05d416707b4eb8ba2616958ffe40a7cacf438296", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.045700000000000005 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0145, - 0.0126, - 0.012, - 0.0119, - 0.0117 - ], - "median_ms": 0.012, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0155, - 0.016, - 0.0158, - 0.016, - 0.0154 - ], - "median_ms": 0.0158, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0065, - 0.0068, - 0.0064, - 0.0065, - 0.006 - ], - "median_ms": 0.0065, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0062, - 0.0061, - 0.0061, - 0.0064, - 0.0071 - ], - "median_ms": 0.0062, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0052, - 0.0054, - 0.0054, - 0.0049, - 0.0051 - ], - "median_ms": 0.0052, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "baseline", - "trial": "trial-002", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-002/result.json", - "source_sha256": "c167e1d49bb7a2381287fba0ce1d716ca55c3469a90c1f53b95d86049464d67b", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.0453 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0134, - 0.0127, - 0.0115, - 0.0112, - 0.0104 - ], - "median_ms": 0.0115, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0156, - 0.0148, - 0.0149, - 0.0152, - 0.015 - ], - "median_ms": 0.015, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0059, - 0.0156, - 0.0093, - 0.0074, - 0.0064 - ], - "median_ms": 0.0074, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0066, - 0.0061, - 0.0057, - 0.0063, - 0.0073 - ], - "median_ms": 0.0063, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0055, - 0.0053, - 0.005, - 0.0051, - 0.005 - ], - "median_ms": 0.0051, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "baseline", - "trial": "trial-003", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/baseline/trial-003/result.json", - "source_sha256": "fd967164167654667c47ad3934ffa4dd54993f01cf5c0f0681d87d2653f8c951", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.04800000000000001 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.016, - 0.0145, - 0.0134, - 0.0123, - 0.0123 - ], - "median_ms": 0.0134, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0162, - 0.0167, - 0.0159, - 0.017, - 0.0157 - ], - "median_ms": 0.0162, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0066, - 0.0075, - 0.0063, - 0.0066, - 0.0063 - ], - "median_ms": 0.0066, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0065, - 0.0064, - 0.0062, - 0.0062, - 0.0067 - ], - "median_ms": 0.0064, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0055, - 0.0054, - 0.0053, - 0.0054, - 0.0052 - ], - "median_ms": 0.0054, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-001", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-001/result.json", - "source_sha256": "5a886688a35a8f033a09da2390ac5375212efe9c7c965c26e8509fb11bd19daa", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.1256 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0146, - 0.0129, - 0.0122, - 0.0343, - 0.0158 - ], - "median_ms": 0.0146, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0149, - 0.0147, - 0.1118, - 0.061, - 0.0512 - ], - "median_ms": 0.0512, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0208, - 0.0191, - 0.0219, - 0.0218, - 0.0213 - ], - "median_ms": 0.0213, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0211, - 0.0205, - 0.0197, - 0.021, - 0.0253 - ], - "median_ms": 0.021, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0184, - 0.0175, - 0.0164, - 0.0175, - 0.0163 - ], - "median_ms": 0.0175, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-002", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-002/result.json", - "source_sha256": "80db25ef61f6ee371214574d4003f0dc561a9c73f8884dfb969036a55f6ad908", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.0436 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0148, - 0.014, - 0.0121, - 0.0124, - 0.0116 - ], - "median_ms": 0.0124, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0144, - 0.0136, - 0.0137, - 0.015, - 0.0133 - ], - "median_ms": 0.0137, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0074, - 0.0068, - 0.0071, - 0.0057, - 0.0056 - ], - "median_ms": 0.0068, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0068, - 0.0059, - 0.0056, - 0.0057, - 0.0068 - ], - "median_ms": 0.0059, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.005, - 0.0053, - 0.0048, - 0.0048, - 0.0048 - ], - "median_ms": 0.0048, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-003", - "source_result": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc/candidate/trial-003/result.json", - "source_sha256": "26671faef9a4cb403d1c731747143218a5da07f83f69eb34c3cb492bdc3ebc41", - "status": "completed", - "passed": true, - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.0439 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0143, - 0.0128, - 0.0127, - 0.0126, - 0.0119 - ], - "median_ms": 0.0127, - "sample_input_sha256": [ - "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0138, - 0.0135, - 0.0138, - 0.0142, - 0.0135 - ], - "median_ms": 0.0138, - "sample_input_sha256": [ - "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0065, - 0.0063, - 0.0061, - 0.006, - 0.0062 - ], - "median_ms": 0.0062, - "sample_input_sha256": [ - "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0064, - 0.0058, - 0.0056, - 0.0058, - 0.0066 - ], - "median_ms": 0.0058, - "sample_input_sha256": [ - "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0055, - 0.0054, - 0.0054, - 0.0051, - 0.0051 - ], - "median_ms": 0.0054, - "sample_input_sha256": [ - "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - } - ], - "evaluation": { - "verdict": "PASS", - "reasons": [], - "metric_key": "metrics.suite_latency_ms", - "metric_unit": "ms", - "comparison_scope": "rmsnorm-cpu:synchronous-compute-call", - "direction": "minimize", - "min_improvement_pct": 0, - "baseline": { - "values": [ - 0.045700000000000005, - 0.0453, - 0.04800000000000001 - ], - "median": 0.045700000000000005, - "min": 0.0453, - "max": 0.04800000000000001, - "range_pct_of_median": 5.908096280087545, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.1256, - 0.0436, - 0.0439 - ], - "median": 0.0439, - "min": 0.0436, - "max": 0.1256, - "range_pct_of_median": 186.78815489749428, - "excluded_trials": 0 - }, - "improvement_pct": 3.938730853391691, - "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", - "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", - "source_integrity": { - "verified": true, - "reason": "Git source snapshots unchanged", - "command_checks": 7 - }, - "statistical_significance": "not established" - }, - "source_evaluation_sha256": "a488026ab37d256990f631aa1e44e0b9300784de3d9f063c8ca9c385854c0f41", - "profile": { - "path": "replay-profiles/rmsnorm-cpu.txt", - "sha256": "7e566463a8200a0a777902642c4686bcc88aae6e836871b5804b5996f9426064", - "copied_without_byte_changes": true, - "compute_calls": 35 - } - }, - { - "task": "softmax-cpu", - "trials": [ - { - "side": "baseline", - "trial": "trial-001", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-001/result.json", - "source_sha256": "e9138fab5651d7c7f54d4dd55a058d7df47e599f63e2c140479c8cf3a9f3b963", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.044000000000000004 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0145, - 0.0141, - 0.0129, - 0.0139, - 0.0139 - ], - "median_ms": 0.0139, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0148, - 0.0151, - 0.0153, - 0.0148, - 0.0148 - ], - "median_ms": 0.0148, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0058, - 0.006, - 0.0059, - 0.0056, - 0.0059 - ], - "median_ms": 0.0059, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0051, - 0.0052, - 0.0052, - 0.0051, - 0.0053 - ], - "median_ms": 0.0052, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0043, - 0.0042, - 0.0041, - 0.0042, - 0.0043 - ], - "median_ms": 0.0042, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "baseline", - "trial": "trial-002", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-002/result.json", - "source_sha256": "128420def0c712683758bed15189b6cf97ef2ca26c2a79bc95cdfbe3bbf50170", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.045500000000000006 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0146, - 0.014, - 0.0133, - 0.0152, - 0.0137 - ], - "median_ms": 0.014, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0162, - 0.0167, - 0.0165, - 0.0167, - 0.0166 - ], - "median_ms": 0.0166, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0056, - 0.0061, - 0.0056, - 0.0056, - 0.0058 - ], - "median_ms": 0.0056, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0056, - 0.0048, - 0.0052, - 0.0052 - ], - "median_ms": 0.0052, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0047, - 0.004, - 0.0043, - 0.0041, - 0.0041 - ], - "median_ms": 0.0041, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "baseline", - "trial": "trial-003", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/baseline/trial-003/result.json", - "source_sha256": "cb51e799d2270c94c1a552fc00dcaf832fcf7d8afca3f1c53584e27d4bd45129", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "19ebd40f558705986eaad7c1dc49e97faeb0056530689e9697ff1e05b13fd7ab", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.1093 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0332, - 0.0326, - 0.0302, - 0.0324, - 0.0305 - ], - "median_ms": 0.0324, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0341, - 0.0344, - 0.0343, - 0.0335, - 0.0343 - ], - "median_ms": 0.0343, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0157, - 0.0163, - 0.0154, - 0.0314, - 0.0191 - ], - "median_ms": 0.0163, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.014, - 0.015, - 0.0144, - 0.0152, - 0.0156 - ], - "median_ms": 0.015, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0127, - 0.0123, - 0.0113, - 0.0113, - 0.011 - ], - "median_ms": 0.0113, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-001", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-001/result.json", - "source_sha256": "3dd12e1fd31a06c3bcf07a42317b2caf8db41afb4406a3dc5ff210a15c724b35", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.0426 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0136, - 0.0133, - 0.0125, - 0.0119, - 0.0119 - ], - "median_ms": 0.0125, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.015, - 0.0147, - 0.0146, - 0.0152, - 0.0144 - ], - "median_ms": 0.0147, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0058, - 0.0059, - 0.0058, - 0.0056, - 0.0058 - ], - "median_ms": 0.0058, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0053, - 0.005, - 0.005, - 0.0055, - 0.0054 - ], - "median_ms": 0.0053, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0041, - 0.0039, - 0.0108, - 0.005, - 0.0043 - ], - "median_ms": 0.0043, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-002", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-002/result.json", - "source_sha256": "232c9351a68fa3776912588492578172618bb3a396d2bf1c66159f393a7c1f7a", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.0422 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0137, - 0.013, - 0.0128, - 0.0136, - 0.0125 - ], - "median_ms": 0.013, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0148, - 0.014, - 0.0146, - 0.0153, - 0.0143 - ], - "median_ms": 0.0146, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0055, - 0.0057, - 0.0055, - 0.0055, - 0.0056 - ], - "median_ms": 0.0055, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0051, - 0.0054, - 0.0051, - 0.0051, - 0.0053 - ], - "median_ms": 0.0051, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0041, - 0.0039, - 0.0041, - 0.004, - 0.004 - ], - "median_ms": 0.004, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - }, - { - "side": "candidate", - "trial": "trial-003", - "source_result": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9/candidate/trial-003/result.json", - "source_sha256": "5c937be24fb41fa0960c7beceac63b83eef26f4bb8d85706875a96b2a2c7994b", - "status": "completed", - "passed": true, - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "measurement": { - "aggregation": "sum-of-per-case-medians", - "boundary": "synchronous-compute-call", - "case_ids": [ - "normal", - "strided", - "large", - "tiny", - "singleton" - ], - "input_sequence": "seeded-distinct-per-call-v1", - "key": "metrics.suite_latency_ms", - "samples": 5, - "unit": "ms", - "warmups": 2 - }, - "metrics": { - "suite_latency_ms": 0.044800000000000006 - }, - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0143, - 0.0172, - 0.0136, - 0.0144, - 0.014 - ], - "median_ms": 0.0143, - "sample_input_sha256": [ - "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0" - ] - }, - { - "id": "strided", - "samples_ms": [ - 0.0156, - 0.0153, - 0.0151, - 0.0154, - 0.0147 - ], - "median_ms": 0.0153, - "sample_input_sha256": [ - "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37" - ] - }, - { - "id": "large", - "samples_ms": [ - 0.0059, - 0.0062, - 0.0058, - 0.006, - 0.006 - ], - "median_ms": 0.006, - "sample_input_sha256": [ - "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf" - ] - }, - { - "id": "tiny", - "samples_ms": [ - 0.0052, - 0.0053, - 0.0052, - 0.005, - 0.0056 - ], - "median_ms": 0.0052, - "sample_input_sha256": [ - "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4" - ] - }, - { - "id": "singleton", - "samples_ms": [ - 0.0042, - 0.004, - 0.0039, - 0.004, - 0.004 - ], - "median_ms": 0.004, - "sample_input_sha256": [ - "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200" - ] - } - ], - "assertions_passed": 164, - "planned_and_executed_input_sequence_matches_summary": true - } - ], - "evaluation": { - "verdict": "PASS", - "reasons": [], - "metric_key": "metrics.suite_latency_ms", - "metric_unit": "ms", - "comparison_scope": "softmax-cpu:synchronous-compute-call", - "direction": "minimize", - "min_improvement_pct": 0, - "baseline": { - "values": [ - 0.044000000000000004, - 0.045500000000000006, - 0.1093 - ], - "median": 0.045500000000000006, - "min": 0.044000000000000004, - "max": 0.1093, - "range_pct_of_median": 143.5164835164835, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.0426, - 0.0422, - 0.044800000000000006 - ], - "median": 0.0426, - "min": 0.0422, - "max": 0.044800000000000006, - "range_pct_of_median": 6.1032863849765375, - "excluded_trials": 0 - }, - "improvement_pct": 6.373626373626387, - "measurement_method": "Alternating AB/BA sequential trials; median and range are descriptive only.", - "quality_contract": "Exit zero covers only the supplied validation command; inspect its logs and coverage.", - "source_integrity": { - "verified": true, - "reason": "Git source snapshots unchanged", - "command_checks": 7 - }, - "statistical_significance": "not established" - }, - "source_evaluation_sha256": "2015ce893c327c7c5c65800d1f7b1e81c630e94c4e8dca0b98b513c573e462c7", - "profile": { - "path": "replay-profiles/softmax-cpu.txt", - "sha256": "a0d40fe469c8bb794dd2a7c9a918fc4e0d98c0f0f1fb378e96ea21e6547e37d1", - "copied_without_byte_changes": true, - "compute_calls": 35 - } - } - ] -} diff --git a/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt b/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt deleted file mode 100644 index 37dd11e88..000000000 --- a/docs/plans/issue-252/replay-profiles/rmsnorm-cpu.txt +++ /dev/null @@ -1,19 +0,0 @@ - 525 function calls in 0.000 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 35 0.000 0.000 0.000 0.000 implementation.py:6(compute) - 35 0.000 0.000 0.000 0.000 fromnumeric.py:3699(mean) - 35 0.000 0.000 0.000 0.000 _methods.py:115(_mean) - 35 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} - 35 0.000 0.000 0.000 0.000 _methods.py:73(_count_reduce_items) - 70 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} - 70 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass} - 70 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} - 35 0.000 0.000 0.000 0.000 fromnumeric.py:3694(_mean_dispatcher) - 35 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} - 35 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} - 35 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - - diff --git a/docs/plans/issue-252/replay-profiles/softmax-cpu.txt b/docs/plans/issue-252/replay-profiles/softmax-cpu.txt deleted file mode 100644 index 5ce9b4d3d..000000000 --- a/docs/plans/issue-252/replay-profiles/softmax-cpu.txt +++ /dev/null @@ -1,19 +0,0 @@ - 595 function calls in 0.000 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 35 0.000 0.000 0.000 0.000 implementation.py:6(compute) - 70 0.000 0.000 0.000 0.000 fromnumeric.py:66(_wrapreduction) - 35 0.000 0.000 0.000 0.000 fromnumeric.py:3011(max) - 70 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} - 35 0.000 0.000 0.000 0.000 fromnumeric.py:2304(sum) - 70 0.000 0.000 0.000 0.000 fromnumeric.py:67() - 70 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} - 70 0.000 0.000 0.000 0.000 {method 'items' of 'dict' objects} - 35 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} - 35 0.000 0.000 0.000 0.000 fromnumeric.py:2299(_sum_dispatcher) - 35 0.000 0.000 0.000 0.000 fromnumeric.py:3006(_max_dispatcher) - 35 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - - diff --git a/docs/plans/issue-252/replay-validation-summary.json b/docs/plans/issue-252/replay-validation-summary.json deleted file mode 100644 index 94df1658e..000000000 --- a/docs/plans/issue-252/replay-validation-summary.json +++ /dev/null @@ -1,1255 +0,0 @@ -{ - "scope": "local CPU integration with authored controls", - "checks_succeeded": true, - "campaign_acceptance": "INCONCLUSIVE", - "agent_or_skill_benefit": "not measured", - "accelerator_qualification": "not run", - "tasks": [ - { - "task": "rmsnorm-cpu", - "adapter": "numpy-axis1-v2", - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "2f5006180baf11c9a9623006768bad776f5fc38ee281ef36edb3cec7cdfdcafc", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "correctness": "PASS", - "profile_status": "completed", - "skilleval_good": "PASS", - "skilleval_bad": "FAIL", - "bad_measurement_status": "failed", - "measurement_trials": 6, - "measured_calls": 150, - "source_integrity": true, - "descriptive_workbench_verdict": "PASS", - "workbench_run": "rmsnorm-cpu/workbench/20261103T155814Z-bd60e491cc", - "all_consumer_identities_match": true, - "all_workloads_match": true, - "input_sequence": [ - { - "id": "normal", - "input_sha256": "bfb0c12b819449a751bd50f84ad4e79a6ce0a84f430ce4f62b2e81f3a73463f4", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "normal", - "input_sha256": "cf2888897320784dc66658e689680430c618691b7a2de9a134f9558ba7a0126c", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "normal", - "input_sha256": "51a6cac6657663a2ab66b41f37cba11829be73d68df9e906e927666233d89e1a", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "12f9092d31c8d7d0153acf8079592b7ea1f1d1558ae6b50f5af9ec0bd0e56ce9", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "297705faef78ed9b8a8d51f6eff03fcba6526feb9bb77c33aa837a32c4cf75c0", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "68ba7eb4bbccf67b5758645d05f7a85e004d92da1a8be1e8d39e0aeca7f7ce2c", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "ca3502d85d4700396578ccbf4638632bd23f603d791589837331b38ec4ec55cf", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "0cf53d712ace6cdd013fc50c2fa4ef1e9a84d36cedec99fd1f9d3e3e271475fd", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "strided", - "input_sha256": "597416a7eadfc8f71919803a2b09509b05793a65bb778718e224071f88b0e583", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "strided", - "input_sha256": "705550de58cc58bfe2ce1f830a65d543976c2f71543486aa0a983088a3786430", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "0bbd05e24e6de4443792eb690faec6afaba4c65ceac5bcd753fa2f926d5e6375", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "03d3174b17cb53cd0436a8408ea98de17210cb7874afe6b668f30c540d6c914f", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "93eac27d07f54549fd25311b063b637ef354554181f4e6faa192011b65aa57bd", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "68331b9f3c40076f596c078c9e7b1c89028feee70ecd3642fbe545a7710b7e5b", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "5c28285af6ccbe887e6354fb953aab049c81b31b0503b51ab8d9c61196472126", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "large", - "input_sha256": "a89e3333ecc48304b7a6bcb33c03c1ab4f29570abfe066b529603a0bd0f0a082", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "large", - "input_sha256": "f36c9462377ab6c85ac6f68612c7a1c58ab8a846061148ea888587fddb4d5e4d", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "c32613e6ae17d46edd7d4f73527e062b1447078c8df04efd2d794b1f44153f31", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "4739f5f8a1fd07c2f63b2f97ec3c65e5d113e54a4dcedc74bb725898e60c6060", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "bb893bbfa7115d0a41c431b1b4111f2a0310df6f768cb8e29a87e4167f958345", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "708d9ada93034d9b004f3186f6c3bc1e901ffbf437211c1ab154f4b5387860ba", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "6a3b8e2783bd417a366bbe5cc38a65c0effe57848d11d0058289e8281ff530f5", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "tiny", - "input_sha256": "527e17491f351e73097806f3e5046060dec073b42881fb8ac0e8e770abf43689", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "tiny", - "input_sha256": "506d98ed53de0d20d52a5942c3067e279b921ca129b57208b83ceafb47dc4b25", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "5f249c72a4c7117dabcbd2dd2d1b7a2cabe3f87b06c49ebb620956ac29265bdd", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "b56fbbc49d0413e07e8c9aea36f26a57d4abc388c3c3fedc8d22d203e6668a10", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "bbf84b16454a3d4ad3de7ac8575eaafd547262a1760514508a7904d5d1a824ec", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "89ece9030dff7498f71151d54449d68d5cf3d8d59c7c836420620d075eb2450f", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "68835074c397f1aa5e920e28416092ffbd7d28bd392528e2da6986e75edbdf1a", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "singleton", - "input_sha256": "2a2a58087a23241da691f034142ccc3319ce9992f950004fc4f974fe00234577", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "singleton", - "input_sha256": "464f72712d23c5b975837e0202084eeefaf2ba52afba221ce1f8f7efd0918539", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "944a49da54546c2c9ef02c69319a85ae375162b878b07263ee5915afc6dee7c9", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "655b8711036934bd9719e2381e7fced7b30152bda63b10b646d698086251e7ea", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "2ab0e8b098f645c5373f8e61498c934897c187fb39898b42d9f41db2d4450552", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "fdc4a30d5b12857410f0846fcb0c42371a020409b57aad1c80a9cbad55b6a5f6", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ], - "call_index": 6, - "phase": "measurement" - } - ], - "all_input_sequences_match": true, - "all_valid_executed_calls_match": true, - "expected_measured_calls": 150 - }, - { - "task": "softmax-cpu", - "adapter": "numpy-axis1-v2", - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "88b56ab74d5dfd449e7f0a908df4ebc64c88adcb0be8ece0ace924de382b372a", - "grader_sha256": "0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407", - "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "correctness": "PASS", - "profile_status": "completed", - "skilleval_good": "PASS", - "skilleval_bad": "FAIL", - "bad_measurement_status": "failed", - "measurement_trials": 6, - "measured_calls": 150, - "source_integrity": true, - "descriptive_workbench_verdict": "PASS", - "workbench_run": "softmax-cpu/workbench/20261103T155842Z-9466b60fa9", - "all_consumer_identities_match": true, - "all_workloads_match": true, - "input_sequence": [ - { - "id": "normal", - "input_sha256": "c25b8959c8e773366dfa4dd26d41366e74d72258ced92ba8ecc6be7555112100", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "normal", - "input_sha256": "adcf65bddfc618eafd449d562205d1b2ab8dbdd24bd2fb3868b3f26d3c6bcce1", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "normal", - "input_sha256": "78bfd0e8bfa10afbffe87f26d4c710897fb136cbfb7b13ad73b972d0147f309b", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "9b43630dbbb0ae53a4c0cdb5583f0ae2be837d7a7da9d90f9d27fba2ed7ba77f", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "8f787cdc0e3cedeb2c2fe76b0fd900aa49f28501223f615bf803d5c75942160c", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "6527163cb056b965d7fc5847083a5815e4d7303d0c33f4296c4a85ad249617eb", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "normal", - "input_sha256": "b76ac149343dc83a7f95bf9dd6e0ce7be2959c503e6d22f94c68043c43b748d0", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 1028, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "768e1830be4762059c5b81b967bf098d187efc162a3b3cd5b12ea0dd16b878d9", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "strided", - "input_sha256": "ace3f9750c6d76101c01a30ed1237c64fb4f3937daddec762d10f1b17f6345d9", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "strided", - "input_sha256": "dcd2a3deded615bc4ee4c934c5d8781d96c4c2cc2f8326a452eb8e00351546f0", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "00a0dc1c4c7da3ec6f26bdf0c35a420189c6558cd9eacc648cbf6fa8d3769cd0", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "82520f59795b6a807d273a8ba01f32dd2c69453cba7770b64f0bd2cb89b3f4f8", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "f32aa5df4959debd6def00d24037967ee64dded4920b3d1fd2c04180f191395b", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "strided", - "input_sha256": "45067806d3546e89d7a10b9b617e50e68f9d2f92bd14ae658a6ebfcd51ec5c37", - "shape": [ - 16, - 257 - ], - "dtype": "float32", - "strides": [ - 2056, - 8 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "4e82ca66db9606b5566293f2edc5eea1e9df6649e56f2112f6f2e9e938d8438c", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "large", - "input_sha256": "af087bae1e27a00565a87dfa7113ebc69b26c01903362fc1ab9549c577ded188", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "large", - "input_sha256": "b5ae111c8be7a197ab3ab72a59c69beecd87b18ef36446939aa7b3fa4834834a", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "fbd8f4570afde9b79dd09298256acf8e79f04290eff642543acb01e1502d2c99", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "191f57996b6ed1226c55254f22e32818e07602a454ae138b2913611cbf02187c", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "2d42f9307851b1a6173a3c2c4ebe7e8e438aee880050390c535f610bb1138021", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "large", - "input_sha256": "f69f0bc72652225e55bad8d41fed79fe341de1a2d1cc8ec1646c1210da883edf", - "shape": [ - 3, - 31 - ], - "dtype": "float32", - "strides": [ - 124, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "bf4febd2050386a0849e2ce1be8d75657f3346c903bcf1b0a3d2d743bbea80ab", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "tiny", - "input_sha256": "5d962ceb305b5ddbc5776378338a91ab3dabc64f5245a31aa4aecdd17642769e", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "tiny", - "input_sha256": "3fe6bfb8d005793e54db5a1ccb2e24cdaad209e2ddc91c110f49c013ffd8f944", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "13fb1d6d82d253db4548d77a71e7c9c54ca21691965fa753da89b00cc20eb5be", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "96e6fb5b0fcb2568103b2c250b613636e4dfa839deba002cf8234e246d72d6c9", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "27506b1ca2344d32a03d995f375753fb8dd0ddd4939ccc771594658077f08299", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "tiny", - "input_sha256": "d29ae435fc0c1c9bb32c59810098138b0c56e738a527d32a50391c5a2edda7c4", - "shape": [ - 2, - 17 - ], - "dtype": "float32", - "strides": [ - 68, - 4 - ], - "call_index": 6, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "e1cfcaed0914dbc73942f399349276d1f2788d07ff18cd095d89183a6adf53af", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 0, - "phase": "warmup" - }, - { - "id": "singleton", - "input_sha256": "4cf5dbc59fe7b8cdf8b14fdefce446edd0a50982d37dbb5c0681872338ee500c", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 1, - "phase": "warmup" - }, - { - "id": "singleton", - "input_sha256": "6d6d4ca219d17c233d601a71cd900ca65158815fc88572c7f97791013d53c9ee", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 2, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "7beaee4b5d6ffe22a955b399456a8396e604547983df2aef8eab43cae84df769", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 3, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "d9706bf157d4b33a98a426b510c9ba2a0d43b983b9331a2f5696b204be8fc4c5", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 4, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "b0a367d4b838550486dc3e540ad707ff3fe9c016dd56efdcbf0398761c083c74", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 5, - "phase": "measurement" - }, - { - "id": "singleton", - "input_sha256": "5d47f80ea8679048808669d912c4d904be07ea6609e412d6243550ccca01c200", - "shape": [ - 2, - 1 - ], - "dtype": "float32", - "strides": [ - 4, - 4 - ], - "call_index": 6, - "phase": "measurement" - } - ], - "all_input_sequences_match": true, - "all_valid_executed_calls_match": true, - "expected_measured_calls": 150 - } - ] -} diff --git a/docs/plans/issue-252/replay-validation.md b/docs/plans/issue-252/replay-validation.md deleted file mode 100644 index 47be485ed..000000000 --- a/docs/plans/issue-252/replay-validation.md +++ /dev/null @@ -1,52 +0,0 @@ -# v2：按 RSI 测量边界修复输出重放 - -当前 RSI 的候选比较要求测量实际实现的计算，不能让最终输出缓存冒充优化。[PR review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4456416093) 指出 v1 预检、预热和计时复用同一输入，正确的 memoizing 实现可能只在预检计算一次。这是任务测量语义的缺陷，与有没有 CPU/GPU、是否有独立 oracle 无关。 - -## 修复与版本边界 - -- 适配器更新为 `numpy-axis1-v2`；每个任务显式声明 `seeded-distinct-per-call-v1` 输入协议和五个计时 case。全零输入仍检查正确性，但不计入性能分数。 -- 执行前生成全部输入、逐输入独立 oracle 和预定顺序。输入对象一直保留，字节不与预检、预热和其他采样重复；baseline 与 candidate 使用同一序列。 -- `check`、`measure`、`profile` 检查同一序列。结果区分计划的 `input_sequence` 与实际 `executed_calls`，每个计时样本绑定输入摘要；失败只留下实际执行的前缀。 -- 对按 id/完整内容缓存的正确实现，不禁止代码写缓存，而是验证本序列没有命中旧结果。按 shape/layout 直接重放旧答案的实现会被变化输入拒绝，且不能输出性能指标。 -- 输入和 oracle 生成在计时之外；计时仍只包围 compute。该机制不提供恶意代码隔离或隐藏测试，公开输入仍可能被过拟合。 - -任务、grader 身份和指标的五 case 聚合均已变化。v1 的 [集成记录](validation.md) 和 [用途对照](usefulness.md) 完整保留，不改写、不拼入 v2 的分数。正确性与性能排序仍分别判断。 - -## 实际验证 - -**版本边界：本页记录的是 2026-11-03 UTC 的既有 v2 运行，grader 为 `0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407`，对应 ce84506e 发布的测量实现，ac03ae23 未改变这些代码。** 后续 [CLI 状态修复](review.md)改变了 grader 目录中的 `cli.py`，因此新提交有新 grader 身份；计算、oracle 和计时文件不变，但本页 300 个样本不能称为修复后新 head 的运行证据。新 head 的专项回归单独记录。 - -2026-11-03（UTC），Windows / Python 3.11.18 / NumPy 2.4.6。针对 v2 的专项回归 **40 项通过**，包括真实 baseline/candidate、缓存命中、旧输出重放、逐次输入配对、边界协议、漂移、超时、Workbench 和 SkillEval verifier。另行运行的 CLI catalog 和 SkillEval 相关回归也通过；Ruff 和 diff 检查通过。 - -开发时首轮回归暴露了 `SeedSequence` 的 NumPy 命名空间错误，修正为 `np.random.SeedSequence` 后完整重跑通过；未把首轮失败归因于环境或既有基线。日志保留在仓库外的本地 outputs 中。 - -随后单独执行现有复现工具： - -```bash -python tools/engineering_benchmark_validate.py \ - --repo --output-dir --repetitions 3 -``` - -这是固定示例候选的协议/消费者验证，不是新的性能优化搜索。工具在执行前冻结任务、候选及 RunSpec；实际原始目录名为 `inferrsi-252-replay-v2-validation`。 - -| 检查 | Softmax CPU | RMSNorm CPU | -| --- | --- | --- | -| 正确性范围 | 6 个基础 case + 35 次变化输入调用 | 相同 | -| 正确实现独立断言 | 164 项通过 | 164 项通过 | -| 正确/全零错误产物在 SkillEval | PASS / FAIL | PASS / FAIL | -| 错误测量 | 拒绝，无可用指标 | 拒绝，无可用指标 | -| 实际 Workbench trial / 计时样本 | 6 / 150 | 6 / 150 | -| 独立 profile | 完成 | 完成 | -| 所有消费者 task/grader/runtime 与基础输入 | 一致 | 一致 | -| 计划输入序列与正确实现的实际调用序列 | 一致 | 一致 | -| Workbench 源码完整性 | 通过 | 通过 | - -[原始可公开汇总](replay-validation-summary.json) 保留每次变化输入的摘要及身份、数量和原始 verdict，未包含私人路径。后续 review 从同一次本地记录只读导出了[全部 300 个计时样本及原始聚合](replay-measurements.json)、[RMSNorm profile](replay-profiles/rmsnorm-cpu.txt) 和 [Softmax profile](replay-profiles/softmax-cpu.txt)。原汇总保持逐字节不变；补充文件记录来源摘要、每个 trial 的原始结果哈希、样本输入哈希和完整 evaluation，未重跑、删慢样本或更改 verdict。未公开完整执行日志和含本地路径的运行文件；原始文件哈希只是来源标识，不代表读者已获得那些完整文件。 - -导出时逐项核对 12 个 trial 的 300 个样本、每 case 中位数、每 trial 中位数之和、Workbench 两侧聚合，以及计划/实际输入配对。两次 Workbench 描述性 verdict 都为 PASS，未将其解释为可靠性能提升；本次没有完成 v2 同码噪声资格验证，也没有建立统计显著性。原始 RMSNorm candidate trial 值为 0.1256、0.0436、0.0439 ms，Softmax baseline 为 0.044000000000000004、0.045500000000000006、0.1093 ms，完整保留其中的离散值。profile 的文本精度下时间多显示 0.000，只支持调用结构检查，不支持瓶颈定位结论。 - -## 对 RSI 的实际含义 - -现在普通 id/content 缓存不能因预检复用而得到重放时间，错误的旧输出也不能成为优化证据。这提高了 Workbench 和离线 verifier 共同消费的任务语义可靠性。 - -这仍不是“RSI 已经自动变好”的证据。可靠性能排序、正式 SkillEval 两臂产物交接、真实技能/搜索收益及目标硬件验收仍未完成；调用关系及下一步验收沿 [RSI 接入审计](rsi-integration.md) 组织，不另建执行器或搜索控制器。 diff --git a/docs/plans/issue-252/review.md b/docs/plans/issue-252/review.md deleted file mode 100644 index 3eed3f280..000000000 --- a/docs/plans/issue-252/review.md +++ /dev/null @@ -1,80 +0,0 @@ -# #252 本轮审查记录与阅读入口 - -本页固定记录 `9454672f` 这一轮审查。之后用户要求继续正式接入，新增实现与验证见[正式 SkillEval 接入](formal-integration.md)；下文“未接入”是当时状态，不能代替当前接入说明。 - -旧四项分工及 Issue 状态同样保留为该轮历史；当前已按用户委托收敛为 #336 任务库和 #337 接入，见[当前总览](README.md)。历史审查记录不表示旧 Issue 仍需独立跟踪。 - -本轮继续审查 PR #338，并准备供首次接触 InferRSI 的读者使用的[完整导读](walkthrough.md)。结论是：发现并修复一个 CLI 失败状态传播缺陷，补齐公开可检查的既有 v2 样本/profile，修正文档的历史版本指向。PR 仍保持 Draft；这些工作不代表维护者已经接受范围或批准合并。 - -## 审查对象和覆盖 - -审查起点为 `ac03ae23ce0bb5864a55c2c97d5f3d8d3b550d39`，对照原始基线 `663f3b327998526a13aa31e72ad729025c18983c`，共 42 个变更文件。审查分工与本轮修复复核均在同一工作树进行；新导读、样本导出和修复属于本轮增量。 - -| 范围 | 具体检查 | 结果 | -| --- | --- | --- | -| 任务契约、worker、NumPy 评测、两套任务包 | manifest 拒绝路径、输入上界、身份冻结、独立 oracle、axis=1/4D/跨步布局、变化输入、逐调用检查、profile/计时边界 | 未发现额外需修复缺陷；原 40 项专项测试独立重跑通过 | -| consumers、离线 verifier、CLI、接线和打包 | 真实 executor/RunSpec/run_trial 接入、结果与状态、包内资源、验证脚本 | 发现 R1；其余未发现新的实质问题 | -| 文档与证据 | 四项交付含义、RSI 调用关系、历史版本、声称完成的范围、样本可检查性 | 发现 R2/R3 并修正；保留正式集成与验收缺口 | -| 本轮 R1 修复 | 独立复核 diff、读取真实状态文件、退出码与既有 run 命令一致性 | 修复复核未发现新问题；新增实际执行回归通过 | - -“未发现”描述本轮检查结果，不是对所有输入、设备或恶意源码的保证。代码入口仍限定已审核可信实现。 - -## 已处理的问题 - -### R1 · P2：compare 在失败时仍返回退出码 0 - -**触发：** 通过 `inferrsi benchmarks compare` 运行返回全零的错误 Softmax 候选。Workbench 实际记录 `state=failed`、`verdict=INCONCLUSIVE`；旧 CLI 仅返回运行目录和范围信息，最终进程退出码却为 0。 - -**影响：** 脚本或 Agent 如果以退出码判断成功，会把失败比较误当成功。这与已有 `inferrsi run` 的退出约定不一致。 - -**修复：** benchmark CLI 读取并原样返回保留的 `status.json`；顶层 CLI 按实际 verdict 返回 PASS=0、FAIL=1、INCONCLUSIVE=2。不修改 Workbench 的判定规则，不把未完成比较改写为数值 FAIL。 - -**验证：** 三个新回归均执行真实 CLI/Workbench，并检查响应与磁盘状态及 evaluation 一致： - -| 情况 | 状态与 verdict | 退出码 | -| --- | --- | --- | -| 正确候选完成比较，但无法达到 100% 延迟下降的测试门槛 | completed / FAIL | 1 | -| 全零错误候选导致测量失败 | failed / INCONCLUSIVE | 2 | -| 极短预算在执行前耗尽 | timeout / INCONCLUSIVE | 2 | - -测试中的 100% 和极短预算是为确定性触发分支设置的测试条件，不是性能验收标准。`grade` 对错误产物本身仍可报告正确性 FAIL；不完整的 A/B 比较则是 INCONCLUSIVE。 - -### R2：v2 可公开记录只有数量和身份，没有原始计时 - -**影响：** 读者无法从仓库字节复算 300 个样本的聚合或查看 profile。 - -**处理：** 从原始本地 v2 运行只读导出 [replay-measurements.json](replay-measurements.json) 和两份 [RMSNorm](replay-profiles/rmsnorm-cpu.txt) / [Softmax](replay-profiles/softmax-cpu.txt) profile。保留全部 12 个 trial、300 个样本、逐样本输入哈希、原始 evaluation 与来源哈希；profile 逐字节复制，既有 summary 不改写。不重复运行，不筛除离散值，不声称公开了未附带的完整日志。 - -**核验：** 样本数、每 case 中位数、trial 指标、Workbench 两侧聚合、输入序列与旧汇总逐项一致。补充材料提高可检查性，不会使原本未建立的测量资格自动成立。 - -### R3：当前接入文档仍指向历史 v1 验证 - -**影响：** 读者可能以为当前工具可以生成旧协议的 360 个样本，或将 v1 和 v2 结果合用。 - -**处理：** 子项②/④指向 v2 记录，历史安装证据单独标明；v1 页面固定到原始实现，说明重放需使用当时的工具、题目和 grader。总览明确主 Agent 作决策，Search Policy 是独立提案。新导读将四个交付步骤与运行时模块分开解释。 - -## 本轮验证及证据边界 - -| 检查 | 结果与边界 | -| --- | --- | -| 修复前核心专项重跑 | ac03ae23 上 40 项通过 | -| 修复后专项回归 | `tests/test_engineering_benchmarks.py` 43 项通过，含 3 个实际 Workbench CLI 回归 | -| 顶层 CLI 回归 | `tests/test_cli_catalog.py` 19 项通过 | -| 静态与文档检查 | Ruff、diff 空白检查、本地 Markdown 目标检查通过；证据导出核验通过 | -| 已有 hosted CI | ac03ae23 的 Linux Python 3.10/3.11 full-check 和 installed-package 通过；[运行记录](https://github.com/InferMatrixLab/InferRSI/actions/runs/41672237031)；不将该 CI 结果冒充本轮新提交检查 | -| 历史 Windows 全量 | 3479 passed、51 skipped、56 failure reports；失败在未修改原始基线复现，原运行仍不绿，本轮未重复整个 Windows 全量 | -| 新性能实验 | 未执行。本轮修复不修改输入/oracle/计算/计时行为；真实 CLI 状态分支通过上述回归验证 | - -**grader 版本变化必须保留：** `grader_digest()` 对整个 benchmark 目录取哈希，因此修改 `cli.py` 也会改变 grader。旧 v2 300 个样本始终属于 `0b75b5bbe7fc048293a6c8b4c2e9a35d1cbf63c27b16b420f3f49da3efb57407`；本轮修复后的测试身份为 `34928e6956f54aab565a4d48ea68f74ee72aa91578f8ce63fd013934c71b4d55`。不能把旧样本重新绑定成新提交的运行记录。 - -外部 reviewer 在 [r4459426716](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4459426716) 确认了 ac03ae23 的缓存重放修复；该确认明确只覆盖这一问题，不能描述为完整 PR approval。本轮独立分工 review 也不替代维护者的项目验收。 - -## 仍需分别解决的事项 - -- 首批 CPU 任务及接口范围的维护者 accept/revise/reject 决定；原生父子 Issue 关系和标签由有权限的维护者协调。 -- 同码噪声与可分辨性能排序；现有 Workbench 的描述性 PASS/FAIL 不足以支持稳定优化收益。 -- `tools.skill_eval_compare` 的正式任务适配、产物交接与受支持隔离；当前只是底层保留产物 verifier。 -- 真实旧/新 skill 的受控 Agent 实验及资源成本；固定作者产物不能证明技能收益。 -- 硬件确定后的同步、计时、真实负载与设备/服务验收。 - -下一步工作沿已有 RSI 入口推进，详见 [接入审计](rsi-integration.md)。四个子 Issue 保持打开；本轮不作完成、合并或可靠性能提升的声明。 diff --git a/docs/plans/issue-252/rsi-integration.md b/docs/plans/issue-252/rsi-integration.md deleted file mode 100644 index e66bb270d..000000000 --- a/docs/plans/issue-252/rsi-integration.md +++ /dev/null @@ -1,63 +0,0 @@ -# #252 在现有 InferRSI 中怎样发挥作用 - -验收起点是“谁调用任务包，拿结果作什么决定”。两个算子能运行、两个底层消费者复用 oracle，只完成组件级验证。它们尚不能证明 InferRSI 的优化者因此变得更有效。 - -## 两条现有路径，共用一份任务定义 - -| 路径 | 变化的对象 | #252 的职责 | 结果影响的决定 | -| --- | --- | --- | --- | -| 引擎优化：主 Agent → Workbench | 引擎的代码或配置 | 定义适用的任务、输入、oracle、诊断与测量边界 | 该实现是否正确；是否有足够证据保留它；下一轮需要什么诊断 | -| 优化者评估：SkillEval | skill、知识或 harness 的候选版本 | 用相同任务和独立 oracle 检查两边实际产出的实现 | 新指导是否帮助完成任务；质量和资源成本如何变化 | - -两条路径中改变的对象不能混淆。引擎代码变快，不等于某个 skill 有效；skill 版本比较也不能通过替换任务或判分规则制造进步。整体 agent/harness 收益仍由 #248 的受控实验回答，#249 负责并行候选搜索的相关范围。 - -已有 [harness-improvement controller](../../harness-improvement.md) 可以消费其支持的 SkillEval grades，进行 development 选择、重跑和一次 holdout 接受/拒绝。本次 benchmark 不能假定已经接入它。不要为接入 #252 再造一套总控制器或执行平台。 - -## 对照真实入口审计当前状态 - -| 入口 | 现在已经接通什么 | 尚未接通什么 | -| --- | --- | --- | -| `benchmarks plan/compare` → `TaskPackage.runspec` → `run_experiment` | 同一任务生成既有 RunSpec，由 Workbench 执行配对比较并保留证据 | 自动从用户目标/profile 选出适用任务；服务与目标硬件验收 | -| `benchmarks check/profile/measure` | 分开的数值校验、按需 CPU/NPU 诊断、测量操作 | 真实 Agent 产物到最终评测的完整接入；profiling 不作为默认门禁 | -| `skill_verifier.task_descriptor` → `skill_evaluation.run_trial` | 对 controller 已审核的保留实现运行真实 verifier；答案提交该实现的 SHA | Agent 根据不同技能实际产出两份实现的比较 | -| `tools.skill_eval_compare` | 新增 `engineering-benchmark-v1`：共享题目/两臂 skill 冻结、实际文件留存、隔离计算与父进程 oracle、配对报告重验；支持现有 local stage/capture | 当前仅正确性；真实受控 Agent/skill 收益、性能排序、provider generation 模式未接入 | -| `tools.skill_eval_improve` | 已有面向支持的离线 audit 任务的选择、重跑和 holdout 流程 | 共享工程基准尚未接入该控制器的任务/grader 契约 | - -依据为 [CLI 适配](../../../src/inferrsi/harness/benchmarks/cli.py)、[保留产物 verifier](../../../src/inferrsi/harness/benchmarks/skill_verifier.py)、[SkillEval 比较入口](../../../tools/skill_eval_compare.py)、[现有 engineering adapter](../../../src/inferrsi/harness/skill_eval_engineering.py) 和 [改进入口](../../../tools/skill_eval_improve.py)。主 Agent 的调查与代码修改职责见[现有插件入口](../../../plugins/inferrsi/skills/inferrsi/SKILL.md)。 - -## 一次引擎优化中应该怎样使用 - -1. **从目标和验收条件开始。** 主 Agent 明确模型、硬件、工作负载、基线和质量要求，再确认该任务是否覆盖所需语义、dtype、布局和环境。名称相同的算子任务不自动代表真实模型或服务。 -2. **冻结任务与实验条件。** 在候选尝试前固定任务、oracle、输入序列、版本、比较条件和预算。候选只改允许的实现；不同时修改裁判或基线。 -3. **在允许的工具和预算内自主优化。** Agent 可阅读代码、查阅允许使用的知识、运行 profiling、修改和测试，自行选择适用方法。评测不强制每次检索知识或调用 profiler；实际调用和消耗应留存，未做诊断时不声称已经观测到瓶颈。 -4. **先挡住错误，再测量。** 数值不正确的候选不能获得可用于选择的性能分数。计时必须测到实际计算；预检不能给后续同输入缓存重放预热。测量规则的修改产生新的任务/grader 身份，不重写旧证据。 -5. **区分原始报告与优化决定。** 保留 Workbench 原始 PASS/FAIL/INCONCLUSIVE；主 Agent 同时判断证据是否足以支持保留/放弃。若同码对照无法分辨，决策是“测量不足，先修协议”，不能把某次 FAIL 归纳成整个优化机制无效。 -6. **记录对下一轮有用的事实。** 保留候选、环境、任务、正确性失败、原始样本及未解决的诊断。经验整理与晋升走现有流程；不从一次算子结果自动推广成服务收益或通用技能。 - -## 按 RSI 需求补证的顺序 - -### A. 先使任务适合被优化者消费 - -要求 oracle 独立、候选修改边界清楚、实际计算不被预检缓存替代、输入在消费者之间配对一致、测量失败不产生可用性能证据。这是 #252/#337 的基础交付，不是另一个 search-policy 项目。 - -[缓存重放 review](https://github.com/InferMatrixLab/InferRSI/pull/338#discussion_r4456416093) 暴露了这一层的具体缺口，现已通过 [v2 变化输入协议及回归](replay-validation.md)修复。此前的普通冒烟漏检对照证明了正确性检查用途；同码噪声与可靠性能排序仍须单独验证。 - -### B. 证明结果确实进入优化决定 - -最小演练应沿已有入口保留“任务与条件 → Agent 实际优化 → 产物 → check → compare → 结果与消耗记录”链，并至少包含：错误产物被拒绝、正确但证据不足的产物不被当作改进、负结果留下适用条件。Profiling 是可选诊断，不是通用链路的必经阶段；已执行工具资格试点的冻结规则不因此改变。既有 [RMSNorm 对照](usefulness.md) 提供了部分输入与结果；真实 Agent 到最终产物评测和整轮消耗的接入仍未完成，不能据此宣布完整闭环验收。 - -### C. 接到已有 SkillEval 两臂入口 - -已沿 `skill_eval_compare` 扩展 `engineering-benchmark-v1`，支持 update 模式的旧/新 skill 两臂，保留冻结、模型/预算、日志与报告。两边分别交付 `implementation.py`，控制器留存实际字节；候选在既有 bubblewrap 边界内计算，父进程使用共享输入与 oracle 独立检查。`skill_eval_local` 也支持 stage/capture；完整使用方法和验证分层见[正式接入说明](formal-integration.md)。 - -原有可信 CPU worker 仍不能直接接未经审核的模型代码；新 formal adapter 为产物 replay 使用隔离路径，没有宿主回退。当前只输出正确性，不使用候选自报 latency；真实 Agent 执行权限、技能收益和性能排序另行验证。固定作者脚本只证明通路和裁判行为，不能证明技能帮助。 - -### D. 最后评估优化者本身是否改善 - -实际 agent 实验要固定任务集、模型、总预算和对照，记录所有尝试、质量、可分辨的性能与资源使用。若用于选择可复用技能，还要有适当的独立验证任务；同一道公开题换一个 seed 不自动等于独立 holdout。先明确新 adapter 支持的范围，再评估能否被既有 harness-improvement controller 消费。 - -这一层沿 #88/#248 及已有 controller 的职责推进，不把整个搜索控制器、自动知识晋升或服务验收塞进 #252。 - -## 当前结论 - -共享任务与底层消费者复用已有实际证据，输入重放缺陷已修复；后续正式两臂 adapter 也已实现产物交接与独立正确性检查。真实 Agent/skill 收益、可靠性能排序、provider generation 和改进控制器接入仍未完成。具体平台软件对照结果见正式接入记录，不能用接口实现代替系统收益验收。PR 继续作为 Draft 评审，不另建编排系统。 diff --git a/docs/plans/issue-252/task-expansion/01-operators.md b/docs/plans/issue-252/task-expansion/01-operators.md deleted file mode 100644 index 381e631f3..000000000 --- a/docs/plans/issue-252/task-expansion/01-operators.md +++ /dev/null @@ -1,37 +0,0 @@ -# 算子计算：当前在 NPU 上跑通 RMSNorm - -**当前题目：在 Ascend NPU 的给定输入、权重和容差下，比较 affine RMSNorm 的基础运算与融合实现，同时保留计算结果。** - -状态：`rmsnorm-npu` 已在 Ascend 910B1 完成真实正确性、同步计时、CPU/NPU profile、A/A 和候选比较，见[实测记录](../npu-validation.md)。资格限已记录 runtime、float32 与 workload，不能外推整类算子或其他硬件。`softmax-cpu` 保留为软件控制，`rmsnorm-cpu` 已移除；[正式正确性接入](../formal-integration.md)目前仍是 CPU 路径。 - -## 为什么先在 NPU 做这个任务 - -RMSNorm 有清晰的数值语义、基础运算基线和 torch_npu 融合候选，能在一个受限问题上检验真实设备执行、独立 oracle、同步计时和结果交接。先完成这条 NPU 通路，再扩展题类；更多 CPU 数学样例不能替代设备验收。 - -在现有 SearchSpace 中关联 Operator 的 Optimize/Verify。NPU profile 必须实际生成并解释；任务标签不证明 Agent 具有 Diagnose 能力。设备结果只提升该环境和工作负载的证据，不能自动提高整个 Accelerator 层的成熟度。 - -## 固定来源与复用决定 - -| 来源 | 具体借鉴 | 适用边界 | -| --- | --- | --- | -| [Ascend op-plugin RMSNorm](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/op_plugin/ops/opapi/RmsNormKernelOpApi.cpp) 与 [对应测试](https://github.com/Ascend/op-plugin/blob/9f23f00b1ebe54cca612b0537bc582dba409a0af/test/test_custom_ops/test_npu_rms_norm.py) | 真实 affine 算子 API；`9f23f00`，BSD-3-Clause | 本轮先声明二维 contiguous float32，运行时支持与性能另做设备验证；不导入上游 runner | -| [KernelBench Softmax](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/23_Softmax.py)、[RMSNorm](https://github.com/ScalingIntelligence/KernelBench/blob/423217d9fda91e0c2d67e4a43bf62f96f6d104f1/KernelBench/level1/36_RMSNorm_.py) | 既有 CPU 任务的数学题意；`423217d`，MIT，原包保留许可证 | 新 NPU 题单独定义二维 affine RMSNorm，不继承旧四维 axis=1、无 affine 或上游 GPU 资格 | -| [AMD AgentKernelArena task v2](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) | semantics、candidate、baseline、reference、workload 与 evaluation 的分工 | 作为协议设计参考；不导入其 GPU/Agent 执行平台或声称当前 CPU 实现满足其设备资格 | -| [BackendBench](https://github.com/meta-pytorch/BackendBench/blob/2a8d7e19e2c13c789ff2dceb44883ea8ea04dab4/README.md) | 边界输入/布局检查与模型来源 shape 的区分 | 作为扩展输入的设计参考；未复制 OpInfo/FACTO 或 TorchBench 运行体系 | - -后两项本轮仅引用既有[源码调研](../01-research.md)，没有复制其代码；未来复制具体文件时按其文件及依赖许可保留归属。 - -## Agent 做什么、工具检查什么 - -- NPU 改动面是 `implementation.py` 的 `compute(x, weight, *, eps)`；不能修改 oracle、输入分布、容差或计时器。 -- 首版明确二维 contiguous float32，输入/权重保持及变化输入需检查；独立 CPU double oracle 不复用设备归约。 -- CPU/NPU profiling 单独运行，不将诊断开销计入优化分数；缺少设备 trace 如实记录。 -- Workbench 比较已审核可信源码；正式 SkillEval adapter 当前只支持 CPU 隔离 replay，不接收 NPU 题。 - -## 下一步资格实验 - -在真实 NPU 完成同码/no-op、错误但快的实现和基础算子/融合候选对照，确认计时信号可区分；保留原始样本与版本，不把 CPU 历史样本重绑到新 grader。 - -本轮先验证选定 NPU 任务的执行与测量协议。后续从实际模型 profile 选择代表瓶颈，再决定扩展 reduction、fusion、attention 或其他算子。每个设备任务独立定义同步、编译、warmup、dtype、数值检查和计时边界；不能只给 CPU manifest 换一个 backend 字符串。 - -该类任务的成功只支持相应算子/环境结论。模型或服务收益仍需对应工作负载的端到端验收。 diff --git a/docs/plans/issue-252/task-expansion/02-host-runtime.md b/docs/plans/issue-252/task-expansion/02-host-runtime.md deleted file mode 100644 index a5db83890..000000000 --- a/docs/plans/issue-252/task-expansion/02-host-runtime.md +++ /dev/null @@ -1,77 +0,0 @@ -# 候选类别 02：CPU 侧数据处理与运行时开销 - -状态：2026-11-04 来源检查与任务设计；尚未导入题包、构建环境或运行性能实验。用户已将当前试点指定为 NPU，先完成 [RMSNorm NPU 通路](../npu-pilot.md)；本页保留为后续 CPU 仓库补丁方向，不再是下一项必须交付的试点。 - -## 要测什么 - -让 Agent 在真实仓库中发现并减少推理前后的 CPU 开销，同时保持输入输出语义。 -若后续开展这一类别，首题建议选择 **大量 added tokens 时的 Tokenizer.encode**。 -它关联现有 SearchSpace 中的主机侧处理与运行时优化；具体矩阵归属仍需评审，不据此新增已验证覆盖。 -这类题测试真实 repo 的分析和补丁能力；单题结果不代表服务 TTFT 或整个 RSI 能力提升。 - -## 可核查的题源 - -| 对象 | 本轮核对结果 | -| --- | --- | -| benchmark | [GSO](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/README.md)，代码 commit `32d78b331d59ed689b1238789dc20d36576951c0`，MIT | -| 具体任务 | 官方数据中的 `huggingface__tokenizers-076319d`，API 为 `Tokenizer.encode`，4 个测试 | -| 目标仓库 | `huggingface/tokenizers`；目标版本 [LICENSE](https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/LICENSE) 为 Apache-2.0 | -| baseline | `fb837b4adb130dfc89104aa09b1ae223da120278`，由官方 expert commit 的 parent 核对 | -| expert | [`076319d542f87e9f72db379d7e47b29b1ce7bd87`](https://github.com/huggingface/tokenizers/commit/076319d542f87e9f72db379d7e47b29b1ce7bd87)，Aho-Corasick 处理大量 added tokens | -| 数据版本 | 已下载并读取 `gso-bench/gso` revision `c2e4f1a58427cccd15e0e542f136bd204fb19284` 的 `data/test-00000-of-00001.parquet`，102 行中该 task ID 唯一、包含 4 个测试；数据许可声明 MIT | -| 来源记录 | [下载与任务行身份](gso-source-audit.json)保留文件 SHA-256、行选择、canonical JSON 哈希及 base/expert 身份；这是源码检查，不是任务执行证据 | - -初始发现来自实时 datasets-server；随后以固定 revision 的 parquet 完成任务行复核。正式导入还需归档所选资产及许可，锁定最终派生题包身份。 -上游参考改动主要位于 `tokenizers/src/tokenizer/added_vocabulary.rs`，并增加了 Rust 测试。 -Agent 从 baseline 开始修改仓库并提交 patch；expert patch 用于评测资格对照，不默认提供给 Agent。 - -## 输入、正确性和允许改动 - -GSO 的公开问题脚本使用 GPT-2 tokenizer、约 4000 个 added tokens 和多个文本片段；4 个测试还含其他构造及 BERT tokenizer，不能把问题脚本等同于全部测试。性能区间为编码调用。 -当前脚本会下载浮动 `resolve/main/tokenizer.json` 和在线 Gutenberg 语料。导入时必须固定版本与内容哈希，提前准备输入；安装、下载不计入编码耗时。 -数据行的 setup 还包含通用系统安装命令及 `upt upgrade` 拼写问题；本轮未执行这些命令，需要单独整理并验证最小可重建环境。 -任务应固定 tokenizer、added-token 集、文本及线程条件；允许修改实现，不允许替换 tokenizer 定义、测试、输入或参考结果。 - -- 保留 GSO 对 token 序列、token IDs、输出数量及长度的比较。 -- 复用上游 [added_tokens.rs](https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/tokenizers/tests/added_tokens.rs)，覆盖 single_word、lstrip、rstrip、重叠匹配及 offsets。 -- 补充不同文本长度、Unicode、空输入、无命中与多命中的条件；具体检查须经 baseline/expert 双方验证。 -- 基线输出等价检查与固定语义断言并用。基线结果不是完全独立的实现 oracle，不能把相同输出测试称为全语义证明。 -- 同一进程连续处理不同文本，保留输入和结果证据，检查状态缓存或复用结果造成的假改进。 - -## profiling 与测量流程 - -1. 准备固定仓库、Rust/Python 构建工具链与数据，验证 baseline 能重建、测试能通过。 -2. 独立 profiling：定位 Python/Rust 编码调用的时间；需要原生栈时使用支持 Rust 的 profiler，不能仅凭 cProfile 判断 Rust 内部热点。 -3. Agent 写出诊断依据和改法，交付真实 patch；工具审核适用范围并独立验收。 -4. 在声明的 CPU、线程、编译选项和工作负载下重复比较，安装和构建时间单独记录。 -5. 报告每个工作负载的正确性、原始耗时、波动、版本及预算，再解释是否足以保留候选。 - -GSO 的 [执行脚本](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/src/gso/harness/grading/evalscript.py) 对该仓库使用一次外层迭代；不能直接把这一设置当作 RSI 的可靠性能资格。 -重复次数和可接受误差来自本次实验的冻结条件，不照搬排行榜分数、阈值或无测量支持的默认值。 - -## 先证明题目有效，再评 Agent - -| 对照 | 需要检查什么 | 本轮状态 | -| --- | --- | --- | -| baseline 对 baseline（A/A） | 同码波动是否让实验错误判断为改进 | 未运行 | -| baseline 对 expert patch | 已知优化在当前输入和机器上是否仍有稳定信号，并通过正确性检查 | 未运行 | -| 故意错误 patch | 省略 added-token 规则、错误 offsets 或复用前次结果会被拒绝 | 待编写、未运行 | -| Agent 候选 | 在前述资格建立后检查正确性与性能，不以代码改动或测试 PASS 代替优化成功 | 未运行 | - -[跨机器可靠性研究的数据](https://github.com/chenzhi-cz/performance-optimization-benchmark-reliability/blob/44a1cb86a9f765b0375b4cfb479641f3c19f0774/data/gso_cross_machine_valid_tasks.csv)包含此 task ID；这是作者报告的筛选线索，不是我们的本地资格证明。 - -## 怎样接入 InferRSI - -需要的是 **repo-patch adapter**：固定 baseline checkout，接收并留存 patch，构建候选，调用独立测试和测量脚本，产生 Workbench 可读结果。 -此 adapter **尚未实现**；现有 `numpy-axis1-v2` 与 `implementation.py` 入口不能直接承担它。 -优先复用现有 Workbench 的外部命令和证据机制；正式 SkillEval 的 patch 交接、执行隔离与报告复核需要另外适配，不将任意生成代码送入可信宿主 worker。 -Rust 工具链、历史依赖、容器及原生编译让它的接入成本高于当前 NumPy 题，但仍不要求 GPU；实际可重建性待验证。 - -## 借鉴与不采用的边界 - -- [SWE-fficiency TestSpec](https://github.com/swefficiency/swefficiency/blob/12d32a2d6800824a7d84bdb6797b5708e7b7957f/swefficiency/harness/test_spec.py)将 workload、guarding tests 和 profiling 分开，适合作为流程参考；不引入另一套排行榜总分。 -- GSO 的 `tokenizers-fc76ad4` 优化 UnigramTrainer.train，属于训练任务；不能因为仓库与推理相关就纳入本类别。 -- [ISO-Bench 固定数据说明](https://huggingface.co/datasets/ISO-Bench/ISO-Bench/blob/24ef5d04b4993aca5e0d2cfa64b51cb40fc4cb55/README.md)提供更直接的 vLLM/SGLang repo 任务，可作为后续引擎运行时题源。其数据声明 H100／多 GPU 条件，不能据 CPU 局部测试宣称复现。 -- 例如 vLLM [`22d33baca2c0c639cfd45c48e99803e56c3efa74`](https://github.com/vllm-project/vllm/commit/22d33baca2c0c639cfd45c48e99803e56c3efa74)优化单请求的 merge_async_iterators；本轮仅核对官方 diff 身份及实时数据的 H100 标记，未验证该题的性能与 lm-eval 命令。 - -完成标准是固定来源、可重建环境、可靠对照和真实消费者接入均有证据；本页只完成来源选择与设计。 diff --git a/docs/plans/issue-252/task-expansion/03-memory.md b/docs/plans/issue-252/task-expansion/03-memory.md deleted file mode 100644 index 69975a827..000000000 --- a/docs/plans/issue-252/task-expansion/03-memory.md +++ /dev/null @@ -1,64 +0,0 @@ -# 内存与 KV 管理：先复用回收任务，再增加分配任务 - -**任务例子：给定请求结束、保存完成和写入完成事件，Agent 改进待回收队列，既不能提前释放块，也不能遗漏已经可以释放的块。** - -当前状态：设计候选；本次只调查和说明来源，没有新增执行、硬件资格或收益证据。 - -## 放在现有 SearchSpace 的哪里 - -引用既有 [SearchSpace Matrix](../../../searchspace-matrix.md) 的资源与生命周期说明：队列元数据和 CPU 管理开销属于 host 侧；实际设备 KV 容量属于相应加速器资源。这里不创建另一份权威分类，也不因加入任务而提升矩阵成熟度。 - -先区分三个问题：回收逻辑是否正确、管理操作是否更快、真实模型的显存/容量是否改善。CPU 任务只能直接回答前两者；第三个问题需要目标设备和模型实验。 - -## 已有基础与固定来源 - -| 来源 | 可复用内容与身份 | -| --- | --- | -| InferRSI 既有 KV 任务 | [资格说明](../../../kv-reclamation-qualification.md)、[可见契约](../../../../evidence/kv-reclamation-20261030/validator/independent/CONTRACT.md)和[原始证据](../../../../evidence/kv-reclamation-20261030/README.md)；属于既有 #248 工作，不算本次新造题 | -| vLLM | commit `af261bab7f717b6ece30f5ed3213c43974e4a6c4`；[Apache-2.0](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/LICENSE)；[BlockPool](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/vllm/v1/core/block_pool.py)、[延迟回收测试](https://github.com/vllm-project/vllm/blob/af261bab7f717b6ece30f5ed3213c43974e4a6c4/tests/v1/core/test_deferred_block_free.py)提供后续真实语义参考 | -| Mooncake | commit `e5ef0e427a49cd2f8ad5b4356b09435095030439`；[Apache-2.0](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/LICENSE-APACHE)；[分配 benchmark](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/README.md)与[正确性测试](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/tests/offset_allocator_test.cpp)是下一道实际新题候选 | - -既有 KV verifier 的冻结文件 SHA-256 为 `163c21134a8155da2a6469d9dbddb6ea71cb162e14f53e1c4266fe8ce75126f7`。任务、工作负载、补充 oracle 和数值计划各有原记录；复用时逐项检查身份，不能重新解释或改写冻结字节。 - -历史本地隔离不可用，留存结果为 **INCONCLUSIVE**。可信控制直接导入通过与隔离回放是不同证据；之后的新环境结果要另存，不能覆盖旧结果或倒推旧环境通过。 - -## 第一阶段：复用既有合成 KV 回收题 - -- Agent 只修改 `candidate.py` 的 `Reclaimer`；输入事件、oracle、grader 和工作负载受保护。 -- 实现 `apply(event)` 和 `pending()`，保留事件结构校验、请求 generation、重复通知幂等性、无效输入原子性及确定的返回顺序。 -- 回收要求 finish、所需 save_complete 和 processed frontier 三项条件同时满足；允许优化查找与排队数据结构，不允许跳过事件或缓存预定答案。 -- **既有题明确不含共享块、引用计数、前缀匹配、取消、超时、真实线程/异步执行、字节内容或资源容量策略。** 不能借 vLLM 来源把这些写成已覆盖功能。 -- 它是围绕特定知识机制构造的合成近迁移任务，不是历史缺陷复现、自然抽样 holdout，也不是 vLLM 兼容性测试。 - -正确性沿用独立 temporal oracle，逐事件核对回收列表和 pending 状态；原有非法输入、generation 重用、提前释放等控制保持原身份。 - -现有计时是宿主测得的**整条 CPU 任务命令耗时**，包括解释器和 sandbox 启动；不能改称核心队列延迟或显存收益。若另测核心操作，应建立新版本边界和独立资格记录。 - -Profiling 单独分析可信控制的扫描、索引和队列开销，不混入计时，不推断设备 timeline。真实生产者和不可信产物的权限边界仍按原入口处理。 - -## 第二阶段：Mooncake 混合尺寸分配题 - -任务例子：固定内存池容量和 allocate/free 请求序列，改进分配器，使分配更快或更少因碎片失败，同时保持请求完成量和正确的容量不足语义，不发生越界或重叠。 - -- 借鉴 `allocation_strategy_bench` 的 `size_class_churn`、容量预填充和尺寸分布；Agent 修改 allocator/placement 数据结构，不能改请求序列、容量、统计或副本要求。 -- 正确性独立检查地址范围、活动分配不重叠、分配大小、释放恢复和容量守恒；不能靠拒绝困难请求或返回部分副本来获得速度优势。 -- 分别记录完整/部分/失败分配、碎片率、延迟分布和完成量；先定义质量/容量约束，再评价速度，不能只报最快的一项。 -- CPU profiling 查分配、锁和数据结构开销；实际 RSS、设备显存、RDMA 和端到端 KV 命中收益都是不同实验。 -- 上游 [CMake target](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/CMakeLists.txt)链接 `mooncake_store`、cachelib allocator、gflags/glog 等；尚未验证可独立构建，不能承诺零依赖或跨平台立即可跑。 - -## 怎样接入 InferRSI - -先复用 [现有 KV 资格入口](../../../../tools/qualify_kv_reclamation.py)及其独立 oracle/隔离回放，不新建另一套 KV runner。它不是当前 `numpy-axis1-v2` 的一种 operation，也未因列入本文就接上新的正式两臂 adapter。 - -Mooncake 待构建和测量资格明确后，用目标拥有的脚本输出结构化结果，接既有 RunSpec/Workbench；质量脚本和候选分开冻结。不能直接把不可信 C++/Python 交给可信执行入口。 - -若比较 skill/Agent，另固定模型、预算和任务条件，保留两边真实产物及全部尝试；已有合成控制不能证明 skill 收益。 - -## 最小资格实验与采用理由 - -1. KV：核对原冻结身份，在合格 Linux 隔离环境执行既有正/负控制与同码对照，保存新的完整 receipt；历史 INCONCLUSIVE 保持原样。 -2. Mooncake：先证明最小 target 可构建，再固定小规模混合分配 trace，用简单独立区间模型拒绝越界、重叠、漏释放和部分分配伪成功。 -3. 性能：固定容量、预填充、请求量和环境，先检验同码对照；正确但测量不可分辨时保持证据不足，不选速度赢家。 -4. 设备外推：只有实际模型/设备上的容量、质量和性能验收，才能补充显存或服务结论。 - -采用顺序是“复用已有 KV → 资格验证 Mooncake 新题”，因为两者有具体可改对象和明确正确性义务。暂不直接引入完整分布式 KV 服务：部署、网络、模型和存储因素太多，难以在硬件未定时隔离单次改动。 diff --git a/docs/plans/issue-252/task-expansion/04-communication.md b/docs/plans/issue-252/task-expansion/04-communication.md deleted file mode 100644 index dd96f3377..000000000 --- a/docs/plans/issue-252/task-expansion/04-communication.md +++ /dev/null @@ -1,69 +0,0 @@ -# 通信：复用真实测量底座，单独定义可优化实现 - -**任务例子：在固定多卡拓扑、消息大小和数据类型下，Agent 改进 AllReduce 实现或允许的配置，保持所有 rank 的归约结果正确，并减少真实通信耗时。** - -当前状态：设计候选；本次没有新增编译、通信执行或设备资格。硬件未确定，因此只确定来源、候选边界和验收方法，不发布性能结论。 - -## 放在现有 SearchSpace 的哪里 - -引用既有 [SearchSpace Matrix](../../../searchspace-matrix.md) 的 Communication/Network 与相应加速器资源说明，不建立新的权威分类。实际归属取决于瓶颈：同机 GPU 互连实验不能自动充当跨节点网络覆盖。 - -区分通信原语性能、通信/计算重叠和完整服务收益。前者可从单个 collective 开始；重叠必须检查依赖及端到端阶段耗时；服务收益仍需要真实工作负载验收。 - -## 固定来源与采用范围 - -| 来源 | 固定身份、材料与本次决定 | -| --- | --- | -| NVIDIA nccl-tests | commit `afd59abab774a6a4b492eb344bd0c845a0b7bb4d`；[BSD-3-Clause](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/LICENSE.txt)；[README](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/README.md)、[AllReduce](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/src/all_reduce.cu)。作为正确性与真实性能测量底座，不能把跑一次硬件测试等同于给 Agent 一道优化题 | -| MSCCL++ | commit `f5643de98b0c174baa3af2b2e6126f93dd07dc06`；[MIT](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/LICENSE)；[AllReduce candidate/预期结果](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/mscclpp-test/allreduce_test.cu)、[测试框架](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/README.md)。作为可修改通信算法、channel 与 pipeline 的候选来源 | -| MSCCL-tools | commit `030a750fc56e03fb863ffe48268672d84d4cdc7b`；[MIT](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/LICENSE)；[DSL 及 Check 测试](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/tests/test_language.py)。可借鉴通信计划接口和 CPU 符号检查，不作为无设备的性能替身 | - -方法参考：[MSCCLang 论文](https://arxiv.org/abs/2201.11840)解释拓扑与 collective DSL；[MSCCL++ 论文](https://arxiv.org/abs/2504.09014)解释细粒度通信抽象。这里采用接口设计，不复述作者数字为本地结果。 - -## Agent 可以改什么 - -首题在设备明确后从以下两种候选边界选一种，并在实验开始前固定，不能一次混改所有层： - -- **配置题：** 使用固定通信库/二进制，Agent 提交被允许的算法、协议或调度配置。允许项必须经实际版本核对，不能修改 rank 数、消息量、正确性检查或测量命令。 -- **实现题：** 从 MSCCL++ 的指定实现出发，修改通信算法或 pipeline；baseline、输入、预期结果生成和报告器保持受保护。 - -不可用空操作、跳过 rank、减少有效数据、关闭校验或只提交“带宽提高”的文字作为结果。更换通信库、拓扑、数据布局或精度会改变实验条件，需显式新任务/比较身份。 - -## 正确性、测量与 profiling - -正确性需要所有参与 rank 完成，返回与受信参考相符的数据；覆盖不同消息尺寸、边界/非整块输入，以及该实现声明支持的 in-place/out-of-place 模式和数据类型。 - -nccl-tests 有独立校验路径与 `--check`；冻结时不得用零次检查获得合格结果。MSCCL++ AllReduce 测试在 host 生成 expected buffer，可作为适配来源；不能把候选自身返回的通过标记当 oracle。 - -设备测量固定进程/rank 映射、拓扑、软件版本、消息大小、dtype、warmup、重复次数、同步/graph 模式和计时边界。保留原始各次测量及失败，不挑最好一次。 - -报告 latency、algbw、busbw 时保留各自定义；busbw 是 collective 相关归一化，不是任意多 rank 带宽简单相加。CPU launch 时间与设备完成时间不能混用。 - -nccl-tests 此固定版本支持 JSON 和每次 CUDA event 计时；per-iteration 模式与 graph capture 的约束需按原文保留。上游可输出样本不等于我们已验证测量稳定性。 - -Profiling 单独查看各 rank 时间线、同步等待和暴露的通信时间。重叠实验应保留依赖、数据一致性及总阶段耗时，不能把各 stream 时长相加后声称加速。 - -## 无硬件时可以做什么 - -MSCCL-tools 的 CPU `Check()` 根据 chunk 来源和归约成员检查计划语义，可验证 AllGather/AllReduce 调度是否完成规定操作。 - -它不证明真实数据类型数值误差、GPU 执行、跨进程同步或链路性能。计划步数、传输量或模型估计必须标注为模型指标，不能换算成已实测 latency/bandwidth。 - -nccl-tests 的 CUDA/NCCL 运行以及 MSCCL++ 的 CUDA/ROCm 路径均需实际环境；多进程/跨节点场景还涉及 MPI、设备和网络。未具备这些条件时保留“未执行”，不提供 CPU fallback 性能分数。 - -## 怎样接入现有 InferRSI - -使用目标拥有的构建、启动和解析脚本，把正确性命令与 baseline/candidate 测量命令接入既有 RunSpec/Workbench。保留进程退出状态、超时、原始输出、二进制/源码/配置身份及环境记录。 - -通用 Workbench 不会替代多 rank 启停、远端部署、拓扑资格或输出质量检查；这些须由目标脚本明确承担。现有 `numpy-axis1-v2` 和正式内置题 adapter 也不能直接执行通信题。 - -若之后接 SkillEval，两臂使用同一题目和测量规则，保留实际候选及完整成本；设备产物的执行隔离与生产 Agent 权限分别验证，不沿用 CPU sandbox 通过来推断 GPU 隔离。 - -## 最小资格实验与选择理由 - -1. 环境就绪后，运行固定 baseline 及已知正确实现，核对全部 rank、输入/输出和测量边界。 -2. 以漏 rank、错误归约、遗漏尾部数据和缺失输出为负控制，确认独立验收拒绝；超时或设备不可用记证据不足。 -3. 先跑同码/同配置对照，检验噪声和重复性；再比较候选，留存失败和原始样本，不以通用阈值代替具体实验约定。 -4. 重叠与服务收益分别做额外实验，不能由单个 collective 的数字推导。 - -优先采用 nccl-tests 的成熟测量和 MSCCL++ 的明确可改对象，避免重建通信平台。暂不把旧 MSCCL runtime 整体移植为默认依赖，也不在硬件未定时同时实现 CUDA、ROCm、NPU 三条执行链。 diff --git a/docs/plans/issue-252/task-expansion/05-serving.md b/docs/plans/issue-252/task-expansion/05-serving.md deleted file mode 100644 index 57f3145ee..000000000 --- a/docs/plans/issue-252/task-expansion/05-serving.md +++ /dev/null @@ -1,78 +0,0 @@ -# 第五类候选：服务配置与运行时优化 - -**首选任务：固定模型、精度、请求流和硬件资源，让 Agent 优化服务配置，在质量约束下改善服务指标。** -本页是 2026-11-04 的源码调研与任务设计，尚未适配或执行；硬件未定，不报告性能收益或 Agent 能力提升。 -它考察服务配置、批处理和运行时行为，与修改单个 kernel 的题目分开；采用时沿用已有 Workbench，不复制另一套 runner。 - -## 任务要解决什么问题 - -用户给定模型与典型请求流，原配置可能让长输入响应慢、长输出生成慢，或高并发时排队严重。 -Agent 先运行基线和 profiling，再调整声明允许的配置，提交可复现的 launcher；工具独立验收质量和服务性能。 -任务允许直接使用目标项目的 benchmark，不要求真实用户的模型先进入固定题库。 - -| 项目 | 拟定规则 | -| --- | --- | -| 首个场景 | 从长输入、长输出、高负载中选一个，以实际模型与可用环境决定，不一次覆盖全部 | -| 固定内容 | 模型与 tokenizer 身份、权重/精度、输入与到达序列、输出长度规则、质量数据、资源和评测版本 | -| Agent 产物 | 首选 `start_server.sh` 或等价 launcher；若明确允许修改运行时，则单列提交 repo patch | -| 可改范围 | 经任务声明的 batching、prefill、并发或缓存配置；配置题和源码策略题分开比较 | -| 不可改内容 | 受信评测器、请求工作量、质量标准和资源额度；不能少处理请求、截短输出或换模型来获得分数 | -| 基线 | 固定版本服务引擎的明确配置；必须经过相同质量检查和测量流程 | -| 主指标 | 按目标选择 TTFT、decode 吞吐或满足 SLO 的 goodput；方向、单位和阈值由该实验冻结 | -| 辅助指标 | 错误/超时率、完成请求数、实际输出长度、ITL/TPOT、尾延迟、资源使用及原始逐请求记录 | - -## 首选参考：InferenceBench - -固定源码为 `aisa-group/InferenceBench` @ `108ec747fab0bd92fd9de501f62157d074c50ed5`，Apache-2.0。 -[README](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/README.md) 明确：Agent 在固定时间预算内写 launcher，最终在 fresh container 测量，并与 PyTorch 参考比较答案质量。 -它区分输入密集、输出密集、高负载和综合场景，还提供相同时间预算的 random/TPE/SMAC 配置搜索基线。 -这比仅测算子速度更接近 InferRSI 要支持的服务目标；采用的是任务设计和验收分工。 - -本页首版固定模型精度、缩小可调配置，是拟定的派生任务边界。上游允许更开放的框架、量化和运行时选择；以后扩大这些范围时，需要相应质量与资源验收，不能把本页限制说成上游规则。 - -- [场景 A 定义](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/tasks/inference_scenario_a_input_heavy/scenario.json)：输入/输出长度、请求数、并发和负载模式均显式声明。 -- [工作区规则](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/tasks/_shared/workspace_conventions.txt)：launcher 可编辑，测试入口与评测产物由 benchmark 持有。 -- [质量检查](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/inference/quality_gate.py)：核对质量样本完整性、参考准确率与声明门槛；答案质量不是耗时指标的副产品。 -- [运行入口](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/src/eval/inference/runner.py)：分别保留速度、质量及执行完整性结果；未合格结果不能作为有效改进。 - -原生环境需要 H100 80 GB 或 A100、模型和数据缓存、Apptainer；完整流程还有 Claude 裁判审查捷径。 -不把这些平台依赖、付费裁判或论文默认阈值整体搬入 InferRSI。模型权重与数据许可也须另行核对。 -development/evaluation seed 分开可减少调参泄漏，但同一公开任务换 seed 不自动形成独立能力验证集。 - -## 备用与辅助来源 - -| 来源、固定版本及许可 | 具体借鉴 | 不能据此声称什么 | -| --- | --- | --- | -| [llama.cpp / llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) @ `149f5641b931aaf33697d70f9abc90c3e5300247`；MIT | 硬件未定时的 CPU 备用：固定模型，调整 threads、batch、ubatch；复用 warmup、重复样本及 JSON 输出 | 性能 benchmark 自身不提供完整质量 oracle；采用前须补输出/logit 或任务质量验收，CPU 结果不代表加速器 | -| [vllm-evolve](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/README.md) @ `43cc82c2c649699fb166339f85de2c8f79f2db4e`；[LICENSE 正文 Apache-2.0](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/LICENSE) | 窄策略题、SLO goodput、多 seed 比较，以及候选策略实际被加载/调用的检查 | 当前原生后端是 vLLM；simulation、更多 KV/routing 目标仍是 roadmap，不能当成现成通用模拟器 | -| [NVIDIA/SWE-Serve](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/README.md) @ `84d8fd92739a4a76ceddba56e5a70605921c4ad2`；Apache-2.0 | 生产工程任务、独立 verifier、参考修复/no-op 控制；可辅助检验运行时状态和容量逻辑 | 53 题以工程正确性为目标；任务名称含 perf 不代表有速度评分，不能替代本类的性能验收 | - -vllm-evolve 的 [plugin_template.py](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/targets/scheduling/plugin_template.py) 主要按策略结果重排 FCFS waiting deque；priority heap 会跳过。 -`invoked` 只证明调用，`reordered` 才表明 admission 顺序改变；两者都不直接证明收益。其 [evaluator](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/src/vllm_evolve/bench/evaluator.py) 拒绝无法证明插件被调用的性能结果。 -SWE-Serve 的 [adaptive-allocation-safety 评分器](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/tests/score.py) 仅在全部 F2P/P2P 节点通过后给 1 分，无速度评分；CPU 任务也可能要求较大内存和容器空间。 - -## 与现有 RSI 的接入流程 - -1. 在目标环境固定任务、模型、输入、质量标准和资源；准备可审核的服务启动、停止、质量及测量脚本。 -2. 运行基线并独立 profiling：保留队列、prefill/decode 分段或设备 trace，说明所改配置针对哪个观测瓶颈。 -3. Agent 提交 launcher 或声明允许的 patch；记录真实产物身份和全部尝试，不接受文字自报成绩。 -4. 受信入口确认产物实际生效，执行质量检查，再组织正式性能比较；profiling 开销不混入计时。 -5. 将 baseline/candidate/validation 脚本接到已有 Workbench，保留样本和报告；主 Agent 引用证据决定下一步。 - -上述是待适配流程，不能据此声称现有通用 runner 自动先验质量再计时；它当前在测量后执行独立 validation。 -质量预检应显式接入受信脚本；最终质量门槛仍需保留。服务生命周期由审核后的目标脚本处理，不另建执行平台。 -独立执行 Agent 产物的环境与权限仍需资格验证，当前 NumPy 产物 sandbox 不能直接承担任意服务源码。 - -## 必须完成的验证与当前状态 - -| 验证 | 需要保留的证据 | -| --- | --- | -| 质量控制 | 正常基线通过；损坏输出、漏请求、超时或实际输出长度不足被拒绝；基线/参考缺失保持证据不足 | -| 生效检查 | 实际进程、引擎/模型/配置身份与候选匹配；改变 launcher 却没有生效的情况不能记作候选收益 | -| no-op 对照 | 同代码同配置重复比较，检查漂移与噪声；不能因一次测量差异就宣称改进 | -| treatment 对照 | 预先声明一个会改变执行行为的配置，确认行为变化，并报告性能结果；不预设该配置一定更快 | -| 结果使用 | 一份从 profile、假设、产物、质量、计时到主 Agent 决定的完整记录，保留负结果和不确定性 | -| 能力评测 | 另行固定 Agent/skill 条件、预算与任务集；单个 launcher 变快不能直接归因于 skill 或搜索策略 | - -目前完成源码、许可与接口设计调研；未选定模型、硬件、质量数据或性能门槛，未新增 adapter，未运行上述控制。 -下一实施顺序：确定一个可用环境和场景 → 资格验证基线与质量检查 → 接已有 Workbench → 运行上述控制 → 再做真实 Agent 对照。 diff --git a/docs/plans/issue-252/task-expansion/README.md b/docs/plans/issue-252/task-expansion/README.md deleted file mode 100644 index 886ec031e..000000000 --- a/docs/plans/issue-252/task-expansion/README.md +++ /dev/null @@ -1,93 +0,0 @@ -# 按优化方向扩展任务：开源调研与首批选择 - -2026-11-04。本页记录官方源码、任务文件、许可证及论文的调研设计，来源检查不等于执行结果。用户随后明确指定 NPU 试点，当前实施顺序已按下文调整；实际设备证据另见 [NPU 试点](../npu-pilot.md)。以下五类是阅读分组，映射到已有 [SearchSpace](../../../searchspace-matrix.md)，不另建权威分类，也不把“找到来源”记成实测覆盖。 - -## 本轮决定 - -**先在真实 NPU 上跑通 affine RMSNorm 的基线、profiling、候选、独立检查与 Workbench 比较。** CPU Softmax 保留作软件控制；CPU RMSNorm 题包已移除，旧实验只保留历史证据。Tokenizers、KV、通信与服务来源保留为后续扩展。每类先完成一个任务的执行资格，再扩充数量。 - -AgentKernelArena 适合学习 kernel 任务的提交与独立评测；整个 InferRSI 还需要仓库补丁、状态管理、资源配置和服务工作负载。不能把所有任务都变成 `compute(x)`，也不能把功能修复的通过率叫作性能提升。 - -| 类别与任务卡 | 给 Agent 的具体问题 | 优先来源 | 当前状态 / 次序 | -| --- | --- | --- | --- | -| [① 算子计算](01-operators.md) | 在 NPU 上保持 affine RMSNorm 结果并比较基础运算与融合实现 | torch_npu 的实际算子；KernelBench、AgentKernelArena、BackendBench 的评测设计 | **当前试点**；真实设备正确性、profile、同步计时与 A/A 均需验收 | -| [② CPU 热路径与运行时](02-host-runtime.md) | 大量 added tokens 时，保持分词结果不变，降低编码开销 | GSO 的 Tokenizers 历史优化任务；SWE-fficiency 的任务契约 | 后续 CPU 仓库补丁候选，不再作为当前试点；adapter 和历史环境尚未实现 | -| [③ 内存与缓存](03-memory.md) | 正确处理异步完成与 KV 回收；随后优化混合尺寸分配的容量和开销 | 已有 RSI KV 任务、vLLM、Mooncake | 本轮 NPU 通路之后优先评估复用已有任务，保留原资格边界；Mooncake 为独立新题候选 | -| [④ 多卡通信](04-communication.md) | 固定拓扑/消息量下优化 collective 实现或配置 | nccl-tests、MSCCL-tools、MSCCL++ 的任务与测量设计 | 后续需要 NPU 对应通信机制及实际拓扑；CUDA 实现不能直接作为 Ascend 任务 | -| [⑤ 服务配置与调度](05-serving.md) | 固定模型、请求和质量要求下，优化首 token 延迟或有效吞吐 | InferenceBench、llama.cpp、vllm-evolve；Apple RSI-Inference Bench 作引擎扩展参考 | NPU 算子试点后另定模型、服务和质量裁判；已有 GPU/CPU 题源不自动支持 NPU | - -此前提出的“算子、CPU 分词、状态回收”扩展方向继续保留，执行次序已调整为 NPU 优先。状态回收是既有工作，接入不算新造题；RMSNorm 设备试点完成也不代表内存容量、多卡通信或服务端到端已覆盖。 - -## 哪些开源项目更适合借鉴 - -以下采用决定由本次调研提出，不表示维护者已接受扩展范围。固定版本链接用于复查源码；具体任务卡保留更多测试、指标和适配条件。 - -| 来源及已检查内容 | 固定版本 / 许可 | 采用什么；为什么不整体搬入 | -| --- | --- | --- | -| [GSO：任务与实际 grading](https://github.com/gso-bench/gso/blob/32d78b331d59ed689b1238789dc20d36576951c0/src/gso/harness/grading/evalscript.py) | `32d78b3`；代码 MIT，目标 Tokenizers 为 Apache-2.0 | 借真实仓库 base / expert / candidate 三方、输出等价检查与性能 workload；选择推理相关 encode 任务。历史工具链、数据资产与测量资格须重新验证 | -| [SWE-fficiency：TestSpec](https://github.com/swefficiency/swefficiency/blob/12d32a2d6800824a7d84bdb6797b5708e7b7957f/swefficiency/harness/test_spec.py) | `12d32a2`；代码 Apache-2.0，README 声明其他材料 CC-BY-4.0 | 借 correctness tests、performance workload、独立 profiling script 的分工。通用科学 Python 任务不全部适合推理，不导入整套 runner 或总分 | -| [SWE-Serve：adaptive allocation](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/task.toml) 与 [score](https://github.com/NVIDIA/swe-serve/blob/84d8fd92739a4a76ceddba56e5a70605921c4ad2/tasks/ss-sgl-adaptive-allocation-safety/tests/score.py) | `84d8fd9`；Apache-2.0 | 借真实 SGLang 状态/容量修复题与 F2P/P2P。所读题为 correctness 评分，没有速度分；“CPU 任务”仍可能要求较大容器资源 | -| [ISO-Bench](https://github.com/ISO-Bench/ISO-Bench/blob/9cc08ddd1011a34c49b43c22671ae639f16a21d3/README.md) 与 [固定数据卡](https://huggingface.co/datasets/ISO-Bench/ISO-Bench/blob/24ef5d04b4993aca5e0d2cfa64b51cb40fc4cb55/README.md) | 代码 `9cc08dd` Apache-2.0；数据卡 `24ef5d0` CC-BY-4.0 | vLLM/SGLang 性能提交可作为后续真实引擎补丁题源。论文的 54 题与所读数据卡的 106 题是不同版本；具体行来自实时接口，尚未绑定固定 parquet，部分题命令读取失败，不称为已可复现 | -| [Apple RSI-Inference Bench](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/README.md) | `1a38072`；[Apple Sample Code License](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/LICENSE)，不是 MIT/Apache；README 的论文仍为 coming soon | 更贴近整个推理系统：模型接入、实现优化、性能回归修复。优先借真实引擎任务设计；1–8 张 H100/B200、大模型与长执行预算不适合当前轻量首批 | -| [InferenceBench：任务与评测入口](https://github.com/aisa-group/InferenceBench/blob/108ec747fab0bd92fd9de501f62157d074c50ed5/README.md) | `108ec74`；Apache-2.0 | 借服务 launcher、不同负载、独立质量门槛、同预算非 Agent 搜索对照。完整原生流程有指定 GPU、数据缓存及模型裁判，不直接移植平台 | -| [Mooncake：分配 benchmark](https://github.com/kvcache-ai/Mooncake/blob/e5ef0e427a49cd2f8ad5b4356b09435095030439/mooncake-store/benchmarks/README.md) | `e5ef0e4`；Apache-2.0 | 借混合尺寸分配/释放 workload、分配失败与碎片指标，并绑定独立 allocator tests；先验证最小 C++ 构建，不把逻辑容量称为 HBM 节省 | -| [nccl-tests](https://github.com/NVIDIA/nccl-tests/blob/afd59abab774a6a4b492eb344bd0c845a0b7bb4d/README.md)、[MSCCL-tools](https://github.com/microsoft/msccl-tools/blob/030a750fc56e03fb863ffe48268672d84d4cdc7b/tests/test_language.py)、[MSCCL++](https://github.com/microsoft/mscclpp/blob/f5643de98b0c174baa3af2b2e6126f93dd07dc06/test/mscclpp-test/allreduce_test.cu) | `afd59ab` BSD-3-Clause；`030a750` / `f5643de` MIT | 分别提供真实测量、可检查的通信计划、自定义 collective。需要明确候选修改面；硬件测试工具本身不是 Agent 优化题 | -| [llama.cpp：llama-bench](https://github.com/ggml-org/llama.cpp/blob/149f5641b931aaf33697d70f9abc90c3e5300247/tools/llama-bench/README.md) | `149f564`；MIT | 借 CPU prefill/decode、线程/batch 配置与逐次样本。固定模型资产后可作为后续 CPU 服务任务来源；计时工具不自动提供质量 oracle | -| [vllm-evolve：plugin](https://github.com/JiusiServe/vllm-evolve/blob/43cc82c2c649699fb166339f85de2c8f79f2db4e/targets/scheduling/plugin_template.py) | `43cc82c`；实际 LICENSE 为 Apache-2.0 | 借窄 admission 策略入口与实际生效证明。当前主要重排 FCFS waiting deque；不能按 roadmap 声称已有 KV eviction/routing 或模拟平台 | -| [AutoTuneBench](https://github.com/li-ch/autotunebench/blob/64cc8d7ca5d0ee40c4e2189b2c2c8dfb1a2a7c2f/README.md) 与仓内 [已核查参考](../../../../src/inferrsi/data/skill_library/methods/inference-benchmark-design/references/autotunebench-trial-validity.md) | `64cc8d7`；选定代码/数据 MIT，vendor 许可分开 | 借 treatment 是否真正送达、原始记录与资格先于收益的审计方式。既有审查已发现声明与部分执行路径不一致，不把 README 的完整流程当作已验证实现 | - -不优先使用 PIE/MLGym 扩充本轮题数：前者以竞赛程序与模拟 CPU 为主，后者主要优化 ML 质量/奖励，且许可与训练成本另有约束。它们的任务组织经验有价值，但不是当前推理工程性能题的直接替代。 - -## 两个尤其贴近 RSI 的题例 - -**真实仓库热路径：Tokenizers.encode。** 固定原始仓库和 tokenizer 资产，Agent 用 profile 找大量 added tokens 的开销，修改 Rust/Python 代码，提交 patch；裁判检查 token IDs、tokens、offsets 等语义，再比较编码时间。它直接验证 CPU 分词优化，不直接证明服务 TTFT。具体 base/expert 身份及资产冻结缺口见[任务卡](02-host-runtime.md)。 - -**真实引擎回归：SGLang 流式 decode 吞吐下降。** Apple 的 [instruction](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/regression_sglang_stream_decode_tps_c1/instruction.md) 与 [task.toml](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/regression_sglang_stream_decode_tps_c1/task.toml) 给出真实 baseline、流式 workload、输出保持、scored/guard 条件；并允许不同修复机制。这比指定“必须写某个 kernel”更接近 RSI 的瓶颈调查。但该题固定 4×B200 和大模型；其环境清理脚本有枚举 GPU 进程的操作，不能直接移植到共享机器。当前只借设计、保留来源，未执行或复制脚本。 - -Apple 的另一个 [vLLM 源码优化题](https://github.com/apple-aiml-research/ml-rsi-inference-bench/blob/1a38072fbe284234c874356b86c9df666a5a79a3/tasks/optimize_qwen3_6_27b_1xh100_open/instruction.md) 允许热路径、布局、融合等多种改法，但不计纯配置修改；InferenceBench 则允许更开放的服务配置选择。两者不是谁更“完整”，而是考察不同搜索范围。InferRSI 应分别声明代码任务和配置任务，避免同一排行榜混用规则。 - -## 论文怎样影响选择 - -| 一手来源 | 本次采用的结论 | 不由此推导什么 | -| --- | --- | --- | -| [GSO §2.1–2.2](https://arxiv.org/html/2505.23671v3#S2) | 从真实优化提交提取 base、workload、等价检查和 expert control；Agent 提交真实 patch | 历史提交更快不等于我们的环境更快；参考答案不是唯一合法算法 | -| [ISO-Bench](https://arxiv.org/abs/2602.19594) | 从真实推理引擎的优化 PR 中挑代表任务；执行指标与诊断/机制分析分别保留 | LLM 对补丁的相似性/意图评分不能替代数值、质量及性能实测；不要求 Agent 复制人类补丁 | -| [InferenceBench §3、§5](https://arxiv.org/html/2607.20468v1#S3) | 分开 prefill、decode、并发目标；固定质量标准；后续搜索实验加入同预算 random/TPE/SMAC 等对照，检查最终提交是否保留最好有效候选 | 论文中 Agent 或搜索基线的排名不能迁移到 InferRSI；本轮没有执行该实验 | -| [性能 benchmark 可靠性研究 §III–IV](https://arxiv.org/html/2607.01211v2#S3) | 在接入任务前重放参考优化、做同码对照，分开任务有效性与 Agent 成绩；报告分项结果，不急于做一个总分 | 作者跨机结果不代替我们的硬件资格；“能运行”不等于性能信号稳定 | - -软件测试、公开论文、源码检查、真实运行证据分别记录。上述论文/仓库的收益数值没有被用作本次本地结果或默认阈值。 - -## 对接口和流程的具体影响 - -不同类别共享“任务/环境/验收规则固定 → baseline → 候选 → 独立检查 → 比较 → 结果与决定”的过程，profiling 按需使用，候选形式不能只支持一种： - -| 候选形式 | 对应题型 | 与当前实现的关系 | -| --- | --- | --- | -| 一个受限实现文件 | 当前 NPU RMSNorm；已有 CPU 算子和 KV 的 `candidate.py` | NPU 需要自身设备、权重、oracle 与同步协议；NumPy / KV 既有资格不迁移 | -| 固定仓库上的 patch | Tokenizers、Mooncake、引擎回归 | **待实现 repo-patch adapter**；独立构建、受保护测试、完整产物和对应 baseline；不让模型改裁判 | -| 声明的配置或 launcher | llama.cpp / 服务配置 | **待实现对应任务适配**；锁定模型、请求量和设备，检查实际启动/生效的配置及最终服务器 | -| collective 算法或候选库 | 多卡通信 | NPU 方向已定，具体机制/拓扑与多进程适配待研究；符号验证、编译、数据校验、计时分别记录 | - -复用现有 Workbench 的执行、AB/BA 和证据，不把 GSO、Pier、Harbor、InferenceBench 的控制器全部引入。上游 task 包是素材和契约来源；任何新执行环境或多文件候选都需要其自身的产物隔离与验收，现有单文件 sandbox 不能自动覆盖仓库 build/server。 - -按需调用 profiling 时，将诊断与正式计时分开，并保留原始产物。若以后评估“给 Agent profiler 反馈是否有用”，还应保留反馈进入下一次 Agent 输入的证据；生成了一份 profile 本身不足以证明它发挥了作用。 - -## 新任务入库前怎样验证有用 - -1. **来源与可重现性：** 固定仓库、base、参考修复、数据/模型、编译环境和许可证；公开历史参考修复不提供给 Agent。公开题只能作为 development 或受限条件下评测，不能冒充天然未见 holdout。 -2. **裁判区分度：** 正确基线、参考优化、错误但快的变体都实际运行；错误结果不得获得可用于选择的性能分数。 -3. **测量资格：** 同码 A/A 和 no-op 不应被当作进步；参考优化在选定环境确有可分辨信号。保留失败、波动与全部原始样本，无法分辨就暂不用于性能排名。 -4. **目标覆盖：** 计时工作负载、非计时语义边界、guard 工作负载分开说明。不能缩短输入、少处理请求或减少分配来换取虚假加速。 -5. **真实接入：** 源码/编译产物/配置确实进入最终执行，结果可追溯；调用了 profile 则保留诊断。用一条演练证明报告支持“保留/放弃/证据不足”的决定。 -6. **能力评测另做：** 任务资格成立后，固定模型与预算做 Agent/skill/搜索策略对照；记录失败成本、有效尝试数、最终质量和性能。不把一个新任务的通过当成整个 RSI 变强。 - -每个方向分开报告“找到来源、已适配、正确性验证、性能资格、Agent 对照”的状态。第一版不将内存、延迟、带宽或功能通过率压成一个未经论证的总分。 - -## 与两项工作的关系及本轮产物 - -- **#336 任务库：** 承接原 #334 的选题与复用研究；首题 `rmsnorm-npu` 已有工具实测，其他类别按需求逐批研究、实现和验收，原 CPU 两题保留历史身份。 -- **#337 流程接入：** 承接原 #335 的共享接口工作，将实际 Agent 产物接入既有评测并关联消耗；外部项目不必先入题库。正式 SkillEval 仍仅支持它已声明的 CPU 产物，NPU 执行边界尚待解决。 - -repo-patch / launcher / 多进程等形式仍需具体适配；原 #334/#335 关闭表示范围迁移，不表示已经实现这些能力。当前分工与首轮两 PR 安排见[总览](../README.md)。 - -本页保留可复查的来源选择与任务设计，不改写历史证据或 SearchSpace 成熟度。当前 NPU 实施与实测沿 [试点文档](../npu-pilot.md)及原 Issue 跟踪；本页列出的后续来源不因此变成已能运行的新类别。 diff --git a/docs/plans/issue-252/task-expansion/gso-source-audit.json b/docs/plans/issue-252/task-expansion/gso-source-audit.json deleted file mode 100644 index f40e4e18f..000000000 --- a/docs/plans/issue-252/task-expansion/gso-source-audit.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "kind": "source-inspection-only", - "inspected_on": "2026-11-04", - "dataset": "gso-bench/gso", - "dataset_revision": "c2e4f1a58427cccd15e0e542f136bd204fb19284", - "dataset_file": "data/test-00000-of-00001.parquet", - "download_url": "https://huggingface.co/datasets/gso-bench/gso/resolve/c2e4f1a58427cccd15e0e542f136bd204fb19284/data/test-00000-of-00001.parquet", - "dataset_file_bytes": 3843463, - "dataset_file_sha256": "bda458a7b5437c252f6cefdbc896f5f2868de51479e7a221ceda3f3ab74879bc", - "dataset_rows": 102, - "selector": {"instance_id": "huggingface__tokenizers-076319d"}, - "matching_rows": 1, - "selected_row_sha256": "152c078497a9da3e93878655942aa7b0535c39daae3586f800e0b7de02fa3b5f", - "selected_row_hash_encoding": "UTF-8 json.dumps(row, sort_keys=True, ensure_ascii=False, separators=(',', ':')), without trailing newline", - "reader": "pyarrow 20.0.0; parquet read_table(...).to_pylist()", - "task": { - "repo": "huggingface/tokenizers", - "api": "Tokenizer.encode", - "base_commit_expression": "076319d542f87e9f72db379d7e47b29b1ce7bd87^", - "resolved_base_commit": "fb837b4adb130dfc89104aa09b1ae223da120278", - "expert_commit": "076319d542f87e9f72db379d7e47b29b1ce7bd87", - "test_count": 4, - "declared_arch": "x86_64" - }, - "license": { - "dataset_declared": "MIT", - "dataset_card": "https://huggingface.co/datasets/gso-bench/gso/blob/c2e4f1a58427cccd15e0e542f136bd204fb19284/README.md", - "target_code": "Apache-2.0", - "target_license": "https://github.com/huggingface/tokenizers/blob/076319d542f87e9f72db379d7e47b29b1ce7bd87/LICENSE" - }, - "remaining": [ - "Freeze GPT-2 and BERT tokenizer assets currently fetched from resolve/main", - "Freeze the Gutenberg text assets and review redistribution scope", - "Reconstruct and verify the historical build environment instead of executing setup commands blindly", - "Implement the repository-patch adapter and its execution boundary", - "Run baseline/expert correctness, negative controls and same-code measurement qualification" - ], - "execution": { - "setup_commands_run": false, - "benchmark_run": false, - "model_run": false, - "performance_qualified": false - } -} diff --git a/docs/plans/issue-252/usefulness-profiles/einsum.txt b/docs/plans/issue-252/usefulness-profiles/einsum.txt deleted file mode 100644 index ee7a4a8f4..000000000 --- a/docs/plans/issue-252/usefulness-profiles/einsum.txt +++ /dev/null @@ -1,25 +0,0 @@ - 1260 function calls in 0.001 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 42 0.000 0.000 0.001 0.000 implementation.py:2(compute) - 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:515(expand_dims) - 42 0.000 0.000 0.000 0.000 einsumfunc.py:1244(einsum) - 42 0.000 0.000 0.000 0.000 {built-in method numpy._core._multiarray_umath.c_einsum} - 42 0.000 0.000 0.000 0.000 numeric.py:1431(normalize_axis_tuple) - 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:600() - 84 0.000 0.000 0.000 0.000 numeric.py:1480() - 210 0.000 0.000 0.000 0.000 einsumfunc.py:1236(_einsum_dispatcher) - 126 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} - 42 0.000 0.000 0.000 0.000 {method 'reshape' of 'numpy.ndarray' objects} - 84 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} - 126 0.000 0.000 0.000 0.000 {built-in method builtins.len} - 126 0.000 0.000 0.000 0.000 {built-in method builtins.next} - 42 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} - 42 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} - 42 0.000 0.000 0.000 0.000 _shape_base_impl.py:511(_expand_dims_dispatcher) - 42 0.000 0.000 0.000 0.000 {built-in method builtins.iter} - 42 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - - diff --git a/docs/plans/issue-252/usefulness-profiles/unchanged.txt b/docs/plans/issue-252/usefulness-profiles/unchanged.txt deleted file mode 100644 index 9053ff523..000000000 --- a/docs/plans/issue-252/usefulness-profiles/unchanged.txt +++ /dev/null @@ -1,19 +0,0 @@ - 630 function calls in 0.002 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 42 0.000 0.000 0.002 0.000 implementation.py:6(compute) - 42 0.000 0.000 0.001 0.000 fromnumeric.py:3699(mean) - 42 0.001 0.000 0.001 0.000 _methods.py:115(_mean) - 42 0.000 0.000 0.000 0.000 {method 'reduce' of 'numpy.ufunc' objects} - 42 0.000 0.000 0.000 0.000 _methods.py:73(_count_reduce_items) - 84 0.000 0.000 0.000 0.000 {built-in method time.perf_counter_ns} - 84 0.000 0.000 0.000 0.000 {built-in method builtins.isinstance} - 84 0.000 0.000 0.000 0.000 {built-in method builtins.issubclass} - 42 0.000 0.000 0.000 0.000 {built-in method numpy.lib.array_utils.normalize_axis_index} - 42 0.000 0.000 0.000 0.000 fromnumeric.py:3694(_mean_dispatcher) - 42 0.000 0.000 0.000 0.000 {built-in method numpy.asanyarray} - 42 0.000 0.000 0.000 0.000 {method 'disable' of '_lsprof.Profiler' objects} - - diff --git a/docs/plans/issue-252/usefulness-summary.json b/docs/plans/issue-252/usefulness-summary.json deleted file mode 100644 index c0ca64203..000000000 --- a/docs/plans/issue-252/usefulness-summary.json +++ /dev/null @@ -1,2134 +0,0 @@ -{ - "scope": "CPU engineering-use probe with trusted authored candidates", - "plan_sha256": "05fdde5a55f7420569fa85168c39b7873e6a4ed207d05940de5622ab9933b51e", - "script_sha256": "317285d0534f9935d06e1fc488ef35c56a76ab7cba2c2423e5a73ba09799ebc7", - "replay_helper_sha256": "6d7670a62203676de519e893ca5abfaa1d1de6fca94b2b9e136c30382f8959cd", - "smoke_config": { - "seed": 2026, - "shape": [ - 4, - 17, - 9, - 7 - ], - "scale": 4, - "dtype": "float32" - }, - "environment": { - "backend": "cpu", - "python": "3.11.18", - "numpy": "2.4.6", - "os": "Windows", - "machine": "AMD64" - }, - "checks": { - "valid_candidates_accepted": true, - "wrong_candidate_rejected": true, - "wrong_has_no_metric": true, - "profiles_complete": true, - "consumer_identities_match": true, - "actual_workloads_match": true, - "comparison_quality_passed": true, - "comparisons_complete": true - }, - "checks_succeeded": true, - "identities": { - "unchanged": { - "task_id": "rmsnorm-cpu", - "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", - "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", - "candidate_sha256": "c7bc1a3f3ec2e2396a0b2788d1f7b21c059341a7853effaf6f7131c81da292aa", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "einsum": { - "task_id": "rmsnorm-cpu", - "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", - "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", - "candidate_sha256": "8f29ea405865f4493775ca809e37c0dd7f054dbb2b203e6290975b2e9e41b471", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "no-eps": { - "task_id": "rmsnorm-cpu", - "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", - "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", - "candidate_sha256": "9b78a97381b4e5775af022934ea504f6eb6269915adcfb5ffc580fc68693f046", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - } - }, - "comparison": { - "repetitions": 6, - "max_minutes": 2.0, - "min_improvement_pct": 0.0 - }, - "workloads": [ - { - "id": "normal", - "input_sha256": "a931597557d07c29727b4746588ff9f2db5f5a23c67b7148e22bbadf536f6b68", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 4284, - 252, - 28, - 4 - ] - }, - { - "id": "strided", - "input_sha256": "5a1a5521d252443593cf93106a97ab37b39a3ef3d9acde5e50aa6aedc56f912a", - "shape": [ - 4, - 17, - 9, - 7 - ], - "dtype": "float32", - "strides": [ - 8568, - 504, - 56, - 8 - ] - }, - { - "id": "zeros", - "input_sha256": "16d0edc8b7ad7705b23a14058f366ff1c0dfa16a0ad14f741924c308754cf8d1", - "shape": [ - 2, - 3, - 5, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 140, - 28, - 4 - ] - }, - { - "id": "large", - "input_sha256": "4f3eac31387a99ca7259dd058b228f7e3c57d0d349131aa0b5ccb237ffb75660", - "shape": [ - 2, - 5, - 3, - 7 - ], - "dtype": "float32", - "strides": [ - 420, - 84, - 28, - 4 - ] - }, - { - "id": "tiny", - "input_sha256": "558d3471bf22a7b3cb4ef169090e15f3dde3440de6f695c85d04718a5f300cb5", - "shape": [ - 2, - 7, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 420, - 60, - 20, - 4 - ] - }, - { - "id": "singleton", - "input_sha256": "5159fde66d7906ff87e0aabde2557fa06e8294f024b0783ef97a2c23e408ba05", - "shape": [ - 2, - 1, - 3, - 5 - ], - "dtype": "float32", - "strides": [ - 60, - 60, - 20, - 4 - ] - } - ], - "profiles": { - "unchanged": { - "status": "completed", - "diagnostic": { - "kind": "cpu-cprofile", - "artifact": "profile.txt", - "performance_verdict": null, - "limitation": "Instrumented CPU calls; no accelerator timeline" - }, - "has_metric": false, - "artifact_sha256": "7a89e01154b3bb0c0be3e08964d6e5c5c2f6b801889a0cc0c5fdfbebf41f4e7b" - }, - "einsum": { - "status": "completed", - "diagnostic": { - "kind": "cpu-cprofile", - "artifact": "profile.txt", - "performance_verdict": null, - "limitation": "Instrumented CPU calls; no accelerator timeline" - }, - "has_metric": false, - "artifact_sha256": "bdbf5421222ac36d3ef2efd118d8cbd23b15a6b5d2db4beaaedfce1331c0826d" - } - }, - "candidates": { - "unchanged": { - "correctness": "PASS", - "skilleval": "PASS", - "failed_assertions": [] - }, - "einsum": { - "correctness": "PASS", - "skilleval": "PASS", - "failed_assertions": [] - }, - "no-eps": { - "correctness": "FAIL", - "skilleval": "FAIL", - "failed_assertions": [ - "normal.values", - "strided.values", - "zeros.finite", - "zeros.values", - "tiny.values", - "singleton.values" - ] - } - }, - "ordinary_smoke": { - "input_sha256": "0c13945f4562c85d8fd2d9cf450a9e02847b5d34d9dc58f1b54f073c3502a51f", - "candidates": { - "unchanged": { - "passed": true, - "assertions": { - "shape_dtype": true, - "finite": true, - "input_preserved": true, - "values": true - } - }, - "einsum": { - "passed": true, - "assertions": { - "shape_dtype": true, - "finite": true, - "input_preserved": true, - "values": true - } - }, - "no-eps": { - "passed": true, - "assertions": { - "shape_dtype": true, - "finite": true, - "input_preserved": true, - "values": true - } - } - } - }, - "comparisons": { - "unchanged": { - "verdict": "FAIL", - "baseline": { - "values": [ - 0.063, - 0.0484, - 0.046200000000000005, - 0.0459, - 0.0482, - 0.0496 - ], - "median": 0.048299999999999996, - "min": 0.0459, - "max": 0.063, - "range_pct_of_median": 35.40372670807453, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.1169, - 0.11330000000000001, - 0.0622, - 0.0455, - 0.0458, - 0.0832 - ], - "median": 0.0727, - "min": 0.0455, - "max": 0.1169, - "range_pct_of_median": 98.21182943603853, - "excluded_trials": 0 - }, - "improvement_pct": -50.51759834368531, - "statistical_significance": "not established", - "final_quality_passed": true, - "source_integrity": true, - "trials": [ - { - "side": "baseline", - "trial": "trial-001", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0219, - 0.0176, - 0.0151, - 0.0148, - 0.0138 - ], - "median_ms": 0.0151 - }, - { - "id": "strided", - "samples_ms": [ - 0.018, - 0.018, - 0.0175, - 0.0189, - 0.0179 - ], - "median_ms": 0.018 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0089, - 0.0084, - 0.0085, - 0.0086, - 0.0088 - ], - "median_ms": 0.0086 - }, - { - "id": "large", - "samples_ms": [ - 0.009, - 0.0084, - 0.007, - 0.0074, - 0.0081 - ], - "median_ms": 0.0081 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0075, - 0.0073, - 0.0071, - 0.007, - 0.007 - ], - "median_ms": 0.0071 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0065, - 0.0067, - 0.006, - 0.006, - 0.0061 - ], - "median_ms": 0.0061 - } - ], - "metrics": { - "suite_latency_ms": 0.063 - } - }, - { - "side": "baseline", - "trial": "trial-002", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0162, - 0.0141, - 0.0126, - 0.0121, - 0.0105 - ], - "median_ms": 0.0126 - }, - { - "id": "strided", - "samples_ms": [ - 0.0122, - 0.0122, - 0.0124, - 0.0128, - 0.0124 - ], - "median_ms": 0.0124 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0059, - 0.0064, - 0.0061, - 0.0064, - 0.0059 - ], - "median_ms": 0.0061 - }, - { - "id": "large", - "samples_ms": [ - 0.0062, - 0.0061, - 0.0063, - 0.006, - 0.0067 - ], - "median_ms": 0.0062 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0059, - 0.006, - 0.006, - 0.0061 - ], - "median_ms": 0.006 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0049, - 0.0051, - 0.0051, - 0.0052, - 0.0049 - ], - "median_ms": 0.0051 - } - ], - "metrics": { - "suite_latency_ms": 0.0484 - } - }, - { - "side": "baseline", - "trial": "trial-003", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0139, - 0.012, - 0.0111, - 0.0108, - 0.01 - ], - "median_ms": 0.0111 - }, - { - "id": "strided", - "samples_ms": [ - 0.0131, - 0.0127, - 0.0126, - 0.0137, - 0.0123 - ], - "median_ms": 0.0127 - }, - { - "id": "zeros", - "samples_ms": [ - 0.006, - 0.0056, - 0.0057, - 0.0058, - 0.006 - ], - "median_ms": 0.0058 - }, - { - "id": "large", - "samples_ms": [ - 0.006, - 0.0061, - 0.0062, - 0.006, - 0.0064 - ], - "median_ms": 0.0061 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0055, - 0.0055, - 0.0058, - 0.0056 - ], - "median_ms": 0.0056 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0049, - 0.0049, - 0.0052, - 0.005, - 0.0049 - ], - "median_ms": 0.0049 - } - ], - "metrics": { - "suite_latency_ms": 0.046200000000000005 - } - }, - { - "side": "baseline", - "trial": "trial-004", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0137, - 0.012, - 0.0113, - 0.0107, - 0.0097 - ], - "median_ms": 0.0113 - }, - { - "id": "strided", - "samples_ms": [ - 0.0123, - 0.0119, - 0.0123, - 0.013, - 0.0123 - ], - "median_ms": 0.0123 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0059, - 0.0058, - 0.0058, - 0.0055, - 0.0057 - ], - "median_ms": 0.0058 - }, - { - "id": "large", - "samples_ms": [ - 0.006, - 0.006, - 0.0058, - 0.0059, - 0.0065 - ], - "median_ms": 0.006 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0055, - 0.0057, - 0.0056, - 0.0056, - 0.0057 - ], - "median_ms": 0.0056 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0049, - 0.0049, - 0.0049, - 0.0049, - 0.0048 - ], - "median_ms": 0.0049 - } - ], - "metrics": { - "suite_latency_ms": 0.0459 - } - }, - { - "side": "baseline", - "trial": "trial-005", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0147, - 0.0128, - 0.0115, - 0.0115, - 0.0106 - ], - "median_ms": 0.0115 - }, - { - "id": "strided", - "samples_ms": [ - 0.0138, - 0.0246, - 0.0161, - 0.0145, - 0.0132 - ], - "median_ms": 0.0145 - }, - { - "id": "zeros", - "samples_ms": [ - 0.006, - 0.0058, - 0.006, - 0.0058, - 0.0058 - ], - "median_ms": 0.0058 - }, - { - "id": "large", - "samples_ms": [ - 0.006, - 0.0062, - 0.0058, - 0.0058, - 0.0066 - ], - "median_ms": 0.006 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0059, - 0.0056, - 0.0056, - 0.0056, - 0.0055 - ], - "median_ms": 0.0056 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0049, - 0.0049, - 0.0047, - 0.0047, - 0.0048 - ], - "median_ms": 0.0048 - } - ], - "metrics": { - "suite_latency_ms": 0.0482 - } - }, - { - "side": "baseline", - "trial": "trial-006", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0157, - 0.0137, - 0.0123, - 0.0121, - 0.011 - ], - "median_ms": 0.0123 - }, - { - "id": "strided", - "samples_ms": [ - 0.0134, - 0.0133, - 0.0129, - 0.0137, - 0.0128 - ], - "median_ms": 0.0133 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0093, - 0.0075, - 0.0066, - 0.0063, - 0.0061 - ], - "median_ms": 0.0066 - }, - { - "id": "large", - "samples_ms": [ - 0.0064, - 0.0063, - 0.0061, - 0.006, - 0.0073 - ], - "median_ms": 0.0063 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0061, - 0.0061, - 0.006, - 0.0059, - 0.0058 - ], - "median_ms": 0.006 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0052, - 0.0052, - 0.0051, - 0.005, - 0.005 - ], - "median_ms": 0.0051 - } - ], - "metrics": { - "suite_latency_ms": 0.0496 - } - }, - { - "side": "candidate", - "trial": "trial-001", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0363, - 0.0314, - 0.028, - 0.0258, - 0.0246 - ], - "median_ms": 0.028 - }, - { - "id": "strided", - "samples_ms": [ - 0.0398, - 0.034, - 0.0318, - 0.0348, - 0.0331 - ], - "median_ms": 0.034 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0147, - 0.0147, - 0.0144, - 0.0138, - 0.0138 - ], - "median_ms": 0.0144 - }, - { - "id": "large", - "samples_ms": [ - 0.0147, - 0.0148, - 0.0144, - 0.0139, - 0.0163 - ], - "median_ms": 0.0147 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0146, - 0.0143, - 0.0141, - 0.0142, - 0.0139 - ], - "median_ms": 0.0142 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0115, - 0.0116, - 0.0117, - 0.0116, - 0.0117 - ], - "median_ms": 0.0116 - } - ], - "metrics": { - "suite_latency_ms": 0.1169 - } - }, - { - "side": "candidate", - "trial": "trial-002", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0321, - 0.0282, - 0.0256, - 0.0263, - 0.0239 - ], - "median_ms": 0.0263 - }, - { - "id": "strided", - "samples_ms": [ - 0.0313, - 0.0308, - 0.0307, - 0.0319, - 0.0316 - ], - "median_ms": 0.0313 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0154, - 0.0154, - 0.0147, - 0.0143, - 0.0151 - ], - "median_ms": 0.0151 - }, - { - "id": "large", - "samples_ms": [ - 0.0152, - 0.015, - 0.0146, - 0.0149, - 0.0159 - ], - "median_ms": 0.015 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0141, - 0.0138, - 0.014, - 0.0137, - 0.0135 - ], - "median_ms": 0.0138 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0118, - 0.0118, - 0.0125, - 0.0118, - 0.0229 - ], - "median_ms": 0.0118 - } - ], - "metrics": { - "suite_latency_ms": 0.11330000000000001 - } - }, - { - "side": "candidate", - "trial": "trial-003", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0207, - 0.0172, - 0.0153, - 0.0151, - 0.0136 - ], - "median_ms": 0.0153 - }, - { - "id": "strided", - "samples_ms": [ - 0.018, - 0.0173, - 0.0173, - 0.0184, - 0.0173 - ], - "median_ms": 0.0173 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0085, - 0.0081, - 0.0077, - 0.0073, - 0.0076 - ], - "median_ms": 0.0077 - }, - { - "id": "large", - "samples_ms": [ - 0.0077, - 0.0079, - 0.0079, - 0.0075, - 0.0083 - ], - "median_ms": 0.0079 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0076, - 0.0076, - 0.0073, - 0.0072, - 0.0076 - ], - "median_ms": 0.0076 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0066, - 0.0064, - 0.0064, - 0.0063, - 0.0063 - ], - "median_ms": 0.0064 - } - ], - "metrics": { - "suite_latency_ms": 0.0622 - } - }, - { - "side": "candidate", - "trial": "trial-004", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0135, - 0.0119, - 0.0113, - 0.0105, - 0.0103 - ], - "median_ms": 0.0113 - }, - { - "id": "strided", - "samples_ms": [ - 0.0126, - 0.014, - 0.012, - 0.0129, - 0.0125 - ], - "median_ms": 0.0126 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0057, - 0.0059, - 0.0056, - 0.0055, - 0.0056 - ], - "median_ms": 0.0056 - }, - { - "id": "large", - "samples_ms": [ - 0.0054, - 0.0056, - 0.0057, - 0.0056, - 0.0062 - ], - "median_ms": 0.0056 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0056, - 0.0058, - 0.0056, - 0.0054, - 0.0055 - ], - "median_ms": 0.0056 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0048, - 0.0048, - 0.0049, - 0.0049, - 0.0047 - ], - "median_ms": 0.0048 - } - ], - "metrics": { - "suite_latency_ms": 0.0455 - } - }, - { - "side": "candidate", - "trial": "trial-005", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0136, - 0.0116, - 0.0103, - 0.0103, - 0.0096 - ], - "median_ms": 0.0103 - }, - { - "id": "strided", - "samples_ms": [ - 0.0129, - 0.0128, - 0.0127, - 0.0134, - 0.0126 - ], - "median_ms": 0.0128 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0062, - 0.0058, - 0.0057, - 0.0058, - 0.0059 - ], - "median_ms": 0.0058 - }, - { - "id": "large", - "samples_ms": [ - 0.0062, - 0.0063, - 0.0059, - 0.0057, - 0.0068 - ], - "median_ms": 0.0062 - }, - { - "id": "tiny", - "samples_ms": [ - 0.006, - 0.0058, - 0.0059, - 0.0057, - 0.0057 - ], - "median_ms": 0.0058 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0048, - 0.0051, - 0.0049, - 0.005, - 0.0049 - ], - "median_ms": 0.0049 - } - ], - "metrics": { - "suite_latency_ms": 0.0458 - } - }, - { - "side": "candidate", - "trial": "trial-006", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0145, - 0.0124, - 0.0115, - 0.0107, - 0.0564 - ], - "median_ms": 0.0124 - }, - { - "id": "strided", - "samples_ms": [ - 0.0579, - 0.035, - 0.0278, - 0.0288, - 0.0259 - ], - "median_ms": 0.0288 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0113, - 0.0118, - 0.0109, - 0.0109, - 0.0108 - ], - "median_ms": 0.0109 - }, - { - "id": "large", - "samples_ms": [ - 0.0112, - 0.0113, - 0.0105, - 0.011, - 0.0136 - ], - "median_ms": 0.0112 - }, - { - "id": "tiny", - "samples_ms": [ - 0.011, - 0.0107, - 0.0108, - 0.011, - 0.0111 - ], - "median_ms": 0.011 - }, - { - "id": "singleton", - "samples_ms": [ - 0.009, - 0.0089, - 0.0089, - 0.0088, - 0.0089 - ], - "median_ms": 0.0089 - } - ], - "metrics": { - "suite_latency_ms": 0.0832 - } - } - ], - "measured_calls": 360 - }, - "einsum": { - "verdict": "FAIL", - "baseline": { - "values": [ - 0.046099999999999995, - 0.0773, - 0.0537, - 0.04529999999999999, - 0.0463, - 0.0522 - ], - "median": 0.04925, - "min": 0.04529999999999999, - "max": 0.0773, - "range_pct_of_median": 64.9746192893401, - "excluded_trials": 0 - }, - "candidate": { - "values": [ - 0.08750000000000001, - 0.052199999999999996, - 0.0439, - 0.0868, - 0.0891, - 0.0987 - ], - "median": 0.08715, - "min": 0.0439, - "max": 0.0987, - "range_pct_of_median": 62.88009179575443, - "excluded_trials": 0 - }, - "improvement_pct": -76.95431472081219, - "statistical_significance": "not established", - "final_quality_passed": true, - "source_integrity": true, - "trials": [ - { - "side": "baseline", - "trial": "trial-001", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0134, - 0.012, - 0.0104, - 0.0108, - 0.01 - ], - "median_ms": 0.0108 - }, - { - "id": "strided", - "samples_ms": [ - 0.0131, - 0.0126, - 0.0128, - 0.0128, - 0.0128 - ], - "median_ms": 0.0128 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0059, - 0.0061, - 0.0059, - 0.0063, - 0.0055 - ], - "median_ms": 0.0059 - }, - { - "id": "large", - "samples_ms": [ - 0.0056, - 0.006, - 0.0056, - 0.0058, - 0.0066 - ], - "median_ms": 0.0058 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0058, - 0.0057, - 0.0061, - 0.0057 - ], - "median_ms": 0.0058 - }, - { - "id": "singleton", - "samples_ms": [ - 0.005, - 0.005, - 0.0049, - 0.0049, - 0.0052 - ], - "median_ms": 0.005 - } - ], - "metrics": { - "suite_latency_ms": 0.046099999999999995 - } - }, - { - "side": "baseline", - "trial": "trial-002", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0259, - 0.021, - 0.0194, - 0.019, - 0.0167 - ], - "median_ms": 0.0194 - }, - { - "id": "strided", - "samples_ms": [ - 0.0207, - 0.021, - 0.0207, - 0.0225, - 0.0209 - ], - "median_ms": 0.0209 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0102, - 0.0096, - 0.0092, - 0.0096, - 0.0094 - ], - "median_ms": 0.0096 - }, - { - "id": "large", - "samples_ms": [ - 0.0099, - 0.0097, - 0.0096, - 0.0094, - 0.0111 - ], - "median_ms": 0.0097 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0098, - 0.0095, - 0.0094, - 0.0095, - 0.0094 - ], - "median_ms": 0.0095 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0082, - 0.0084, - 0.008, - 0.0082, - 0.0071 - ], - "median_ms": 0.0082 - } - ], - "metrics": { - "suite_latency_ms": 0.0773 - } - }, - { - "side": "baseline", - "trial": "trial-003", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0149, - 0.0127, - 0.0111, - 0.0112, - 0.0106 - ], - "median_ms": 0.0112 - }, - { - "id": "strided", - "samples_ms": [ - 0.0136, - 0.0133, - 0.0133, - 0.0138, - 0.0135 - ], - "median_ms": 0.0135 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0062, - 0.0065, - 0.0061, - 0.0062, - 0.0063 - ], - "median_ms": 0.0062 - }, - { - "id": "large", - "samples_ms": [ - 0.0063, - 0.0063, - 0.0061, - 0.0061, - 0.0075 - ], - "median_ms": 0.0063 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0064, - 0.0062, - 0.0317, - 0.0154, - 0.0112 - ], - "median_ms": 0.0112 - }, - { - "id": "singleton", - "samples_ms": [ - 0.006, - 0.0055, - 0.0053, - 0.0052, - 0.005 - ], - "median_ms": 0.0053 - } - ], - "metrics": { - "suite_latency_ms": 0.0537 - } - }, - { - "side": "baseline", - "trial": "trial-004", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0133, - 0.0121, - 0.0107, - 0.0107, - 0.0098 - ], - "median_ms": 0.0107 - }, - { - "id": "strided", - "samples_ms": [ - 0.0125, - 0.0127, - 0.0121, - 0.0131, - 0.0125 - ], - "median_ms": 0.0125 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0061, - 0.0059, - 0.0056, - 0.0059, - 0.0056 - ], - "median_ms": 0.0059 - }, - { - "id": "large", - "samples_ms": [ - 0.0056, - 0.0059, - 0.0056, - 0.0057, - 0.0064 - ], - "median_ms": 0.0057 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0057, - 0.0058, - 0.0057, - 0.0057, - 0.0057 - ], - "median_ms": 0.0057 - }, - { - "id": "singleton", - "samples_ms": [ - 0.005, - 0.0046, - 0.0048, - 0.0048, - 0.0046 - ], - "median_ms": 0.0048 - } - ], - "metrics": { - "suite_latency_ms": 0.04529999999999999 - } - }, - { - "side": "baseline", - "trial": "trial-005", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0138, - 0.0125, - 0.0109, - 0.0108, - 0.0102 - ], - "median_ms": 0.0109 - }, - { - "id": "strided", - "samples_ms": [ - 0.0128, - 0.0126, - 0.0125, - 0.0131, - 0.0124 - ], - "median_ms": 0.0126 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0061, - 0.006, - 0.0059, - 0.0058, - 0.0058 - ], - "median_ms": 0.0059 - }, - { - "id": "large", - "samples_ms": [ - 0.0062, - 0.0064, - 0.0059, - 0.0057, - 0.0066 - ], - "median_ms": 0.0062 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0062, - 0.0059, - 0.0058, - 0.0058 - ], - "median_ms": 0.0058 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0051, - 0.0052, - 0.0049, - 0.0049, - 0.0049 - ], - "median_ms": 0.0049 - } - ], - "metrics": { - "suite_latency_ms": 0.0463 - } - }, - { - "side": "baseline", - "trial": "trial-006", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0169, - 0.0141, - 0.0134, - 0.0125, - 0.0117 - ], - "median_ms": 0.0134 - }, - { - "id": "strided", - "samples_ms": [ - 0.0129, - 0.0132, - 0.0124, - 0.0137, - 0.0131 - ], - "median_ms": 0.0131 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0064, - 0.0063, - 0.0064, - 0.0061, - 0.0065 - ], - "median_ms": 0.0064 - }, - { - "id": "large", - "samples_ms": [ - 0.0067, - 0.0077, - 0.0065, - 0.0076, - 0.0073 - ], - "median_ms": 0.0073 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0065, - 0.0066, - 0.0065, - 0.0061, - 0.006 - ], - "median_ms": 0.0065 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0053, - 0.0057, - 0.0061, - 0.0055, - 0.0054 - ], - "median_ms": 0.0055 - } - ], - "metrics": { - "suite_latency_ms": 0.0522 - } - }, - { - "side": "candidate", - "trial": "trial-001", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0271, - 0.0242, - 0.0209, - 0.0216, - 0.0203 - ], - "median_ms": 0.0216 - }, - { - "id": "strided", - "samples_ms": [ - 0.0393, - 0.0244, - 0.0233, - 0.0265, - 0.0215 - ], - "median_ms": 0.0244 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0112, - 0.0113, - 0.0109, - 0.0104, - 0.0105 - ], - "median_ms": 0.0109 - }, - { - "id": "large", - "samples_ms": [ - 0.0105, - 0.0105, - 0.0109, - 0.0102, - 0.0126 - ], - "median_ms": 0.0105 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0109, - 0.0118, - 0.0115, - 0.0108, - 0.0109 - ], - "median_ms": 0.0109 - }, - { - "id": "singleton", - "samples_ms": [ - 0.01, - 0.0092, - 0.009, - 0.0088, - 0.0093 - ], - "median_ms": 0.0092 - } - ], - "metrics": { - "suite_latency_ms": 0.08750000000000001 - } - }, - { - "side": "candidate", - "trial": "trial-002", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0176, - 0.0163, - 0.0149, - 0.0146, - 0.0103 - ], - "median_ms": 0.0149 - }, - { - "id": "strided", - "samples_ms": [ - 0.0122, - 0.0126, - 0.0127, - 0.0133, - 0.0126 - ], - "median_ms": 0.0126 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0066, - 0.0063, - 0.0063, - 0.0064, - 0.0063 - ], - "median_ms": 0.0063 - }, - { - "id": "large", - "samples_ms": [ - 0.0062, - 0.0064, - 0.0066, - 0.0064, - 0.0075 - ], - "median_ms": 0.0064 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0065, - 0.0063, - 0.0063, - 0.0062, - 0.0063 - ], - "median_ms": 0.0063 - }, - { - "id": "singleton", - "samples_ms": [ - 0.006, - 0.0059, - 0.0056, - 0.0057, - 0.0055 - ], - "median_ms": 0.0057 - } - ], - "metrics": { - "suite_latency_ms": 0.052199999999999996 - } - }, - { - "side": "candidate", - "trial": "trial-003", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0133, - 0.0117, - 0.011, - 0.0101, - 0.009 - ], - "median_ms": 0.011 - }, - { - "id": "strided", - "samples_ms": [ - 0.011, - 0.0107, - 0.0101, - 0.0109, - 0.0109 - ], - "median_ms": 0.0109 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0059, - 0.0059, - 0.0059, - 0.0061, - 0.0054 - ], - "median_ms": 0.0059 - }, - { - "id": "large", - "samples_ms": [ - 0.0059, - 0.0056, - 0.0054, - 0.0052, - 0.0064 - ], - "median_ms": 0.0056 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0058, - 0.0056, - 0.0054, - 0.0056, - 0.0071 - ], - "median_ms": 0.0056 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0049, - 0.0052, - 0.0052, - 0.0047, - 0.0048 - ], - "median_ms": 0.0049 - } - ], - "metrics": { - "suite_latency_ms": 0.0439 - } - }, - { - "side": "candidate", - "trial": "trial-004", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0269, - 0.0233, - 0.0202, - 0.0199, - 0.0195 - ], - "median_ms": 0.0202 - }, - { - "id": "strided", - "samples_ms": [ - 0.0236, - 0.0229, - 0.0219, - 0.0241, - 0.0231 - ], - "median_ms": 0.0231 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0135, - 0.0132, - 0.0311, - 0.0124, - 0.0121 - ], - "median_ms": 0.0132 - }, - { - "id": "large", - "samples_ms": [ - 0.0102, - 0.0101, - 0.0104, - 0.0102, - 0.0119 - ], - "median_ms": 0.0102 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0102, - 0.0107, - 0.0107, - 0.0121, - 0.0105 - ], - "median_ms": 0.0107 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0098, - 0.0094, - 0.0088, - 0.0096, - 0.0087 - ], - "median_ms": 0.0094 - } - ], - "metrics": { - "suite_latency_ms": 0.0868 - } - }, - { - "side": "candidate", - "trial": "trial-005", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0325, - 0.0266, - 0.0254, - 0.0223, - 0.019 - ], - "median_ms": 0.0254 - }, - { - "id": "strided", - "samples_ms": [ - 0.0217, - 0.0208, - 0.021, - 0.0225, - 0.0212 - ], - "median_ms": 0.0212 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0111, - 0.0106, - 0.0109, - 0.0106, - 0.0106 - ], - "median_ms": 0.0106 - }, - { - "id": "large", - "samples_ms": [ - 0.0107, - 0.0108, - 0.0107, - 0.0104, - 0.0134 - ], - "median_ms": 0.0107 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0111, - 0.0107, - 0.0112, - 0.011, - 0.0111 - ], - "median_ms": 0.0111 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0101, - 0.0101, - 0.0097, - 0.0097, - 0.0101 - ], - "median_ms": 0.0101 - } - ], - "metrics": { - "suite_latency_ms": 0.0891 - } - }, - { - "side": "candidate", - "trial": "trial-006", - "cases": [ - { - "id": "normal", - "samples_ms": [ - 0.0315, - 0.024, - 0.0233, - 0.0205, - 0.0194 - ], - "median_ms": 0.0233 - }, - { - "id": "strided", - "samples_ms": [ - 0.0261, - 0.0272, - 0.0258, - 0.0245, - 0.0264 - ], - "median_ms": 0.0261 - }, - { - "id": "zeros", - "samples_ms": [ - 0.0135, - 0.0132, - 0.0147, - 0.0123, - 0.0126 - ], - "median_ms": 0.0132 - }, - { - "id": "large", - "samples_ms": [ - 0.0132, - 0.013, - 0.0122, - 0.0127, - 0.0149 - ], - "median_ms": 0.013 - }, - { - "id": "tiny", - "samples_ms": [ - 0.0129, - 0.0125, - 0.0122, - 0.0121, - 0.0144 - ], - "median_ms": 0.0125 - }, - { - "id": "singleton", - "samples_ms": [ - 0.0107, - 0.0109, - 0.0106, - 0.0106, - 0.0106 - ], - "median_ms": 0.0106 - } - ], - "metrics": { - "suite_latency_ms": 0.0987 - } - } - ], - "measured_calls": 360 - } - }, - "unchanged_raw_pass": false, - "interpretation": "Raw Workbench verdicts are descriptive; an unchanged PASS is a false-positive warning, not optimization. No speedup claim.", - "campaign_acceptance": "INCONCLUSIVE", - "agent_or_skill_benefit": "not measured", - "accelerator_qualification": "not run" -} diff --git a/docs/plans/issue-252/usefulness.md b/docs/plans/issue-252/usefulness.md deleted file mode 100644 index 168b12b9b..000000000 --- a/docs/plans/issue-252/usefulness.md +++ /dev/null @@ -1,74 +0,0 @@ -# #252：实际工程用途对照 - -本页是 **v1 历史对照**，对应源码 [c06b7255](https://github.com/Bounty-hunter/InferRSI/tree/c06b72555d4643df133c59fe7cead5a91e7b86f5)。下文 720 个样本、脚本摘要和计划均属于该版本。当前已删除 CPU RMSNorm 题包及其专用 fixture；本页命令只能在上述历史 checkout 中执行，[当时的 fixture](https://github.com/Bounty-hunter/InferRSI/blob/c06b72555d4643df133c59fe7cead5a91e7b86f5/tests/fixtures/engineering_benchmark_usefulness.py)仍可追溯。后续修复见 [v2 验证](replay-validation.md)，不修改本页历史结果。 - -本补证回答一个具体问题：共享任务能否帮助使用者识别一个看似合理的错误改动，并让正确候选经过同一套工程检查和实际性能比较？它不以 CI 绿色或某次计时 PASS 作为“有用”的定义。 - -## 预先固定的对照 - -选择既有 `rmsnorm-cpu` 任务，保持六个公开工作负载、axis=1、float32、epsilon、容差、oracle、预热与采样规则不变。所有代码均为审核过的本地实验实现；SkillEval 交接的是保留产物身份，没有调用生成模型。 - -| 实现 | 检查目的 | -| --- | --- | -| unchanged | 逐字复制原版，作为原版对原版的计时对照 | -| no-epsilon | 一个真实容易出现的错误：省掉 epsilon；先观察普通输入冒烟，再交给完整任务及两个消费者 | -| einsum | 用 axis-1 平方和归约替代显式 `x*x` 中间数组，保留 epsilon、类型、布局和输入语义；性能改善只是待检验假设 | - -普通输入冒烟预先定义为 seed 2026、shape `(4, 17, 9, 7)` 的 float32 正态分布乘以 4，使用任务的独立 oracle 和容差。它与完整任务的压力输入不同，结果会如实保留。即使错误实现未通过此冒烟，也不换一组输入重试。 - -先冻结计划、任务、实现、grader/runtime 身份和比较参数，再执行。对 unchanged 与 einsum 各做一次既有 Workbench 比较，按预定顺序运行，所有 trial 均保留。性能阈值只用于展示既有描述性裁决，不作为本补证的功能通过条件，不根据结果选择重跑或修改阈值。 - -## 怎样判断有实际用途 - -1. 正确候选在完整任务和实际 SkillEval verifier 中均通过；两个消费者使用同一任务、实际输入和 oracle。 -2. no-epsilon 在完整任务中被拒绝，错误测量没有可用性能指标；保留具体失败断言。 -3. 正确候选的实际比较保留每个工作负载的原始样本，不能只挑总分或快的 case。 -4. 原版对原版的结果同样公开；若也出现描述性 PASS，必须标明相同实现的计时波动，不能宣称优化收益。 -5. profiling 独立运行且没有性能胜负，保留调用路径及其分辨能力的限制。 - -## 诊断与决策边界 - -已有 CPU profile 只有 42 次记录的 compute 调用，总时间约毫秒量级，文本中的多个耗时显示为 `0.000`。这能说明实际调用了哪些函数，不能证明内存带宽或临时数组分配就是瓶颈。einsum 候选来自源码检查提出的假设，不能写成 profiler 已证实的优化方向。 - -小尺寸公开用例没有建立真实服务代表性。既有 Workbench 使用中位数和阈值，尚未实现统计显著性或多轮选择偏差控制；未改进实现也可能得到描述性 PASS。本补证保留这个事实，不以集成测试通过替代性能验收。 - -## 复现入口与结果 - -两阶段复现入口为 `python -m tests.fixtures.engineering_benchmark_usefulness`：`prepare` 只冻结输入，`run` 要求显式计划摘要并执行一次。使用全新输出目录，参数来自对应 Issue 的预登记记录。完整原始日志留在本地；公开汇总仅发布无私人路径的身份、结果、失败断言和逐 case 样本。 - -实际执行前的条件、身份与摘要见 [#337 预登记](https://github.com/InferMatrixLab/InferRSI/issues/337#issuecomment-6607885902)。本地原始目录名为 `inferrsi-252-usefulness-20261103-v2`；此前的目录只执行过 prepare，没有运行 payload。独立静态检查发现并修复了补证脚本的最终质量门槛缺口，然后重新冻结计划，实际 payload 只执行一次。 - -```bash -python -m tests.fixtures.engineering_benchmark_usefulness prepare \ - --repo --output-dir \ - --repetitions 6 --max-minutes 2 --min-improvement-pct 0 --timeout-seconds 25 -python -m tests.fixtures.engineering_benchmark_usefulness run \ - --output-dir --plan-sha256 -``` - -这些参数来自本次预登记，不是共享的性能验收默认值。新复现的路径或环境会改变计划摘要，应使用该次 prepare 的返回值。 - -## 实测结果与明确决策 - -2026-11-03 在 Windows / Python 3.11.18 / NumPy 2.4.6 上执行。期间没有并行运行本地测试，但宿主机没有专用资源隔离。运行时与任务仍为 PR 初始实现；本补证只新增 fixture 和说明。 - -| 检查 | 实际结果 | 说明 | -| --- | --- | --- | -| no-epsilon 普通输入冒烟 | 通过 | 单个普通输入确实漏掉了错误 | -| no-epsilon 完整任务 / SkillEval | 两边均 FAIL | 拒绝 `normal.values`、`strided.values`、`zeros.finite`、`zeros.values`、`tiny.values`、`singleton.values` 六个断言 | -| no-epsilon 测量 | 拒绝，无性能指标 | 错误实现没有凭借省略计算获得可用分数 | -| unchanged / einsum 正确性 | 两个实现均在两条路径通过 | 两个 Workbench 比较结束时的质量验证也通过 | -| 共享任务和实际输入 | 一致 | 包含正确性、profiling、计时和真实 SkillEval verifier | -| 原版对原版比较 | 12 个 trial，360 个原始样本；原始 verdict 为 FAIL | 报告的 improvement 为 -50.52%，同码对照说明当前测量条件不足以可靠分辨优化 | -| einsum 对原版比较 | 12 个 trial，360 个原始样本；原始 verdict 为 FAIL | 报告的 improvement 为 -76.95%；不能在上述噪声条件下把数值解释为稳定退化或加速 | -| 独立 CPU profiling | 两份均完成，无性能 verdict | 支持调用路径检查，未确认内存瓶颈 | - -**已经证明的用途：** 共享基准抓住了预先指定的普通冒烟漏检，同时让两个消费者用同一套题目和 oracle 识别正确与错误实现，并保留可审查的实验记录。八项功能检查全部通过。 - -**没有通过的用途：** 本机小尺寸协议尚不能支持可靠性能排序。即使正确性通过，也不能据此采用 einsum 候选；本次决定保留原版，不修改任务包默认实现。不重跑挑选好看的结果，也不把两个性能 FAIL 改成 PASS。 - -后续若要将这些任务用于性能搜索，需要单独预登记计时稳定性验证：先使原版对原版具备足够分辨能力，再比较候选；考虑足够长的采样、宿主机干扰、逐 case 噪声和跨轮次选择偏差。这是后续验收项，不是本次已经交付的统计控制器。 - -[公开原始汇总与全部 720 个计时样本](usefulness-summary.json) 为本次 summary 的原始副本；两份已去路径的原始 profile 为 [unchanged](usefulness-profiles/unchanged.txt) 和 [einsum](usefulness-profiles/einsum.txt)。它们保留原始数值与 SHA-256；完整命令、日志和源码快照仍在本地。此次探针代码摘要为 `317285d0534f9935d06e1fc488ef35c56a76ab7cba2c2423e5a73ba09799ebc7`，计划摘要见预登记与 JSON，不因后续文档更新而改写。 - -该检查不证明 agent/skill/search 的收益、GPU/NPU 资格或服务加速，也不实现全流程控制器。 diff --git a/docs/plans/issue-252/validation-summary.json b/docs/plans/issue-252/validation-summary.json deleted file mode 100644 index 5f2d868ec..000000000 --- a/docs/plans/issue-252/validation-summary.json +++ /dev/null @@ -1,53 +0,0 @@ -{ - "scope": "local CPU integration with authored controls", - "checks_succeeded": true, - "campaign_acceptance": "INCONCLUSIVE", - "agent_or_skill_benefit": "not measured", - "accelerator_qualification": "not run", - "tasks": [ - { - "task": "rmsnorm-cpu", - "identity": { - "task_id": "rmsnorm-cpu", - "task_sha256": "903359c46cbbd4059325c2807532449d12beb3eec9931471c84510ba035b1677", - "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", - "candidate_sha256": "28fe2fe3f25fdbd81811d16cf2897d5200c77c0dd5b3b9e65ff7d9e6a23123be", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "correctness": "PASS", - "profile_status": "completed", - "skilleval_good": "PASS", - "skilleval_bad": "FAIL", - "bad_measurement_status": "failed", - "measurement_trials": 6, - "measured_calls": 180, - "source_integrity": true, - "descriptive_workbench_verdict": "FAIL", - "workbench_run": "rmsnorm-cpu/workbench/20261103T141447Z-897e0a2641", - "all_consumer_identities_match": true, - "all_workloads_match": true - }, - { - "task": "softmax-cpu", - "identity": { - "task_id": "softmax-cpu", - "task_sha256": "a0673d210792959bafa60851d2b9e33ded1ab5b7222076fb2858c27d125c6a2d", - "grader_sha256": "e12a40fd37d402668a48f5adacbada6ce864e6aadfa882862f35f767e838b4b0", - "candidate_sha256": "d36d5f8f69a38a0c034f7288c4613132b3b8a1dbf36bfe839690444616a9ff11", - "runtime_sha256": "2141480b7ee5ecc2ccc6bd0032db5c6cd6816403a8f2e790abb187e41b1ab844" - }, - "correctness": "PASS", - "profile_status": "completed", - "skilleval_good": "PASS", - "skilleval_bad": "FAIL", - "bad_measurement_status": "failed", - "measurement_trials": 6, - "measured_calls": 180, - "source_integrity": true, - "descriptive_workbench_verdict": "PASS", - "workbench_run": "softmax-cpu/workbench/20261103T141513Z-717a97f18d", - "all_consumer_identities_match": true, - "all_workloads_match": true - } - ] -} diff --git a/docs/plans/issue-252/validation.md b/docs/plans/issue-252/validation.md deleted file mode 100644 index 9ad9424f6..000000000 --- a/docs/plans/issue-252/validation.md +++ /dev/null @@ -1,58 +0,0 @@ -# #252 本地验证记录 - -本页保留 **v1 历史运行**（六个计时 case、重复输入）的原始记录，不能代表后续 v2 变化输入协议。当前 RSI 接入边界见 [接入审计](rsi-integration.md)，协议修复与新证据见 [v2 验证](replay-validation.md)。旧数值、身份与 verdict 不重写或合并到新结果。 - -验证日期 2026-11-03。实现基于 `663f3b327998526a13aa31e72ad729025c18983c`，对应初始实现 [d0ce9d7](https://github.com/Bounty-hunter/InferRSI/tree/d0ce9d785e1bead9124eed1cf3f6e9f2e0cf05db)；实际运行文件的身份以结果中的内容哈希为准。本记录说明共享任务子集的功能与集成，不是项目维护者的验收决定。 - -## 实际运行结果 - -最终原始目录名为 `inferrsi-252-validation-final`，保留在仓库外的本地 outputs 目录。重放这套 v1 协议须使用上述原始版本的 `tools/engineering_benchmark_validate.py`、任务包和 grader，并核对记录中的文件身份；新执行产生自己的记录，不会重建原始计时值。当前分支的工具执行 v2 协议，不能用于复现本页的样本数量或身份。该次执行使用 Windows、Python 3.11.18、NumPy 2.4.6。 - -[可公开的原始汇总](validation-summary.json) 保留任务/候选/grader/runtime 身份、执行数量、原始 verdict 和覆盖状态，不包含个人路径。完整原始日志和样本仍在本地，公开汇总不能代替独立重放或正式 campaign 验收。 - -| 验证内容 | Softmax CPU | RMSNorm CPU | -| --- | --- | --- | -| workload 数 | 6 | 6 | -| 每次独立正确性断言 | 24 | 24 | -| 正确候选在工程评估中 | 通过 | 通过 | -| 正确候选经过实际 SkillEval run_trial/verifier | 通过 | 通过 | -| 全零错误候选经过同一路径 | 拒绝 | 拒绝 | -| 错误候选请求测量 | 失败，无可用性能指标 | 失败，无可用性能指标 | -| Workbench 配对 trial | 3 baseline + 3 candidate | 3 baseline + 3 candidate | -| 保留的实际计时样本 | 180 | 180 | -| CPU profile | 已生成，42 次 compute 调用 | 已生成，42 次 compute 调用 | -| 所有消费者 task/grader/runtime 身份 | 一致 | 一致 | -| 所有消费者实际输入摘要、shape/dtype/strides | 一致 | 一致 | -| Workbench 源码完整性检查 | 通过 | 通过 | - -`summary.json` 的 `checks_succeeded` 为 true。每个任务目录包含冻结的任务包、候选源码、runspec.json、identity.json、correctness、profile、workbench、skilleval-good、skilleval-bad、wrong-measure。原始样本、命令、stderr、输入摘要和 profile 均保留，可追溯到实际执行。 - -工程评估与 SkillEval 的 PASS 仅表示相应公开用例的数值正确性。Workbench 的比较 verdict 原样保留，包括不满足增益门槛的结果。两次集成运行的性能胜负有变化，且宿主机同时有软件检查活动，不能据此宣布优化收益；本次目的为验证测量通路和失败处理，不是做隔离 CPU 性能实验。统计显著性仍为 not established。 - -## 软件检查 - -- 新增专项回归 `tests/test_engineering_benchmarks.py`：**26 passed**。覆盖真实任务、反例、预检之后变错、task/candidate/grader/runtime 漂移、缺 NumPy、超时、实际 Workbench AB/BA、真实 SkillEval verifier 和错误产物身份。 -- `ruff check src tests tools/engineering_benchmark_validate.py`：通过。 -- `git diff --check`：通过。 -- sdist 和 wheel 构建：通过。Windows 长路径导致的第一次构建失败保留；缩短构建临时目录后成功，没有删减包内容。 -- 全新非 editable wheel 环境，源码目录外运行 `python -I tests/check_installed.py`：通过，包括任务文件、许可证、127 条既有知识记录等安装检查。 -- 同一隔离安装中实际执行 Softmax 独立 oracle：通过。 -- 全量回归使用 Python 3.11、4 个 pytest worker 完成：**3479 passed、51 skipped、56 failed**，另有 281 个通过的 subtest。55 个失败用例产生 56 个失败报告，全部在未修改的原始基线复现。典型原因包括 Windows 的 CRLF 字节差异、符号链接权限、POSIX FIFO/process API、Linux sandbox 和路径假设；没有通过修改无关测试掩盖。 -- 基线对照在独立 detached worktree 执行，确认 import 指向原始基线 `src/inferrsi`；逐个匹配 JUnit 中的失败用例，未出现“本次失败但基线通过”的用例。全量结果仍为非绿色，不能称为所有检查通过。 - -最初仅有 Python 3.14 的环境缺少 tzdata，导致仓库中 6 个已有模块无法收集测试。后续使用仓库 CI 覆盖的 Python 3.11，并在本地测试环境安装 tzdata。未更改项目的时区业务逻辑。26 项专项验证的最终结果来自该 Python 3.11 环境。 - -全量串行尝试在发现原有失败后改用 4 个 worker 完整执行。最终日志为 `inferrsi-252-full.log/xml`；对照为 `inferrsi-252-baseline-all-failures.log/xml`；新增功能专项为 `inferrsi-252-focused.log`。早期失败、构建长路径问题和首次集成运行也保留，没有覆盖为成功。 - -## 四个子项的实际用途 - -后续新增了[实际工程用途对照](usefulness.md)：普通冒烟漏掉的 RMSNorm epsilon 错误被共享任务和 SkillEval 一致拒绝；原版对原版的大幅计时差异同时说明，本机小尺寸协议尚不能支持可靠性能排序。该次公开汇总保留全部 720 个原始样本，不改变本页早先集成运行的结果。 - -1. 调研决定落实在代码边界中：保留 axis=1、独立 oracle、正确性先行、profiling 独立、已有 runner 复用，没有照搬外部搜索平台。 -2. 共享接口会拒绝漂移或未支持环境，避免两个消费者用不同题目版本得出不可比较的结果。 -3. 两个算子确实计算并计时；数值、布局和状态错误会被抓到，未用预制分数代替实现。 -4. Workbench 与 SkillEval 真正执行了同一任务包；逐项比较实际输入摘要验证复用，没有为第二个消费者复制答案。 - -## 本次未建立的结论 - -GPU/NPU 正确性和性能、通信/内存容量任务、真实模型/服务收益、技能或搜索策略收益均未验证。公开开发用例不是 holdout；可信代码执行不等于恶意 agent sandbox。本次公开提交用于评审；尚无独立 reviewer 的签署、维护者对首批子集的正式接受或关闭 #252 的授权，公开追踪入口见总方案。上述差别不影响已经完成的本地实现和功能验证，但不能被省略为“整个 tracking issue 已验收”。 diff --git a/docs/plans/issue-252/walkthrough.md b/docs/plans/issue-252/walkthrough.md deleted file mode 100644 index 74b64290d..000000000 --- a/docs/plans/issue-252/walkthrough.md +++ /dev/null @@ -1,195 +0,0 @@ -# 从零读懂 #252：做了什么，怎样在 InferRSI 中使用 - -这份导读先讲用途，再沿一个 RMSNorm 例子走到代码与结果。总览见 [README](README.md)，本轮审查结论见 [review](review.md)。 - -本页保留早期 CPU 实现、四项分工与当时状态，供追溯使用；当前两项工作、NPU 范围和 PR 安排以[总览](README.md)为准。`rmsnorm-cpu` 已从当前题库删除；下文示例须在[删除前版本](https://github.com/Bounty-hunter/InferRSI/tree/125b06451a487e76a1c20c45abe4e22cd8631d68)中执行，历史实测仍以各节指定版本为准。当前通用流程中 profiling 是可选工具，不按下文演示顺序强制 Agent 执行。 - -**我们给 InferRSI 增加的是 Engineering Benchmarks（共享工程评测任务）：一份题目、一份候选实现，交给固定的独立评测，得到可追溯的正确性、诊断和计时结果。** 主 Agent 根据这些证据决定修改、保留或放弃实现。 - -## 1. 先理解它解决哪一个问题 - -假设你对 Agent 说：“用 InferRSI 优化 RMSNorm。”只有这个名字还不够：沿哪个维度归约？输入是什么精度和布局？可以改哪些文件？怎样判断算错？究竟计时哪一段？ - -没有共享任务时，调用者仍然可以使用已有 Workbench，但需要自己提供这些命令和规则。不同实验若自行定义输入与答案，结果很难直接比较。 - -这次把规则放进一个任务包。当时的 `rmsnorm-cpu` 明确要求：CPU、NumPy、float32、沿 axis=1 归约，保留 epsilon，不修改输入。Agent 只交付 `implementation.py` 中的 `compute(x, *, eps)`。工具检查这个文件的实际行为，并记录是哪一版题目、裁判、源码和运行环境产生了结果。 - -它在 RSI 中的位置是**任务与评测层**。Workbench 继续负责运行实验；主 Agent 继续负责调查、读论文、分析 profile、写代码和决定下一轮。评估 skill 是否有效时，新增的[正式两臂产物 adapter](formal-integration.md)复用这层任务；实际技能收益仍需要受控 Agent 实验。 - -## 2. 四个子任务，具体交付了什么 - -四项分别可以简称为“选题复用、统一规则、交付题包、接入验证”。它们是交付步骤；运行一次优化时不会依次执行这四项。 - -| 子任务 | 实际做的事情 | 交付物 | 对使用者的作用 | 当前边界 | -| --- | --- | --- | --- | --- | -| ① **选题复用** · [#334](https://github.com/InferMatrixLab/InferRSI/issues/334) | 读论文和固定版本源码，对照现有 RSI，决定首批范围 | [调研与选型](01-research.md) | 确定先做 Softmax/RMSNorm，复用任务分工与数学题意，避免重复已有执行器 | CPU 子集是提议，范围待维护者接受 | -| ② **统一规则** · [#335](https://github.com/InferMatrixLab/InferRSI/issues/335) | 定义题目格式、候选入口、执行动作和内容身份 | [契约说明](02-contract.md)、`TaskPackage`、worker、CLI | 不同消费者使用同一题目和裁判；比较中源码或规则漂移会被发现 | 当前只接受 `numpy-axis1-v2`，新设备需要自己的 adapter | -| ③ **交付题包** · [#336](https://github.com/InferMatrixLab/InferRSI/issues/336) | 将两道题变成可运行包，补边界输入与独立答案 | [任务说明](03-tasks.md)、两套 manifest/基线/候选/许可 | Agent 有具体修改对象；错误轴、漏 epsilon、输出重放等错误可被独立检查 | 只覆盖声明的 CPU 输入；不能外推 GPU、通信、容量或服务 | -| ④ **接入验证** · [#337](https://github.com/InferMatrixLab/InferRSI/issues/337) | 生成真实 RunSpec，接现有 Workbench 与底层 SkillEval verifier，运行对照 | [集成说明](04-integration.md)、复现工具、验证记录 | 实验结果进入已有结果目录，错误产物被拒绝，正确实现留下计时证据 | Workbench、底层 verifier 和正式两臂正确性产物入口已实现；可靠性能排序和技能收益未完成 | - -因此，“四项都有代码或文档交付”和“整个 #252 已验收”是两个状态。尤其第④项只完成了表中明确列出的接入层级。 - -## 3. RSI 的两条循环，分别在优化什么 - -| 循环 | 本轮改变什么 | 比较什么 | 谁用结果作决定 | -| --- | --- | --- | --- | -| **优化引擎实现** | `implementation.py` 或真实引擎中的实现 | 原版和候选版是否正确、是否有可分辨的性能差异 | 主 Agent / 实验负责人 | -| **优化优化者** | Agent 使用的 skill、知识或 harness | 两种条件下，Agent 产出的实现质量、速度、成功率和资源成本 | SkillEval 实验负责人及其支持的改进流程 | - -第一条路径已有实际执行。第二条早期只有底层 verifier，后续已补上正式 `skill_eval_compare` 与 local stage/capture：两臂交付真实文件，隔离执行候选，父进程独立检查并形成配对报告。验证使用作者脚本和正确/错误软件对照；没有让两组真实 Agent 分别使用旧/新 skill 生成代码，所以没有 skill 有效性的结论。具体平台验证见[正式接入说明](formal-integration.md)。 - -即便一份事先写好的候选在可靠测量中更快，也不能说明“某个 skill 帮助 Agent 写出了它”。需要先固定任务、模型和预算，再比较 Agent 在两种技能条件下真实交付的文件。 - -### 优化一次实现时，应该怎样运作 - -下面展示主 Agent 应采用的工作顺序。箭头表示数据和决定的传递；目前没有后台控制器自动强制走完整张图。 - -```mermaid -flowchart TD - U[用户目标：优化某个实现] --> A[主 Agent：确定场景与验收条件] - A --> P[主 Agent 调用 profile：取得诊断] - P --> H[主 Agent：结合知识提出假设] - H --> C[主 Agent：修改 implementation.py] - T[冻结任务包：语义、输入、oracle、测量规则] --> K[check：检查实际输出] - T -->|诊断输入与检查规则| P - C -->|候选源码| K - K -->|错误断言| H - K -->|通过后发起比较| W[Workbench：执行原版与候选版] - T --> W - W -->|原始样本、校验结果、版本记录| D[主 Agent：判断证据是否足够] - D -->|需要新诊断或下一轮尝试| P - D -->|保留、拒绝或证据不足| R[保存实现与证据，进入已有后续流程] -``` - -| 图里的框 | 谁调用、输入是什么 | 框里实际做什么 | 交出什么，下一步怎么用 | -| --- | --- | --- | --- | -| 确定场景 | 用户与主 Agent；目标模型、硬件、负载、质量要求 | 判断题目是否覆盖真实需求，冻结实验范围和预算 | 可检查的实验计划；当前 CPU 题不自动覆盖同名 GPU 算子 | -| profile | 主 Agent；基线或候选源码与任务 | 独立运行 CPU cProfile，查看调用结构 | `profile.txt`；用来形成或检查假设，不参与速度分数 | -| 提出假设 | 主 Agent；profile、论文、知识和既往失败 | 解释准备改哪一处、预期为什么有用 | 一项可检验的改动，例如尝试减少中间数组 | -| 修改实现 | 主 Agent；受约束的候选文件 | 修改 `compute`，保持题目语义 | 待审核候选源码；题目、oracle 和基线仍是固定比较输入 | -| check | CLI 或消费者；题目与候选源码 | 用独立参考答案检查基础和变化输入 | `passed`、具体断言、输入身份；错误实现需要修正 | -| Workbench | `compare` 生成 RunSpec 后调用既有 runner | 按 AB/BA 顺序运行原版/候选；各 worker 内先检查，再计时并验证每次输出 | `result.json`、原始样本、`evaluation.json`、`status.json`、源码快照 | -| 判断与保存 | 主 Agent / 负责人；上述完整记录 | 分开判断正确性、测量可信度和任务是否代表真实目标 | 保留、拒绝或证据不足，并保留理由供下一轮使用 | - -AB/BA 是调换执行顺序：一轮先原版再候选，下一轮先候选再原版，减少固定先后顺序的影响。它本身不保证消除噪声。 - -### 谁确保 Agent 按流程走 - -当前有两层约束： - -- **代码强制的局部规则：** manifest/adapter 校验；执行前后身份校验；候选数值检查；检查失败不输出可用性能指标；超时与执行错误保留状态。 -- **由主 Agent 遵循的工作流程：** 先研究与 profiling、冻结假设和验收门槛、根据证据决定下一轮。现有插件入口和工作流文档提供指导，没有 hooks/daemon 强制每次都先做研究或 profile。 - -直接调用 `compare` 不会先自动运行 `profile`。通用 Workbench runner 是先执行 AB/BA，再运行可选 quality command；本次任务 worker 在每个测量进程内部执行正确性检查。不能据此声称通用 runner 已经实现了完整的全局阶段控制。 - -评论中的 **Optimization search policy** 负责“下一步试什么、分多少预算、何时停止”。它消费测量与失败信息。#252 提供任务和评测结果，独立 policy RFC 讨论搜索决策；当前实现没有新的 search controller。真实入口和剩余接入见 [RSI 接入审计](rsi-integration.md)。 - -## 4. 以 RMSNorm 为例，评测器到底在检查什么 - -计算语义是 `y = x / sqrt(mean(x*x, axis=1, keepdims=True) + eps)`。这里 axis=1 很关键：对四维 `(N,C,H,W)` 输入，沿 C 维归约，不能换成常见的“最后一维归约”实现。 - -三个文件/角色各有职责： - -| 角色 | 当前实现 | 用途 | -| --- | --- | --- | -| **baseline** | 任务包中的 NumPy 实现 | 性能比较中的原版 | -| **candidate** | 用户/Agent 交付的 `compute` | 待优化和受检查的实现 | -| **oracle** | 评测器中的标量 double、`math.fsum` 参考算法 | 判断答案是否正确；不拿它的慢速执行时间作为性能基线 | - -“独立 oracle”指算法实现独立，减少基线和候选共享同一错误的风险。Workbench 的 worker 执行已审核可信源码，没有恶意代码隔离。正式 SkillEval 的新 adapter 使用另一条产物隔离路径，在父进程保留 oracle；两条执行边界详见[正式接入说明](formal-integration.md)。 - -六类基础输入覆盖普通值、跨步布局、全零、大数值、极小值和单元素归约。其中 `large` 指数值大，`tiny` 指数值小；不是张量规模大小。极小值可揭示遗漏 epsilon；跨步输入可揭示对内存连续性的错误假设。 - -### 一次调用如何得到结果 - -以当前两道内置题的默认规则为例： - -1. 预先生成全部输入、独立答案和输入摘要。这些准备不计入计算时间。 -2. 检查 6 个基础输入。每次检查 shape/dtype、数值有限、输入未变、答案在容差内。 -3. 对 5 个有变化的 workload，各准备 2 次预热 + 5 次采样，共 35 次调用。每次输入字节不同，所有输入对象保留存活；预检、预热或早先采样不会给后面的同输入缓存命中创造条件。 -4. 每次实际调用后，在计时范围外验证输出。正确性错误立即使这次测量不可用。 -5. `check` 共执行 41 次并产生 164 个断言；`measure` 留下 25 个计时样本；`profile` 单独记录 35 次变化输入调用，不生成性能指标。全零 case 只参与正确性检查。 - -预定输入写入 `input_sequence`，实际调用写入 `executed_calls`。失败可能只完成前缀；有一份完整计划不代表完整执行。每个计时样本还绑定自身的 `sample_input_sha256`。 - -计时范围是同步 `compute` 调用，函数内部申请数组的成本包含在内；进程启动、输入生成、oracle、预热和输出检查不包含在分数内。每个 workload 先取 5 个样本的中位数，再相加得到 `suite_latency_ms`。这是固定小任务集的指标，不能解释成某模型的请求延迟。 - -## 5. 自己从哪里开始看和运行 - -先读任务，再复制候选。当前任务的安装/运行命令见[使用文档](../../engineering-benchmarks.md)。下列 CPU RMSNorm 历史演示仅在上述删除前版本的仓库根目录执行；输出目录必须未存在，重复试验应换新名字。 - -```bash -python -m pip install -e ".[dev,benchmarks]" -inferrsi benchmarks list -inferrsi benchmarks show rmsnorm-cpu -python -c "from pathlib import Path; import shutil; from inferrsi.harness.benchmarks import load_task; p=Path('runs/benchmark-demo'); p.mkdir(parents=True); shutil.copyfile(load_task('rmsnorm-cpu').root / 'candidate.py', p / 'implementation.py')" -inferrsi benchmarks check rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --output-dir runs/benchmark-demo/check --timeout-seconds 30 -inferrsi benchmarks profile rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --output-dir runs/benchmark-demo/profile --timeout-seconds 30 -inferrsi benchmarks plan rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --repo . --output-dir runs/benchmark-demo/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 -inferrsi benchmarks compare rmsnorm-cpu --candidate runs/benchmark-demo/implementation.py --repo . --output-dir runs/benchmark-demo/runs --repetitions 3 --max-minutes 2 --min-improvement-pct 0 -``` - -这是学习接口的演示，数字是演示预算，`min-improvement-pct 0` 不是正式验收门槛。`plan` 只打印 RunSpec；`compare` 重新生成并执行 RunSpec。正式实验必须在搜索前保存并冻结计划，不应把两条命令之间重新生成的内容误当成自动锁定的历史计划。 - -先打开 `runs/benchmark-demo/check/result.json` 看失败断言或通过范围，再打开 `runs/benchmark-demo/profile/profile.txt`。`compare` 返回实际 run 目录和保留状态：PASS 退出 0，FAIL 退出 1，INCONCLUSIVE 退出 2。错误实现若使测量未完成，比较通常为 INCONCLUSIVE；其独立正确性 grade 可为 FAIL。在 run 目录中查看： - -| 文件 | 读它是为了什么 | -| --- | --- | -| `run.lock.json`、`task.lock.json` | 确认比较命令、任务、预算与规则 | -| `baseline/trial-*/result.json`、`candidate/trial-*/result.json` | 查看每题样本、每次输入身份和正确性检查 | -| `evaluation.json` | 查看原始聚合和描述性 verdict，包括统计显著性未建立的声明 | -| `status.json`、各 trial 的 `execution.json` | 确认是否完成，区分数值拒绝、执行失败和预算问题 | -| `source_checks.json` | 确认比较期间源码是否变化 | - -学习后再编辑候选并用新的输出目录验证。示例使用仓库已忽略的 `runs/` 目录。修改受 Git 跟踪的文件会进入 Workbench 源码快照；每次正式实验应保存独立计划和结果。 - -## 6. 证据说明它有什么实际用处 - -本节的 v2 归档来自 2026-11-03 UTC，使用旧 grader `0b75b5b…`，对应 ce84506e 发布、ac03ae23 保留的测量实现。本轮 CLI 修复改变了 grader 身份；输入与计时协议仍为 v2，但这 300 个样本不是修复后新 head 的运行证据。完整身份边界见 [审查记录](review.md)。 - -| 实际做过的检查 | 观察到了什么 | 能支持的结论 | 不能推出什么 | -| --- | --- | --- | --- | -| 历史 v1：RMSNorm 漏 epsilon 的合理错误实现 | 普通输入冒烟通过，完整任务和底层 SkillEval verifier 都拒绝；无性能指标 | 任务边界用例能补普通冒烟的漏洞 | 没有证明所有 RMSNorm 错误都能捕获 | -| 既有 v2 记录：id/content memo 和按 shape 重放旧输出 | 正确 memo 没有旧输入缓存命中；错误重放被拒绝 | 计时不再因预检复用输入而只测旧答案回放 | 没有隐藏题集或防恶意作弊保证 | -| 既有 v2 记录：两题经过真实 Workbench 和底层 verifier | 规则、输入配对一致；保留 12 个 trial、300 个计时样本及两份 profile | 两个消费者实际使用了同一套任务规则 | 没有证明正式 SkillEval 两臂已接通 | -| 历史 v1：同码与 einsum 对照 | 同码也出现很大的时间差，两次原始性能 verdict 均 FAIL | 必须保留负结果并先解决测量可信度 | 不能认定 einsum 稳定变慢，也不能宣布优化成功 | -| 既有 v2 记录：profile | 保留 CPU 调用路径；文本时间精度下多项显示 0.000 | 诊断操作确实独立运行，可检查调用结构 | 不能据此认定内存带宽瓶颈或函数耗时占比 | - -既有 v2 记录的 [300 个原始样本及聚合](replay-measurements.json)、[输入与身份摘要](replay-validation-summary.json)、[RMSNorm profile](replay-profiles/rmsnorm-cpu.txt) 和 [Softmax profile](replay-profiles/softmax-cpu.txt) 已可直接检查。它们来自同一次既有运行，此次整理没有重跑或筛除慢样本。 - -v2 Workbench 的两次原始 verdict 是 PASS，但仍是描述性结果。某组 trial 的范围相对中位数超过 100%，同码噪声资格还未完成，所以我们的决定仍是：**没有证据宣布可靠性能提升。** 历史 v1 的 360/720 个样本不可与 v2 的 300 个样本混合评分。 - -源码 CI 与实验验收也要分开：已记录的 hosted Linux Python 3.10/3.11 全量检查通过；历史 Windows 全量结果仍是 3479 passed、51 skipped、56 failure reports，所有失败在原始基线复现，但该次全量仍不绿。具体 revision 与本轮回归见 [review](review.md)。 - -## 7. 为什么选择这套设计，和 AMD 的项目是什么关系 - -AMD [AgentKernelArena 的固定版本任务契约](https://github.com/AMD-AGI/AgentKernelArena/blob/47c2658a07efc7c2c0f72751781a5b5690471b0b/docs/how-to/add-task.md) 把任务语义、Agent 可修改的实现、性能基线、正确性参考、工作负载和评测操作分清楚。这种分工让人容易理解“给 Agent 什么、要求交什么、谁来验收”。本次借鉴的就是这层职责。 - -| 来源/方案 | 在本次具体落在哪里 | 选择理由 | -| --- | --- | --- | -| AgentKernelArena | 任务包明确输入、候选、baseline、reference、评测 | 给用户和 Agent 明确的工作对象 | -| KernelBench | 两道归约算子的数学题意、axis=1 与来源许可 | 题目真实明确，CPU 适配可在硬件未定时检验共享边界 | -| BackendBench | 边界与布局检查思路 | 提高普通输入之外的错误识别能力 | -| SkillsBench | 题目/答案与技能条件分离 | 让两臂比较使用共同裁判 | -| AlphaEvolve、AIDE | evaluator 与候选搜索策略分开 | #252 保持任务评测职责，搜索策略走独立设计 | -| InferRSI 已有 Workbench/SkillEval | RunSpec、AB/BA、执行状态、verifier | 复用现有运行机制，让任务能在本项目发挥作用 | - -完整论文与固定源码引用、未采用方案见 [01-research](01-research.md)。当前约束下选择“小任务包 + 具体 adapter + 现有执行器”有可验证的交付；它不表示 CPU 小题已足以代表真实推理系统，也不表示完整引入另一套 GPU 平台更适合我们。 - -## 8. 看代码的顺序，以及剩下的工作 - -先看一题的 manifest 和候选，再看执行，再看接入。无需先逐行读历史 JSON。 - -| 顺序 | 入口 | 重点 | -| --- | --- | --- | -| 1 | [历史 RMSNorm task.json](https://github.com/Bounty-hunter/InferRSI/blob/125b06451a487e76a1c20c45abe4e22cd8631d68/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/task.json)、[历史 candidate.py](https://github.com/Bounty-hunter/InferRSI/blob/125b06451a487e76a1c20c45abe4e22cd8631d68/src/inferrsi/data/engineering_benchmarks/rmsnorm-cpu/candidate.py) | 题目规定了什么，Agent 改哪里 | -| 2 | [numpy_task.py](../../../src/inferrsi/harness/benchmarks/numpy_task.py) | 输入、独立 oracle、check、profile 与计时的真实边界 | -| 3 | [contract.py](../../../src/inferrsi/harness/benchmarks/contract.py)、[worker.py](../../../src/inferrsi/harness/benchmarks/worker.py) | manifest 合法性、内容身份、执行前后检查、RunSpec | -| 4 | [consumers.py](../../../src/inferrsi/harness/benchmarks/consumers.py)、[CLI](../../../src/inferrsi/harness/benchmarks/cli.py) | 如何复用执行器，如何呈现状态和结果 | -| 5 | [skill_verifier.py](../../../src/inferrsi/harness/benchmarks/skill_verifier.py) | 已审核产物如何交给现有底层 verifier | -| 6 | [正式 adapter](../../../src/inferrsi/harness/skill_eval_benchmark.py)、[正式接入说明](formal-integration.md) | 两臂产物交接、隔离计算与父进程检查 | -| 7 | [测试](../../../tests/test_engineering_benchmarks.py)、[复现入口](../../../tools/engineering_benchmark_validate.py) | 正确/错误对照和实际消费者验证 | - -下一步优先级是：维护者确认首批范围；建立能分辨差异的 CPU 测量条件与同码对照；在已接入的正式两臂入口上选定受支持的生产者与隔离运行环境，再用受控 Agent 实验检验技能收益。目标硬件确定后，新增设备 adapter、真实负载和对应层级验收。 - -本次 PR 保持 Draft，四个子 Issue 保持打开。我们已经交付了共享任务、真实评测入口及组件证据；正式两臂正确性产物入口已实现，实际平台控制结果见接入记录；可靠性能排序、真实优化者收益、改进控制器和设备/服务收益仍需完成。 diff --git a/docs/skill-eval-comparison.md b/docs/skill-eval-comparison.md index d1aa877b0..3736bce86 100644 --- a/docs/skill-eval-comparison.md +++ b/docs/skill-eval-comparison.md @@ -15,7 +15,7 @@ in **update mode**, with the same prepare/execute/report and local stage/capture interfaces. The trusted parent grades output from a Linux bubblewrap NumPy worker; unsupported sandbox/runtime leaves INCONCLUSIVE without host fallback. This is correctness-only artifact evaluation, not timing or skill-efficacy -qualification. See the [task contract and complete handoff example](plans/issue-252/formal-integration.md). +qualification. See the [task contract and handoff example](engineering-task-artifacts.md). The provider generation mode and offline improvement controller do not support this adapter yet. diff --git a/tools/engineering_benchmark_validate.py b/tools/engineering_benchmark_validate.py index 951eab850..2fb7af8a8 100644 --- a/tools/engineering_benchmark_validate.py +++ b/tools/engineering_benchmark_validate.py @@ -1,4 +1,4 @@ -"""Reproduce the #252 CPU task integration with retained, authored controls. +"""Reproduce CPU task integration with retained, authored controls. No models, hardware allocators or downloaded programs are invoked. Output is a local validation bundle, not campaign acceptance or a skill-benefit experiment.