From 8a682c67f704e7875e6753c32be02d23ddaf7bd9 Mon Sep 17 00:00:00 2001 From: eric8810 Date: Wed, 7 Oct 2026 21:38:14 +0800 Subject: [PATCH 1/3] =?UTF-8?q?feat(npu):=20=E8=BD=BB=E8=8A=AF=E5=88=9D?= =?UTF-8?q?=E9=93=B8=EF=BC=8C=E7=9F=A9=E9=98=B5=E6=88=90=E5=8D=B7=20=C2=B7?= =?UTF-8?q?=20bootstrap=20lightweight=20AMD=20AIE=20compilation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/amd-aie-compilation-results.json | 197 +++++++++++++ docs/amd-aie-development.md | 184 ++++++++++++ docs/amd-npu-acceleration.md | 2 + docs/amd-npu-lightweight-plan.md | 187 ++++++++++++ tools/npu/build_iree.py | 188 +++++++++++++ tools/npu/compile_iree_hotspot.py | 173 ++++++++++++ tools/npu/export_iree_hotspot.py | 266 ++++++++++++++++++ tools/npu/export_iree_model.py | 94 +++++++ tools/npu/fetch_iree_source.py | 59 ++++ tools/npu/fetch_peano.py | 79 ++++++ tools/npu/import_iree_model.py | 123 ++++++++ tools/npu/iree-frontend.requirements.txt | 9 + tools/npu/iree-source.lock.json | 48 ++++ .../npu/patches/check-matmul-map-ranks.patch | 44 +++ tools/npu/transform_iree_depthwise.py | 87 ++++++ 15 files changed, 1740 insertions(+) create mode 100644 docs/amd-aie-compilation-results.json create mode 100644 docs/amd-aie-development.md create mode 100644 docs/amd-npu-lightweight-plan.md create mode 100644 tools/npu/build_iree.py create mode 100644 tools/npu/compile_iree_hotspot.py create mode 100644 tools/npu/export_iree_hotspot.py create mode 100644 tools/npu/export_iree_model.py create mode 100644 tools/npu/fetch_iree_source.py create mode 100644 tools/npu/fetch_peano.py create mode 100644 tools/npu/import_iree_model.py create mode 100644 tools/npu/iree-frontend.requirements.txt create mode 100644 tools/npu/iree-source.lock.json create mode 100644 tools/npu/patches/check-matmul-map-ranks.patch create mode 100644 tools/npu/transform_iree_depthwise.py diff --git a/docs/amd-aie-compilation-results.json b/docs/amd-aie-compilation-results.json new file mode 100644 index 0000000..728b42e --- /dev/null +++ b/docs/amd-aie-compilation-results.json @@ -0,0 +1,197 @@ +{ + "schemaVersion": "1.0", + "stage": "development", + "date": "2026-10-07", + "sourceModelSha256": "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + "sourceLockSha256": "a58136fc60e8eb5023798c3e8d3a7014abbeb2f3107f6a727c5b8f1f0cd0bf4a", + "compiler": { + "sourceCommits": { + "amdAie": "fddfec1be6ceefbdb890079d957947dfa1fe0848", + "third_party/aie-rt": "a6196ebd905ceaf92cabc2e594017fa11b9e4323", + "third_party/bootgen": "92e09bf37ea17d7b1f0e102a2548f27fb768651c", + "third_party/iree": "26083307b8753a11870b8caf590afea15c9429fa", + "third_party/mlir-air": "ca2448548d66d7e8e05bacb9e81e7df48a9ef729", + "third_party/openssl": "0e04e101507df752500093ff4b04cce5a898abaa", + "iree/third_party/benchmark": "192ef10025eb2c4cdd392bc502f0c852196baa48", + "iree/third_party/llvm-project": "8be29edc20dfb19245ab8cf49669c80125ece4f5", + "iree/third_party/flatcc": "9362cd00f0007d8cbee7bff86e90fb4b6b227ff3", + "iree/third_party/musl": "3f701faace7addc75d16dea8a6cd769fa5b3f260" + }, + "localPatches": [ + { + "path": "patches/check-matmul-map-ranks.patch", + "sha256": "5a0a9b3efe57cc4d257b8fd9d08f59a4c158a07c71a7242d21b812314ee9fb05", + "reason": "Reject scalar/rank-one generic maps before matmul dimension access" + } + ], + "localSourceDiffSha256": "b62c2c5415db8739e8799ca8f1f28658c1e27229ad0897fafce505150d6cb6f9", + "binaries": [ + { + "path": "tools/iree-compile", + "bytes": 6256, + "sha256": "a751017528d0e96e3331e119cfa87c9705e802a3fcc27c68b4a74f8afbe7e3f3" + }, + { + "path": "tools/iree-opt", + "bytes": 6248, + "sha256": "5b7123611e66efbbc527c0dfb0ac225a1597193cf4bbdb69022fdc4c169366b4" + }, + { + "path": "lib/libIREECompiler.so", + "bytes": 271528912, + "sha256": "352f1d6e4c7f3945f7fbbdf09255598cad153ec3afbac52f353241559cbc4408" + } + ] + }, + "runtimeBinaries": [ + { + "path": "tools/iree-run-module", + "bytes": 1843488, + "sha256": "b269f29d707f00a426f09a2fb962806b61e7c3a4edabe7da4b13d9057f8ee3f5" + }, + { + "path": "tools/iree-dump-module", + "bytes": 462336, + "sha256": "83eb65201baa9071dd14af67e265b70b0b3239b76bcf6bb7835dd5ddb4e8d34b" + } + ], + "target": "npu4", + "deviceHAL": "amdxdna", + "coreRows": 2, + "coreCols": 2, + "deviceValidated": false, + "numericsValidated": false, + "fullRecognitionSupported": false, + "reports": { + "matrixTargets": { + "compileStage": "targets", + "status": "failed", + "kernelMode": "peano-bf16", + "kernelPrecision": "bf16 converted to bfp16ebs8 inside Peano matmul microkernel", + "results": [ + { + "variant": "projection-matmul", + "status": "compiled", + "returnCode": 0, + "artifact": { + "path": "projection-matmul.mlir", + "bytes": 1248515, + "sha256": "e25ab9d86cb7e14fcf56e09f8a27bf915fe31e074526a2825650c18419c15a4d" + }, + "log": { + "path": "projection-matmul.log", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + { + "variant": "projection-matmul-padded", + "status": "compiled", + "returnCode": 0, + "artifact": { + "path": "projection-matmul-padded.mlir", + "bytes": 1248480, + "sha256": "be7d04f2b328b244ac74aa475abbb5b59e0e8059395fcd99714b8ead5a62a256" + }, + "log": { + "path": "projection-matmul-padded.log", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + }, + { + "variant": "projection", + "status": "failed", + "returnCode": 1, + "log": { + "path": "projection.log", + "bytes": 5614, + "sha256": "96adc90f3fe65b0a682cfdb697640621073dced26f9b590f4d88289ddc6ff593" + } + }, + { + "variant": "mlp-residual", + "status": "failed", + "returnCode": 1, + "log": { + "path": "mlp-residual.log", + "bytes": 8236, + "sha256": "00db9558a947f4b6c22fbc4b5e28e1998ac0b7bfd93f4365b336c09cbb11f4f2" + } + } + ] + }, + "matrixBinary": { + "compileStage": "binary", + "status": "compiled", + "kernelMode": "peano-bf16", + "kernelPrecision": "bf16 converted to bfp16ebs8 inside Peano matmul microkernel", + "results": [ + { + "variant": "projection-matmul-padded", + "status": "compiled", + "returnCode": 0, + "artifact": { + "path": "projection-matmul-padded.vmfb", + "bytes": 612008, + "sha256": "c1d76da61b859a4aade426fa562eb997f32c73b40242cc359a07cf7d1f3613de" + }, + "log": { + "path": "projection-matmul-padded.log", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + ] + }, + "graphSources": { + "compileStage": "sources", + "status": "compiled", + "kernelMode": "vector", + "kernelPrecision": "source IR types", + "results": [ + { + "variant": "recognition", + "status": "compiled", + "returnCode": 0, + "artifact": { + "path": "recognition.mlir", + "bytes": 42484239, + "sha256": "2bb0403b4a48ad345e4837e37515b633a9b38d044d8aaa19f75d2d58c008b5dc" + }, + "log": { + "path": "recognition.log", + "bytes": 0, + "sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + } + } + ] + }, + "graphTargets": { + "compileStage": "targets", + "status": "failed", + "kernelMode": "vector", + "kernelPrecision": "source IR types", + "results": [ + { + "variant": "recognition", + "status": "crashed", + "returnCode": -11, + "log": { + "path": "recognition.log", + "bytes": 97, + "sha256": "beeae0292954fe1578e7f14762b2437daa2c88904dfa257b5362f332a124faf2" + }, + "signal": 11 + } + ] + } + }, + "remainingWork": [ + "layout and precision conversion dispatch lowering or coarse CPU/NPU partition", + "reduce full graph PackOp/packTranspose crash", + "evaluate BFP16 precision changes", + "share weights across widths", + "native backend and optional package integration" + ] +} diff --git a/docs/amd-aie-development.md b/docs/amd-aie-development.md new file mode 100644 index 0000000..8948612 --- /dev/null +++ b/docs/amd-aie-development.md @@ -0,0 +1,184 @@ +# AMD AIE 开发与子图编译 + +状态：开发工具链与子图导出已实现，尚未接入发布后端。此流程不执行设备推理。 +总体选型与后续阶段见 [轻量部署方案](amd-npu-lightweight-plan.md)。 +本轮工具链身份、编译状态与产物大小/哈希见 [编译结果清单](amd-aie-compilation-results.json)。 + +## 构建输入 + +`tools/npu/iree-source.lock.json` 锁定 AMD AIE、IREE、LLVM 与必要子模块提交，以及 Peano wheel 的大小和 SHA256。 +工具只读取已锁定的 Small 0.3.4 识别模型，拒绝源模型哈希不符。 +编译工具和输出保存在 `.cache/amd-aie/`；不进入主 package 或当前 AMD 支持包。 + +## 准备源码 + +以下 checkout 发生在独立开发源码目录。工具按锁文件获取所需子模块： + +```sh +python tools/npu/fetch_iree_source.py \ + --output-dir .cache/amd-aie/source +``` + +本机需要 CMake、Ninja、Clang/Clang++、Git、Python、Perl、Make 与相关系统开发头文件。 +上游构建还会准备 OpenSSL。Peano 只用于构建，下载与解包命令为： + +```sh +python tools/npu/fetch_peano.py \ + --package-cache .cache/amd-aie/downloads \ + --output-dir .cache/amd-aie/peano +``` + +已有缓存时可加 `--offline`。输出目录必须不存在；工具检查归档身份、拒绝重复路径和链接，并保留执行权限。 + +## 构建编译器 + +```sh +python tools/npu/build_iree.py \ + --source-dir .cache/amd-aie/source \ + --peano-dir .cache/amd-aie/peano/llvm-aie \ + --build-dir .cache/amd-aie/toolchain-build \ + --jobs 2 +``` + +构建源树提交必须与锁文件相符。编译器启用 AMD AIE 和 LLVM CPU 后端；关闭单元测试、设备测试、示例及 Python bindings。 +上游 CMake 会原地修改 aie-rt、bootgen 与 mlir-air；锁文件同时固定这些配置补丁的 diff 哈希，拒绝其他已跟踪源码改动。 +本地 matmul 索引映射边界补丁也按补丁文件与最终源码 diff 双重锁定，由构建工具应用。 +`toolchain-build.json` 记录固定提交、配置/构建命令及状态。失败后可在同一个构建目录继续增量构建。 +`--configure-only` 仅生成构建配置，不能用于后续子图编译的成功证明。 +增量构建在配置命令、源码提交与主机工具一致时复用 CMake 配置，避免上游配置步骤反复改写头文件造成大规模重编译。 + +## 导出真实识别热点块 + +导出对象为 `Conv.49 → bias → GELU → Conv.50 → bias → residual`。 +FP32 ONNX 保留原始子图；MLIR 将 1×1 卷积映射为矩阵乘，转换为 NHWC，使用 BF16 操作数与 FP32 累加。 +GELU 保留 Erf 公式，数值变化尚未验证。 + +导出环境需要 numpy 与 onnx；可使用现有模型构建 Python 环境。 + +```sh +python tools/npu/export_iree_hotspot.py \ + --source-model models/generated/ppocrv6-small-onnx-20260714.2/rec/inference.onnx \ + --output-dir .cache/amd-aie/hotspot-384 \ + --width 384 +``` + +输出包括： + +- `reference-fp32.onnx`：对应源模型的独立 FP32 子图。 +- `projection.mlir`：首个投影层和 bias，用于定位基础映射缺口。 +- `projection-matmul.mlir`：真实首层的矩阵核心，BF16 矩阵输入、FP32 输出；布局、输入转换和 bias 在接口之外。 +- `projection-matmul-padded.mlir`：矩阵核心按 128 行向上补齐；调用方补零，使用后裁切有效行。 +- `mlp-residual.mlir`：完整 MLP 残差块。 +- `weights-fp32.npz`：原始矩阵权重与 bias。 +- `hotspot-manifest.json`：源模型/产物哈希、宽度、形状、节点及精度转换记录。 + +宽度 384 时块输入输出为 `[1,384,3,96]`。可使用既有 20 桶内的其他宽度，但每次使用新的输出目录。 + +## 完整识别图前端 + +前端单独使用 Python 3.12 环境，依赖不会进入运行时包： + +```sh +python3.12 -m venv .cache/amd-aie/host-compiler-venv +.cache/amd-aie/host-compiler-venv/bin/pip install \ + -r tools/npu/iree-frontend.requirements.txt +.cache/amd-aie/host-compiler-venv/bin/python tools/npu/export_iree_model.py \ + --source-model models/generated/ppocrv6-small-onnx-20260714.2/rec/inference.onnx \ + --output-dir .cache/amd-aie/model-384 --width 384 +python tools/npu/import_iree_model.py \ + --model-dir .cache/amd-aie/model-384 \ + --frontend-environment .cache/amd-aie/host-compiler-venv \ + --output-dir .cache/amd-aie/model-384-imported +``` + +导出工具固定输入尺寸、折叠已知形状查询，再用 ONNX 官方转换器将 opset 11 转为 17。 +导入流程保留 FP32，将图降为 core Linalg，把普通卷积和深度卷积改为 NHWC，保持模型接口不变。 +宽度 384 的输入为 `[1,3,48,384]`，输出为 `[1,48,18710]`；已完成 43 个普通卷积和 14 个深度卷积转换。 +池化等其他操作仍可能使用 NCHW。`import-report.json` 保存步骤、日志、工具身份和输出哈希。 + +标准前端版本 3.12.0 比锁定的 AMD IREE 源码新。输出文本 IR 尚需 AMD 编译器接受； +不能据此认定两版 VMFB 或运行时兼容。完整图已通过锁定 AMD 编译器的 `--stage sources`， +生成 executable sources，证明前端文本 IR 可被该版本接受；尚未通过 AMD 内核 lowering。 +完整 FP32 图已用标准 IREE CPU 后端编译，产物为 +21,344,236 字节，但未执行推理，也未验证精度。该产物不作为 AMD 模型发布。 + +## 独立运行时构建 + +```sh +python tools/npu/build_iree.py \ + --source-dir .cache/amd-aie/source \ + --peano-dir .cache/amd-aie/peano/llvm-aie \ + --build-dir .cache/amd-aie/runtime-build \ + --runtime-only --jobs 2 +``` + +运行时与编译器使用不同构建目录。当前 Linux x86_64 Release 构建的 `iree-run-module` +为 1,843,488 字节，已链接 amdxdna 驱动；ELF 动态依赖为 libm、libuuid、libstdc++、libgcc_s、libc。 +它不依赖 vendor SDK 或 XRT。这只是独立执行工具的大小，完整支持包还需 addon、模型及其他运行时文件。 +本阶段未运行该工具访问设备。 + +## 编译与保留诊断 + +先进行目标 IR lowering： + +```sh +python tools/npu/compile_iree_hotspot.py \ + --hotspot-dir .cache/amd-aie/hotspot-384 \ + --toolchain-build-dir .cache/amd-aie/toolchain-build \ + --peano-dir .cache/amd-aie/peano/llvm-aie \ + --output-dir .cache/amd-aie/compile-384-targets \ + --stage targets +``` + +工具要求完成固定版本构建，并检查输入 MLIR 与 manifest 的哈希。 +首期显式指定 `npu4`、2×2 核阵列、`amdxdna` HAL、objectFifo 与 pack-peel，不使用 vendor ORT、XRT HAL 或设备探测。 +输出 `compile-report.json` 及每个子图的日志。编译失败或超时返回非零状态，保留诊断；不能将失败描述为 CPU 回退后编译成功。 + +目标 IR 完成后，使用新的输出目录及 `--stage binary` 尝试生成 VMFB。 +`--variant` 可单独选择完整块、投影层或矩阵核心；`--timeout` 限制每次编译的秒数。 +VMFB 才属于后续加载候选；目标 MLIR 不作为部署模型打包。 + +同一工具支持把 `--hotspot-dir` 换成 `--model-dir .cache/amd-aie/model-384-imported`， +尝试完整 FP32 图的 AMD 编译，并保留 `recognition.log`。此时不选择热点 `--variant`； +图导入成功不预设 AMD 编译成功，失败不会自动改用 CPU 后端。 + +### 已编译的诊断矩阵核心 + +宽度 384 的原始矩阵为 `288×384 · 384×768`。直接目标编译可通过，但二进制链接需要上游未提供的 +`48×32×128` 微内核实例。显式补齐输入到 `384×384`、保留原权重后可编译成 AMD VMFB： + +```sh +python tools/npu/compile_iree_hotspot.py \ + --hotspot-dir .cache/amd-aie/hotspot-384 \ + --toolchain-build-dir .cache/amd-aie/toolchain-build \ + --peano-dir .cache/amd-aie/peano/llvm-aie \ + --output-dir .cache/amd-aie/compile-384-matmul-binary \ + --variant projection-matmul-padded --kernel-mode peano-bf16 --stage binary +``` + +补丁后通过开发脚本生成的产物为 612,008 字节。该文件只覆盖矩阵核心，不包含 bias、GELU、残差、输入布局转换或裁切。 +调用方须补零最后 96 行，裁切输出至前 288 行，再执行原 bias。 +Peano 微内核内部将 BF16 转为 `bfp16ebs8` 块浮点后计算；这项精度变化尚未验证。 +普通向量 lowering 当前不支持 npu4 的 BF16 矩阵指令类型，因此此处显式启用微内核，并关闭 Chess。 +`--core-rows` / `--core-cols` 可改变阵列大小，默认 2×2 来避开本例更大阵列的分块及内存问题。 + +完整投影层在布局转换/FP32→BF16 的独立 dispatch 处报 `Unhandled pass pipeline in setRootConfig`。 +这些诊断不能作为完整识别模型已支持的结论；后续仍需处理 CPU/NPU 分区或补齐对应 lowering。 + +完整 FP32 图进入 AMD lowering 后发生 SIGSEGV；systemd-coredump 回溯指向 +`AffineMap::getResult → AMDAIE::isMatmul → initAIELaunchConfig`。 +同一时段内核日志没有 OOM 记录，仍有充足可用内存。 +源码中 matmul 识别器按无符号 `nResults - 2` 访问索引映射，尚未检查结果数； +补上边界检查后，整图越过该阶段，随后在 `PackOp::createTransposedClone → packTranspose → AMDAIEPackAndTransposePass` +再次发生 SIGSEGV。第二处崩溃与具体触发算子仍待缩减样例确认。 + +## 结果解释 + +- 导出完成：证明生成了固定源模型的开发输入。 +- executable sources 完成：证明前端 IR 和公共图 lowering 被锁定编译器接受，不证明 AMD 内核可编译。 +- 目标 lowering 完成：证明指定编译路径接受了该输入。 +- 二进制完成：证明生成了加载候选，不证明设备兼容或数值正确。 +- `deviceValidated` 与 `numericsValidated` 在该流程中始终为 `false`。 + +后续依据实际错误决定布局/分块改造、算子补齐或调整分区，不能为了通过编译删除 GELU、残差或更改字典类别。 +此开发入口不替换现有 `amdnpu` 后端，不改变默认 provider，不修改发布包和版本号。 diff --git a/docs/amd-npu-acceleration.md b/docs/amd-npu-acceleration.md index 37804f2..f655e86 100644 --- a/docs/amd-npu-acceleration.md +++ b/docs/amd-npu-acceleration.md @@ -1,5 +1,7 @@ # AMD NPU 加速技术方案 +后续轻量部署改造的选型、实施阶段和发布约束见 [AMD NPU 轻量部署改造方案](amd-npu-lightweight-plan.md)。该方案尚未实施，当前文档描述既有 Ryzen AI vendor 后端。 + 更新时间：2026-10-06。状态：Linux x64 glibc 后端与打包改造已实现，默认关闭、配置显式开启；0.5.9 独立支持包的构建输入已包含 Ryzen AI 1.8 的原生部署库与 Small 0.3.4 的 20 个真实 BF16 context。无需真机即可完成编译，尚未执行 AMD 设备推理。正式 npm 发布仍待完成。 ## 范围与模型路线 diff --git a/docs/amd-npu-lightweight-plan.md b/docs/amd-npu-lightweight-plan.md new file mode 100644 index 0000000..2314017 --- /dev/null +++ b/docs/amd-npu-lightweight-plan.md @@ -0,0 +1,187 @@ +# AMD NPU 轻量部署改造方案 + +日期：2026-10-07。状态：P0/P1 开发进行中；构建、子图导出和编译报告工具已实现，完整后端尚未接入。 + +## 1. 目标与结论 + +将 AMD NPU 独立支持包从当前约 548 MB 下载、1.39 GB 解包的规模，改造为只包含执行运行时、OCR 编译产物和必要许可文件的部署包。LLVM、Peano、模型导入及编译工具留在构建环境。 + +**首选路线：评估 IREE AMD AIE 提前编译 Small 识别模型，使用对应的 IREE amdxdna 运行时。** 如编译缺口过大，第二路线采用 HRX + IRON 实现 OCR 专用内核与调度。xdna-engine 用作算法和内核参考；不整体接入 FastFlowLM。 + +本方案先完成编译可行性评估，再确定最终后端。现有算子代码和轻量运行库的存在，并不证明完整 PP-OCR 图已支持。 + +## 2. 范围与约束 + +- 首期：Linux x64 glibc、XDNA2、PP-OCRv6 Small 识别模型；检测继续使用 CPU。 +- 模型基线：Small 0.3.4，识别模型 SHA256 为 `5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634`。 +- 优先使用 BF16 计算，保持 FP32 输入输出及原有 CTC 字典、解码行为。首期不引入 INT8 校准路线。 +- 保留 20 个识别宽度：320、384、480、544、576、608、704、736、832、960、1056、1184、1248、1376、1600、1984、2240、2560、2880、3200。 +- AMD 默认关闭，继续通过 `execution.provider: "amdnpu"` 显式启用，支持包由用户单独安装；Auto 不选择 AMD。 +- 维护者已取消真机验收作为发布前置条件。设备兼容性、数值结果与性能的未验证状态必须如实记录。 +- Windows、musl、PHX/HPT、其他模型规模及检测模型的 NPU 改造不纳入首期。 + +新运行时的支持范围须重新确认。现有 vendor 后端的 STX/KRK 范围不能直接继承为新后端的设备验证结论。 + +## 3. 调研依据 + +### 3.1 固定源码与发布包 + +以下为本次调查使用的固定版本，而非随 `main` 更新的构建输入。 + +| 项目 | 源码提交或发布版本 | 用途 | +|---|---|---| +| [IREE AMD AIE](https://github.com/nod-ai/iree-amd-aie/tree/fddfec1be6ceefbdb890079d957947dfa1fe0848) | `fddfec1be6ceefbdb890079d957947dfa1fe0848` | 首选图编译器与配套运行时 | +| [IRON / MLIR-AIE](https://github.com/Xilinx/mlir-aie/tree/a75319190bdb1ca83703433b2aa8f8f52bc80565) | `a75319190bdb1ca83703433b2aa8f8f52bc80565` | 定制内核、数据流及 HRX 接入参考 | +| [xdna-engine](https://github.com/atassis/xdna-engine/tree/013b3e31c4832d99808c5dc5592dbf872433a33b) | `013b3e31c4832d99808c5dc5592dbf872433a33b` | GEMM、卷积与调度参考 | +| [FastFlowLM](https://github.com/ROCm/FastFlowLM/tree/8d6768dab2fd8616ea3b814f611ff01a0758681f) | `8d6768dab2fd8616ea3b814f611ff01a0758681f`；部署包 1.0.7 | 专用模型执行方案比较 | +| [HRX Linux 发布包](https://github.com/jtuyls/hrx/releases/tag/flm-hrx-amdxdna-v2026.09.02) | `flm-hrx-amdxdna-v2026.09.02` | 轻量执行库体积及 C ABI 检查 | + +HRX Linux 归档 SHA256：`c273a92ab69c3280d6bdbf4f6e4c6317fde821d4dcc53b45e82a1dacb7d4a2de`，与 IRON 锁文件一致。正式实现时还需锁定 IREE、Peano、导入工具及子模块版本，形成完整工具链清单。 + +### 3.2 本地测量 + +体积采用十进制 MB；下表均不包含系统驱动和模型权重，除明确标注的现有 AMD 支持包。 + +| 产物 | 下载大小 | 解包或库大小 | 含义 | +|---|---:|---:|---| +| 当前 AMD npm 支持包 | 548.19 MB | 1,389.33 MB | 含 vendor 部署库、编译 context 和原生 addon | +| HRX Linux 归档 | 0.683 MB | 核心库 1.824 MB | 只提供执行层，不提供完整 OCR 模型执行器 | +| FastFlowLM 1.0.7 Ubuntu 24.04 deb | 31.77 MB | 包内文件 227.59 MB | 多模型 CLI、引擎与内核；仍依赖系统 XRT 等 | + +FastFlowLM 的约 17 MB 宣传值不用于本方案预算。HRX 的小体积也不能直接作为最终 OCR 支持包体积。 + +### 3.3 模型特征 + +宽度 384 的识别静态图含 57 个 Conv、13 个 MatMul 和 3 个 Softmax。卷积包括 depthwise、`stride=[2,1]`、1×7 核；另有池化、归一化及 GELU 的 Erf 展开。 + +其中 39 个 Conv 为 1×1。在已推断形状的卷积中，1×1 占约 90.9% 的 MAC，适合优先研究 GEMM 映射。该比例仅为卷积计算量估计，未计入仍含符号维度的 MatMul，不代表端到端时延占比或加速比。 + +## 4. 方案比较与选择 + +| 路线 | 可复用能力 | 主要缺口 | 决策 | +|---|---|---|---| +| IREE AMD AIE + 对应运行时 | BF16 普通卷积、depthwise、矩阵乘、Softmax 编译配置；图编译基础设施 | 导入、布局转换、分块对齐、整图融合及 CPU/NPU 分区 | 优先评估 | +| HRX + IRON 专用执行器 | 小型 C ABI、设备缓冲区和预编译内核派发 | 自建模型执行计划、权重布局与缺失内核 | 第二路线 | +| xdna-engine 整体接入 | 部分手写 AIE 内核与模型调度 | Rust/XRT 集成；现成超分辨率调度无法表达完整识别图 | 参考组件 | +| FastFlowLM 整体接入 | 已有特定模型引擎 | 无现成 PP-OCR 导入路径，需另写模型引擎 | 不采用 | + +IREE 的选型依据是减少模型映射工作量。其源码仍标记 early-phase，且存在具体限制： + +1. 普通卷积路径要求 NHWC；现有 NCHW 模型需要转换布局。 +2. depthwise 分块代码注明尺寸及切分检查尚不完善。 +3. 矩阵维度需满足指令与分块粒度；最终 `[120,18710]` 分类权重需要确认 padding、裁切或其他分块方案。 +4. 归一化、激活及跨算子融合仍需按实际识别图逐项确认。 + +依据见固定版本的 [KernelDispatch.cpp](https://github.com/nod-ai/iree-amd-aie/blob/fddfec1be6ceefbdb890079d957947dfa1fe0848/compiler/plugins/target/AMD-AIE/iree-amd-aie/Transforms/KernelDispatch.cpp) 及其 `Transforms/test/lowering_strategy_*` 文件。编译检查输入的存在不等于本项目模型已经编译成功。 + +**IREE 与 HRX 的产物格式不能预设互换。** 当前 IREE amdxdna 使用 `amdaie-pdi-fb`，所调查 HRX 接口使用 XADX/xclbin 与 transaction 描述。首选路线使用配套 IREE 运行时；如改为 HRX，需要独立验证产物转换和加载路径。现有 vendor EPContext/.rai 也不能直接替换运行库后继续使用。 + +## 5. 目标架构 + +```mermaid +flowchart TD + A[锁定的 Small ONNX 与字典] --> B[构建环境：导入、布局转换、BF16 与图优化] + B --> C[IREE AMD AIE 提前编译] + C --> D[运行时、模型产物、清单、许可与 SBOM] + D --> E[独立 AMD npm 支持包] + E --> F[显式选择 amdnpu] + F --> G[C++ InferenceSession 适配层] + G --> H[IREE amdxdna 执行运行时] + H --> I[系统 amdxdna 驱动与 NPU] + G --> J[声明的 CPU 分区] +``` + +### 构建与产物 + +- 构建工具链在隔离目录或 CI 中运行；用户安装时不编译模型、不下载 SDK、不执行驱动安装。 +- 编译前记录模型哈希、转换规则、算子清单、布局和精度变化。 +- 编译后记录每个宽度的输入输出、目标设备、执行格式、运行时版本和文件哈希。 +- 优先设计共享权重与按宽度区分的执行计划，降低重复权重开销。是否可行由编译器及加载格式验证；如不可共享，必须单独报告重复体积。 +- 默认平台包和主 package 不携带 AMD、OpenVINO 或新编译工具链。 + +### 原生接入 + +- 实现现有 `InferenceSession` 接口，保留 FP32 输入输出及原有 CTC 解码。 +- 首期保持 batch=1、20 个宽度桶和 CPU detector。 +- 使用会话内持久缓冲区，尽量让融合块之间的激活留在设备；避免逐算子 CPU/NPU 往返。 +- 粗粒度分区方案须明确 CPU 算子、传输点和同步点，不能仅根据算子 MAC 决定分区。 +- manifest 新增内部后端标识和执行格式版本，区分 vendor 与 IREE/HRX 产物，拒绝错配。 +- 公开 provider 名称保持 `amdnpu`；请求 provider、实际执行后端、CPU 分区及验证状态进入诊断信息。 +- 资源释放顺序明确为任务完成、缓冲区、可执行产物、会话和运行库；运行中不得卸载库。 + +### 错误和回退 + +- 库、模型或清单哈希不符直接报错，不静默回退。 +- 缺包、无设备、权限不足和驱动不兼容分别提供明确原因。 +- CPU 分区与整会话回退分别遵循现有配置。显式请求 AMD 后，实际在 CPU 执行时必须在诊断中体现。 +- 推理中断、同步超时及后续桶加载失败应报告错误，不伪装成成功的 NPU 推理。 + +## 6. 体积预算 + +建议将 **AMD 支持包下载 ≤64 MiB、解包 ≤128 MiB** 作为本轮工程目标；这是待实测的设计预算，不是已经实现的大小承诺。 + +| 组成 | 压缩预算 | +|---|---:| +| 执行运行时及原生 addon | 16 MiB | +| OCR 权重 | 16 MiB | +| 全部宽度的内核与执行计划 | 28 MiB | +| 清单、许可和其他必要文件 | 4 MiB | + +产物组装阶段输出逐文件体积、依赖闭包和重复权重统计。超出预算时先定位运行库、权重重复、内核重复或调试信息，不以运行时联网下载隐藏安装成本。64 MiB 目标如无法满足，须给出测量依据后修订。 + +主 package 的 39 MB 模型要求单独保留。当前 Small 0.3.4 模型 tgz 实测约 52.53 MB，已有超出该要求的问题；此项需另行处理，不能通过拆分 NPU 包宣称已经满足。 + +## 7. 实施阶段与决策点 + +| 阶段 | 工作内容 | 交付与退出条件 | +|---|---|---| +| P0：锁定构建基础 | 固定完整工具链与源模型；建立算子、形状、许可和体积清单 | 构建输入可追溯，导入路线明确；列出缺失工具与算子 | +| P1：热点子图编译 | 提取宽度 384 的连续 1×1 卷积融合块，完成 BF16 和 NHWC 转换 | 生成可加载格式的产物，记录布局、缓冲区、算子覆盖和体积；只编译成功不宣称数值或设备正确 | +| P2：关键缺口评估 | 覆盖 depthwise、stride、1×7、归一化、激活及分类头对齐 | 决定整图、粗粒度分区或转 HRX；归档不能编译的最小问题样例 | +| P3：原生后端与完整桶 | 实现会话、缓冲区、错误语义；生成全部 20 桶 | 产物符合统一接口，CPU 分区显式，设备目标和格式版本锁定 | +| P4：独立包与发布准备 | 依赖裁剪、许可/SBOM、hash manifest、体积门禁及迁移说明 | 输出可复核候选包；发布材料明确尚无设备验证与性能结论 | + +P2 是主要选型决策点：如果关键算子无法编译、需要大量模型专用编译器修改，或只能形成频繁主机往返的分区，转向 HRX + IRON。改走第二路线时保留已有模型分析和转换规则，重新编译内核及实现调度；不预设旧执行产物可复用。 + +本方案不设未经评估的工期。P1/P2 完成后才能根据实际缺口安排后续工作。 + +## 8. 验收计划 + +本节描述后续实施时应提供的证据，本次方案编写未执行这些检查。 + +### 无真机可完成 + +- 完整工具链、源模型、转换图和所有部署产物可追溯，形状、字典类别与哈希匹配。 +- 转换后的图在 CPU 参考执行中与原始模型比较张量、文本与置信度；记录误差分布，误差阈值在 P0 锁定。 +- 分类 padding 必须在 Softmax/CTC 前正确屏蔽或裁除补齐类别，保证仍为 18,710 类。 +- 编译路径覆盖全部宽度，公开列出未下放的 CPU 算子和设备目标。 +- 包内动态依赖闭包、RPATH、版本与格式检查、错误分支、资源管理、默认禁用及独立安装行为符合约定。 +- 用真实候选包记录压缩/解包体积，确认不包含编译器和 SDK 开发工具。 + +CPU 参考比较只能证明转换图的软件语义，不能证明编译后的 NPU 内核数值正确。没有软件模拟或设备执行证据的部分，明确标为未验证。 + +### 不阻塞发布的设备证据 + +设备加载、驱动/固件适配、编译内核数值、冷/热时延、功耗及实际加速比列为后续可选补充。候选包和发布说明保持 `deviceValidated=false`，不宣称特定设备已经验证或一定加速。 + +## 9. 发布与迁移 + +- 延续独立 AMD 支持包，Auto 默认行为及显式启用方式保持一致。 +- 新产物使用新的锁文件身份、清单及缓存命名空间，避免复用 vendor context 或旧缓存。 +- 首次切换前保存上一可用包的版本和完整哈希；回滚通过安装明确的旧包版本完成。 +- 同一包版本只绑定一种确定的产物集合；不在用户机器上按网络或设备状态替换包内容。 +- 更新 AMD 文档、构建与发布文档、CHANGELOG、安装示例及 release notes，列出运行时、模型、驱动要求和未验证项。 +- 实现需要新版本时默认递增补丁版本；本方案本身不修改版本号、不触发发布。 + +## 10. 当前状态 + +已完成固定源码下载、发布包解包、ELF/头文件检查和识别模型静态结构分析。开发入口见 [AMD AIE 开发与子图编译](amd-aie-development.md)：已实现固定工具链构建、真实识别热点导出、完整图导入与编译报告工具。 +宽度 384 的完整 FP32 图已完成静态形状处理、opset 转换和 57 个卷积的布局转换，标准 IREE CPU 编译产物约 21.34 MB；未执行推理。 +同版源码的独立 amdxdna 执行工具已构建，约 1.84 MB，动态依赖不含 vendor SDK 或 XRT。它并非最终支持包。 +完整后端、数值比较与 NPU 执行尚未完成。 + +首轮实际 AMD 编译已生成宽度 384 的补齐投影矩阵核心 VMFB（612,008 字节），完整图通过 executable sources 阶段。 +完整热点块仍受布局/精度转换 dispatch 限制；整图后端 lowering 仍在打包转置阶段崩溃。 +Peano 微内核内部使用 BFP16，因此不能将该候选描述为已验证的纯 BF16 实现。 + +下一项可执行工作为补齐 P1/P2：缩减打包转置崩溃样例，确定粗粒度分区与算子 lowering，并处理跨宽度权重共享。 diff --git a/tools/npu/build_iree.py b/tools/npu/build_iree.py new file mode 100644 index 0000000..1704159 --- /dev/null +++ b/tools/npu/build_iree.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Build the pinned AMD AIE compiler without device access or test targets. + +Source checkout/submodule population and checksum-verified Peano extraction +are prerequisites. All outputs remain development tooling, never npm payload. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess + +LOCK = Path(__file__).with_name("iree-source.lock.json") + + +def head(path: Path, configure_diff=None) -> str: + changes = subprocess.check_output( + ["git", "-C", str(path), "status", "--porcelain", "--untracked-files=no", "--ignore-submodules=all"], + text=True) + if changes: + diff = subprocess.check_output(["git", "-C", str(path), "diff", "--binary", "--no-ext-diff", + "--no-color", "--ignore-submodules=all", "--src-prefix=a/", "--dst-prefix=b/", "HEAD", "--"]) + if configure_diff is None or hashlib.sha256(diff).hexdigest() != configure_diff: + raise ValueError(f"source contains unexpected tracked modifications: {path}") + return subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + + +def apply_source_patches(source: Path, lock: dict) -> None: + patches = [] + for record in lock.get("localPatches", []): + patch = LOCK.parent / record["path"] + with patch.open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != record["sha256"]: + raise ValueError("local compiler patch hash mismatch") + patches.append(str(patch.resolve())) + dirty = subprocess.check_output(["git", "-C", str(source), "status", "--porcelain", + "--untracked-files=no", "--ignore-submodules=all"], text=True) + if patches and not dirty: + subprocess.run(["git", "-C", str(source), "apply", "--check", *patches], check=True) + subprocess.run(["git", "-C", str(source), "apply", *patches], check=True) + head(source, lock.get("localSourceDiffSha256")) + + +def build(source: Path, peano: Path, output: Path, jobs: int, configure_only=False, + runtime_only=False) -> dict: + if jobs < 1: + raise ValueError("jobs must be positive") + source, peano, output = source.resolve(), peano.resolve(), output.resolve() + if output == source or output.is_relative_to(source): + raise ValueError("build output must be separate from the source tree") + lock = json.loads(LOCK.read_text("utf-8")) + if head(source, lock.get("localSourceDiffSha256")) != lock["amdAie"]["commit"]: + raise ValueError("AMD AIE source commit mismatch") + apply_source_patches(source, lock) + revisions = {"amdAie": head(source, lock.get("localSourceDiffSha256"))} + for relative, expected in lock["amdAie"]["submodules"].items(): + actual = head(source / relative, lock.get("upstreamConfigureDiffSha256", {}).get(relative)) + if actual != expected: + raise ValueError(f"source submodule commit mismatch: {relative}") + revisions[relative] = actual + iree = source / "third_party/iree" + for relative, expected in lock["ireeSubmodules"].items(): + actual = head(iree / relative) + if actual != expected: + raise ValueError(f"IREE submodule commit mismatch: {relative}") + revisions[f"iree/{relative}"] = actual + # Archive extraction should preserve executable modes. + if not os.access(peano / "bin/clang", os.X_OK): + raise ValueError("Peano clang is missing or not executable") + provenance = json.loads((peano / "toolchain-source.json").read_text("utf-8")) + if provenance["source"] != lock["peano"]: + raise ValueError("Peano source identity mismatch; use fetch_peano.py") + if [b["path"] for b in provenance["binaries"]] != ["bin/clang", "bin/ld.lld"]: + raise ValueError("Peano provenance must include clang and ld.lld") + for binary in provenance["binaries"]: + if binary["path"] not in ("bin/clang", "bin/ld.lld"): + raise ValueError("unexpected Peano binary record") + with (peano / binary["path"]).open("rb") as stream: + if hashlib.file_digest(stream, "sha256").hexdigest() != binary["sha256"]: + raise ValueError("Peano binary hash mismatch") + for tool in ("cmake", "ninja", "clang", "clang++"): + if shutil.which(tool) is None: + raise ValueError(f"required build tool missing: {tool}") + output.mkdir(parents=True, exist_ok=True) + kind = "runtime" if runtime_only else "compiler" + previous_path = output / "toolchain-build.json" + previous = json.loads(previous_path.read_text("utf-8")) if previous_path.exists() else {} + if previous and previous.get("kind", "compiler") != kind: + raise ValueError("compiler and runtime builds require separate output directories") + flags = ["-GNinja", "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_C_COMPILER=clang", "-DCMAKE_CXX_COMPILER=clang++", + "-DIREE_BUILD_TESTS=OFF", "-DIREE_BUILD_SAMPLES=OFF", + "-DIREE_BUILD_PYTHON_BINDINGS=OFF", + "-DIREE_ENABLE_ASSERTIONS=OFF", "-DIREE_ERROR_ON_MISSING_SUBMODULES=OFF", + "-DIREE_INPUT_STABLEHLO=OFF", "-DIREE_INPUT_TORCH=OFF", "-DIREE_INPUT_TOSA=OFF", + "-DIREE_TARGET_BACKEND_DEFAULTS=OFF", "-DIREE_TARGET_BACKEND_LLVM_CPU=ON", + "-DIREE_HAL_DRIVER_DEFAULTS=OFF", "-DIREE_HAL_DRIVER_LOCAL_TASK=ON", + "-DIREE_HAL_DRIVER_LOCAL_SYNC=ON", "-DIREE_EXTERNAL_HAL_DRIVERS=amdxdna", + "-DENABLE_AMDXDNA_CTS_TESTS=OFF", "-DENABLE_AMDXDNA_AIE_EXECUTABLE_TESTS=OFF", + "-DIREE_DEFAULT_CPU_LLVM_TARGETS=X86", "-DLLVM_TARGETS_TO_BUILD=X86", "-DLLVM_INCLUDE_TESTS=OFF", + "-DLLVM_INCLUDE_BENCHMARKS=OFF", "-DLLVM_ENABLE_ASSERTIONS=OFF", + "-DLLVM_PARALLEL_LINK_JOBS=1", + f"-DIREE_CMAKE_PLUGIN_PATHS={source}", f"-DPEANO_INSTALL_DIR={peano}", + f"-DOPENSSL_BUILD_HASH={lock['openssl']['sha256']}"] + linker = peano / "bin/ld.lld" + if os.access(linker, os.X_OK): + for linker_kind in ("EXE", "SHARED", "MODULE"): + flags.append(f"-DCMAKE_{linker_kind}_LINKER_FLAGS=-fuse-ld={linker}") + if runtime_only: + flags.append("-DIREE_BUILD_COMPILER=OFF") + configure = ["cmake", "-S", str(iree), "-B", str(output), *flags] + names = ("iree-run-module", "iree-dump-module") if runtime_only else ("iree-compile", "iree-opt") + compile_command = ["cmake", "--build", str(output), "--target", *names, + "--parallel", str(jobs)] + manifest = {"schemaVersion": "1.0", "stage": "development", "kind": "runtime" if runtime_only else "compiler", + "sourceCommits": revisions, + "upstreamConfigureDiffSha256": lock.get("upstreamConfigureDiffSha256", {}), + "localPatches": lock.get("localPatches", []), + "localSourceDiffSha256": lock.get("localSourceDiffSha256"), + "peanoVersion": lock["peano"]["version"], "openssl": lock["openssl"], "configureCommand": configure, + "buildCommand": compile_command, "status": "configuring", "deviceValidated": False, + "hostTools": {tool: subprocess.check_output([tool, "--version"], text=True).splitlines()[0] + for tool in ("cmake", "ninja", "clang")}} + report = output / "toolchain-build.json" + reuse_configuration = (not configure_only and (output / "CMakeCache.txt").is_file() and + previous.get("configureCommand") == configure and + previous.get("sourceCommits") == revisions and + previous.get("hostTools") == manifest["hostTools"] and + (previous.get("configurationCompleted") or + previous.get("status") in ("built", "building", "configured"))) + manifest["configurationReused"] = bool(reuse_configuration) + + def write(): + report.write_text(json.dumps(manifest, indent=2) + "\n", "utf-8") + + write() + try: + if not reuse_configuration: + subprocess.run(configure, check=True) + # Upstream CMake patches these checkouts in place. Accept only the + # exact changes produced by the locked configuration, including on + # subsequent incremental builds. + for relative in lock["amdAie"]["submodules"]: + head(source / relative, lock.get("upstreamConfigureDiffSha256", {}).get(relative)) + manifest["configurationCompleted"] = True + manifest["status"] = "configured" if configure_only else "building" + write() + if not configure_only: + subprocess.run(compile_command, check=True) + manifest["binaries"] = [] + paths = [f"tools/{name}" for name in names] + if not runtime_only: + paths.append("lib/libIREECompiler.so") + for relative in paths: + path = output / relative + with path.open("rb") as stream: + identity = hashlib.file_digest(stream, "sha256").hexdigest() + manifest["binaries"].append({"path": relative, "bytes": path.stat().st_size, + "sha256": identity}) + manifest["status"] = "built" + write() + except subprocess.CalledProcessError as exc: + manifest.update(status="failed", returnCode=exc.returncode) + write() + raise + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir", type=Path, required=True) + parser.add_argument("--peano-dir", type=Path, required=True) + parser.add_argument("--build-dir", type=Path, required=True) + parser.add_argument("--jobs", type=int, default=2) + parser.add_argument("--configure-only", action="store_true") + parser.add_argument("--runtime-only", action="store_true", + help="build standalone execution tools in a separate directory") + args = parser.parse_args() + result = build(args.source_dir, args.peano_dir, args.build_dir, args.jobs, args.configure_only, args.runtime_only) + print(f"AMD AIE {result['kind']}: {result['status']}") + + +if __name__ == "__main__": + main() diff --git a/tools/npu/compile_iree_hotspot.py b/tools/npu/compile_iree_hotspot.py new file mode 100644 index 0000000..83113be --- /dev/null +++ b/tools/npu/compile_iree_hotspot.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Compile source-bound OCR development IR and retain diagnostics on failure.""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +from pathlib import Path +import signal +import subprocess +import sys + +LOCK = Path(__file__).with_name("iree-source.lock.json") + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def compile_hotspot(hotspot: Path, toolchain: Path, peano: Path, output: Path, + stage="targets", timeout=600, variant="all", imported_model=False, + kernel_mode="vector", core_rows=2, core_cols=2) -> dict: + if timeout <= 0: + raise ValueError("timeout must be positive") + if not 1 <= core_rows <= 4 or not 1 <= core_cols <= 8: + raise ValueError("npu4 core array must fit 4 rows and 8 columns") + hotspot, toolchain, peano, output = (p.resolve() for p in (hotspot, toolchain, peano, output)) + if output.exists(): + raise ValueError("output exists; select a fresh directory") + lock = json.loads(LOCK.read_text("utf-8")) + manifest_path = hotspot / ("import-report.json" if imported_model else "hotspot-manifest.json") + manifest = json.loads(manifest_path.read_text("utf-8")) + if imported_model and (manifest.get("status") != "imported" or variant != "all"): + raise ValueError("full graph requires a completed import and no hotspot variant selection") + build = json.loads((toolchain / "toolchain-build.json").read_text("utf-8")) + if (build.get("kind") != "compiler" or build["status"] != "built" or + build["sourceCommits"]["amdAie"] != lock["amdAie"]["commit"] or + build["sourceCommits"]["third_party/iree"] != lock["amdAie"]["submodules"]["third_party/iree"]): + raise ValueError("compiler must come from the completed locked build") + if (build.get("localPatches", []) != lock.get("localPatches", []) or + build.get("localSourceDiffSha256") != lock.get("localSourceDiffSha256") or + build.get("upstreamConfigureDiffSha256") != lock.get("upstreamConfigureDiffSha256")): + raise ValueError("compiler source patch provenance mismatch") + if manifest["sourceModelSha256"] != lock["sourceModelSha256"]: + raise ValueError("hotspot source model identity mismatch") + compiler = toolchain / "tools/iree-compile" + if not compiler.is_file(): + raise ValueError("iree-compile is missing") + binary = next(item for item in build["binaries"] if item["path"] == "tools/iree-compile") + if compiler.stat().st_size != binary["bytes"] or digest(compiler) != binary["sha256"]: + raise ValueError("compiler binary identity mismatch") + library = toolchain / "lib/libIREECompiler.so" + library_record = next(item for item in build["binaries"] if item["path"] == "lib/libIREECompiler.so") + if library.stat().st_size != library_record["bytes"] or digest(library) != library_record["sha256"]: + raise ValueError("compiler library identity mismatch") + peano_source = json.loads((peano / "toolchain-source.json").read_text("utf-8")) + if peano_source["source"] != lock["peano"]: + raise ValueError("Peano source identity mismatch") + for name in ("bin/clang", "bin/ld.lld"): + item = next(item for item in peano_source["binaries"] if item["path"] == name) + if digest(peano / name) != item["sha256"]: + raise ValueError("Peano binary identity mismatch") + records = ({manifest["artifact"]["path"]: manifest["artifact"]} if imported_model else + {item["path"]: item for item in manifest["files"]}) + variants = (("recognition",) if imported_model else + (("projection-matmul", "projection-matmul-padded", "projection", "mlp-residual") if variant == "all" else (variant,))) + sources = {name: hotspot / ("model.mlir" if imported_model else f"{name}.mlir") for name in variants} + for name in variants: + source = sources[name] + record = records[source.name] + if source.stat().st_size != record["bytes"] or digest(source) != record["sha256"]: + raise ValueError(f"IR identity mismatch: {name}") + output.mkdir(parents=True) + report = {"schemaVersion": "1.0", "stage": "development", "compileStage": stage, + "width": manifest["width"], "sourceModelSha256": manifest["sourceModelSha256"], + "inputKind": "recognition-graph" if imported_model else "hotspot", + "inputManifestSha256": digest(manifest_path), + "sourceLockSha256": digest(LOCK), + "compilerSha256": digest(compiler), "sourceCommits": build["sourceCommits"], + "compilerLibrarySha256": digest(library), + "target": "npu4", "deviceHAL": "amdxdna", "deviceValidated": False, + "coreRows": core_rows, "coreCols": core_cols, "kernelMode": kernel_mode, + "kernelPrecision": ("bf16 converted to bfp16ebs8 inside Peano matmul microkernel" + if kernel_mode == "peano-bf16" else "source IR types"), + "numericsValidated": False, "deployableArtifactProduced": False, "results": []} + report_path = output / "compile-report.json" + + def save(): + report_path.write_text(json.dumps(report, indent=2) + "\n", "utf-8") + + save() + for name in variants: + extension = "vmfb" if stage == "binary" else "mlir" + artifact = output / f"{name}.{extension}" + command = [str(compiler), str(sources[name]), + "--iree-hal-target-backends=amd-aie", "--iree-amdaie-target-device=npu4", + "--iree-amdaie-device-hal=amdxdna", + f"--iree-amdaie-num-rows={core_rows}", f"--iree-amdaie-num-cols={core_cols}", + "--iree-amdaie-lower-to-aie-pipeline=objectFifo", + "--iree-amdaie-tile-pipeline=pack-peel", + f"--iree-amd-aie-peano-install-dir={peano}", + "--iree-hal-memoization=false", "--iree-hal-indirect-command-buffers=false", + f"--iree-hal-dump-executable-files-to={output / (name + '-executables')}", + "-o", str(artifact)] + if stage in ("sources", "targets"): + command.append(f"--compile-to=executable-{stage}") + if kernel_mode == "peano-bf16": + command.extend(["--iree-amdaie-enable-ukernels=matmul", "--iree-amd-aie-enable-chess-for-ukernel=false"]) + entry = {"variant": name, "command": command, "status": "compiling"} + report["results"].append(entry) + save() + log = output / f"{name}.log" + with log.open("wb") as stream: + process = subprocess.Popen(command, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + try: + code = process.wait(timeout=timeout) + entry.update(returnCode=code, status="compiled" if code == 0 else "failed") + if code < 0: + entry.update(status="crashed", signal=-code) + except subprocess.TimeoutExpired: + # Peano/bootgen children belong to this compilation, too. + os.killpg(process.pid, signal.SIGKILL) + process.wait() + entry["status"] = "timeout" + except BaseException: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + entry["status"] = "interrupted" + report["status"] = "interrupted" + save() + raise + entry["log"] = {"path": log.name, "bytes": log.stat().st_size, "sha256": digest(log)} + if entry["status"] == "compiled": + if not artifact.is_file() or artifact.stat().st_size == 0: + entry.update(status="failed", reason="compiler produced no artifact") + else: + entry["artifact"] = {"path": artifact.name, "bytes": artifact.stat().st_size, + "sha256": digest(artifact)} + save() + report["status"] = "compiled" if all(x["status"] == "compiled" for x in report["results"]) else "failed" + # Lowered textual IR is not a deployable model, even when lowering succeeds. + report["deployableArtifactProduced"] = stage == "binary" and report["status"] == "compiled" + save() + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + inputs = parser.add_mutually_exclusive_group(required=True) + inputs.add_argument("--hotspot-dir", type=Path) + inputs.add_argument("--model-dir", type=Path, help="directory produced by import_iree_model.py") + parser.add_argument("--toolchain-build-dir", type=Path, required=True) + parser.add_argument("--peano-dir", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--stage", choices=("sources", "targets", "binary"), default="targets") + parser.add_argument("--variant", choices=("projection-matmul", "projection-matmul-padded", "projection", "mlp-residual", "all"), default="all") + parser.add_argument("--kernel-mode", choices=("vector", "peano-bf16"), default="vector") + parser.add_argument("--core-rows", type=int, default=2) + parser.add_argument("--core-cols", type=int, default=2) + parser.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + result = compile_hotspot(args.model_dir or args.hotspot_dir, args.toolchain_build_dir, args.peano_dir, + args.output_dir, args.stage, args.timeout, args.variant, args.model_dir is not None, + args.kernel_mode, args.core_rows, args.core_cols) + print(f"AMD AIE {result['inputKind']}: {result['status']}; device/numerics remain unvalidated") + sys.exit(0 if result["status"] == "compiled" else 1) + + +if __name__ == "__main__": + main() diff --git a/tools/npu/export_iree_hotspot.py b/tools/npu/export_iree_hotspot.py new file mode 100644 index 0000000..52732e3 --- /dev/null +++ b/tools/npu/export_iree_hotspot.py @@ -0,0 +1,266 @@ +#!/usr/bin/env python3 +"""Export a real Small recognition MLP block for ahead-of-time compilation. + +This development tool does not execute inference or alter release packages. +FP32 ONNX is preserved as the reference graph; MLIR uses BF16 matmul operands, +FP32 accumulation, bias, GELU and residual. Numerical parity is not implied. +""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +LOCK = Path(__file__).with_name("iree-source.lock.json") +WIDTHS = (320, 384, 480, 544, 576, 608, 704, 736, 832, 960, 1056, 1184, + 1248, 1376, 1600, 1984, 2240, 2560, 2880, 3200) + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def tensor_type(shape, dtype="f32") -> str: + return f"tensor<{'x'.join(str(x) for x in shape)}x{dtype}>" + + +def bf16_bits(values): + import numpy as np + words = np.asarray(values, dtype="> 16) & 1)) >> 16).astype(" str: + import numpy as np + bits = bf16_bits(values) if dtype == "bf16" else np.asarray(values, dtype=" : {tensor_type(values.shape, dtype)}' + + +def mlir_block(shape, weights, biases, full: bool) -> str: + """Lower 1x1 Conv -> bias -> exact GELU -> 1x1 Conv -> bias -> residual.""" + n, k, h, w = shape + m, hidden = n * h * w, weights[0].shape[1] + nchw, nhwc = tensor_type(shape), tensor_type((n, h, w, k)) + x32, x16 = tensor_type((m, k)), tensor_type((m, k), "bf16") + h32, h16 = tensor_type((m, hidden)), tensor_type((m, hidden), "bf16") + b1, b2 = tensor_type((hidden,)), tensor_type((k,)) + lines = [ + "// Source-bound development IR; not a qualified model artifact.", + "#id = affine_map<(d0, d1) -> (d0, d1)>", + "#bias = affine_map<(d0, d1) -> (d1)>", + f"module {{\n func.func @{'mlp_residual' if full else 'projection'}(%input: {nchw}) -> {nchw if full else h32} {{", + f" %w1 = arith.constant {dense(weights[0], 'bf16')}", + f" %b1 = arith.constant {dense(biases[0])}", + " %zero = arith.constant 0.0 : f32", + f" %layout_empty = tensor.empty() : {nhwc}", + f" %nhwc = linalg.transpose ins(%input : {nchw}) outs(%layout_empty : {nhwc}) permutation = [0, 2, 3, 1]", + f" %flat = tensor.collapse_shape %nhwc [[0, 1, 2], [3]] : {nhwc} into {x32}", + ] + + def cast(name, source, src_type, dst_type): + lines.extend([ + f" %{name}_empty = tensor.empty() : {dst_type}", + f' %{name} = linalg.generic {{indexing_maps = [#id, #id], iterator_types = ["parallel", "parallel"]}} ins(%{source} : {src_type}) outs(%{name}_empty : {dst_type}) {{', + " ^bb0(%a: f32, %unused: bf16):", + " %v = arith.truncf %a : f32 to bf16", + " linalg.yield %v : bf16", + f" }} -> {dst_type}", + ]) + + def matmul(name, source, weight, src_type, weight_type, result_type): + lines.extend([ + f" %{name}_empty = tensor.empty() : {result_type}", + f" %{name}_zero = linalg.fill ins(%zero : f32) outs(%{name}_empty : {result_type}) -> {result_type}", + f" %{name} = linalg.matmul ins(%{source}, %{weight} : {src_type}, {weight_type}) outs(%{name}_zero : {result_type}) -> {result_type}", + ]) + + def bias(name, source, bias_name, source_type, bias_type): + lines.extend([ + f" %{name}_empty = tensor.empty() : {source_type}", + f' %{name} = linalg.generic {{indexing_maps = [#id, #bias, #id], iterator_types = ["parallel", "parallel"]}} ins(%{source}, %{bias_name} : {source_type}, {bias_type}) outs(%{name}_empty : {source_type}) {{', + " ^bb0(%a: f32, %b: f32, %unused: f32):", + " %v = arith.addf %a, %b : f32", + " linalg.yield %v : f32", + f" }} -> {source_type}", + ]) + + cast("x_bf16", "flat", x32, x16) + matmul("projected", "x_bf16", "w1", x16, tensor_type(weights[0].shape, "bf16"), h32) + bias("biased", "projected", "b1", h32, b1) + if not full: + lines.extend([f" return %biased : {h32}", " }", "}"]) + return "\n".join(lines) + "\n" + lines.extend([ + f" %w2 = arith.constant {dense(weights[1], 'bf16')}", + f" %b2 = arith.constant {dense(biases[1])}", + " %sqrt2 = arith.constant 1.4142135623730951 : f32", + " %half = arith.constant 0.5 : f32", + " %one = arith.constant 1.0 : f32", + f" %gelu_empty = tensor.empty() : {h32}", + f' %gelu = linalg.generic {{indexing_maps = [#id, #id], iterator_types = ["parallel", "parallel"]}} ins(%biased : {h32}) outs(%gelu_empty : {h32}) {{', + " ^bb0(%a: f32, %unused: f32):", + " %scaled = arith.divf %a, %sqrt2 : f32", + " %erf = math.erf %scaled : f32", + " %plus = arith.addf %erf, %one : f32", + " %product = arith.mulf %a, %plus : f32", + " %v = arith.mulf %product, %half : f32", + " linalg.yield %v : f32", + f" }} -> {h32}", + ]) + cast("hidden_bf16", "gelu", h32, h16) + matmul("contracted", "hidden_bf16", "w2", h16, tensor_type(weights[1].shape, "bf16"), x32) + bias("final_biased", "contracted", "b2", x32, b2) + lines.extend([ + f" %res_empty = tensor.empty() : {x32}", + f' %res = linalg.generic {{indexing_maps = [#id, #id, #id], iterator_types = ["parallel", "parallel"]}} ins(%final_biased, %flat : {x32}, {x32}) outs(%res_empty : {x32}) {{', + " ^bb0(%a: f32, %b: f32, %unused: f32):", + " %v = arith.addf %a, %b : f32", + " linalg.yield %v : f32", + f" }} -> {x32}", + f" %expanded = tensor.expand_shape %res [[0, 1, 2], [3]] output_shape [{n}, {h}, {w}, {k}] : {x32} into {nhwc}", + f" %out_empty = tensor.empty() : {nchw}", + f" %out = linalg.transpose ins(%expanded : {nhwc}) outs(%out_empty : {nchw}) permutation = [0, 3, 1, 2]", + f" return %out : {nchw}", " }", "}", + ]) + return "\n".join(lines) + "\n" + + +def mlir_projection_matmul(shape, weight, padded=False) -> str: + """Isolate the real first Conv's matrix core with an explicit BF16 ABI. + + Layout conversion, input rounding and bias remain outside this diagnostic + kernel. This is not a replacement for either complete exported block. + """ + n, k, h, w = shape + rows = ((n * h * w + 127) // 128) * 128 if padded else n * h * w + lhs = tensor_type((rows, k), "bf16") + rhs = tensor_type(weight.shape, "bf16") + result = tensor_type((rows, weight.shape[1])) + return "\n".join([ + "// Diagnostic matrix core only; external layout/cast/bias required.", + "module {", + f" func.func @projection_matmul(%input: {lhs}) -> {result} {{", + f" %weight = arith.constant {dense(weight, 'bf16')}", + " %zero = arith.constant 0.0 : f32", + f" %empty = tensor.empty() : {result}", + f" %initial = linalg.fill ins(%zero : f32) outs(%empty : {result}) -> {result}", + f" %out = linalg.matmul ins(%input, %weight : {lhs}, {rhs}) outs(%initial : {result}) -> {result}", + f" return %out : {result}", " }", "}", "", + ]) + + +def export(source: Path, output: Path, width: int) -> dict: + import numpy as np + import onnx + from onnx import numpy_helper + lock = json.loads(LOCK.read_text("utf-8")) + if digest(source) != lock["sourceModelSha256"]: + raise ValueError("recognition model does not match the locked Small 0.3.4 source") + if width not in WIDTHS: + raise ValueError("width must belong to the locked recognition bucket contract") + if output.exists(): + raise ValueError("output exists; select a fresh directory") + model = onnx.load(source) + for d, value in zip(model.graph.input[0].type.tensor_type.shape.dim, (1, 3, 48, width)): + d.ClearField("dim_param") + d.dim_value = value + model = onnx.shape_inference.infer_shapes(model) + by_name = {n.name: n for n in model.graph.node} + first, second, residual = (by_name[k] for k in ("Conv.49", "Conv.50", "Add.144")) + start, end = first.input[0], residual.output[0] + shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim] + for v in (*model.graph.input, *model.graph.value_info, *model.graph.output)} + shape = shapes[start] + if shape != [1, 384, 3, width // 4] or shapes[end] != shape: + raise ValueError("hotspot shape differs from the source-bound contract") + constants = {t.name: numpy_helper.to_array(t) for t in model.graph.initializer} + producers = {v: n for n in model.graph.node for v in n.output} + + def constant(name): + if name in constants: + return constants[name] + node = producers.get(name) + if node is None: + raise ValueError(f"unresolved constant: {name}") + attrs = {a.name: onnx.helper.get_attribute_value(a) for a in node.attribute} + if node.op_type == "Constant": + value = numpy_helper.to_array(attrs["value"]) + elif node.op_type == "Identity": + value = constant(node.input[0]) + elif node.op_type == "Reshape": + value = constant(node.input[0]).reshape(tuple(int(x) for x in constant(node.input[1]))) + else: + raise ValueError(f"unsupported constant producer: {node.op_type}") + constants[name] = value + return value + + weights, biases = [], [] + for conv, bias_node in ((first, by_name["Add.138"]), (second, by_name["Add.142"])): + attrs = {a.name: onnx.helper.get_attribute_value(a) for a in conv.attribute} + weight = constant(conv.input[1]) + if (weight.shape[2:] != (1, 1) or attrs.get("group", 1) != 1 or + list(attrs.get("strides", [1, 1])) != [1, 1] or + any(attrs.get("pads", [0, 0, 0, 0]))): + raise ValueError("hotspot must be dense stride-one 1x1 convolution") + weights.append(weight[:, :, 0, 0].T.copy()) + bias_input = next(v for v in bias_node.input if v != conv.output[0]) + value = constant(bias_input) + if value.shape != (1, weight.shape[0], 1, 1): + raise ValueError("unexpected convolution bias broadcast shape") + biases.append(value.reshape(-1).copy()) + output.mkdir(parents=True) + reference = onnx.utils.Extractor(model).extract_model([start], [end]) + onnx.save(reference, output / "reference-fp32.onnx") + for name, full in (("projection", False), ("mlp-residual", True)): + (output / f"{name}.mlir").write_text(mlir_block(shape, weights, biases, full), "utf-8") + (output / "projection-matmul.mlir").write_text(mlir_projection_matmul(shape, weights[0]), "utf-8") + (output / "projection-matmul-padded.mlir").write_text(mlir_projection_matmul(shape, weights[0], True), "utf-8") + np.savez(output / "weights-fp32.npz", w1=weights[0], b1=biases[0], w2=weights[1], b2=biases[1]) + files = [{"path": p.name, "bytes": p.stat().st_size, "sha256": digest(p)} + for p in sorted(output.iterdir())] + manifest = {"schemaVersion": "1.0", "stage": "development", "deviceValidated": False, + "numericsValidated": False, "sourceModelSha256": digest(source), + "width": width, "input": {"name": start, "shape": shape, "type": "float32"}, + "output": {"name": end, "shape": shape, "type": "float32"}, + "variants": { + "projection-matmul-padded": { + "inputShape": [((shape[0] * shape[2] * shape[3] + 127) // 128) * 128, shape[1]], + "outputShape": [((shape[0] * shape[2] * shape[3] + 127) // 128) * 128, weights[0].shape[1]], + "inputType": "bfloat16", "outputType": "float32", + "validRows": shape[0] * shape[2] * shape[3], + "callerRequirements": ["zero-fill-extra-input-rows", "crop-extra-output-rows"], + "excludedOperations": ["layout-conversion", "input-rounding", "bias"]}, + "projection-matmul": {"inputShape": [shape[0] * shape[2] * shape[3], shape[1]], + "outputShape": [shape[0] * shape[2] * shape[3], weights[0].shape[1]], + "inputType": "bfloat16", "outputType": "float32", + "excludedOperations": ["layout-conversion", "input-rounding", "bias"]}, + "projection": {"inputShape": shape, + "outputShape": [shape[0] * shape[2] * shape[3], weights[0].shape[1]], + "interfaceType": "float32"}, + "mlp-residual": {"inputShape": shape, "outputShape": shape, + "interfaceType": "float32"}}, + "sourceNodes": [n.name for n in reference.graph.node], + "precision": {"operands": "bf16", "accumulation": "float32", "gelu": "erf-float32"}, + "transformations": ["NCHW-to-NHWC", "1x1-conv-to-matmul", "bf16-round-nearest-even"], + "files": files} + (output / "hotspot-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", "utf-8") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-model", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--width", type=int, default=384, choices=WIDTHS) + args = parser.parse_args() + result = export(args.source_model, args.output_dir, args.width) + print(f"Exported source-bound hotspot: width={result['width']}, {len(result['sourceNodes'])} nodes") + + +if __name__ == "__main__": + main() diff --git a/tools/npu/export_iree_model.py b/tools/npu/export_iree_model.py new file mode 100644 index 0000000..23175da --- /dev/null +++ b/tools/npu/export_iree_model.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Specialize a source-bound recognition graph for the IREE ONNX importer. + +Only static Shape queries are folded; ONNX's version converter preserves the +opset migration. This is preparation for compilation, not numerical/device +qualification. The original model and npm packages are never overwritten. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +try: + from .export_iree_hotspot import LOCK, WIDTHS, digest +except ImportError: + from export_iree_hotspot import LOCK, WIDTHS, digest + + +def export(source: Path, output: Path, width: int) -> dict: + import onnx + from onnx import helper + lock = json.loads(LOCK.read_text("utf-8")) + if digest(source) != lock["sourceModelSha256"]: + raise ValueError("source recognition model identity mismatch") + if width not in WIDTHS: + raise ValueError("width must belong to the locked recognition bucket contract") + if output.exists(): + raise ValueError("output exists; select a fresh directory") + model = onnx.load(source) + if len(model.graph.input) != 1 or len(model.graph.output) != 1: + raise ValueError("recognition graph must have one input and output") + dims = model.graph.input[0].type.tensor_type.shape.dim + if len(dims) != 4: + raise ValueError("recognition input must be NCHW") + for dimension, value in zip(dims, (1, 3, 48, width)): + dimension.ClearField("dim_param") + dimension.dim_value = value + folded = [] + # Remove stale inferred intermediates before specializing the static input. + # Each iteration may make another Shape input statically known. + while True: + model.graph.ClearField("value_info") + model = onnx.shape_inference.infer_shapes(model, data_prop=True) + shapes = {v.name: [d.dim_value for d in v.type.tensor_type.shape.dim] + for v in (*model.graph.input, *model.graph.value_info, *model.graph.output)} + changed = False + for node in model.graph.node: + if node.op_type != "Shape": + continue + shape = shapes.get(node.input[0], []) + if not shape or any(d <= 0 for d in shape): + continue + attrs = {a.name: helper.get_attribute_value(a) for a in node.attribute} + selected = shape[attrs.get("start", 0):attrs.get("end", len(shape))] + folded.append({"node": node.name, "sourceTensor": node.input[0], "value": selected}) + node.ClearField("input") + node.ClearField("attribute") + node.op_type = "Constant" + tensor = helper.make_tensor("", onnx.TensorProto.INT64, [len(selected)], selected) + node.attribute.append(helper.make_attribute("value", tensor)) + changed = True + if not changed: + break + original_opsets = [{"domain": x.domain, "version": x.version} for x in model.opset_import] + model = onnx.version_converter.convert_version(model, 17) + model.graph.ClearField("value_info") + model = onnx.shape_inference.infer_shapes(model, data_prop=True) + output.mkdir(parents=True) + artifact = output / "recognition-opset17.onnx" + onnx.save(model, artifact) + manifest = {"schemaVersion": "1.0", "stage": "development", "width": width, + "sourceModelSha256": digest(source), "originalOpsets": original_opsets, + "targetOpset": 17, "shapeQueriesFolded": folded, + "inputShape": [1, 3, 48, width], "expectedOutputShape": [1, width // 8, 18710], + "precision": "float32", "numericsValidated": False, "deviceValidated": False, + "files": [{"path": artifact.name, "bytes": artifact.stat().st_size, + "sha256": digest(artifact)}]} + (output / "model-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", "utf-8") + return manifest + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-model", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--width", type=int, choices=WIDTHS, default=384) + args = parser.parse_args() + result = export(args.source_model, args.output_dir, args.width) + print(f"Specialized recognition width={result['width']}; folded {len(result['shapeQueriesFolded'])} Shape nodes") + + +if __name__ == "__main__": + main() diff --git a/tools/npu/fetch_iree_source.py b/tools/npu/fetch_iree_source.py new file mode 100644 index 0000000..1f3fe50 --- /dev/null +++ b/tools/npu/fetch_iree_source.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +"""Populate the fixed AMD AIE source tree and only required submodules.""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import subprocess + +LOCK = Path(__file__).with_name("iree-source.lock.json") + + +def head(path: Path) -> str: + return subprocess.check_output(["git", "-C", str(path), "rev-parse", "HEAD"], text=True).strip() + + +def fetch(output: Path, offline=False, jobs=3) -> Path: + if jobs < 1: + raise ValueError("jobs must be positive") + lock = json.loads(LOCK.read_text("utf-8")) + output = output.resolve() + if not output.exists(): + if offline: + raise ValueError("source checkout is absent from offline cache") + output.mkdir(parents=True) + subprocess.run(["git", "init", str(output)], check=True) + subprocess.run(["git", "-C", str(output), "remote", "add", "origin", + lock["amdAie"]["repository"]], check=True) + subprocess.run(["git", "-C", str(output), "fetch", "--depth", "1", "origin", + lock["amdAie"]["commit"]], check=True) + subprocess.run(["git", "-C", str(output), "checkout", "--detach", "FETCH_HEAD"], check=True) + if not (output / ".git").is_dir() or head(output) != lock["amdAie"]["commit"]: + raise ValueError("source output must be a standalone checkout of the locked commit") + roots = [(output, lock["amdAie"]["submodules"]), + (output / "third_party/iree", lock["ireeSubmodules"])] + for root, revisions in roots: + if not offline: + # Parent checkout pins these gitlinks. No recursive unused SDKs, + # driver installers, test suites, or release writes are invoked. + subprocess.run(["git", "-C", str(root), "submodule", "update", "--init", + "--depth", "1", "--jobs", str(jobs), *revisions], check=True) + for relative, expected in revisions.items(): + path = root / relative + if not (path / ".git").exists() or head(path) != expected: + raise ValueError(f"source submodule identity mismatch: {relative}") + return output + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--offline", action="store_true") + parser.add_argument("--jobs", type=int, default=3) + args = parser.parse_args() + print(fetch(args.output_dir, args.offline, args.jobs)) + + +if __name__ == "__main__": + main() diff --git a/tools/npu/fetch_peano.py b/tools/npu/fetch_peano.py new file mode 100644 index 0000000..4a9f1cb --- /dev/null +++ b/tools/npu/fetch_peano.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Extract a hash-locked build-only Peano toolchain, preserving executable modes.""" +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path, PurePosixPath +import shutil +import stat +import urllib.request +import zipfile + +LOCK = Path(__file__).with_name("iree-source.lock.json") + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def fetch(cache: Path, output: Path, offline=False) -> Path: + package = json.loads(LOCK.read_text("utf-8"))["peano"] + if output.exists(): + raise ValueError("toolchain output exists; select a fresh directory") + cache.mkdir(parents=True, exist_ok=True) + archive = cache / package["filename"] + if not archive.is_file(): + if offline: + raise ValueError("locked Peano wheel is absent from offline cache") + temporary = archive.with_suffix(".partial") + with urllib.request.urlopen(package["url"], timeout=60) as source, temporary.open("wb") as target: + shutil.copyfileobj(source, target) + if temporary.stat().st_size != package["bytes"] or digest(temporary) != package["sha256"]: + raise ValueError("downloaded Peano wheel identity mismatch") + temporary.replace(archive) + if archive.stat().st_size != package["bytes"] or digest(archive) != package["sha256"]: + raise ValueError("Peano wheel identity mismatch") + with zipfile.ZipFile(archive) as wheel: + entries = wheel.infolist() + names = [item.filename for item in entries] + if len(names) != len(set(names)): + raise ValueError("Peano archive contains duplicate paths") + for item in entries: + relative = PurePosixPath(item.filename) + if relative.is_absolute() or ".." in relative.parts or "\\" in item.filename: + raise ValueError("unsafe Peano archive path") + if stat.S_ISLNK(item.external_attr >> 16): + raise ValueError("symbolic links are unsupported in the Peano archive") + output.mkdir(parents=True) + for item in entries: + if not item.filename.startswith("llvm-aie/") or item.is_dir(): + continue + target = output / item.filename + target.parent.mkdir(parents=True, exist_ok=True) + with wheel.open(item) as source, target.open("wb") as destination: + shutil.copyfileobj(source, destination) + target.chmod(0o755 if item.external_attr >> 16 & 0o111 else 0o644) + root = output / "llvm-aie" + if not (root / "bin/clang").is_file(): + raise ValueError("locked toolchain lacks clang") + record = {"schemaVersion": "1.0", "stage": "build-only", "source": package, + "binaries": [{"path": f"bin/{name}", "sha256": digest(root / "bin" / name)} + for name in ("clang", "ld.lld")]} + (root / "toolchain-source.json").write_text(json.dumps(record, indent=2) + "\n", "utf-8") + return root + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package-cache", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--offline", action="store_true") + args = parser.parse_args() + print(fetch(args.package_cache, args.output_dir, args.offline)) + + +if __name__ == "__main__": + main() diff --git a/tools/npu/import_iree_model.py b/tools/npu/import_iree_model.py new file mode 100644 index 0000000..bb0353f --- /dev/null +++ b/tools/npu/import_iree_model.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Import a specialized OCR graph into core Linalg and channels-last convs. + +The build-only frontend is separate from the pinned AMD compiler. Its output +must still be accepted by that compiler before claiming AMD support. +""" +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import sys + +try: + from .export_iree_hotspot import LOCK, digest +except ImportError: + from export_iree_hotspot import LOCK, digest + + +def import_model(model: Path, environment: Path, output: Path, timeout=600) -> dict: + if timeout <= 0: + raise ValueError("timeout must be positive") + model, environment, output = (p.resolve() for p in (model, environment, output)) + if output.exists(): + raise ValueError("output exists; select a fresh directory") + lock = json.loads(LOCK.read_text("utf-8")) + manifest = json.loads((model / "model-manifest.json").read_text("utf-8")) + if manifest["sourceModelSha256"] != lock["sourceModelSha256"]: + raise ValueError("model source identity mismatch") + source = model / "recognition-opset17.onnx" + record = next(x for x in manifest["files"] if x["path"] == source.name) + if source.stat().st_size != record["bytes"] or digest(source) != record["sha256"]: + raise ValueError("specialized model identity mismatch") + python, optimizer = environment / "bin/python", environment / "bin/iree-opt" + requirements = Path(__file__).with_name("iree-frontend.requirements.txt") + required = dict(line.strip().split("==", 1) for line in requirements.read_text("utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#")) + versions = json.loads(subprocess.check_output( + [str(python), "-c", "import importlib.metadata as m,json,sys; print(json.dumps({x:m.version(x) for x in json.loads(sys.argv[1])}))", + json.dumps(list(required))], text=True)) + if versions != required: + raise ValueError("frontend versions differ from iree-frontend.requirements.txt") + compiler_library = Path(subprocess.check_output( + [str(python), "-c", "import pathlib,iree.compiler; print(pathlib.Path(iree.compiler.__file__).parent/'_mlir_libs'/'libIREECompiler.so')"], + text=True).strip()) + output.mkdir(parents=True) + raw, core = output / "imported.mlir", output / "core-linalg.mlir" + conv, depthwise, final = (output / name for name in + ("channels-last.mlir", "depthwise-channels-last.mlir", "model.mlir")) + commands = [ + ("onnx-import", [str(python), "-m", "iree.compiler.tools.import_onnx", str(source), "-o", str(raw)]), + ("core-linalg", [str(optimizer), str(raw), "--torch-onnx-to-torch-backend-pipeline", + "--torch-backend-to-linalg-on-tensors-backend-pipeline", "-o", str(core)]), + ("conv-layout", [str(optimizer), str(core), "--iree-preprocessing-convert-conv-to-channels-last", + "--canonicalize", "--cse", "-o", str(conv)]), + ("depthwise-layout", [str(python), str(Path(__file__).with_name("transform_iree_depthwise.py").resolve()), + "--input", str(conv), "--output", str(depthwise)]), + ("canonicalize", [str(optimizer), str(depthwise), "--canonicalize", "--cse", "-o", str(final)]), + ] + report = {"schemaVersion": "1.0", "stage": "development", "status": "importing", + "sourceModelSha256": manifest["sourceModelSha256"], "modelManifestSha256": digest(model / "model-manifest.json"), + "width": manifest["width"], "frontendVersions": versions, + "frontendRequirementsSha256": digest(requirements), + "frontendCompilerLibrarySha256": digest(compiler_library), + "frontendOptimizerSha256": digest(optimizer), "numericsValidated": False, + "deviceValidated": False, "amdCompilerAccepted": False, "steps": []} + path = output / "import-report.json" + + def save(): + path.write_text(json.dumps(report, indent=2) + "\n", "utf-8") + + save() + for name, command in commands: + step = {"name": name, "command": command, "status": "running"} + report["steps"].append(step) + save() + log = output / f"{name}.log" + with log.open("wb") as stream: + process = subprocess.Popen(command, stdout=stream, stderr=subprocess.STDOUT, start_new_session=True) + try: + code = process.wait(timeout=timeout) + step.update(returnCode=code, status="completed" if code == 0 else "failed") + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + step["status"] = "timeout" + except BaseException: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + step["status"] = "interrupted" + report["status"] = "interrupted" + save() + raise + step["log"] = {"path": log.name, "bytes": log.stat().st_size, "sha256": digest(log)} + if step["status"] != "completed": + report["status"] = "failed" + save() + return report + save() + report["status"] = "imported" + report["artifact"] = {"path": final.name, "bytes": final.stat().st_size, "sha256": digest(final)} + save() + return report + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model-dir", type=Path, required=True) + parser.add_argument("--frontend-environment", type=Path, required=True) + parser.add_argument("--output-dir", type=Path, required=True) + parser.add_argument("--timeout", type=int, default=600) + args = parser.parse_args() + result = import_model(args.model_dir, args.frontend_environment, args.output_dir, args.timeout) + print(f"Recognition graph: {result['status']}; AMD compiler acceptance remains unverified") + sys.exit(0 if result["status"] == "imported" else 1) + + +if __name__ == "__main__": + main() diff --git a/tools/npu/iree-frontend.requirements.txt b/tools/npu/iree-frontend.requirements.txt new file mode 100644 index 0000000..c0401e9 --- /dev/null +++ b/tools/npu/iree-frontend.requirements.txt @@ -0,0 +1,9 @@ +# Build-only frontend environment; Python 3.12 on Linux x64. +iree-base-compiler==3.12.0 +onnx==1.21.0 +numpy==2.5.3 +ml_dtypes==0.6.0 +protobuf==7.36.2 +sympy==1.14.0 +mpmath==1.3.0 +typing_extensions==4.16.0 diff --git a/tools/npu/iree-source.lock.json b/tools/npu/iree-source.lock.json new file mode 100644 index 0000000..8547e4d --- /dev/null +++ b/tools/npu/iree-source.lock.json @@ -0,0 +1,48 @@ +{ + "schemaVersion": "1.0", + "stage": "development", + "sourceModelSha256": "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + "amdAie": { + "repository": "https://github.com/nod-ai/iree-amd-aie.git", + "commit": "fddfec1be6ceefbdb890079d957947dfa1fe0848", + "submodules": { + "third_party/aie-rt": "a6196ebd905ceaf92cabc2e594017fa11b9e4323", + "third_party/bootgen": "92e09bf37ea17d7b1f0e102a2548f27fb768651c", + "third_party/iree": "26083307b8753a11870b8caf590afea15c9429fa", + "third_party/mlir-air": "ca2448548d66d7e8e05bacb9e81e7df48a9ef729", + "third_party/openssl": "0e04e101507df752500093ff4b04cce5a898abaa" + } + }, + "ireeSubmodules": { + "third_party/benchmark": "192ef10025eb2c4cdd392bc502f0c852196baa48", + "third_party/llvm-project": "8be29edc20dfb19245ab8cf49669c80125ece4f5", + "third_party/flatcc": "9362cd00f0007d8cbee7bff86e90fb4b6b227ff3", + "third_party/musl": "3f701faace7addc75d16dea8a6cd769fa5b3f260" + }, + "openssl": { + "version": "1.1.1k", + "url": "https://mirror.viaduck.org/openssl/openssl-1.1.1k.tar.gz", + "bytes": 9823400, + "sha256": "892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5" + }, + "peano": { + "version": "19.0.0.2025052701+31d2aa6e", + "filename": "llvm_aie-19.0.0.2025052701+31d2aa6e-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://github.com/Xilinx/llvm-aie/releases/download/nightly-20240501-20260527/llvm_aie-19.0.0.2025052701%2B31d2aa6e-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "bytes": 136087766, + "sha256": "0baa997e0c5742d7dcbafa2c86642031f446850d1c82d8f2f32ca69427b674da" + }, + "upstreamConfigureDiffSha256": { + "third_party/aie-rt": "5ddbb02c3fcc8c832a6c50920e88498e552bf1b559e14fcc5fcb8a9784ee5e97", + "third_party/bootgen": "95dcde671c20411290ad1af9bd55da75c3d52d54eac2e0d996feb86298c40ea7", + "third_party/mlir-air": "7f79b8f80a7b890e92401b58a01d8efafc95ca43cd74bf0f77a0bdc8b4e38ff0" + }, + "localPatches": [ + { + "path": "patches/check-matmul-map-ranks.patch", + "sha256": "5a0a9b3efe57cc4d257b8fd9d08f59a4c158a07c71a7242d21b812314ee9fb05", + "reason": "Reject scalar/rank-one generic maps before matmul dimension access" + } + ], + "localSourceDiffSha256": "b62c2c5415db8739e8799ca8f1f28658c1e27229ad0897fafce505150d6cb6f9" +} diff --git a/tools/npu/patches/check-matmul-map-ranks.patch b/tools/npu/patches/check-matmul-map-ranks.patch new file mode 100644 index 0000000..19d1d75 --- /dev/null +++ b/tools/npu/patches/check-matmul-map-ranks.patch @@ -0,0 +1,44 @@ +--- a/compiler/plugins/target/AMD-AIE/iree-amd-aie/Transforms/Utils/AMDAIEUtils.cpp ++++ b/compiler/plugins/target/AMD-AIE/iree-amd-aie/Transforms/Utils/AMDAIEUtils.cpp +@@ -238,6 +238,13 @@ + // Step 2. Check the innermost 3 dimensions 'parallel, parallel, reduction' + // map exactly to a matmul. + ArrayAttr maps = linalgOp.getIndexingMaps(); ++ // Generic reductions can have scalar or rank-one indexing maps. ++ // Check before getAffineMapDim subtracts two from an unsigned rank. ++ if (maps.size() < 3) return false; ++ for (unsigned i = 0; i < 3; ++i) { ++ if (cast(maps[i]).getValue().getNumResults() < 2) ++ return false; ++ } + uint32_t A = 0, B = 1, C = 2; + bool isMatmul = + getAffineMapDim(maps, A, 0) == getAffineMapDim(maps, C, 0) && // M +@@ -267,6 +274,13 @@ + // Step 2. Check the innermost 3 dimensions 'parallel, parallel, reduction' + // map exactly to a matmul_transpose_a. + ArrayAttr maps = linalgOp.getIndexingMaps(); ++ // Generic reductions can have scalar or rank-one indexing maps. ++ // Check before getAffineMapDim subtracts two from an unsigned rank. ++ if (maps.size() < 3) return false; ++ for (unsigned i = 0; i < 3; ++i) { ++ if (cast(maps[i]).getValue().getNumResults() < 2) ++ return false; ++ } + uint32_t A = 0, B = 1, C = 2; + bool isATransposed = + getAffineMapDim(maps, A, 1) == getAffineMapDim(maps, C, 0) && // M +@@ -296,6 +310,13 @@ + // Step 2. Check the innermost 3 dimensions 'parallel, parallel, reduction' + // map exactly to a matmul_transpose_a. + ArrayAttr maps = linalgOp.getIndexingMaps(); ++ // Generic reductions can have scalar or rank-one indexing maps. ++ // Check before getAffineMapDim subtracts two from an unsigned rank. ++ if (maps.size() < 3) return false; ++ for (unsigned i = 0; i < 3; ++i) { ++ if (cast(maps[i]).getValue().getNumResults() < 2) ++ return false; ++ } + uint32_t A = 0, B = 1, C = 2; + bool isBTransposed = + getAffineMapDim(maps, A, 0) == getAffineMapDim(maps, C, 0) && // M diff --git a/tools/npu/transform_iree_depthwise.py b/tools/npu/transform_iree_depthwise.py new file mode 100644 index 0000000..b6790fb --- /dev/null +++ b/tools/npu/transform_iree_depthwise.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Convert static depthwise Linalg convolutions from NCHW/CHW to NHWC/HWC. + +Run inside the IREE frontend environment. Ordinary convolutions are handled +by IREE's channels-last preprocessing pass. Scalar computation is preserved. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def transform(source: Path, output: Path) -> int: + from iree.compiler import ir + if output.exists(): + raise ValueError("output exists; select a fresh IR path") + with ir.Context(), ir.Location.unknown(): + module = ir.Module.parse(source.read_text("utf-8")) + candidates = [] + + def walk(operation): + if operation.name == "linalg.depthwise_conv_2d_nchw_chw": + candidates.append(operation) + for region in operation.regions: + for block in region.blocks: + for child in block.operations: + walk(child.operation) + + walk(module.operation) + + def transpose(value, permutation): + source_type = ir.RankedTensorType(value.type) + if any(d < 0 for d in source_type.shape): + raise ValueError("depthwise layout transformation requires static tensor shapes") + result_type = ir.RankedTensorType.get([source_type.shape[i] for i in permutation], + source_type.element_type) + empty = ir.Operation.create("tensor.empty", results=[result_type]) + op = ir.Operation.create("linalg.transpose", operands=[value, empty.results[0]], + results=[result_type], regions=1, + attributes={"permutation": ir.DenseI64ArrayAttr.get(permutation), + "operandSegmentSizes": ir.DenseI32ArrayAttr.get([1, 1])}) + body = op.regions[0].blocks.append(source_type.element_type, source_type.element_type) + with ir.InsertionPoint(body): + ir.Operation.create("linalg.yield", operands=[body.arguments[0]]) + return op.results[0] + + for original in candidates: + if len(original.operands) != 3 or len(original.results) != 1 or len(original.regions) != 1: + raise ValueError("unsupported depthwise operation contract") + if any(ir.RankedTensorType(v.type).encoding is not None for v in original.operands): + raise ValueError("encoded tensors require a dedicated layout conversion") + # Move the scalar region intact; its computation does not depend + # on the enclosing tensor dimension order. + with ir.InsertionPoint(original): + image = transpose(original.operands[0], [0, 2, 3, 1]) + weight = transpose(original.operands[1], [1, 2, 0]) + initial = transpose(original.operands[2], [0, 2, 3, 1]) + attrs = {key: value for key, value in original.attributes.items() + if key != "linalg.memoized_indexing_maps"} + replacement = ir.Operation.create("linalg.depthwise_conv_2d_nhwc_hwc", + operands=[image, weight, initial], + results=[initial.type], attributes=attrs, regions=1) + for block in list(original.regions[0].blocks): + block.append_to(replacement.regions[0]) + result = transpose(replacement.results[0], [0, 3, 1, 2]) + if result.type != original.results[0].type: + raise ValueError("layout transformation changed the external tensor type") + original.results[0].replace_all_uses_with(result) + original.erase() + module.operation.verify() + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(str(module) + "\n", "utf-8") + return len(candidates) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + count = transform(args.input, args.output) + print(json.dumps({"depthwiseConverted": count, "numericsValidated": False, "deviceValidated": False})) + + +if __name__ == "__main__": + main() From 60124a68a54157ba89c348b589713702e6c392ab Mon Sep 17 00:00:00 2001 From: eric8810 Date: Wed, 7 Oct 2026 22:51:23 +0800 Subject: [PATCH 2/3] =?UTF-8?q?feat(npu):=20=E8=BD=BB=E8=8A=AF=E8=B4=AF?= =?UTF-8?q?=E5=8D=B7=EF=BC=8C=E5=8F=8C=E8=B7=AF=E5=90=8C=E5=BD=92=20=C2=B7?= =?UTF-8?q?=20complete=20lightweight=20AMD=20recognition=20backend?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/npu-native.yml | 24 +- .github/workflows/npu-support-release.yml | 20 +- CHANGELOG.md | 12 +- CMakeLists.txt | 2 +- docs/amd-aie-deployment-results.json | 570 ++++++++++++++++++++++ docs/amd-aie-development.md | 67 ++- docs/amd-npu-acceleration.md | 67 ++- docs/amd-npu-lightweight-plan.md | 27 +- docs/npu-runtime-release.md | 27 +- docs/releases/npm-0.5.9.en.md | 27 +- docs/releases/npm-0.5.9.md | 29 +- packages/runtime/src/npu-descriptor.cjs | 35 +- src/core/engine.cpp | 2 +- src/inference/amdnpu/backend.cpp | 133 ++++- src/inference/amdnpu/backend.hpp | 4 +- src/inference/amdnpu/runtime_api.h | 25 + tools/npm_release.py | 4 +- tools/npu/build_amdaie.py | 67 +++ tools/npu/build_amdaie_runtime.py | 68 +++ tools/npu/compile_iree_recognition.py | 146 ++++++ tools/npu/import_amdaie.py | 94 ++++ tools/npu/package_support.py | 4 +- tools/npu/partition_iree_model.py | 152 ++++++ tools/npu/record_support_pack.py | 37 ++ tools/npu/route_iree_dispatches.py | 89 ++++ tools/npu/runtime/CMakeLists.txt | 36 ++ tools/npu/runtime/exports.map | 1 + tools/npu/runtime/merge_parameters.cc | 97 ++++ tools/npu/runtime/runtime.cc | 258 ++++++++++ tools/npu/sdk.py | 40 +- 30 files changed, 2067 insertions(+), 97 deletions(-) create mode 100644 docs/amd-aie-deployment-results.json create mode 100644 src/inference/amdnpu/runtime_api.h create mode 100644 tools/npu/build_amdaie.py create mode 100644 tools/npu/build_amdaie_runtime.py create mode 100644 tools/npu/compile_iree_recognition.py create mode 100644 tools/npu/import_amdaie.py create mode 100644 tools/npu/partition_iree_model.py create mode 100644 tools/npu/record_support_pack.py create mode 100644 tools/npu/route_iree_dispatches.py create mode 100644 tools/npu/runtime/CMakeLists.txt create mode 100644 tools/npu/runtime/exports.map create mode 100644 tools/npu/runtime/merge_parameters.cc create mode 100644 tools/npu/runtime/runtime.cc diff --git a/.github/workflows/npu-native.yml b/.github/workflows/npu-native.yml index 94c25dd..297769c 100644 --- a/.github/workflows/npu-native.yml +++ b/.github/workflows/npu-native.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: inputs: amd_sdk_run_id: - description: Optional run containing amdnpu-candidate-sdk imported from Ryzen AI 1.8 + description: Optional run containing an amdnpu-candidate-sdk; otherwise build lightweight IREE support required: false default: "" type: string @@ -20,11 +20,8 @@ permissions: jobs: build: - # Draft release assets are visible only to tokens with push access. - permissions: - contents: write - actions: read runs-on: ubuntu-24.04 + timeout-minutes: 180 steps: - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 @@ -32,7 +29,16 @@ jobs: node-version: "22" - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: "3.11" + python-version: "3.12" + - name: Install AMD build dependencies + if: inputs.amd_sdk_run_id == '' + run: sudo apt-get update && sudo apt-get install -y clang lld ninja-build uuid-dev + - name: Cache pinned AMD build tools and model artifacts + if: inputs.amd_sdk_run_id == '' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: .cache/amd-aie + key: amd-aie-${{ runner.os }}-${{ hashFiles('tools/npu/iree-source.lock.json', 'tools/npu/iree-frontend.requirements.txt', 'tools/npu/*iree*.py', 'tools/npu/*amdaie*.py', 'tools/npu/runtime/*', 'src/inference/amdnpu/runtime_api.h') }} - name: Assemble pinned Intel native SDK run: python tools/npu/build_openvino.py --package-cache .cache/npu --output-dir dist/npu-sdk/openvino - name: Import acquired AMD candidate @@ -43,11 +49,9 @@ jobs: path: dist/npu-sdk/amdnpu run-id: ${{ inputs.amd_sdk_run_id }} github-token: ${{ github.token }} - - name: Fetch locked AMD deployment runtime and compiled recognition models + - name: Build lightweight AMD runtime and all recognition buckets if: inputs.amd_sdk_run_id == '' - env: - GH_TOKEN: ${{ github.token }} - run: python tools/npu/fetch_amdnpu.py --package-cache .cache/npu --output-dir dist/npu-sdk/amdnpu + run: python tools/npu/build_amdaie.py --work-dir .cache/amd-aie --output-dir dist/npu-sdk/amdnpu --jobs 2 - name: Build the native addon with packaged NPU SDKs shell: bash run: | diff --git a/.github/workflows/npu-support-release.yml b/.github/workflows/npu-support-release.yml index e572179..af052ff 100644 --- a/.github/workflows/npu-support-release.yml +++ b/.github/workflows/npu-support-release.yml @@ -19,10 +19,8 @@ permissions: jobs: build: - # The AMD deployment archive remains a draft asset until v0.5.9 ships. - permissions: - contents: write runs-on: ubuntu-24.04 + timeout-minutes: 180 strategy: fail-fast: false matrix: @@ -34,10 +32,18 @@ jobs: node-version: '22' - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 with: - python-version: '3.11' + python-version: '3.12' + - name: Install AMD build dependencies + if: matrix.provider == 'amdnpu' + run: sudo apt-get update && sudo apt-get install -y clang lld ninja-build uuid-dev + - name: Cache pinned AMD build tools and model artifacts + if: matrix.provider == 'amdnpu' + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 + with: + path: .cache/amd-aie + key: amd-aie-${{ runner.os }}-${{ hashFiles('tools/npu/iree-source.lock.json', 'tools/npu/iree-frontend.requirements.txt', 'tools/npu/*iree*.py', 'tools/npu/*amdaie*.py', 'tools/npu/runtime/*', 'src/inference/amdnpu/runtime_api.h') }} - name: Prepare the selected provider only env: - GH_TOKEN: ${{ github.token }} PROVIDER: ${{ matrix.provider }} RELEASE_VERSION: ${{ inputs.version }} run: | @@ -46,7 +52,7 @@ jobs: if [[ "$PROVIDER" == openvino ]]; then python tools/npu/build_openvino.py --package-cache .cache/npu --output-dir dist/sdk else - python tools/npu/fetch_amdnpu.py --package-cache .cache/npu --output-dir dist/sdk + python tools/npu/build_amdaie.py --work-dir .cache/amd-aie --output-dir dist/sdk --jobs 2 fi - name: Build a separate CPU and NPU addon env: @@ -73,6 +79,8 @@ jobs: --sdk-dir dist/sdk --output-dir dist/package mkdir -p dist/tarballs npm pack ./dist/package --ignore-scripts --json --pack-destination dist/tarballs > dist/tarballs/pack-manifest.json + python tools/npu/record_support_pack.py --package-dir dist/package \ + --pack-manifest dist/tarballs/pack-manifest.json --output-file dist/tarballs/support-size-report.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: npu-support-${{ matrix.provider }}-${{ inputs.version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index c75738a..55a1f21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,14 +21,18 @@ Release preparation; npm publication and GitHub Release are pending. pinned OpenVINO 2026.4.0 libraries; default platform packages do not. See [Intel NPU 加速技术方案](docs/intel-npu-acceleration.md). - Added a separately installed AMD NPU backend (`provider: "amdnpu"`) for Linux x64 - glibc and Ryzen AI STX/KRK devices. It isolates the vendor ORT runtime, - loads 20 immutable BF16 recognition contexts, and runs detection on CPU. + glibc and Ryzen AI STX/KRK devices. The default deployment now uses a small + IREE C ABI runtime, 20 CPU/NPU recognition modules and one shared weight archive. + It offloads 19 dense pointwise convolutions, splits wide matrices to fit DMA + limits, keeps other recognition operators on IREE CPU, and runs detection on ORT CPU. Added model compilation and SDK import tools, descriptor validation, license/SBOM staging, and candidate/release workflows. AMD is excluded from Auto and enabled only through `execution.provider: "amdnpu"` (or `--provider amdnpu`). NPU device reports are optional for release; - `deviceValidated` remains false. The optional AMD support package includes Ryzen AI 1.8 - deployment libraries and real compiled contexts bound to Small 0.3.4. + `deviceValidated` remains false. BF16 operands use BFP16ebs8 microkernels; + numerical parity and hardware execution remain unverified. Deployment artifacts + are bound to Small 0.3.4, with no vendor SDK, compiler, Python or XRT in the AMD payload. + Legacy vendor deployments retain their separate ORT API/namespace path. Neither support package is automatically installed or selected by Auto. The existing Small model bundle remains unchanged. See [AMD NPU 加速技术方案](docs/amd-npu-acceleration.md) and [NPU SDK 构建与发布](docs/npu-runtime-release.md). diff --git a/CMakeLists.txt b/CMakeLists.txt index 53bd1b9..550fcd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ set(LIGHT_OCR_OPENVINO_SDK_DIR "" CACHE PATH set(LIGHT_OCR_OPENVINO_C_LIBRARY "" CACHE FILEPATH "OpenVINO C runtime loaded by direct C++ callers; defaults to /lib/libopenvino_c.so") set(LIGHT_OCR_AMDNPU_SDK_DIR "" CACHE PATH - "Verified Ryzen AI runtime SDK with precompiled BF16 recognition buckets (Linux x64 glibc)") + "Verified AMD runtime deployment with precompiled recognition buckets (Linux x64 glibc)") if(LIGHT_OCR_ENABLE_SANITIZERS AND LIGHT_OCR_ENABLE_THREAD_SANITIZER) message(FATAL_ERROR "Address/Undefined sanitizers and ThreadSanitizer require separate builds") diff --git a/docs/amd-aie-deployment-results.json b/docs/amd-aie-deployment-results.json new file mode 100644 index 0000000..a8338ae --- /dev/null +++ b/docs/amd-aie-deployment-results.json @@ -0,0 +1,570 @@ +{ + "schemaVersion": "1.0", + "date": "2026-10-07", + "status": "built-and-packaged-unqualified", + "releaseVersion": "0.5.9", + "pullRequest": "https://github.com/arcships/light-ocr/pull/71", + "sourceModelSha256": "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + "sourceLockSha256": "a58136fc60e8eb5023798c3e8d3a7014abbeb2f3107f6a727c5b8f1f0cd0bf4a", + "sourceCommits": { + "amdAie": "fddfec1be6ceefbdb890079d957947dfa1fe0848", + "third_party/aie-rt": "a6196ebd905ceaf92cabc2e594017fa11b9e4323", + "third_party/bootgen": "92e09bf37ea17d7b1f0e102a2548f27fb768651c", + "third_party/iree": "26083307b8753a11870b8caf590afea15c9429fa", + "third_party/mlir-air": "ca2448548d66d7e8e05bacb9e81e7df48a9ef729", + "third_party/openssl": "0e04e101507df752500093ff4b04cce5a898abaa", + "iree/third_party/benchmark": "192ef10025eb2c4cdd392bc502f0c852196baa48", + "iree/third_party/llvm-project": "8be29edc20dfb19245ab8cf49669c80125ece4f5", + "iree/third_party/flatcc": "9362cd00f0007d8cbee7bff86e90fb4b6b227ff3", + "iree/third_party/musl": "3f701faace7addc75d16dea8a6cd769fa5b3f260" + }, + "compilerBinaries": [ + { + "path": "tools/iree-compile", + "bytes": 6256, + "sha256": "a751017528d0e96e3331e119cfa87c9705e802a3fcc27c68b4a74f8afbe7e3f3" + }, + { + "path": "tools/iree-opt", + "bytes": 6248, + "sha256": "5b7123611e66efbbc527c0dfb0ac225a1597193cf4bbdb69022fdc4c169366b4" + }, + { + "path": "lib/libIREECompiler.so", + "bytes": 271528912, + "sha256": "352f1d6e4c7f3945f7fbbdf09255598cad153ec3afbac52f353241559cbc4408" + } + ], + "compilerLocalPatches": [ + { + "path": "patches/check-matmul-map-ranks.patch", + "sha256": "5a0a9b3efe57cc4d257b8fd9d08f59a4c158a07c71a7242d21b812314ee9fb05", + "reason": "Reject scalar/rank-one generic maps before matmul dimension access" + } + ], + "runtimeBinaries": [ + { + "path": "liblight_ocr_amdaie.so.1.0.0", + "bytes": 1255312, + "sha256": "6eab7ad3cb2cacbb856a91c6fe83a45111c4a0d475a360d880cbf46867d1d24d" + }, + { + "path": "merge-aie-parameters", + "bytes": 351832, + "sha256": "761b201c532bc59129a700a9825e2193e9ea95ea3088af79c104d8ef495c0393" + } + ], + "runtimeWrapperSources": [ + { + "path": "tools/npu/runtime/CMakeLists.txt", + "sha256": "ea05adf9538d051375f28b3f6ba9241da73478d958f07b3366b7deadcd8ac3c8" + }, + { + "path": "tools/npu/runtime/runtime.cc", + "sha256": "6444f88e7a2076f03d19ef55517a83dec3e552985ede13613e5625e230de5896" + }, + { + "path": "tools/npu/runtime/merge_parameters.cc", + "sha256": "604a02e66419a2a60623640d3a4295dc2cbf0c4b425f2af5c3052b56bca9d2ed" + }, + { + "path": "tools/npu/runtime/exports.map", + "sha256": "0d627d7d3fadc191b930af21c7636c06cb1f93ade86c01659b236519c3a389ff" + }, + { + "path": "src/inference/amdnpu/runtime_api.h", + "sha256": "3830626cb7d76c2043deca5e2c6d3183f5921c433a66b37b321d62e23496b8ce" + } + ], + "buildScripts": { + "partitionScriptSha256AtBuildStart": "7772b5208e54e517c6a1a32703a28734254b9c27d0522ea8347fb8d549bb86ad", + "partitionScriptSha256AtPackaging": "cc6bb8059460517223ad6ac419c2012e1c0e53209d13da472c3e6f53525a9c79", + "routeScriptSha256": "87d00b8a45dda0863e1f40cc88d17c27fd29d54e2fdb9b70449dc397f4e39915", + "parameterMergerSha256": "761b201c532bc59129a700a9825e2193e9ea95ea3088af79c104d8ef495c0393", + "note": "Only explanatory comments in the partition script changed during the all-bucket build; transformation code was unchanged." + }, + "recognition": { + "completeRecognitionContract": true, + "input": "1x3x48xwidth FP32 NCHW", + "output": "1x(width/8)x18710 FP32", + "widths": [ + 320, + 384, + 480, + 544, + 576, + 608, + 704, + 736, + 832, + 960, + 1056, + 1184, + 1248, + 1376, + 1600, + 1984, + 2240, + 2560, + 2880, + 3200 + ], + "npuPointwiseConvolutions": 19, + "maxMatmulRowsPerChunk": 3840, + "precision": "bf16-bfp16ebs8", + "cpuPartitionRequired": true, + "cpuOperators": [ + "layout", + "padding", + "FP32-to-BF16", + "bias", + "GELU", + "residual", + "other convolutions", + "attention", + "classifier", + "softmax" + ], + "buckets": [ + { + "width": 320, + "artifact": { + "path": "recognition-320.vmfb", + "bytes": 335923, + "sha256": "5adf1290b28d07a77300c470c2fab7fa8586abbc5a3aec8a9c3a1cb1951125f2" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 384, + "artifact": { + "path": "recognition-384.vmfb", + "bytes": 326755, + "sha256": "4915f3a06ccc84767f350e8f54c04c6ad411d29398795e39d9e9c0914ff4566d" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 480, + "artifact": { + "path": "recognition-480.vmfb", + "bytes": 330099, + "sha256": "6ababc13508264d07ba8c3965bb7d294bc8b01e7b5a9b576d92bafa1bb6322f5" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 544, + "artifact": { + "path": "recognition-544.vmfb", + "bytes": 354067, + "sha256": "e2ae7972fb2c5144c812a467e321aa633fb481c736f0214349b194518811af26" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 576, + "artifact": { + "path": "recognition-576.vmfb", + "bytes": 350291, + "sha256": "b00af3cb84b3e32df05117c8f7076ad315ec2d0b80c2440d24dfbb3b2dbe0a25" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 608, + "artifact": { + "path": "recognition-608.vmfb", + "bytes": 354819, + "sha256": "da831af19083aa5a1d72adad7b299ec107d179233c1f56145ffaa42c3b432108" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 704, + "artifact": { + "path": "recognition-704.vmfb", + "bytes": 362659, + "sha256": "c5ef383733b7f41b8977915c47026dc82bb581d132cd36d493daf5165c105b39" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 736, + "artifact": { + "path": "recognition-736.vmfb", + "bytes": 362451, + "sha256": "30d88d8bd02f7d31e9c431acf9bca392c409e4f52c73f0f962d9df38bc1bb0cd" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 832, + "artifact": { + "path": "recognition-832.vmfb", + "bytes": 362099, + "sha256": "3c486bf905ae5851544e430ad3d838aa18bc715af4f33489ea4a444020ca7769" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 960, + "artifact": { + "path": "recognition-960.vmfb", + "bytes": 368979, + "sha256": "e2efabb676755bae4db687c3d7fd1e4c841213988c2af9a6e7f7fa3434a52cc6" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1056, + "artifact": { + "path": "recognition-1056.vmfb", + "bytes": 373779, + "sha256": "407dca271db1ca75667b6dac4138e3c81f4828fe482bc90ca5e750a45611fd5c" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1184, + "artifact": { + "path": "recognition-1184.vmfb", + "bytes": 374483, + "sha256": "c9938ac92dcc5fc59ff5c05639934ee933285ad7c474681f069ed582ef18416f" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1248, + "artifact": { + "path": "recognition-1248.vmfb", + "bytes": 387539, + "sha256": "ef8bdc7874ef20db99347e156ed8b024b7278ea0839f165f1b165dee7dd3b12a" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1376, + "artifact": { + "path": "recognition-1376.vmfb", + "bytes": 386771, + "sha256": "dc04274717eeab9871839b2a5fb90fe26632a7f11a4ef0b7c08e674066fccba9" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1600, + "artifact": { + "path": "recognition-1600.vmfb", + "bytes": 399571, + "sha256": "099a968bf8b96ff8b04d064a441d757c2b5d9b7da56f2f93c558a7d211596dc5" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 1984, + "artifact": { + "path": "recognition-1984.vmfb", + "bytes": 410915, + "sha256": "c8700a867763b7c6971f8b6f0501571837f1612d9c4a5599f75e3d99cb58ac2a" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 2240, + "artifact": { + "path": "recognition-2240.vmfb", + "bytes": 419587, + "sha256": "a588ef13fc079dc457565296d3dc90555ce386e0e2296472224b76803c16ac17" + }, + "dispatches": { + "cpu": 138, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 2560, + "artifact": { + "path": "recognition-2560.vmfb", + "bytes": 367267, + "sha256": "12c0a37d97b3e095cb39c9475e26a0655aa7350b575907ba582d6362fc959ff1" + }, + "dispatches": { + "cpu": 120, + "npu": 19 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 2880, + "artifact": { + "path": "recognition-2880.vmfb", + "bytes": 475187, + "sha256": "7d0748a9f19c5ed91c1c2e71835f13d15aaf2e1f8e3a16655566177dd774c135" + }, + "dispatches": { + "cpu": 138, + "npu": 32 + }, + "npuPointwiseConvolutions": 19 + }, + { + "width": 3200, + "artifact": { + "path": "recognition-3200.vmfb", + "bytes": 487315, + "sha256": "8a82155a29a33878301d3577ab504806ac1b8980e66f0706daf639cfa0ce78c6" + }, + "dispatches": { + "cpu": 138, + "npu": 32 + }, + "npuPointwiseConvolutions": 19 + } + ], + "sharedParameters": { + "path": "recognition.irpa", + "bytes": 17145856, + "sha256": "50230a7f2bdfa89464c08e3c18a7aa45bd241ee32fd0c9b6ac728861565506ba" + }, + "modelArtifactBytes": 24736412 + }, + "sdk": { + "artifactSetSha256": "20c2673971c16287a27ff5f43c2ffd3fee2ef02661efd04e5f6014b43b75bd6f", + "artifactBytes": 25992524, + "artifacts": [ + { + "path": "deployment.json", + "bytes": 800, + "sha256": "4c12d98ed80a13ffa010b06de4bc741ad20ee47518639f3076d95da78bce9f08" + }, + { + "path": "lib/liblight_ocr_amdaie.so.1", + "bytes": 1255312, + "sha256": "6eab7ad3cb2cacbb856a91c6fe83a45111c4a0d475a360d880cbf46867d1d24d" + }, + { + "path": "models/recognition-1056.vmfb", + "bytes": 373779, + "sha256": "407dca271db1ca75667b6dac4138e3c81f4828fe482bc90ca5e750a45611fd5c" + }, + { + "path": "models/recognition-1184.vmfb", + "bytes": 374483, + "sha256": "c9938ac92dcc5fc59ff5c05639934ee933285ad7c474681f069ed582ef18416f" + }, + { + "path": "models/recognition-1248.vmfb", + "bytes": 387539, + "sha256": "ef8bdc7874ef20db99347e156ed8b024b7278ea0839f165f1b165dee7dd3b12a" + }, + { + "path": "models/recognition-1376.vmfb", + "bytes": 386771, + "sha256": "dc04274717eeab9871839b2a5fb90fe26632a7f11a4ef0b7c08e674066fccba9" + }, + { + "path": "models/recognition-1600.vmfb", + "bytes": 399571, + "sha256": "099a968bf8b96ff8b04d064a441d757c2b5d9b7da56f2f93c558a7d211596dc5" + }, + { + "path": "models/recognition-1984.vmfb", + "bytes": 410915, + "sha256": "c8700a867763b7c6971f8b6f0501571837f1612d9c4a5599f75e3d99cb58ac2a" + }, + { + "path": "models/recognition-2240.vmfb", + "bytes": 419587, + "sha256": "a588ef13fc079dc457565296d3dc90555ce386e0e2296472224b76803c16ac17" + }, + { + "path": "models/recognition-2560.vmfb", + "bytes": 367267, + "sha256": "12c0a37d97b3e095cb39c9475e26a0655aa7350b575907ba582d6362fc959ff1" + }, + { + "path": "models/recognition-2880.vmfb", + "bytes": 475187, + "sha256": "7d0748a9f19c5ed91c1c2e71835f13d15aaf2e1f8e3a16655566177dd774c135" + }, + { + "path": "models/recognition-320.vmfb", + "bytes": 335923, + "sha256": "5adf1290b28d07a77300c470c2fab7fa8586abbc5a3aec8a9c3a1cb1951125f2" + }, + { + "path": "models/recognition-3200.vmfb", + "bytes": 487315, + "sha256": "8a82155a29a33878301d3577ab504806ac1b8980e66f0706daf639cfa0ce78c6" + }, + { + "path": "models/recognition-384.vmfb", + "bytes": 326755, + "sha256": "4915f3a06ccc84767f350e8f54c04c6ad411d29398795e39d9e9c0914ff4566d" + }, + { + "path": "models/recognition-480.vmfb", + "bytes": 330099, + "sha256": "6ababc13508264d07ba8c3965bb7d294bc8b01e7b5a9b576d92bafa1bb6322f5" + }, + { + "path": "models/recognition-544.vmfb", + "bytes": 354067, + "sha256": "e2ae7972fb2c5144c812a467e321aa633fb481c736f0214349b194518811af26" + }, + { + "path": "models/recognition-576.vmfb", + "bytes": 350291, + "sha256": "b00af3cb84b3e32df05117c8f7076ad315ec2d0b80c2440d24dfbb3b2dbe0a25" + }, + { + "path": "models/recognition-608.vmfb", + "bytes": 354819, + "sha256": "da831af19083aa5a1d72adad7b299ec107d179233c1f56145ffaa42c3b432108" + }, + { + "path": "models/recognition-704.vmfb", + "bytes": 362659, + "sha256": "c5ef383733b7f41b8977915c47026dc82bb581d132cd36d493daf5165c105b39" + }, + { + "path": "models/recognition-736.vmfb", + "bytes": 362451, + "sha256": "30d88d8bd02f7d31e9c431acf9bca392c409e4f52c73f0f962d9df38bc1bb0cd" + }, + { + "path": "models/recognition-832.vmfb", + "bytes": 362099, + "sha256": "3c486bf905ae5851544e430ad3d838aa18bc715af4f33489ea4a444020ca7769" + }, + { + "path": "models/recognition-960.vmfb", + "bytes": 368979, + "sha256": "e2efabb676755bae4db687c3d7fd1e4c841213988c2af9a6e7f7fa3434a52cc6" + }, + { + "path": "models/recognition.irpa", + "bytes": 17145856, + "sha256": "50230a7f2bdfa89464c08e3c18a7aa45bd241ee32fd0c9b6ac728861565506ba" + } + ], + "systemLibraries": [ + "libc.so.6", + "libgcc_s.so.1", + "libm.so.6", + "libstdc++.so.6", + "libuuid.so.1" + ] + }, + "npm": { + "package": "@arcships/light-ocr-amdnpu-linux-x64-gnu", + "version": "0.5.9", + "archive": { + "path": "arcships-light-ocr-amdnpu-linux-x64-gnu-0.5.9.tgz", + "bytes": 27869026, + "sha256": "2bee7052a4dbfcfea8338e1fa06e42061fc77b9d4aee21d52794f9f28cc4e8b8" + }, + "unpackedBytes": 59690023, + "fileCount": 50, + "lightweightAmd": true, + "deviceValidated": false, + "numericsValidated": false, + "budget": { + "compressedBytes": 67108864, + "unpackedBytes": 134217728 + } + }, + "historicalVendorPackage": { + "compressedBytes": 548609874, + "unpackedBytes": 1390267314 + }, + "scope": { + "detector": "ORT CPU", + "recognition": "IREE CPU/NPU partitions", + "amdDefaultEnabled": false, + "includedInAuto": false, + "includedInBasePackage": false, + "modelTier": "Small 0.3.4", + "deviceTarget": "XDNA2 npu4; STX/KRK compatibility not device-qualified" + }, + "validation": { + "testsRun": false, + "numericsValidated": false, + "deviceValidated": false, + "performanceMeasured": false, + "reason": "Maintainer requested complete development without validation.", + "performed": [ + "source export/import", + "20-bucket compilation", + "runtime and addon builds", + "SDK assembly", + "npm packaging and size/hash inventory" + ] + } +} diff --git a/docs/amd-aie-development.md b/docs/amd-aie-development.md index 8948612..c4b2bef 100644 --- a/docs/amd-aie-development.md +++ b/docs/amd-aie-development.md @@ -1,14 +1,75 @@ # AMD AIE 开发与子图编译 -状态：开发工具链与子图导出已实现，尚未接入发布后端。此流程不执行设备推理。 +状态：完整识别分区后端、全部宽度桶编译、共享权重和独立包构建已实现。此流程不执行设备推理。 总体选型与后续阶段见 [轻量部署方案](amd-npu-lightweight-plan.md)。 -本轮工具链身份、编译状态与产物大小/哈希见 [编译结果清单](amd-aie-compilation-results.json)。 +完整部署产物和包大小见 [部署结果清单](amd-aie-deployment-results.json)。 +早期子图调查见 [初期编译结果](amd-aie-compilation-results.json)，不能用其中的阶段状态代替当前后端状态。 + +## 完整部署入口 + +Python 3.12、CMake ≥3.26、Clang、Ninja、Git、Perl、Make 和 uuid 开发头文件为构建依赖。 +以下命令准备锁定源码、Peano、前端环境和原始 Small 模型，构建编译器与执行库，生成全部 20 桶并组装 SDK： + +```sh +python3.12 tools/npu/build_amdaie.py \ + --work-dir .cache/amd-aie --output-dir dist/amdaie-sdk --jobs 2 +``` + +输出目录必须不存在。完整模型缓存按锁文件、前端要求和转换脚本身份区分；运行库单独增量构建。 +安装用户不需要 Python、编译器、Ryzen AI SDK、XRT 或联网模型编译；仍需可用的系统 amdxdna 驱动、固件和 STX/KRK NPU。 +CI 在 Ubuntu 24.04 构建，系统动态依赖包括 libuuid、libstdc++、libgcc_s、libm、libc。 + +### 分区与数据流 + +- 完整图输入 `[1,3,48,width]`，输出 `[1,width/8,18710]`，保留 FP32 ABI 和原有 CTC。 +- 19 个输入/输出通道均 ≥128 的 1×1、stride=1 卷积转换成 BF16 矩阵乘。M/N/K 补齐到 128 的倍数，计算后裁切有效空间和通道。 +- 布局转换、补零、FP32→BF16、bias、GELU、残差、其他卷积、attention、分类头和 Softmax 使用 IREE CPU；检测继续用 ORT CPU。 +- Peano 微内核把 BF16 转为 BFP16ebs8 后计算，累加输出为 FP32。不能宣称纯 BF16 数值、推理正确或已有加速比。 +- `partition_iree_model.py` 隔离矩阵计算并阻止零初始化跨 dispatch 共享；`route_iree_dispatches.py` 在 Flow 形成后赋予明确 CPU/NPU affinity，避免早期 affinity 被优化丢失。 +- 每个 VMFB 同时含 CPU ELF、AMD PDI 和跨设备调度。CPU 可以映射 AMD host-only BO；共享分配策略选择 NPU 分配器，双方的映射、缓存同步和 fence 由 HAL 管理。 +- 权重使用内容哈希作为参数 key；构建工具合并相同参数，所有桶共用 `models/recognition.irpa`。编译器、Peano、各桶的临时 IRPA 和原始 ONNX 不随 AMD 包分发。 + +### 原生运行接口 + +`src/inference/amdnpu/runtime_api.h` 定义版本化 C ABI；运行库只导出 `LightOcrAieGetApi`，IREE 和驱动符号隐藏。 +会话按需加载宽度桶；运行库持有权重、CPU/NPU 设备、参数模块和 VM context。 +原生 `AmdNpuSession` 串行执行共享引擎调用，核对库、部署配置、共享权重和模型的字节数及 SHA256。 +关闭时先释放全部桶会话，再释放引擎；库保持映射，避免卸载线程局部状态。 + +新配置 `target=IREEAMDAIE`、`runtimeAbi=1`，与既有 VAIML context 明确区分。 +旧 vendor 部署仍由其独立 ORT API/namespace 加载。两种部署共享公开 provider `amdnpu`，新包实际 provider 链为 `IREEAMDAIE → IREECPU`。 +`precision` 诊断为 `bf16-bfp16ebs8`，`deviceValidated=false`。 + +### 独立支持包 + +```sh +cmake -S . -B build-amdaie -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DLIGHT_OCR_DEPENDENCY_CACHE_DIR="$PWD/.cache/dependencies" \ + -DLIGHT_OCR_AMDNPU_SDK_DIR="$PWD/dist/amdaie-sdk" \ + -DLIGHT_OCR_BUILD_NODE=ON -DLIGHT_OCR_BUILD_TESTS=OFF -DLIGHT_OCR_BUILD_TOOLS=OFF \ + -DLIGHT_OCR_NODE_INCLUDE_DIR=/path/to/node/include/node +cmake --build build-amdaie --parallel 2 +python tools/generate_release_metadata.py --build-dir build-amdaie \ + --output-dir reports/amdaie --platform-id linux-x64 --model-free +python tools/npu/package_support.py --provider amdnpu --build-dir build-amdaie \ + --metadata-dir reports/amdaie --sdk-dir dist/amdaie-sdk --output-dir dist/amdaie-package +npm pack ./dist/amdaie-package --ignore-scripts --pack-destination dist +``` + +AMD 不自动安装、不参与 Auto。显式配置 `execution.provider="amdnpu"` 后才加载；`cpuPartition=forbid` 拒绝分区模型。 +`npu-support-release.yml` 和 `npu-native.yml` 默认使用轻量源码构建入口，保留 Intel 路线。 +按维护者要求，本轮只执行生成部署产物所需的构建与编译，没有运行测试、数值比较或设备推理。 + +## 早期编译调查（历史记录） + +以下记录说明从整图直接下放转为明确 CPU/NPU 分区的原因。未完成的是早期整图全部 NPU 的 lowering，不是当前完整分区后端。 ## 构建输入 `tools/npu/iree-source.lock.json` 锁定 AMD AIE、IREE、LLVM 与必要子模块提交，以及 Peano wheel 的大小和 SHA256。 工具只读取已锁定的 Small 0.3.4 识别模型，拒绝源模型哈希不符。 -编译工具和输出保存在 `.cache/amd-aie/`；不进入主 package 或当前 AMD 支持包。 +编译工具保存在 `.cache/amd-aie/`，不进入运行时包。AMD 支持包只装入运行库、全部 VMFB、单份共享 IRPA、清单与许可证。 ## 准备源码 diff --git a/docs/amd-npu-acceleration.md b/docs/amd-npu-acceleration.md index f655e86..5ec7083 100644 --- a/docs/amd-npu-acceleration.md +++ b/docs/amd-npu-acceleration.md @@ -1,38 +1,59 @@ # AMD NPU 加速技术方案 -后续轻量部署改造的选型、实施阶段和发布约束见 [AMD NPU 轻量部署改造方案](amd-npu-lightweight-plan.md)。该方案尚未实施，当前文档描述既有 Ryzen AI vendor 后端。 +更新时间：2026-10-07。完整 CPU/NPU 分区后端、20 个识别宽度桶、共享权重和独立支持包构建已实现。AMD 默认关闭，用户通过配置显式开启；未运行测试、数值比较或 AMD 设备推理。 -更新时间：2026-10-06。状态：Linux x64 glibc 后端与打包改造已实现，默认关闭、配置显式开启；0.5.9 独立支持包的构建输入已包含 Ryzen AI 1.8 的原生部署库与 Small 0.3.4 的 20 个真实 BF16 context。无需真机即可完成编译，尚未执行 AMD 设备推理。正式 npm 发布仍待完成。 +## 使用方式 -## 范围与模型路线 +```sh +npm install @arcships/light-ocr @arcships/light-ocr-amdnpu-linux-x64-gnu +``` -首期针对 STX/KRK（Ryzen AI 300、Max 等 XDNA2），使用 BF16 识别模型，检测保持 ORT CPU。PHX/HPT 的 INT8 路线、Windows、musl 均未实现。`precision` 仅接受 `auto`；`cpuPartition=forbid` 不支持此路线。 +在原有引擎配置中设置 `execution: { provider: "amdnpu" }`，或使用 CLI `--provider amdnpu`、C++ `ExecutionProvider::amdnpu`。 +支持包需与 Core 版本匹配；0.5.9 仍处于发布准备阶段。 -Ryzen AI 1.8 的 Linux 文档支持 CNN BF16/INT8 编译和执行。旧版预研把 release notes 中 LLM 的 Linux 模型生成限制推广到了 CNN OCR，现予纠正。[AMD Linux 文档](https://ryzenai.docs.amd.com/en/latest/linux.html)、[Release Notes](https://ryzenai.docs.amd.com/en/latest/relnotes.html)。 +AMD 不参与 Auto，支持包不会自动安装。系统需可读写的 STX/KRK NPU 设备节点、amdxdna 驱动和固件；CI 构建环境为 Ubuntu 24.04 Linux x64 glibc。 +不需要完整 Ryzen AI SDK、XRT、Python 或用户机器上的模型编译。PHX/HPT、Windows、arm64、musl 尚未支持。 -BF16 路线由 VAIML 编译 FP32 ONNX，生成可嵌入权重与微码的 EPContext；无需引入 INT8 校准集。C++ 部署使用预编译 BF16 context，保留 FP32 输入输出。VitisAI 自动将未支持的算子分配给 CPU，诊断因此明确包含 CPU provider。[AMD 模型编译与部署](https://ryzenai.docs.amd.com/en/latest/modelrun.html)。 +## 模型与分区 -## 已完成代码 +识别模型固定绑定 Small 0.3.4 的原始 ONNX SHA256，保留 batch=1、20 个宽度桶和 FP32 输入输出： +`[1,3,48,width] → [1,width/8,18710]`。原有字典和 CTC 解码继续使用。 -- `src/inference/amdnpu/backend.cpp`：通过独立 glibc link namespace 装载 AMD 自带 ORT C API 20，避免绑定到本进程的 CPU/WebGPU ORT；AMD runtime 对象始终由对应 API 释放。 -- 校验源识别模型哈希、VAIML 配置和 20 个静态宽度桶；识别形状为 `[1,3,48,width]`，批次为 1。编译结果的字典类别数必须与 bundle 一致。 -- 设备检查要求可读写的 AMD STX/KRK `/dev/accel` 节点；无设备返回 `adapter_unavailable`。库或模型哈希不符属于不可恢复错误，不能静默跳到 CPU。 -- detector 使用 CPU，recognition 使用 `VitisAIExecutionProvider` 和其允许的 CPU 分区。只有首次会话及对应桶实际创建成功后才选中该候选；后续推理失败直接报告错误。 -- `tools/npu/compile_amdnpu.py`：在 vendor SDK Python 环境编译所有桶，要求存在嵌入式 VitisAI EPContext，并检查输入输出类型与形状；编译过程不执行模型推理，不要求真机验收。 -- `tools/npu/import_amdnpu.py`：从已取得的 SDK 部署目录导入原生库，设置包内 RPATH、检查 ELF 依赖闭包、记录变更前后哈希，并锁定模型、配置及许可证。 -- CMake、Node descriptor 2.1、JS loader、npm staging、SBOM 和候选/正式发布流程均已接入。构建入口与报告格式见 [NPU SDK 构建与发布](npu-runtime-release.md)。 +| 计算 | 实际执行 | +| --- | --- | +| 19 个输入/输出通道均 ≥128 的 1×1、stride=1 卷积矩阵核心 | IREE AMD AIE / XDNA2 NPU | +| 布局转换、补零、精度转换、bias、GELU、残差 | IREE CPU | +| 其他卷积、attention、分类头和 Softmax | IREE CPU | +| 检测 | ONNX Runtime CPU | -## 默认关闭与发布策略 +矩阵 M/N/K 按 128 补齐并裁切；单次 dispatch 最多 3840 行，较宽桶分段执行，规避固定编译器的宽矩阵 shim DMA 维度 lowering 越界。 +BF16 操作数在 Peano 微内核内转换为 BFP16ebs8，FP32 累加；精度诊断为 `bf16-bfp16ebs8`。 +这是完整识别分区图，没有纯 NPU、数值正确或性能提升的实测结论。 +`precision` 请求仍只接受 `auto`；`cpuPartition=forbid` 会拒绝此路线。 -AMD 始终排除在 Auto 候选列表之外；用户通过 `execution.provider: "amdnpu"`、CLI `--provider amdnpu` 或 C++ 的 `ExecutionProvider::amdnpu` 显式开启。0.5.9 独立 AMD 支持包附带 vendor SDK 与匹配的 Small 识别模型，普通平台包不包含它们；运行时未配置 AMD 时不装载 vendor ORT。 +## 独立包内容 -按维护者决定，NPU 真机验收不再阻塞发布。SDK 导入的 `qualificationOnly: true` 仅记录尚无设备证据，平台包可随其他已满足条件的运行时一起发布；`deviceValidated` 保持 `false`。库存、模型哈希、依赖闭包和许可证校验继续执行，设备报告可选。 +- 一个小型原生 C ABI 执行库，IREE 与 AMD 驱动符号隐藏。 +- 20 个 VMFB，内含 CPU ELF、AMD PDI 和跨设备执行计划。 +- 一份共享 IRPA 权重：参数按内容哈希命名，相同权重跨宽度只存一次。 +- 匹配 addon、CPU ORT、部署配置、完整哈希库存、许可证和 SBOM。 -## 尚未验证的设备行为（可选后续工作） +编译器、Peano、原始 ONNX、临时权重归档和 vendor SDK 库不进入 AMD 包。主 package 与普通平台包不携带这些 AMD 产物。 +逐文件大小及包大小见 [部署结果清单](amd-aie-deployment-results.json)。 -1. 在 STX/KRK 主机确认独立 namespace 装载、XRT/amdxdna 驱动及 VitisAI C API 20 协同工作。 -2. 比较已编译 20 桶模型与 FP32 CPU goldens，记录文本、框、置信度及 BF16 误差。 -3. 测量冷/热延迟、CPU 时间和 RSS，覆盖关闭/重建、权限不足、驱动兼容及 WebGPU 共存。 +## 原生行为与诊断 -SDK 的原始与打包后哈希、依赖闭包、许可证/第三方 notices 和源模型绑定已随部署包记录；这不等同于真机验收或独立法律审阅。 -这些设备行为尚无实测结论，不再作为源码合入或 AMD 显式开启支持发布的前置条件。 +库、配置、共享权重和桶模型均按字节数/SHA256 校验；哈希错误直接报告，不能静默回退。 +没有可访问 NPU 时返回 `adapter_unavailable`。后续桶按需加载；推理和创建错误保持原有错误/回退策略。 + +内部配置为 `target=IREEAMDAIE`、`runtimeAbi=1`；实际识别 provider 链为 `IREEAMDAIE → IREECPU`，检测为 `CPUExecutionProvider`。 +会话关闭先释放模型 context，再释放参数模块与设备；动态库保持映射。 +`deviceValidated=false`，且没有数值验证。按维护者决定，验证不阻塞此次开发交付与显式启用支持的发布准备。 + +## 构建与兼容 + +入口、锁定工具链与分区实现见 [完整部署开发说明](amd-aie-development.md)；独立包发布流程见 [NPU SDK 构建与发布](npu-runtime-release.md)。 +选型、体积预算和实际实施状态见 [轻量部署方案](amd-npu-lightweight-plan.md)。 + +旧 VAIML/EPContext 部署仍可通过既有 `fetch_amdnpu.py` / `import_amdnpu.py` 工具准备；原生层保留 vendor ORT 独立 API/namespace 路径。 +新发布工作流默认从锁定源码构建 IREE 轻量部署，不复用旧 context，也不要求取得专有 SDK。 diff --git a/docs/amd-npu-lightweight-plan.md b/docs/amd-npu-lightweight-plan.md index 2314017..7b00932 100644 --- a/docs/amd-npu-lightweight-plan.md +++ b/docs/amd-npu-lightweight-plan.md @@ -1,6 +1,6 @@ # AMD NPU 轻量部署改造方案 -日期：2026-10-07。状态：P0/P1 开发进行中；构建、子图导出和编译报告工具已实现，完整后端尚未接入。 +日期：2026-10-07。状态：完整 CPU/NPU 分区后端、20 桶、共享权重及独立支持包流程已实现；不做推理验证。 ## 1. 目标与结论 @@ -8,7 +8,7 @@ **首选路线：评估 IREE AMD AIE 提前编译 Small 识别模型，使用对应的 IREE amdxdna 运行时。** 如编译缺口过大，第二路线采用 HRX + IRON 实现 OCR 专用内核与调度。xdna-engine 用作算法和内核参考；不整体接入 FastFlowLM。 -本方案先完成编译可行性评估，再确定最终后端。现有算子代码和轻量运行库的存在，并不证明完整 PP-OCR 图已支持。 +本方案先完成编译可行性评估，再实现完整 CPU/NPU 分区后端。下文保留路线调查与阶段规划；最终实现和部署产物见文末状态及结果清单。 ## 2. 范围与约束 @@ -96,14 +96,14 @@ flowchart TD - 构建工具链在隔离目录或 CI 中运行；用户安装时不编译模型、不下载 SDK、不执行驱动安装。 - 编译前记录模型哈希、转换规则、算子清单、布局和精度变化。 - 编译后记录每个宽度的输入输出、目标设备、执行格式、运行时版本和文件哈希。 -- 优先设计共享权重与按宽度区分的执行计划，降低重复权重开销。是否可行由编译器及加载格式验证；如不可共享，必须单独报告重复体积。 +- 已实现内容哈希参数 key、按宽度 VMFB 与单份共享 IRPA，减少重复权重。逐文件大小随部署清单记录。 - 默认平台包和主 package 不携带 AMD、OpenVINO 或新编译工具链。 ### 原生接入 - 实现现有 `InferenceSession` 接口，保留 FP32 输入输出及原有 CTC 解码。 - 首期保持 batch=1、20 个宽度桶和 CPU detector。 -- 使用会话内持久缓冲区，尽量让融合块之间的激活留在设备；避免逐算子 CPU/NPU 往返。 +- 使用 IREE VM/HAL 管理激活、映射和同步；19 个卷积矩阵核心下放 NPU，宽桶拆成多个 dispatch，其余算子由 CPU 执行。分区性能尚无测量。 - 粗粒度分区方案须明确 CPU 算子、传输点和同步点，不能仅根据算子 MAC 决定分区。 - manifest 新增内部后端标识和执行格式版本，区分 vendor 与 IREE/HRX 产物，拒绝错配。 - 公开 provider 名称保持 `amdnpu`；请求 provider、实际执行后端、CPU 分区及验证状态进入诊断信息。 @@ -118,7 +118,7 @@ flowchart TD ## 6. 体积预算 -建议将 **AMD 支持包下载 ≤64 MiB、解包 ≤128 MiB** 作为本轮工程目标；这是待实测的设计预算，不是已经实现的大小承诺。 +**AMD 支持包下载 ≤64 MiB、解包 ≤128 MiB** 已作为构建门禁；实际候选体积记录在部署结果清单中。 | 组成 | 压缩预算 | |---|---:| @@ -143,11 +143,11 @@ flowchart TD P2 是主要选型决策点：如果关键算子无法编译、需要大量模型专用编译器修改，或只能形成频繁主机往返的分区，转向 HRX + IRON。改走第二路线时保留已有模型分析和转换规则，重新编译内核及实现调度；不预设旧执行产物可复用。 -本方案不设未经评估的工期。P1/P2 完成后才能根据实际缺口安排后续工作。 +实际实现选择明确 CPU/NPU 分区：保持完整识别接口，下放 19 个较大 pointwise 卷积，其他识别算子和检测均留在 CPU。 ## 8. 验收计划 -本节描述后续实施时应提供的证据，本次方案编写未执行这些检查。 +维护者已明确要求本轮不做验证。仅生成构建产物、库存、身份及体积记录；以下数值、设备和行为检查均为可选后续工作，不阻塞此次交付。 ### 无真机可完成 @@ -175,13 +175,10 @@ CPU 参考比较只能证明转换图的软件语义，不能证明编译后的 ## 10. 当前状态 -已完成固定源码下载、发布包解包、ELF/头文件检查和识别模型静态结构分析。开发入口见 [AMD AIE 开发与子图编译](amd-aie-development.md)：已实现固定工具链构建、真实识别热点导出、完整图导入与编译报告工具。 -宽度 384 的完整 FP32 图已完成静态形状处理、opset 转换和 57 个卷积的布局转换，标准 IREE CPU 编译产物约 21.34 MB；未执行推理。 -同版源码的独立 amdxdna 执行工具已构建，约 1.84 MB，动态依赖不含 vendor SDK 或 XRT。它并非最终支持包。 -完整后端、数值比较与 NPU 执行尚未完成。 +P0–P4 的构建、后端、完整桶和独立封装代码均已实现。IREE 整图全部 NPU lowering 的限制通过明确分区处理：19 个 BF16/BFP16 矩阵运算交给 NPU，CPU 保留其余算子、布局转换与分类头。 -首轮实际 AMD 编译已生成宽度 384 的补齐投影矩阵核心 VMFB（612,008 字节），完整图通过 executable sources 阶段。 -完整热点块仍受布局/精度转换 dispatch 限制；整图后端 lowering 仍在打包转置阶段崩溃。 -Peano 微内核内部使用 BFP16，因此不能将该候选描述为已验证的纯 BF16 实现。 +完整入口为 `tools/npu/build_amdaie.py`。同一锁定源码构建编译器和运行库，生成全部 20 个 VMFB 和一份共享 IRPA；不会复用 vendor context。 +原生后端和 descriptor 已识别新 ABI、配置和权重身份；CI 默认构建轻量 AMD SDK，独立包带许可证、SBOM 和哈希库存。 -下一项可执行工作为补齐 P1/P2：缩减打包转置崩溃样例，确定粗粒度分区与算子 lowering，并处理跨宽度权重共享。 +实际产物大小见 [部署结果清单](amd-aie-deployment-results.json)，构建步骤见 [完整部署开发说明](amd-aie-development.md)。 +没有执行测试、CPU 数值比较或 NPU 推理，没有精度和性能结论。`deviceValidated=false`，不作为合入和显式启用发布的阻塞项。 diff --git a/docs/npu-runtime-release.md b/docs/npu-runtime-release.md index 1d87911..99d23cf 100644 --- a/docs/npu-runtime-release.md +++ b/docs/npu-runtime-release.md @@ -1,6 +1,6 @@ # NPU SDK 构建与发布 -当前实现覆盖 Linux x64 glibc。Intel 使用 OpenVINO C API；AMD 使用独立装载的 Ryzen AI ORT 与预编译 BF16 识别模型。安装后的 OCR 进程无需 Python，系统仍需厂商 NPU 驱动，AMD 路线使用 Ryzen AI 1.8 支持的 Ubuntu 24.04/glibc 环境。0.5.9 将两套 NPU 分别放入用户单独安装的支持包；普通平台包不包含这些库和 context，默认模型制品不变。 +当前实现覆盖 Linux x64 glibc。Intel 使用 OpenVINO C API；AMD 默认使用轻量 IREE C ABI 运行库、20 个 CPU/NPU 识别模块和单份共享 IRPA 权重。安装后的 OCR 进程无需 Python，系统仍需厂商 NPU 驱动，AMD CI 使用 Ubuntu 24.04/glibc 环境。0.5.9 将两套 NPU 分别放入用户单独安装的支持包；普通平台包不包含这些库和 context，默认模型制品不变。 ## Intel 候选 SDK @@ -13,9 +13,28 @@ python tools/npu/build_openvino.py \ 当前 driver/compiler floor 来自本机 OpenVINO 设备属性的数值，尚未覆盖其他代设备。调整配置会要求重新绑定审阅报告。 -## AMD 候选 SDK +## AMD 轻量候选 SDK -常规发布直接取得已锁定的部署包，不需要安装完整 SDK： +常规构建不需要专有 Ryzen AI SDK，入口使用 Python 3.12： + +```bash +python3.12 tools/npu/build_amdaie.py \ + --work-dir .cache/amd-aie --output-dir dist/npu-sdk/amdnpu --jobs 2 +``` + +工具准备锁定源码、Peano 和导入依赖，同版源码构建编译器/运行库，再编译全部 20 桶。 +19 个较大 pointwise 卷积矩阵核心使用 NPU；其余识别算子与检测使用 CPU。宽矩阵分段，单次 NPU dispatch 不超过 3840 行。 + +部署目录仅包含小型运行库、VMFB、单份共享 IRPA、JSON 清单和许可文件。用户安装时不需要编译器、完整 SDK、XRT 或 Python。 +配置与 descriptor 使用 `IREEAMDAIE` 标识，原生 ABI 固定为 1；权重、模型和运行库均绑定字节数及 SHA256。 +CPU 与 NPU 的实际分区、BF16→BFP16ebs8 精度变化及未验证状态明确记录。 + +`npu-support-release.yml` 默认使用此入口生成独立 AMD 包；`npu-native.yml` 在未提供 SDK artifact 时也使用轻量源码构建。 +完整命令与部署结果见 [AMD AIE 开发说明](amd-aie-development.md)。本轮没有运行测试、数值比较或设备推理。 + +### 旧 vendor 部署兼容路径 + +以下工具仅供已有 VAIML/EPContext 部署保留和回滚，不是新支持包的默认构建输入： ```bash python tools/npu/fetch_amdnpu.py \ @@ -82,7 +101,7 @@ const engine = await createEngine({ }); ``` -模型无关 runtime 还需 `bundlePath`。两套支持包都是 optional peer，不随普通安装自动下载；显式 provider 才选择对应独立 addon，Auto 保持基础包策略。缺少包时报告 `unsupported_capability`，没有静默回退。AMD 的 20 个 context 绑定 Small 0.3.4。 +模型无关 runtime 还需 `bundlePath`。两套支持包都是 optional peer，不随普通安装自动下载；显式 provider 才选择对应独立 addon，Auto 保持基础包策略。缺少包时报告 `unsupported_capability`，没有静默回退。AMD 的 20 个识别模块绑定 Small 0.3.4。 支持包内包含原生依赖和产物身份；终端用户无需完整 SDK/Python，仍需系统 NPU 驱动。真机报告不作为发布前置，`deviceValidated` 保持 false。 diff --git a/docs/releases/npm-0.5.9.en.md b/docs/releases/npm-0.5.9.en.md index 9fc08f2..c025d56 100644 --- a/docs/releases/npm-0.5.9.en.md +++ b/docs/releases/npm-0.5.9.en.md @@ -20,10 +20,10 @@ NPU implementation: [PR #67](https://github.com/arcships/light-ocr/pull/67), mer ## Changes -- The default Small package retains its existing approximately 39 MB model promise and immutable model package 0.3.4. No AMD/OpenVINO libraries or compiled NPU contexts enter the default dependency closure. +- The default Small package retains immutable model package 0.3.4. The existing size requirement is 39 MB; the measured model archive is currently about 52.53 MB, and this AMD change does not resolve that difference. No AMD/OpenVINO libraries or compiled NPU contexts enter the default dependency closure. - Eight base native packages retain Apple/WebGPU/CPU policies. NPU support is separately installed as `@arcships/light-ocr-openvino-linux-x64-gnu@0.5.9` or `@arcships/light-ocr-amdnpu-linux-x64-gnu@0.5.9`. - These are optional peers, without automatic installation. Explicit `execution.provider` selects the matching separate addon; Auto does not select either NPU. Vendor drivers are required; device inference remains unqualified. -- AMD includes 20 actual BF16 contexts bound to Small 0.3.4 and is substantially larger. Other tiers need their own compiled contexts. +- AMD uses a small IREE C ABI runtime, 20 CPU/NPU VMFB modules and one shared IRPA bound to Small 0.3.4. Other tiers need their own compiled modules. ## Release sequence @@ -34,15 +34,32 @@ NPU implementation: [PR #67](https://github.com/arcships/light-ocr/pull/67), mer 5. Publish the base closure through its existing next/latest sequence; publish independent support packages through their own workflow under next. Document and previews remain on next. 6. Update the draft target, publication identities and dates before publishing v0.5.9. +## Lightweight AMD preparation + +[PR #71](https://github.com/arcships/light-ocr/pull/71) implements the complete IREE CPU/NPU recognition backend, all 20 buckets and shared weights. Nineteen dense pointwise convolution matrix cores run on the NPU; other recognition operators and detection remain on CPU. BF16 operands use BFP16ebs8 microkernels, without numerical or performance evidence. + +`npu-support-release.yml` builds pinned sources through `build_amdaie.py`. The optional package contains a small C ABI library, 20 VMFBs and one shared IRPA, with licenses, SBOM and hashes. Compiler tools, Peano, original ONNX, the vendor SDK, Python and XRT are excluded. AMD remains explicitly installed and selected, outside Auto. + +The actual npm archive has a 64 MiB download / 128 MiB unpacked budget. CI records `support-size-report.json`; local bytes and SHA256 are in [the deployment report](../amd-aie-deployment-results.json). Version 0.5.9 remains unreleased. Only builds and packaging were performed; tests, numerical comparisons and hardware acceptance were omitted at the maintainer's request. + + +Local lightweight candidate, including the addon, base dependencies, 20 buckets and shared weights: + +| Artifact | Compressed bytes | Unpacked bytes | +| --- | ---: | ---: | +| Independent AMD npm package `0.5.9` | 27,869,026 | 59,690,023 | + +npm SHA256: `2bee7052a4dbfcfea8338e1fa06e42061fc77b9d4aee21d52794f9f28cc4e8b8`. Download size is about 94.9% smaller than the historical vendor package. Runtime: 1,255,312 bytes; shared weights: 17,145,856 bytes; all 20 VMFBs: 7,590,556 bytes. The release workflow will record its own artifact identities. + ## Evidence and rejected candidate The previous candidate run `37480252872` passed eight-platform build/smoke, but incorrectly bundled both NPUs into Linux x64: 577,384,672 compressed bytes and 1,455,176,275 unpacked bytes. **Do not publish this candidate; rebuild from corrected main.** -Source NPU build run `37469626258` succeeded. All 20 AMD contexts were actually compiled without device inference. Original SDK provenance and deployment archive identity remain locked in `amdnpu-source.lock.json` and `amdnpu.lock.json`; the approximately 508 MiB archive supplies only the independently installed AMD support package. +Source NPU build run `37469626258` succeeded. All 20 AMD contexts were actually compiled without device inference. Original SDK provenance and deployment archive identity remain locked in `amdnpu-source.lock.json` and `amdnpu.lock.json`; the approximately 508 MiB vendor archive remains a historical input. New workflows build lightweight IREE deployments from sources pinned by `iree-source.lock.json`. The 13 new base identities were vacant. Final corrected tarball identities and both support-package builds are recorded after generation. -## Local split-package build evidence +## Local split-package build evidence (historical vendor candidate) All three addons (base CPU, independent OpenVINO and independent AMD) built and staged separately. Base inventory contains CPU only, no NPU directories, with 33,091,890 native bytes; this local baseline excludes the production WebGPU/PDF attachments. Both independent support tarballs were generated: @@ -57,7 +74,7 @@ These are local packaging artifacts, not the corrected eight-platform release ru Restore stable tags using the archived 0.5.8 publication artifact (run `36013862673`): Small/native `0.5.8`, runtime `0.1.8`. Handle compatibility/preview tags separately; never overwrite or delete published package versions. -## AMD deployment artifact +## AMD deployment artifact (historical vendor archive) - Archive: `amdnpu-1.8.0-linux-x64-ppocrv6-small-0.3.4-opt-in-v1.tar.gz` - Size: `532,631,585` bytes (~508 MiB); unpacked SDK inventory: `1,356,635,270` bytes (~1.26 GiB). This substantially increases the Linux x64 npm package size. diff --git a/docs/releases/npm-0.5.9.md b/docs/releases/npm-0.5.9.md index 52f0391..53555f6 100644 --- a/docs/releases/npm-0.5.9.md +++ b/docs/releases/npm-0.5.9.md @@ -23,10 +23,10 @@ ## 用户可见变化 -- 主包保持原来的约 39 MB Small 模型承诺：模型包 `0.3.4` 及其锁定制品不变，不加入 AMD/OpenVINO 运行库或 AMD 编译模型。 +- 主包不加入 AMD/OpenVINO 运行库或 AMD 编译模型，Small 模型包 `0.3.4` 及锁定制品不变。39 MB 是既有体积要求；模型 tgz 目前实测约 52.53 MB，本次 AMD 改造不宣称已解决该差额。 - 默认 runtime 的八个平台依赖均保持基础运行时，Auto 沿用 Apple/WebGPU/CPU 路线。 - 两套 NPU 独立分发：`@arcships/light-ocr-openvino-linux-x64-gnu@0.5.9`、`@arcships/light-ocr-amdnpu-linux-x64-gnu@0.5.9`。用户单独安装后，配置 `execution.provider` 为 `openvino` 或 `amdnpu`。optional peer 不会自动安装；安装后也不进入默认 Auto。 -- 支持包各自包含匹配的 addon、原生依赖、哈希、许可证与 SBOM；AMD 包另外含 Small 0.3.4 的 20 个真实 BF16 context，体积较大。系统仍需厂商驱动，无需终端用户安装完整 SDK/Python。 +- 支持包各自包含匹配的 addon、原生依赖、哈希、许可证与 SBOM；AMD 包使用轻量 IREE C ABI 运行库、20 个 CPU/NPU VMFB 和单份共享 IRPA 权重，绑定 Small 0.3.4。系统仍需厂商驱动，无需终端用户安装完整 SDK/Python。 - 不要求 NPU 真机验收，`deviceValidated: false`；Windows、arm64、musl 不含这两套支持包。 ## Release 操作 @@ -38,15 +38,34 @@ 5. 按既有正式流程发布基础包到 `next`，完成 registry 回装后晋升 stable Small/runtime/native。独立支持包通过其自身 workflow 发布到 `next`，Document/preview 继续保留 `next`。 6. 将草稿 target 指向最终 main 提交，补录实际发布身份、日期和正式制品链接，再公开 `v0.5.9`。 +## 轻量 AMD 构建与发布准备 + +[PR #71](https://github.com/arcships/light-ocr/pull/71) 接入完整 IREE CPU/NPU 分区后端：19 个较大 pointwise 卷积矩阵核心由 NPU 执行，其余识别算子和检测保留 CPU。BF16 操作数使用 BFP16ebs8 微内核，没有数值和性能实测结论。 + +`npu-support-release.yml` 使用 `build_amdaie.py` 从固定源码生成小型 C ABI 库、20 个 VMFB 和一份共享 IRPA；编译器、Peano、原始 ONNX、完整 vendor SDK、Python 与 XRT 均不进入支持包。运行时库、配置、权重及全部桶模型继续校验字节数和 SHA256。 + +真实 npm 压缩包受下载 ≤64 MiB、解包 ≤128 MiB 的门禁约束，结果写入 workflow artifact 中的 `support-size-report.json`。本地候选的逐文件身份、大小与包 SHA256 见 [轻量 AMD 部署结果](../amd-aie-deployment-results.json)。 + + +本地轻量候选（含 addon、基础依赖、20 桶与共享权重）： + +| 制品 | 下载 bytes | 解包 bytes | +| --- | ---: | ---: | +| AMD 独立 npm 包 `0.5.9` | 27,869,026 | 59,690,023 | + +npm SHA256：`2bee7052a4dbfcfea8338e1fa06e42061fc77b9d4aee21d52794f9f28cc4e8b8`。相比历史 vendor 包，下载体积减少约 94.9%。运行库 1,255,312 bytes，单份共享权重 17,145,856 bytes，全部 20 个 VMFB 合计 7,590,556 bytes。该包为本地候选，正式工作流会重新记录其制品身份。 + +当前版本仍为补丁发布 `0.5.9`。本轮仅完成代码、构建和封装，没有 npm 发布或公开 Release；按维护者要求未运行测试、数值比较或真机验收。AMD 继续单独安装、显式启用、排除在 Auto 之外。 + ## 当前证据与废弃候选 - 实现 PR #67、版本准备 PR #68 已合并；此次拆包修正仍待合并。 - 旧候选 run `37480252872` 八平台构建及 smoke 虽通过，但 Linux x64 误含两套 NPU，单包 `577,384,672` bytes、解压 `1,455,176,275` bytes。**该候选不用于正式发布，须从拆包后的 main 重建。** - 双 NPU 源码构建 run `37469626258` 通过；AMD 官方 SDK 已取得，20 桶实际编译，无 AMD 设备推理证据。 -- 原始 AMD SDK 身份见 `tools/npu/amdnpu-source.lock.json`；部署输入见 `tools/npu/amdnpu.lock.json`。约 508 MiB 的部署归档仍作为独立 AMD 支持包的构建输入，不进入普通包。 +- 原始 AMD SDK 身份见 `tools/npu/amdnpu-source.lock.json`；部署输入见 `tools/npu/amdnpu.lock.json`。约 508 MiB 的 vendor 部署归档保留为历史输入；新工作流默认用 `iree-source.lock.json` 固定的源码构建轻量 IREE 部署。 - 13 个基础新 package identity 的空位检查已通过；两个新 NPU 支持包须独立发布。最终拆包候选的 tarball 和 run 在生成后补录。 -## 本地拆包构建证据 +## 本地拆包构建证据（历史 vendor 候选） 三个 addon（基础 CPU、独立 OpenVINO、独立 AMD）均已分别构建并完成 staging。基础库存仅含 CPU provider，无 NPU 目录，native 文件共 `33,091,890` bytes（此处不含正式 WebGPU/PDF 分发附件）。两个独立支持包已实际生成 tarball： @@ -61,7 +80,7 @@ 已发布 npm 版本不可覆盖。若 0.5.9 晋升后需要回滚，使用 0.5.8 的已归档正式发布 artifact（run `36013862673`）恢复 stable Small/native `0.5.8` 与 runtime `0.1.8` 的 `latest` 标签。Document 与 preview 标签单独处理；不删除用户已安装的 0.5.9 产物。 -## AMD 部署制品 +## AMD 部署制品（历史 vendor 归档） - 归档：`amdnpu-1.8.0-linux-x64-ppocrv6-small-0.3.4-opt-in-v1.tar.gz` - 大小：`532,631,585` bytes（约 508 MiB）；SDK 库存共 `1,356,635,270` bytes（约 1.26 GiB），Linux x64 npm 包因此显著增大。 diff --git a/packages/runtime/src/npu-descriptor.cjs b/packages/runtime/src/npu-descriptor.cjs index be4cc34..4392be5 100644 --- a/packages/runtime/src/npu-descriptor.cjs +++ b/packages/runtime/src/npu-descriptor.cjs @@ -1,6 +1,7 @@ 'use strict'; const path = require('node:path'); +const fs = require('node:fs'); const WIDTHS = Object.freeze([320, 384, 480, 544, 576, 608, 704, 736, 832, 960, 1056, 1184, 1248, 1376, 1600, 1984, 2240, 2560, 2880, 3200]); @@ -17,8 +18,8 @@ function validateNpuProviders(descriptor, root, { exactKeys, verifyArtifact, sam } exactKeys(provider, ['runtimeProvider', 'providerVersion', 'qualificationId', 'providerLibrary', 'configuration', 'artifacts'], `providers.${name}`); - const expected = name === 'openvino' ? 'OpenVINO' : 'VitisAIExecutionProvider'; - if (provider.runtimeProvider !== expected || typeof provider.providerVersion !== 'string' || + const expected = name === 'openvino' ? ['OpenVINO'] : ['VitisAIExecutionProvider', 'IREEAMDAIE']; + if (!expected.includes(provider.runtimeProvider) || typeof provider.providerVersion !== 'string' || !provider.providerVersion || typeof provider.qualificationId !== 'string' || !provider.qualificationId || !Array.isArray(provider.artifacts) || !provider.artifacts.length) { fail('Invalid NPU provider identity'); @@ -66,7 +67,9 @@ function validateNpuProviders(descriptor, root, { exactKeys, verifyArtifact, sam if (typeof configuration.sourceModelSha256 !== 'string' || !/^[0-9a-f]{64}$/.test(configuration.sourceModelSha256) || !Array.isArray(configuration.recognitionModels) || configuration.recognitionModels.length !== WIDTHS.length || - !path.basename(library).startsWith('libonnxruntime.so')) { + !(provider.runtimeProvider === 'IREEAMDAIE' + ? path.basename(library) === 'liblight_ocr_amdaie.so.1' + : path.basename(library).startsWith('libonnxruntime.so'))) { fail('Invalid AMD NPU recognition model identity'); } const modelPaths = new Set(); @@ -82,6 +85,32 @@ function validateNpuProviders(descriptor, root, { exactKeys, verifyArtifact, sam path: modelPath, bytes: model.artifact.bytes, sha256: model.artifact.sha256, }) }); }); + if (provider.runtimeProvider === 'IREEAMDAIE') { + let compiler; + try { compiler = JSON.parse(fs.readFileSync(configPath, 'utf8')); } + catch { fail('Invalid AMD AIE deployment configuration'); } + if (compiler.target !== 'IREEAMDAIE' || compiler.runtimeAbi !== 1 || + compiler.device !== 'npu4' || compiler.sourceModelSha256 !== configuration.sourceModelSha256 || + compiler.parameterScope !== 'recognition' || compiler.precision !== 'bf16-bfp16ebs8' || + compiler.cpuPartitionRequired !== true || typeof compiler.runtimeVersion !== 'string' || + !compiler.runtimeVersion || !Array.isArray(compiler.widths) || + compiler.widths.length !== WIDTHS.length || compiler.widths.some((width, i) => width !== WIDTHS[i]) || + configuration.recognitionModels.some((item) => !item.artifact.path.endsWith('.vmfb'))) { + fail('AMD AIE deployment contract differs from the locked recognition model'); + } + const parameters = compiler.parameters; + if (!parameters || typeof parameters.path !== 'string' || !parameters.path || + parameters.path.includes('\\') || path.posix.isAbsolute(parameters.path) || + parameters.path.split('/').some((component) => !component || component === '.' || component === '..')) { + fail('Invalid AMD shared parameter path'); + } + const artifact = { ...parameters, + path: path.posix.join(path.posix.dirname(configuration.compilerConfiguration.path), parameters.path) }; + verifyArtifact(root, artifact, 'AMD shared recognition parameters'); + if (!provider.artifacts.some((item) => sameArtifact(item, artifact))) { + fail('AMD shared recognition parameters are outside the runtime inventory'); + } + } nativeConfiguration = Object.freeze({ sourceModelSha256: configuration.sourceModelSha256, compilerConfiguration: Object.freeze({ path: configPath, bytes: configuration.compilerConfiguration.bytes, sha256: configuration.compilerConfiguration.sha256 }), diff --git a/src/core/engine.cpp b/src/core/engine.cpp index f70c54b..c4f26e6 100644 --- a/src/core/engine.cpp +++ b/src/core/engine.cpp @@ -1474,7 +1474,7 @@ Result> internal::EngineFactory::create( : selected_provider == ExecutionProvider::openvino ? "OpenVINO" : selected_provider == ExecutionProvider::amdnpu - ? "VitisAIExecutionProvider" + ? recognition->execution_info().actual_provider_chain.front() : selected_provider == ExecutionProvider::webgpu ? "WebGpuExecutionProvider" : "CPUExecutionProvider"; diff --git a/src/inference/amdnpu/backend.cpp b/src/inference/amdnpu/backend.cpp index 245d3ea..f0fa17b 100644 --- a/src/inference/amdnpu/backend.cpp +++ b/src/inference/amdnpu/backend.cpp @@ -15,7 +15,9 @@ #include #include #include +#include +#include "inference/amdnpu/runtime_api.h" #include "inference/openvino/backend.hpp" #include "util/sha256.hpp" @@ -148,6 +150,61 @@ std::vector tensor_shape(const OrtApi& api, const OrtTensorTypeAnd } return shape; } + +struct AieRuntime { + const LightOcrAieApiV1* api = nullptr; +}; + +std::shared_ptr aie_runtime_for(const RuntimeArtifact& artifact) { + static std::mutex mutex; + static auto* runtimes = new std::map>; + const std::lock_guard lock(mutex); + (void)read_artifact(artifact); + const auto key = artifact.path + ":" + artifact.sha256; + const auto found = runtimes->find(key); + if (found != runtimes->end()) return found->second; + // Only the versioned C entry point is exported. IREE and AMD driver symbols + // are hidden, so this library can coexist with the CPU ORT in this namespace. + void* handle = ::dlopen(artifact.path.c_str(), RTLD_NOW | RTLD_LOCAL); + if (!handle) { + const char* error = ::dlerror(); + throw SetupError(CreationReason::unrecoverable_load_failed, + error ? error : "Cannot load the lightweight AMD runtime"); + } + const auto get_api = reinterpret_cast( + ::dlsym(handle, "LightOcrAieGetApi")); + const auto* api = get_api ? get_api(1) : nullptr; + if (!api || api->abi_version != 1 || api->struct_size != sizeof(LightOcrAieApiV1) || + !api->create || !api->load || !api->run || !api->destroy_session || !api->destroy_engine) { + ::dlclose(handle); + throw SetupError(CreationReason::provider_abi_mismatch, "AMD AIE runtime C ABI mismatch"); + } + // Keep mapped libraries alive, as with the vendor backend. Engine/session + // resources are still released individually when the OCR engine closes. + auto runtime = std::make_shared(); + runtime->api = api; + runtimes->emplace(key, runtime); + return runtime; +} + +RuntimeArtifact parameter_artifact(const RuntimeArtifact& configuration, + const nlohmann::json& compiler) { + const auto& record = compiler.at("parameters"); + const auto relative = fs::u8path(record.at("path").get()); + if (relative.empty() || relative.is_absolute()) + throw SetupError(CreationReason::package_corrupt, "Invalid AMD parameter archive path"); + for (const auto& component : relative) { + if (component == "." || component == "..") + throw SetupError(CreationReason::package_corrupt, "Invalid AMD parameter archive path"); + } + const auto path = fs::u8path(configuration.path).parent_path() / relative; + for (auto parent = path.parent_path(); parent != parent.root_path(); parent = parent.parent_path()) { + if (fs::is_symlink(fs::symlink_status(parent))) + throw SetupError(CreationReason::package_corrupt, "AMD parameter archive uses a symlink parent"); + } + return RuntimeArtifact{path.string(), record.at("bytes").get(), + record.at("sha256").get()}; +} } // namespace struct AmdNpuSession::State { @@ -157,12 +214,37 @@ struct AmdNpuSession::State { std::string output; }; std::shared_ptr runtime; + std::shared_ptr aie_runtime; + void* aie_engine = nullptr; + std::map aie_sessions; + std::mutex aie_mutex; std::vector models; std::string compiler_configuration; std::map sessions; std::size_t classes = 0; ~State() { for (auto& entry : sessions) runtime->api->ReleaseSession(entry.second.value); + if (aie_runtime) { + for (auto& entry : aie_sessions) aie_runtime->api->destroy_session(entry.second); + aie_runtime->api->destroy_engine(aie_engine); + } + } + void* aie_session_for(std::uint32_t width) { + const auto existing = aie_sessions.find(width); + if (existing != aie_sessions.end()) return existing->second; + const auto model = std::find_if(models.begin(), models.end(), [width](const auto& item) { + return item.width == width; + }); + if (model == models.end()) throw std::runtime_error("AMD NPU width has no compiled recognition module"); + const auto bytes = read_artifact(model->artifact); + void* session = nullptr; + char message[2048] = {}; + if (aie_runtime->api->load(aie_engine, bytes.data(), bytes.size(), width, + &session, message, sizeof(message))) + throw std::runtime_error(message); + try { aie_sessions.emplace(width, session); } + catch (...) { aie_runtime->api->destroy_session(session); throw; } + return session; } Session& session_for(std::uint32_t width) { const auto existing = sessions.find(width); @@ -248,17 +330,37 @@ Result> AmdNpuSession::create( } } (void)read_artifact(config.amdnpu_runtime); - (void)read_artifact(config.amdnpu_compiler_configuration); + const auto configuration_bytes = read_artifact(config.amdnpu_compiler_configuration); + const auto compiler = nlohmann::json::parse(configuration_bytes.begin(), configuration_bytes.end()); + const bool lightweight = compiler.value("target", std::string{}) == "IREEAMDAIE"; + std::vector parameters; + if (lightweight) { + if (classes != 18710 || compiler.value("runtimeAbi", 0) != 1 || + compiler.value("sourceModelSha256", std::string{}) != config.model_sha256 || + compiler.value("device", std::string{}) != "npu4" || + compiler.value("parameterScope", std::string{}) != "recognition") + return fail(CreationReason::model_compute_unsupported, "AMD AIE deployment contract differs from this model"); + parameters = read_artifact(parameter_artifact(config.amdnpu_compiler_configuration, compiler)); + } if (!stx_npu_available()) { return fail(CreationReason::adapter_unavailable, "No accessible AMD STX/KRK NPU on this host"); } auto state = std::make_unique(); - state->runtime = runtime_for(config.amdnpu_runtime); state->models = config.amdnpu_recognition_models; state->compiler_configuration = config.amdnpu_compiler_configuration.path; state->classes = classes; try { - (void)state->session_for(widths.front()); + if (lightweight) { + state->aie_runtime = aie_runtime_for(config.amdnpu_runtime); + char message[2048] = {}; + if (state->aie_runtime->api->create(parameters.data(), parameters.size(), + &state->aie_engine, message, sizeof(message))) + throw SetupError(CreationReason::unrecoverable_load_failed, message); + (void)state->aie_session_for(widths.front()); + } else { + state->runtime = runtime_for(config.amdnpu_runtime); + (void)state->session_for(widths.front()); + } } catch (const SetupError&) { throw; } catch (const std::exception& error) { @@ -266,16 +368,18 @@ Result> AmdNpuSession::create( } SessionExecutionInfo info; info.requested_provider = config.requested_provider_override.empty() ? "amdnpu" : config.requested_provider_override; - info.actual_provider_chain = {"VitisAIExecutionProvider", "CPUExecutionProvider"}; + info.actual_provider_chain = lightweight + ? std::vector{"IREEAMDAIE", "IREECPU"} + : std::vector{"VitisAIExecutionProvider", "CPUExecutionProvider"}; info.device = "npu:AMD XDNA2"; info.device_family = "STX/KRK"; info.operating_system = "linux"; - info.precision = "bf16"; - info.shape_policy = "nchw-static-amd-bf16-buckets-v1"; + info.precision = lightweight ? "bf16-bfp16ebs8" : "bf16"; + info.shape_policy = lightweight ? "nchw-static-amd-aie-buckets-v1" : "nchw-static-amd-bf16-buckets-v1"; info.model_id = config.model_id; info.model_sha256 = config.model_sha256; - info.runtime = "Ryzen AI ONNX Runtime"; - info.runtime_version = state->runtime->version; + info.runtime = lightweight ? "IREE AMD AIE" : "Ryzen AI ONNX Runtime"; + info.runtime_version = lightweight ? compiler.at("runtimeVersion").get() : state->runtime->version; info.model_cache_status = "precompiled"; info.qualification_id = config.qualification_id; info.device_validated = config.npu_device_validated; @@ -295,6 +399,19 @@ Result AmdNpuSession::run(const std::vector& values, values.size() != static_cast(144 * shape[3])) { throw std::runtime_error("AMD NPU input does not match a recognition bucket"); } + if (state_->aie_runtime) { + const std::lock_guard lock(state_->aie_mutex); + const auto width = static_cast(shape[3]); + void* session = state_->aie_session_for(width); + const auto count = static_cast(width / 8) * state_->classes; + auto output = std::make_shared>(count); + char message[2048] = {}; + if (state_->aie_runtime->api->run(session, values.data(), values.size(), + output->data(), output->size(), message, sizeof(message))) + throw std::runtime_error(message); + return Result::success(TensorOutput(output, output->data(), + {1, static_cast(width / 8), static_cast(state_->classes)}, count)); + } auto& session = state_->session_for(static_cast(shape[3])); const auto& api = *state_->runtime->api; OrtMemoryInfo* raw_memory = nullptr; diff --git a/src/inference/amdnpu/backend.hpp b/src/inference/amdnpu/backend.hpp index 1af93b7..a55c888 100644 --- a/src/inference/amdnpu/backend.hpp +++ b/src/inference/amdnpu/backend.hpp @@ -4,8 +4,8 @@ namespace light_ocr::internal { -// Ryzen AI owns a separate ORT C API table. No Ort:: wrapper or global ORT -// API pointer may be used for objects allocated by this runtime. +// Lightweight deployments use the versioned IREE C ABI. Legacy Ryzen AI +// deployments retain their separate ORT C API table and namespace. class AmdNpuSession final : public InferenceSession { public: ~AmdNpuSession() noexcept override; diff --git a/src/inference/amdnpu/runtime_api.h b/src/inference/amdnpu/runtime_api.h new file mode 100644 index 0000000..8901a79 --- /dev/null +++ b/src/inference/amdnpu/runtime_api.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +#ifdef __cplusplus +extern "C" { +#endif + +// A versioned C boundary keeps IREE symbols and types out of the addon. +// All buffers are borrowed for a call; create/load copy their input bytes. +// Sessions must be destroyed before their engine. Calls are serialized by +// the native backend; an engine/session is not concurrently callable. +typedef struct LightOcrAieApiV1 { + uint32_t abi_version; + uint32_t struct_size; + int (*create)(const uint8_t*, size_t, void**, char*, size_t); + int (*load)(void*, const uint8_t*, size_t, uint32_t, void**, char*, size_t); + int (*run)(void*, const float*, size_t, float*, size_t, char*, size_t); + void (*destroy_session)(void*); + void (*destroy_engine)(void*); +} LightOcrAieApiV1; + +const LightOcrAieApiV1* LightOcrAieGetApi(uint32_t version); +#ifdef __cplusplus +} +#endif diff --git a/tools/npm_release.py b/tools/npm_release.py index ed23f89..f45fee7 100644 --- a/tools/npm_release.py +++ b/tools/npm_release.py @@ -429,7 +429,9 @@ def validate_runtime_descriptor( provider_id not in provider_names or not isinstance(provider, dict) or set(provider) != expected_fields - or provider.get("runtimeProvider") != provider_names[provider_id] + or provider.get("runtimeProvider") not in ( + {"VitisAIExecutionProvider", "IREEAMDAIE"} if provider_id == "amdnpu" + else {provider_names[provider_id]}) or not isinstance(provider.get("qualificationId"), str) or not provider["qualificationId"] or not isinstance(provider.get("artifacts"), list) diff --git a/tools/npu/build_amdaie.py b/tools/npu/build_amdaie.py new file mode 100644 index 0000000..62c6662 --- /dev/null +++ b/tools/npu/build_amdaie.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Reproducibly build the optional AMD SDK without the proprietary Ryzen SDK.""" +from __future__ import annotations +import argparse +import hashlib +import json +from pathlib import Path +import subprocess +import sys + +import build_amdaie_runtime +import build_iree +import compile_iree_recognition +import fetch_iree_source +import fetch_peano +import import_amdaie +from export_iree_hotspot import LOCK, digest +from sdk import recognition_widths, verify_record + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("--work-dir",type=Path,required=True) + parser.add_argument("--output-dir",type=Path,required=True) + parser.add_argument("--jobs",type=int,default=2) + args=parser.parse_args() + if sys.version_info[:2]!=(3,12):parser.error("build frontend requires Python 3.12") + if args.jobs<1:parser.error("jobs must be positive") + if args.output_dir.exists():parser.error("select a fresh SDK output directory") + work=args.work_dir.resolve();work.mkdir(parents=True,exist_ok=True) + tools=Path(__file__).parent.resolve();repo=tools.parents[1] + source=fetch_iree_source.fetch(work/"source",jobs=args.jobs) + peano=work/"peano/llvm-aie" + if not peano.exists():peano=fetch_peano.fetch(work/"downloads",work/"peano") + build_iree.build(source,peano,work/"toolchain-build",args.jobs) + frontend=work/"frontend" + if not frontend.exists():subprocess.run([sys.executable,"-m","venv",str(frontend)],check=True) + subprocess.run([str(frontend/"bin/python"),"-m","pip","install","-r", + str(tools/"iree-frontend.requirements.txt")],check=True) + bundle=repo/"models/generated/ppocrv6-small-onnx-20260714.2" + if not bundle.exists(): + subprocess.run([sys.executable,str(repo/"tools/bootstrap_models.py"),"--tier","small", + "--output",str(bundle),"--cache-dir",str(work/"model-downloads")],check=True) + runtime=work/"deployment-runtime-build" + build_amdaie_runtime.build(source,runtime,args.jobs) + inputs=[LOCK,tools/"iree-frontend.requirements.txt"]+[tools/name for name in + ("compile_iree_recognition.py","partition_iree_model.py","route_iree_dispatches.py", + "export_iree_model.py","import_iree_model.py","runtime/merge_parameters.cc")] + key=hashlib.sha256("".join(digest(p) for p in inputs).encode()).hexdigest()[:16] + models=work/("recognition-"+key) + report_path=models/"compile-report.json" + if report_path.exists(): + report=json.loads(report_path.read_text()) + if (report.get("status")!="compiled" or report.get("widths")!=recognition_widths() or + report.get("sourceLockSha256")!=digest(LOCK) or + report.get("parameterMergerSha256")!=digest(runtime/"merge-aie-parameters")): + raise ValueError(f"incomplete or stale model cache: {models}; move it aside before rebuilding") + for item in report["buckets"]:verify_record(item["artifact"],models/"artifacts") + verify_record(report["parameters"],models/"artifacts") + else: + compile_iree_recognition.compile_models(bundle/"rec/inference.onnx",frontend, + work/"toolchain-build",peano,runtime/"merge-aie-parameters",models) + result=import_amdaie.assemble(source,runtime,models,bundle,args.output_dir) + print(f"Optional AMD SDK: {sum(r['bytes'] for r in result['artifacts'])} bytes; no inference executed") + + +if __name__=="__main__":main() diff --git a/tools/npu/build_amdaie_runtime.py b/tools/npu/build_amdaie_runtime.py new file mode 100644 index 0000000..78382cd --- /dev/null +++ b/tools/npu/build_amdaie_runtime.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Build the small C ABI runtime and the build-only parameter assembler.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import shutil +import subprocess +from build_iree import head +from export_iree_hotspot import LOCK, digest + + +def build(source: Path, output: Path, jobs=2) -> dict: + source,output=source.resolve(),output.resolve() + lock=json.loads(LOCK.read_text()) + if head(source,lock.get("localSourceDiffSha256"))!=lock["amdAie"]["commit"]: + raise ValueError("AMD AIE source commit mismatch") + revisions={"amdAie":lock["amdAie"]["commit"]} + for relative,revision in lock["amdAie"]["submodules"].items(): + if head(source/relative,lock.get("upstreamConfigureDiffSha256",{}).get(relative))!=revision: + raise ValueError("AMD AIE submodule source mismatch") + revisions[relative]=revision + for relative,revision in lock["ireeSubmodules"].items(): + if head(source/"third_party/iree"/relative)!=revision: + raise ValueError("IREE submodule source mismatch") + revisions["iree/"+relative]=revision + if jobs<1:raise ValueError("jobs must be positive") + root=Path(__file__).parent.resolve() + sources=[root/"runtime"/name for name in ("CMakeLists.txt","runtime.cc","merge_parameters.cc","exports.map")] + sources.append(root.parents[1]/"src/inference/amdnpu/runtime_api.h") + records=[{"path":p.relative_to(root.parents[1]).as_posix(),"sha256":digest(p)} for p in sources] + configure=["cmake","-S",str(root/"runtime"),"-B",str(output),"-GNinja", + "-DCMAKE_BUILD_TYPE=Release","-DCMAKE_C_COMPILER=clang","-DCMAKE_CXX_COMPILER=clang++", + f"-DLIGHT_OCR_IREE_SOURCE={source/'third_party/iree'}", + f"-DLIGHT_OCR_AMD_AIE_SOURCE={source}","-DIREE_INPUT_STABLEHLO=OFF", + "-DIREE_INPUT_TORCH=OFF","-DIREE_INPUT_TOSA=OFF"] + output.mkdir(parents=True,exist_ok=True) + manifest_path=output/"runtime-build.json" + manifest={"schemaVersion":"1.0","status":"building","sourceCommits":revisions, + "sourceLockSha256":digest(LOCK),"wrapperSources":records,"configureCommand":configure, + "deviceValidated":False,"numericsValidated":False} + manifest_path.write_text(json.dumps(manifest,indent=2)+"\n") + try: + subprocess.run(configure,check=True) + subprocess.run(["cmake","--build",str(output),"--target","light_ocr_amdaie", + "merge-aie-parameters","--parallel",str(jobs)],check=True) + library=output/"liblight_ocr_amdaie.so.1.0.0" + subprocess.run(["strip","--strip-unneeded",str(library)],check=True) + manifest["binaries"]=[{"path":path.name,"bytes":path.stat().st_size,"sha256":digest(path)} + for path in (library,output/"merge-aie-parameters")] + manifest["status"]="built" + except BaseException: + manifest["status"]="failed";raise + finally:manifest_path.write_text(json.dumps(manifest,indent=2)+"\n") + return manifest + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source-dir",type=Path,required=True) + parser.add_argument("--output-dir",type=Path,required=True) + parser.add_argument("--jobs",type=int,default=2) + args=parser.parse_args() + result=build(args.source_dir,args.output_dir,args.jobs) + print(f"AMD C ABI runtime: {result['binaries'][0]['bytes']} bytes") + + +if __name__=="__main__":main() diff --git a/tools/npu/compile_iree_recognition.py b/tools/npu/compile_iree_recognition.py new file mode 100644 index 0000000..14b7eae --- /dev/null +++ b/tools/npu/compile_iree_recognition.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Build all Small recognition width buckets with explicit CPU/AMD partitions. + +Only deployable VMFBs and one shared IRPA are written under artifacts/. Host +tools and compiler intermediates remain in work/. No inference or tests run. +""" +from __future__ import annotations +import argparse +import json +import os +from pathlib import Path +import shutil +import signal +import subprocess +import sys + +from export_iree_hotspot import LOCK, WIDTHS, digest + + +def compile_models(source, frontend, toolchain, peano, merger, output, + widths=WIDTHS, timeout=900, keep_intermediates=False): + source,frontend,toolchain,peano,merger,output=(p.resolve() for p in + (source,frontend,toolchain,peano,merger,output)) + lock=json.loads(LOCK.read_text()) + if digest(source)!=lock["sourceModelSha256"]: + raise ValueError("source recognition model differs from the lock") + build=json.loads((toolchain/"toolchain-build.json").read_text()) + if (build.get("status")!="built" or build.get("kind")!="compiler" or + build.get("localPatches")!=lock.get("localPatches") or + build.get("localSourceDiffSha256")!=lock.get("localSourceDiffSha256") or + build["sourceCommits"]["amdAie"]!=lock["amdAie"]["commit"] or + build["sourceCommits"]["third_party/iree"]!=lock["amdAie"]["submodules"]["third_party/iree"]): + raise ValueError("compiler source identity mismatch") + for name in ("tools/iree-compile","tools/iree-opt","lib/libIREECompiler.so"): + record=next(item for item in build["binaries"] if item["path"]==name) + if (toolchain/name).stat().st_size!=record["bytes"] or digest(toolchain/name)!=record["sha256"]: + raise ValueError("compiler binary identity mismatch") + provenance=json.loads((peano/"toolchain-source.json").read_text()) + if provenance["source"]!=lock["peano"]: + raise ValueError("Peano source identity mismatch") + for record in provenance["binaries"]: + if digest(peano/record["path"])!=record["sha256"]: + raise ValueError("Peano executable identity mismatch") + if output.exists(): + raise ValueError("select a fresh output directory") + if not widths or any(w not in WIDTHS for w in widths) or len(set(widths))!=len(widths): + raise ValueError("invalid recognition widths") + output.mkdir(parents=True) + artifacts=output/"artifacts"; artifacts.mkdir() + tools=Path(__file__).parent.resolve() + python=frontend/"bin/python" + compiler=toolchain/"tools/iree-compile" + optimizer=toolchain/"tools/iree-opt" + report={"schemaVersion":"1.0","status":"building","sourceModelSha256":digest(source), + "sourceLockSha256":digest(LOCK),"sourceCommits":build["sourceCommits"], + "compilerBinaries":build["binaries"],"localPatches":build["localPatches"], + "partitionScriptSha256":digest(tools/"partition_iree_model.py"), + "routeScriptSha256":digest(tools/"route_iree_dispatches.py"), + "parameterMergerSha256":digest(merger),"widths":list(widths), + "precision":"bf16-bfp16ebs8","cpuPartitionRequired":True, + "numericsValidated":False,"deviceValidated":False,"buckets":[]} + report_path=output/"compile-report.json" + def save():report_path.write_text(json.dumps(report,indent=2)+"\n") + def run(command, log): + with log.open("wb") as stream: + process=subprocess.Popen(list(map(str,command)),stdout=stream,stderr=subprocess.STDOUT, + start_new_session=True) + try: + code=process.wait(timeout=timeout) + if code:raise RuntimeError(f"build command failed ({code}); see {log}") + except BaseException: + if process.poll() is None: + os.killpg(process.pid,signal.SIGKILL);process.wait() + raise + def record(path):return {"path":path.relative_to(artifacts).as_posix(), + "bytes":path.stat().st_size,"sha256":digest(path)} + save() + archives=[] + try: + for width in widths: + work=output/"work"/str(width);work.mkdir(parents=True) + model, imported=work/"model",work/"imported" + entry={"width":width,"status":"building"};report["buckets"].append(entry);save() + print(f"Building recognition width={width}",flush=True) + run([python,tools/"export_iree_model.py","--source-model",source, + "--output-dir",model,"--width",width],work/"export.log") + run([sys.executable,tools/"import_iree_model.py","--model-dir",model, + "--frontend-environment",frontend,"--output-dir",imported, + "--timeout",timeout],work/"import.log") + partitioned,flow,routed,parameters=(work/name for name in + ("partitioned.mlir","flow.mlir","routed.mlir","parameters.mlir")) + run([python,tools/"partition_iree_model.py","--input",imported/"core-linalg.mlir", + "--output",partitioned,"--report",work/"partitions.json"],work/"partition.log") + partition=json.loads((work/"partitions.json").read_text()) + run([compiler,partitioned,"--iree-hal-target-device=cpu=local", + "--iree-hal-local-target-device-backends=llvm-cpu","--iree-llvmcpu-target-cpu=generic", + "--compile-to=flow","-o",flow],work/"flow.log") + run([sys.executable,tools/"route_iree_dispatches.py","--input",flow,"--output",routed, + "--report",work/"routing.json","--expected-npu-dispatches", + partition["npuDispatches"]],work/"route.log") + archive=work/"parameters.irpa";archives.append(archive) + run([optimizer,routed,f"--iree-io-export-parameters=path=recognition={archive} minimum-size=256", + "--canonicalize","--cse","-o",parameters],work/"parameters.log") + artifact=artifacts/f"recognition-{width}.vmfb" + run([compiler,parameters,"--compile-from=flow", + "--iree-hal-local-target-device-backends=llvm-cpu","--iree-llvmcpu-target-cpu=generic", + "--iree-amdaie-target-device=npu4","--iree-amdaie-device-hal=amdxdna", + "--iree-amdaie-num-cols=2","--iree-amdaie-num-rows=2", + "--iree-amdaie-enable-ukernels=matmul","--iree-amd-aie-enable-chess-for-ukernel=false", + f"--iree-amd-aie-peano-install-dir={peano}","--iree-hal-memoization=false", + "--iree-hal-indirect-command-buffers=false","--iree-amdaie-lower-to-aie-pipeline=objectFifo", + "--iree-amdaie-tile-pipeline=pack-peel","-o",artifact],work/"binary.log") + entry.update(status="compiled",artifact=record(artifact),partition=partition, + routing=json.loads((work/"routing.json").read_text())) + save() + if not keep_intermediates: + shutil.rmtree(model);shutil.rmtree(imported) + for path in (partitioned,flow,routed,parameters):path.unlink() + shared=artifacts/"recognition.irpa" + run([merger,shared,*archives],output/"merge-parameters.log") + report.update(status="compiled",parameters=record(shared), + completeRecognitionContract=list(widths)==list(WIDTHS), + artifactBytes=sum(e["artifact"]["bytes"] for e in report["buckets"])+shared.stat().st_size) + save() + if not keep_intermediates: + for path in archives:path.unlink() + except BaseException: + report["status"]="failed";save();raise + print(f"Compiled {len(widths)} width buckets; deployment models {report['artifactBytes']} bytes",flush=True) + return report + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + for name in ("source-model","frontend-environment","toolchain-dir","peano-dir","parameter-merger","output-dir"): + parser.add_argument("--"+name,type=Path,required=True) + parser.add_argument("--width",type=int,choices=WIDTHS,action="append") + parser.add_argument("--timeout",type=int,default=900) + parser.add_argument("--keep-intermediates",action="store_true") + a=parser.parse_args() + if a.timeout<=0:parser.error("timeout must be positive") + compile_models(a.source_model,a.frontend_environment,a.toolchain_dir,a.peano_dir, + a.parameter_merger,a.output_dir,a.width or WIDTHS,a.timeout,a.keep_intermediates) + + +if __name__=="__main__":main() diff --git a/tools/npu/import_amdaie.py b/tools/npu/import_amdaie.py new file mode 100644 index 0000000..915b8de --- /dev/null +++ b/tools/npu/import_amdaie.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Assemble an opt-in AMD deployment SDK from compiled runtime/model artifacts.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import re +import shutil +import subprocess +from export_iree_hotspot import LOCK, digest +from sdk import artifact_set, record, recognition_widths, validate_sdk, verify_record + + +def assemble(source, runtime, models, bundle, output): + source,runtime,models,bundle,output=(p.resolve() for p in (source,runtime,models,bundle,output)) + lock=json.loads(LOCK.read_text()) + compiled=json.loads((models/"compile-report.json").read_text()) + built=json.loads((runtime/"runtime-build.json").read_text()) + if (compiled.get("status")!="compiled" or compiled.get("completeRecognitionContract") is not True or + compiled.get("sourceModelSha256")!=lock["sourceModelSha256"] or + compiled.get("sourceLockSha256")!=digest(LOCK) or + [item["width"] for item in compiled["buckets"]]!=recognition_widths() or + built.get("status")!="built" or built.get("sourceLockSha256")!=digest(LOCK) or + built.get("sourceCommits")!=compiled.get("sourceCommits")): + raise ValueError("deployment requires complete, identically pinned runtime and model builds") + if output.exists():raise ValueError("select a fresh SDK output directory") + library_record=next(item for item in built["binaries"] if item["path"]=="liblight_ocr_amdaie.so.1.0.0") + library_source=verify_record(library_record,runtime) + dynamic=subprocess.check_output(["readelf","-d",str(library_source)],text=True) + needed=set(re.findall(r'\(NEEDED\).*\[([^\]]+)\]',dynamic)) + system={"libc.so.6","libstdc++.so.6","libgcc_s.so.1","libm.so.6","libdl.so.2", + "libpthread.so.0","librt.so.1","ld-linux-x86-64.so.2","libuuid.so.1"} + if needed-system:raise ValueError(f"runtime depends on non-system libraries: {needed-system}") + output.mkdir(parents=True) + library=output/"lib/liblight_ocr_amdaie.so.1";library.parent.mkdir() + shutil.copyfile(library_source,library) + model_dir=output/"models";model_dir.mkdir() + records=[] + for item in compiled["buckets"]: + if (item["partition"]["npuPointwiseConvolutions"]!=19 or + item["routing"]["dispatches"]["npu"]!=item["partition"]["npuDispatches"] or + item["routing"]["dispatches"]["cpu"]<=0): + raise ValueError("recognition model lacks the expected NPU/CPU partitions") + original=verify_record(item["artifact"],models/"artifacts") + destination=model_dir/original.name;shutil.copyfile(original,destination) + records.append({"width":item["width"],"artifact":record(destination,output)}) + parameters=verify_record(compiled["parameters"],models/"artifacts") + shared=model_dir/"recognition.irpa";shutil.copyfile(parameters,shared) + version="iree-amd-aie-"+lock["amdAie"]["commit"][:12] + configuration={"schemaVersion":"1.0","target":"IREEAMDAIE","runtimeAbi":1, + "runtimeVersion":version,"sourceModelSha256":lock["sourceModelSha256"], + "device":"npu4","parameterScope":"recognition","precision":"bf16-bfp16ebs8", + "cpuPartitionRequired":True,"widths":recognition_widths(),"parameters":record(shared,output), + "npuPointwiseConvolutions":19,"deviceValidated":False,"numericsValidated":False} + config=output/"deployment.json";config.write_text(json.dumps(configuration,indent=2)+"\n") + licenses=output/"licenses";licenses.mkdir() + license_sources={"AMD-AIE-LICENSE":source/"LICENSE", + "IREE-LICENSE":source/"third_party/iree/LICENSE", + "LLVM-LICENSE":source/"third_party/iree/third_party/llvm-project/LICENSE.TXT", + "flatcc-LICENSE":source/"third_party/iree/third_party/flatcc/LICENSE", + "flatcc-NOTICE":source/"third_party/iree/third_party/flatcc/NOTICE"} + for name,path in license_sources.items():shutil.copyfile(path,licenses/name) + for path in sorted((bundle/"LICENSES").iterdir()): + if path.is_file():shutil.copyfile(path,licenses/("model-"+path.name)) + artifacts=[record(path,output) for path in sorted(output.rglob("*")) + if path.is_file() and path.parent!=licenses] + value={"schemaVersion":"1.0","provider":"amdnpu","platformId":"linux-x64", + "providerVersion":version,"qualificationId":f"amdnpu-aie-{lock['amdAie']['commit'][:12]}-opt-in-v1", + "qualificationOnly":True,"runtimeLibrary":library.relative_to(output).as_posix(), + "configuration":{"sourceModelSha256":lock["sourceModelSha256"],"recognitionModels":records, + "compilerConfiguration":record(config,output)}, + "artifacts":artifacts,"artifactSetSha256":artifact_set(artifacts), + "files":[record(p,output) for p in sorted(output.rglob("*")) if p.is_file()], + "licenses":[record(p,output) for p in sorted(licenses.iterdir())], + "source":{"repository":lock["amdAie"]["repository"],"commit":lock["amdAie"]["commit"], + "runtimeBuildSha256":digest(runtime/"runtime-build.json"), + "modelBuildSha256":digest(models/"compile-report.json"), + "sourceLockSha256":digest(LOCK),"systemLibraries":sorted(needed)}, + "deviceValidated":False,"numericsValidated":False} + (output/"sdk-manifest.json").write_text(json.dumps(value,indent=2)+"\n") + validate_sdk(output,"amdnpu") + return value + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + for name in ("source-dir","runtime-build-dir","models-dir","model-bundle-dir","output-dir"): + parser.add_argument("--"+name,type=Path,required=True) + a=parser.parse_args() + result=assemble(a.source_dir,a.runtime_build_dir,a.models_dir,a.model_bundle_dir,a.output_dir) + print(f"AMD lightweight SDK: {sum(p['bytes'] for p in result['artifacts'])} bytes") + + +if __name__=="__main__":main() diff --git a/tools/npu/package_support.py b/tools/npu/package_support.py index 047bc95..387d9d7 100644 --- a/tools/npu/package_support.py +++ b/tools/npu/package_support.py @@ -41,7 +41,9 @@ def main() -> None: It is excluded from the default dependency closure and contains a separate NPU addon. Use `execution: {{ provider: "{provider}" }}` to select it. Vendor drivers are required. Hardware inference has not been qualified. -''' + ('AMD contexts are bound to Small 0.3.4; Tiny/Medium require their own compiled contexts.\n' +''' + ('AMD recognition modules are bound to Small 0.3.4; Tiny/Medium require their own compiled modules.\n' + 'Lightweight IREE deployments contain 20 CPU/NPU modules and one shared weight archive.\n' + 'BF16 pointwise convolutions use BFP16ebs8 microkernels; other recognition operators and detection run on CPU.\n' if provider == 'amdnpu' else 'OpenVINO compiles the selected ONNX models through the installed NPU driver.\n')) release.artifact_hashes(output, name, release.CORE_VERSION) print(name) diff --git a/tools/npu/partition_iree_model.py b/tools/npu/partition_iree_model.py new file mode 100644 index 0000000..fc9fa5e --- /dev/null +++ b/tools/npu/partition_iree_model.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Partition dense pointwise convolutions for the pinned AMD BF16 microkernel. + +Run in the locked frontend environment. Layouts, padding, activation, other +convolutions and the classifier retain CPU FP32 execution. No inference runs. +""" +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +def matrix_region(shapes): + image, weight, result = shapes + b, k, h, w = image + n = weight[0] + m = b * h * w + pm, pk, pn = (((v + 127) // 128) * 128 for v in (m, k, n)) + def t(dims, dtype="f32"): + return "tensor<" + "x".join(map(str, dims)) + "x" + dtype + ">" + src, wt, dst = map(t, shapes) + nhwc, whwc, flat, wf = t((b,h,w,k)), t((1,1,k,n)), t((m,k)), t((k,n)) + padded, pw = t((pm,pk)), t((pk,pn)) + a16, b16, c32, crop = t((pm,pk),"bf16"), t((pk,pn),"bf16"), t((pm,pn)), t((m,n)) + lines = ["#id = affine_map<(d0, d1) -> (d0, d1)>", "module {", + f"func.func @region(%image: {src}, %weight: {wt}, %initial: {dst}) -> {dst} {{", + "%zero = arith.constant 0.0 : f32"] + def layout(name, value, source, target, perm): + lines.extend([f"%{name}_empty = tensor.empty() : {target}", + f"%{name} = linalg.transpose ins(%{value} : {source}) outs(%{name}_empty : {target}) permutation = {perm}"]) + def pad(name, value, source, target, high): + lines.extend([f"%{name} = tensor.pad %{value} low[0, 0] high{high} {{", + "^bb0(%i: index, %j: index):", "tensor.yield %zero : f32", + f"}} : {source} to {target}"]) + def cast(name, value, source, target): + lines.extend([f"%{name}_empty = tensor.empty() : {target}", + f'%{name} = linalg.generic {{indexing_maps = [#id, #id], iterator_types = ["parallel", "parallel"]}} ins(%{value} : {source}) outs(%{name}_empty : {target}) {{', + "^bb0(%a: f32, %unused: bf16):", "%v = arith.truncf %a : f32 to bf16", + "linalg.yield %v : bf16", f"}} -> {target}"]) + layout("nhwc","image",src,nhwc,[0,2,3,1]) + layout("whwc","weight",wt,whwc,[2,3,1,0]) + lines.extend([f"%flat = tensor.collapse_shape %nhwc [[0, 1, 2], [3]] : {nhwc} into {flat}", + f"%wf = tensor.collapse_shape %whwc [[0, 1, 2], [3]] : {whwc} into {wf}"]) + pad("padded","flat",flat,padded,[pm-m,pk-k]) + pad("pw","wf",wf,pw,[pk-k,pn-n]) + cast("a16","padded",padded,a16) + cast("b16","pw",pw,b16) + lines.append(f"%b = util.optimization_barrier %b16 : {b16}") + # The pinned shim DMA lowering retains too many address dimensions for + # wide weight-repeat schedules. Use the largest row count compiled by + # this model on 2x2 tiles, and split wider tensors at the CPU boundary. + chunks = [] + for offset in range(0,pm,3840): + rows=min(3840,pm-offset) + index=len(chunks) + left,result_type=t((rows,pk),"bf16"),t((rows,pn)) + lines.extend([f"%slice{index} = tensor.extract_slice %a16[{offset}, 0] [{rows}, {pk}] [1, 1] : {a16} to {left}", + f"%a{index} = util.optimization_barrier %slice{index} : {left}", + f"%mm{index} = flow.dispatch.region -> ({result_type}) {{", + f"%region_zero{index} = util.optimization_barrier %zero : f32", + f"%empty{index} = tensor.empty() : {result_type}", + f"%zeros{index} = linalg.fill ins(%region_zero{index} : f32) outs(%empty{index} : {result_type}) -> {result_type}", + f"%product{index} = linalg.matmul ins(%a{index}, %b : {left}, {b16}) outs(%zeros{index} : {result_type}) -> {result_type}", + f"flow.return %product{index} : {result_type}", "}", + f"%barrier{index} = util.optimization_barrier %mm{index} : {result_type}"]) + chunks.append((offset,rows,result_type)) + if len(chunks)==1: + merged="%barrier0" + else: + lines.append(f"%assembled = tensor.empty() : {c32}") + merged="%assembled" + for index,(offset,rows,result_type) in enumerate(chunks): + lines.append(f"%joined{index} = tensor.insert_slice %barrier{index} into {merged}[{offset}, 0] [{rows}, {pn}] [1, 1] : {result_type} into {c32}") + merged=f"%joined{index}" + lines.extend([ + f"%crop = tensor.extract_slice {merged}[0, 0] [{m}, {n}] [1, 1] : {c32} to {crop}", + f"%expanded = tensor.expand_shape %crop [[0, 1, 2], [3]] output_shape [{b}, {h}, {w}, {n}] : {crop} into {t((b,h,w,n))}"]) + layout("output","expanded",t((b,h,w,n)),dst,[0,3,1,2]) + lines.extend([f"%sum_empty = tensor.empty() : {dst}", + f"%sum = linalg.add ins(%output, %initial : {dst}, {dst}) outs(%sum_empty : {dst}) -> {dst}", + f"return %sum : {dst}", "}", "}"]) + return "\n".join(lines), {"originalShapes": shapes,"paddedMatmul": [pm,pn,pk], + "npuDispatches":len(chunks),"maximumDispatchRows":3840} + + +def transform(source: Path, output: Path) -> dict: + from iree.compiler import ir + if output.exists(): + raise ValueError("output already exists") + selected = [] + with ir.Context(), ir.Location.unknown(): + # bootgen treats whitespace in executable file names as separators. + # Give the locked model a stable public entry point and safe symbols. + module = ir.Module.parse(source.read_text("utf-8").replace( + "PaddlePaddle Graph in PIR mode", "recognition")) + candidates = [] + def walk(op): + if op.name == "linalg.conv_2d_nchw_fchw": + shapes = [list(ir.RankedTensorType(v.type).shape) for v in op.operands] + if shapes[1][2:] == [1,1] and shapes[0][1] >= 128 and shapes[1][0] >= 128: + candidates.append((op,shapes)) + for region in op.regions: + for block in region.blocks: + for child in block.operations: + walk(child.operation) + walk(module.operation) + for original, shapes in candidates: + if any(d <= 0 for shape in shapes for d in shape): + raise ValueError("static positive dimensions are required") + if shapes[0][0] != shapes[2][0] or shapes[0][2:] != shapes[2][2:]: + raise ValueError("pointwise spatial dimensions differ") + for key in ("strides","dilations"): + if any(int(v) != 1 for v in ir.DenseIntElementsAttr(original.attributes[key])): + raise ValueError("only unit-stride pointwise convolutions are supported") + if any(str(ir.RankedTensorType(v.type).element_type) != "f32" for v in original.operands): + raise ValueError("source must retain FP32 tensors") + fragment, record = matrix_region(shapes) + helper = ir.Module.parse(fragment) + block = helper.body.operations[0].regions[0].blocks[0] + for argument, operand in zip(block.arguments, original.operands): + argument.replace_all_uses_with(operand) + operations = list(block.operations) + value = operations[-1].operands[0] + for operation in operations[:-1]: + operation.operation.move_before(original) + original.results[0].replace_all_uses_with(value) + original.erase() + selected.append(record) + if not selected: + raise ValueError("no NPU partitions found") + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(str(module)+"\n", "utf-8") + return {"precision":"bf16-bfp16ebs8", "npuPointwiseConvolutions":len(selected), + "npuDispatches":sum(p["npuDispatches"] for p in selected), + "partitions":selected, "cpuPartitionRequired":True, + "numericsValidated":False, "deviceValidated":False} + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input",type=Path,required=True) + parser.add_argument("--output",type=Path,required=True) + parser.add_argument("--report",type=Path,required=True) + args=parser.parse_args() + report=transform(args.input,args.output) + args.report.write_text(json.dumps(report,indent=2)+"\n") + print(f"Partitioned {report['npuPointwiseConvolutions']} dense pointwise convolutions") + + +if __name__=="__main__": + main() diff --git a/tools/npu/record_support_pack.py b/tools/npu/record_support_pack.py new file mode 100644 index 0000000..4b81d6e --- /dev/null +++ b/tools/npu/record_support_pack.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Record an actual npm archive and enforce the lightweight AMD size budget.""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +from sdk import digest + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("--package-dir",type=Path,required=True) + parser.add_argument("--pack-manifest",type=Path,required=True) + parser.add_argument("--output-file",type=Path,required=True) + a=parser.parse_args() + packed=json.loads(a.pack_manifest.read_text()) + if not isinstance(packed,list) or len(packed)!=1:raise ValueError("expected one npm archive") + item=packed[0] + filename=item["filename"] + if Path(filename).name!=filename:raise ValueError("unsafe npm archive name") + archive=a.pack_manifest.parent/filename + if archive.stat().st_size!=item["size"]:raise ValueError("npm archive size differs from manifest") + descriptor=json.loads((a.package_dir/"native/runtime-descriptor.json").read_text()) + lightweight=descriptor["providers"].get("amdnpu",{}).get("runtimeProvider")=="IREEAMDAIE" + report={"package":item["name"],"version":item["version"], + "archive":{"path":filename,"bytes":archive.stat().st_size,"sha256":digest(archive)}, + "unpackedBytes":item["unpackedSize"],"fileCount":len(item["files"]), + "lightweightAmd":lightweight,"deviceValidated":False,"numericsValidated":False} + if lightweight: + report["budget"]={"compressedBytes":64*1024*1024,"unpackedBytes":128*1024*1024} + if item["size"]>report["budget"]["compressedBytes"] or item["unpackedSize"]>report["budget"]["unpackedBytes"]: + raise ValueError("lightweight AMD package exceeds its 64 MiB/128 MiB budget") + a.output_file.write_text(json.dumps(report,indent=2)+"\n") + print(f"{item['name']}: {item['size']} compressed / {item['unpackedSize']} unpacked bytes") + + +if __name__=="__main__":main() diff --git a/tools/npu/route_iree_dispatches.py b/tools/npu/route_iree_dispatches.py new file mode 100644 index 0000000..0905ffd --- /dev/null +++ b/tools/npu/route_iree_dispatches.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Assign affinities after Flow formation and name shared weights by content. + +Pre-Flow affinities can be lost during dispatch creation. Only the BF16 +matmuls inserted by partition_iree_model.py are assigned to the AMD device. +""" +from __future__ import annotations +import argparse +import hashlib +import json +from pathlib import Path +import re + + +def route(source: Path, output: Path, expected_npu_dispatches=19) -> dict: + if output.exists(): + raise ValueError("output already exists") + lines = source.read_text("utf-8").splitlines() + out, renames = [], {} + counts = {"cpu":0,"npu":0} + kernel = False + for line in lines: + if line.startswith("module "): + line = ('module attributes {stream.affinity.default = #hal.device.affinity<@npu>, ' + 'stream.topology = #hal.device.topology @npu = {transparent_access = true})]>} {') + out.append(line) + out.append(' util.global private @npu = #hal.device.target<"amdxdna", ' + '[#hal.executable.target<"amd-aie", "amdaie-pdi-fb", ' + '{num_cols = 2 : i32, num_rows = 2 : i32, target_device = "npu4", ' + 'ukernels = "matmul"}>]> : !hal.device') + continue + # The scalar barrier prevents CSE/constant hoisting from moving the + # zero accumulator out of its dispatch. Dispatch formation is now + # complete, so it is safe to remove this scalar-only barrier. + if "func.func " in line: + kernel = "_bf16xbf16xf32" in line + if kernel and "util.optimization_barrier" in line: + match = re.fullmatch(r'\s*(%[\w]+) = util.optimization_barrier (%[\w]+) : f32',line) + if not match: + raise ValueError("unexpected optimization barrier in NPU dispatch") + barrier, replacement = match.groups() + # Resolve only subsequent uses within this function. + renames[barrier] = replacement + continue + if kernel: + for old, new in renames.items(): + if old.startswith("%"): + line = re.sub(re.escape(old)+r'(?![\w])',new,line) + if line.strip() == "return": + kernel = False + renames = {k:v for k,v in renames.items() if not k.startswith("%")} + if " = flow.dispatch " in line: + device = "npu" if "_bf16xbf16xf32" in line else "cpu" + counts[device] += 1 + if "stream.affinity" in line or ") : " not in line: + raise ValueError("unexpected dispatch syntax") + line = line.replace(") : ",f") {{stream.affinity = #hal.device.affinity<@{device}>}} : ",1) + match = re.match(r'\s*util.global private @([\w]+)(?: \{[^}]+\})? = (dense<.*)',line) + if match: + old, value = match.groups() + renames[old] = "weight_"+hashlib.sha256(value.encode()).hexdigest() + if re.search(r'tensor<[^>]*xbf16>',value): + line = line.replace('#hal.device.affinity<@cpu>','#hal.device.affinity<@npu>') + out.append(line) + if counts["npu"] != expected_npu_dispatches or counts["cpu"] == 0: + raise ValueError(f"source graph partition contract changed: {counts}") + text = "\n".join(out)+"\n" + text = re.sub(r'@([\w]+)',lambda m:"@"+renames.get(m[1],m[1]),text) + output.parent.mkdir(parents=True,exist_ok=True) + output.write_text(text,"utf-8") + return {"dispatches":counts,"sharedWeightKeys":len(renames), + "cpuPartitionRequired":True,"deviceValidated":False,"numericsValidated":False} + + +def main(): + parser=argparse.ArgumentParser(description=__doc__) + parser.add_argument("--input",type=Path,required=True) + parser.add_argument("--output",type=Path,required=True) + parser.add_argument("--report",type=Path,required=True) + parser.add_argument("--expected-npu-dispatches",type=int,default=19) + args=parser.parse_args() + report=route(args.input,args.output,args.expected_npu_dispatches) + args.report.write_text(json.dumps(report,indent=2)+"\n") + print(json.dumps(report)) + + +if __name__=="__main__": + main() diff --git a/tools/npu/runtime/CMakeLists.txt b/tools/npu/runtime/CMakeLists.txt new file mode 100644 index 0000000..c24570c --- /dev/null +++ b/tools/npu/runtime/CMakeLists.txt @@ -0,0 +1,36 @@ +cmake_minimum_required(VERSION 3.26) +project(light_ocr_aie_runtime LANGUAGES C CXX) +if(NOT LIGHT_OCR_IREE_SOURCE OR NOT LIGHT_OCR_AMD_AIE_SOURCE) + message(FATAL_ERROR "Pinned IREE and AMD AIE source paths are required") +endif() +set(CMAKE_POSITION_INDEPENDENT_CODE ON) +set(CMAKE_CXX_STANDARD 17) +set(IREE_BUILD_COMPILER OFF CACHE BOOL "" FORCE) +set(IREE_BUILD_TESTS OFF CACHE BOOL "" FORCE) +set(IREE_BUILD_SAMPLES OFF CACHE BOOL "" FORCE) +set(IREE_BUILD_PYTHON_BINDINGS OFF CACHE BOOL "" FORCE) +set(IREE_HAL_DRIVER_DEFAULTS OFF CACHE BOOL "" FORCE) +set(IREE_HAL_DRIVER_LOCAL_SYNC ON CACHE BOOL "" FORCE) +set(IREE_HAL_DRIVER_LOCAL_TASK OFF CACHE BOOL "" FORCE) +set(IREE_EXTERNAL_HAL_DRIVERS amdxdna CACHE STRING "" FORCE) +set(IREE_CMAKE_PLUGIN_PATHS "${LIGHT_OCR_AMD_AIE_SOURCE}" CACHE STRING "" FORCE) +set(ENABLE_AMDXDNA_CTS_TESTS OFF CACHE BOOL "" FORCE) +set(ENABLE_AMDXDNA_AIE_EXECUTABLE_TESTS OFF CACHE BOOL "" FORCE) +set(IREE_ENABLE_ASSERTIONS OFF CACHE BOOL "" FORCE) +set(IREE_ERROR_ON_MISSING_SUBMODULES OFF CACHE BOOL "" FORCE) +add_subdirectory("${LIGHT_OCR_IREE_SOURCE}" iree) +add_library(light_ocr_amdaie SHARED runtime.cc) +target_include_directories(light_ocr_amdaie PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/../../../src") +target_link_libraries(light_ocr_amdaie PRIVATE + iree::base iree::hal + iree::hal::drivers::local_sync::registration + iree-amd-aie::driver::amdxdna::registration + iree::io::formats::parser_registry iree::io::parameter_index_provider + iree::modules::hal iree::modules::io::parameters iree::vm::bytecode::module) +target_compile_options(light_ocr_amdaie PRIVATE -fvisibility=hidden -ffunction-sections -fdata-sections) +target_link_options(light_ocr_amdaie PRIVATE + "-Wl,--gc-sections" "-Wl,--exclude-libs,ALL" "-Wl,--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map") +set_target_properties(light_ocr_amdaie PROPERTIES VERSION 1.0.0 SOVERSION 1) +add_executable(merge-aie-parameters merge_parameters.cc) +target_link_libraries(merge-aie-parameters PRIVATE iree::base + iree::io::formats::irpa iree::io::formats::parser_registry) diff --git a/tools/npu/runtime/exports.map b/tools/npu/runtime/exports.map new file mode 100644 index 0000000..4c8b539 --- /dev/null +++ b/tools/npu/runtime/exports.map @@ -0,0 +1 @@ +{ global: LightOcrAieGetApi; local: *; }; diff --git a/tools/npu/runtime/merge_parameters.cc b/tools/npu/runtime/merge_parameters.cc new file mode 100644 index 0000000..a520de6 --- /dev/null +++ b/tools/npu/runtime/merge_parameters.cc @@ -0,0 +1,97 @@ +// Build-only IRPA assembler. Never included in a deployment package. +#include +#include +#include +#include +#include +#include +#include +#include "iree/base/api.h" +#include "iree/io/file_handle.h" +#include "iree/io/formats/irpa/irpa_builder.h" +#include "iree/io/formats/parser_registry.h" +#include "iree/io/parameter_index.h" + +namespace { +void check(iree_status_t status) { + if (iree_status_is_ok(status)) return; + char text[2048] = {}; iree_host_size_t length = 0; + iree_status_format(status,sizeof(text),text,&length); + iree_status_ignore(status); + throw std::runtime_error(text); +} +iree_status_t open_output(void* user, iree_io_physical_offset_t offset, + iree_io_physical_size_t length, iree_io_file_handle_t** file) { + return iree_io_file_handle_create(IREE_IO_FILE_MODE_READ|IREE_IO_FILE_MODE_WRITE, + iree_make_cstring_view(static_cast(user)),offset+length, + iree_allocator_system(),file); +} +} + +int main(int argc,char** argv) { + if (argc < 3) { std::cerr << "Usage: merge-aie-parameters OUTPUT INPUT...\n"; return 2; } + iree_io_parameter_index_t *merged=nullptr,*built=nullptr; + std::vector> archives; + struct Entry { uint64_t size; std::vector bytes; }; + std::map seen; + try { + archives.reserve(argc-2); + check(iree_io_parameter_index_create(iree_allocator_system(),&merged)); + for (int a=2;a(input),std::istreambuf_iterator()); + auto& bytes=archives.back(); + iree_io_file_handle_t* file=nullptr; + iree_io_parameter_index_t* index=nullptr; + auto status=iree_io_file_handle_wrap_host_allocation(IREE_IO_FILE_ACCESS_READ, + iree_make_byte_span(bytes.data(),bytes.size()), + iree_io_file_handle_release_callback_null(),iree_allocator_system(),&file); + if (iree_status_is_ok(status)) status=iree_io_parameter_index_create(iree_allocator_system(),&index); + if (iree_status_is_ok(status)) status=iree_io_parse_file_index(IREE_SV("irpa"),file,index,iree_allocator_system()); + if (iree_status_is_ok(status)) { + for (iree_host_size_t i=0;ikey.data,entry->key.size); + std::vector value; + if (entry->type==IREE_IO_PARAMETER_INDEX_ENTRY_STORAGE_TYPE_FILE) { + auto offset=entry->storage.file.offset; + if (offset>bytes.size() || entry->length>bytes.size()-offset) { + status=iree_make_status(IREE_STATUS_OUT_OF_RANGE,"Invalid parameter range");break; + } + value.assign(bytes.begin()+offset,bytes.begin()+offset+entry->length); + } else { + value.assign(entry->storage.splat.pattern, + entry->storage.splat.pattern+entry->storage.splat.pattern_length); + } + const auto found=seen.find(key); + if (found!=seen.end()) { + if (found->second.size!=entry->length || found->second.bytes!=value) { + status=iree_make_status(IREE_STATUS_INVALID_ARGUMENT,"Conflicting parameter key");break; + } + } else { + seen.emplace(key,Entry{entry->length,std::move(value)}); + status=iree_io_parameter_index_add(merged,entry); + if (!iree_status_is_ok(status)) break; + } + } + } + iree_io_parameter_index_release(index); + iree_io_file_handle_release(file); + check(status); + } + check(iree_io_parameter_index_create(iree_allocator_system(),&built)); + iree_io_parameter_archive_file_open_callback_t callback={open_output,argv[1]}; + check(iree_io_build_parameter_archive(merged,built,callback,0,iree_allocator_system())); + std::cout << "Shared parameters: " << seen.size() << '\n'; + iree_io_parameter_index_release(built); + iree_io_parameter_index_release(merged); + return 0; + } catch (const std::exception& e) { + iree_io_parameter_index_release(built); + iree_io_parameter_index_release(merged); + std::cerr << e.what() << '\n';return 1; + } +} diff --git a/tools/npu/runtime/runtime.cc b/tools/npu/runtime/runtime.cc new file mode 100644 index 0000000..967d439 --- /dev/null +++ b/tools/npu/runtime/runtime.cc @@ -0,0 +1,258 @@ +#include "inference/amdnpu/runtime_api.h" + +#include +#include +#include +#include +#include + +#include "iree/base/api.h" +#include "iree/async/frontier_tracker.h" +#include "iree/async/util/proactor_pool.h" +#include "iree/hal/api.h" +#include "iree/hal/buffer_view_util.h" +#include "iree/hal/device_group.h" +#include "iree/hal/drivers/local_sync/registration/driver_module.h" +#include "iree/io/file_handle.h" +#include "iree/io/formats/parser_registry.h" +#include "iree/io/parameter_index.h" +#include "iree/io/parameter_index_provider.h" +#include "iree/modules/hal/module.h" +#include "iree/modules/io/parameters/module.h" +#include "iree/vm/api.h" +#include "iree/vm/bytecode/module.h" +#include "iree-amd-aie/driver/amdxdna/registration/driver_module.h" + +namespace { +constexpr size_t kClasses = 18710; +iree_allocator_t allocator() { return iree_allocator_system(); } + +void check(iree_status_t status) { + if (iree_status_is_ok(status)) return; + char message[2048] = {}; + iree_host_size_t length = 0; + iree_status_format(status, sizeof(message), message, &length); + iree_status_ignore(status); + throw std::runtime_error(message[0] ? message : "IREE AMD operation failed"); +} + +template +int boundary(char* error, size_t capacity, F&& work) noexcept { + if (error && capacity) error[0] = 0; + try { work(); return 0; } + catch (const std::exception& e) { + if (error && capacity) { + std::strncpy(error, e.what(), capacity - 1); + error[capacity - 1] = 0; + } + } catch (...) { + if (error && capacity) { + std::strncpy(error, "Unknown IREE AMD runtime failure", capacity - 1); + error[capacity - 1] = 0; + } + } + return 1; +} + +struct Engine { + std::vector weights; + iree_vm_instance_t* instance = nullptr; + iree_hal_driver_registry_t* registry = nullptr; + iree_hal_device_t* cpu = nullptr; + iree_hal_device_t* npu = nullptr; + iree_async_proactor_pool_t* proactor_pool = nullptr; + iree_async_frontier_tracker_t* frontier_tracker = nullptr; + iree_hal_device_group_t* group = nullptr; + iree_vm_module_t* hal = nullptr; + iree_vm_module_t* parameters = nullptr; + ~Engine() { + iree_vm_module_release(parameters); + iree_vm_module_release(hal); + iree_hal_device_group_release(group); + iree_hal_device_release(npu); + iree_hal_device_release(cpu); + iree_async_frontier_tracker_release(frontier_tracker); + iree_async_proactor_pool_release(proactor_pool); + iree_hal_driver_registry_free(registry); + iree_vm_instance_release(instance); + } +}; + +struct Session { + Engine* engine = nullptr; + uint32_t width = 0; + std::vector bytecode; + iree_vm_module_t* model = nullptr; + iree_vm_context_t* context = nullptr; + iree_vm_function_t function = {}; + ~Session() { + iree_vm_context_release(context); + iree_vm_module_release(model); + } +}; + +iree_status_t choose_allocator( + void* user, iree_host_size_t count, + const iree_hal_device_queue_affinity_pair_t* pairs, + iree_hal_memory_type_t, iree_hal_buffer_usage_t, + iree_hal_module_device_allocator_select_flags_t, + iree_host_size_t* selection) { + // CPU kernels can map AMD's host-only BOs. AMD dispatches cannot consume + // local CPU allocations: shared resources must originate on the NPU. + auto* engine = static_cast(user); + *selection = 0; + for (iree_host_size_t i = 0; i < count; ++i) + if (pairs[i].device == engine->npu) { *selection = i; break; } + return iree_ok_status(); +} + +void create_device(Engine& engine, const char* name, iree_hal_device_t** output) { + iree_hal_driver_t* driver = nullptr; + check(iree_hal_driver_registry_try_create(engine.registry, + iree_make_cstring_view(name), allocator(), &driver)); + auto params = iree_hal_device_create_params_default(); + params.proactor_pool = engine.proactor_pool; + auto status = iree_hal_driver_create_default_device(driver, ¶ms, allocator(), output); + iree_hal_driver_release(driver); + check(status); +} + +int create(const uint8_t* data, size_t size, void** output, char* error, size_t capacity) { + if (output) *output = nullptr; + return boundary(error, capacity, [&] { + if (!data || !size || !output) throw std::invalid_argument("Missing parameter archive"); + auto engine = std::make_unique(); + engine->weights.assign(data, data + size); + check(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT, allocator(), &engine->instance)); + check(iree_hal_module_register_all_types(engine->instance)); + check(iree_hal_driver_registry_allocate(allocator(), &engine->registry)); + check(iree_hal_local_sync_driver_module_register(engine->registry)); + check(iree_hal_amdxdna_driver_module_register(engine->registry)); + check(iree_async_proactor_pool_create(1, nullptr, + iree_async_proactor_pool_options_default(), allocator(), &engine->proactor_pool)); + check(iree_async_frontier_tracker_create(iree_async_frontier_tracker_options_default(), + allocator(), &engine->frontier_tracker)); + create_device(*engine, "local-sync", &engine->cpu); + create_device(*engine, "amdxdna", &engine->npu); + iree_hal_device_group_builder_t builder; + iree_hal_device_group_builder_initialize(&builder, engine->frontier_tracker); + check(iree_hal_device_group_builder_add_device(&builder, engine->cpu)); + check(iree_hal_device_group_builder_add_device(&builder, engine->npu)); + check(iree_hal_device_group_builder_finalize(&builder, allocator(), &engine->group)); + auto policy = iree_hal_module_device_policy_default(); + policy.allocator_select = {choose_allocator, engine.get()}; + check(iree_hal_module_create(engine->instance, policy, engine->group, + IREE_HAL_MODULE_FLAG_SYNCHRONOUS, iree_hal_module_debug_sink_null(), + allocator(), &engine->hal)); + iree_io_file_handle_t* file = nullptr; + iree_io_parameter_index_t* index = nullptr; + iree_io_parameter_provider_t* provider = nullptr; + // Locals are released on both success and failure. The parameter module + // retains its provider/index/file, whose byte span is owned by Engine. + auto status = iree_io_file_handle_wrap_host_allocation( + IREE_IO_FILE_ACCESS_READ, iree_make_byte_span(engine->weights.data(), size), + iree_io_file_handle_release_callback_null(), allocator(), &file); + if (iree_status_is_ok(status)) status = iree_io_parameter_index_create(allocator(), &index); + if (iree_status_is_ok(status)) status = iree_io_parse_file_index(IREE_SV("irpa"), file, index, allocator()); + if (iree_status_is_ok(status)) status = iree_io_parameter_index_provider_create( + IREE_SV("recognition"), index, + IREE_IO_PARAMETER_INDEX_PROVIDER_DEFAULT_MAX_CONCURRENT_OPERATIONS, allocator(), &provider); + if (iree_status_is_ok(status)) status = iree_io_parameters_module_create( + engine->instance, 1, &provider, allocator(), &engine->parameters); + iree_io_parameter_provider_release(provider); + iree_io_parameter_index_release(index); + iree_io_file_handle_release(file); + check(status); + *output = engine.release(); + }); +} + +int load(void* handle, const uint8_t* data, size_t size, uint32_t width, + void** output, char* error, size_t capacity) { + if (output) *output = nullptr; + return boundary(error, capacity, [&] { + if (!handle || !data || !size || !output || !width || width % 8) + throw std::invalid_argument("Invalid recognition module"); + auto session = std::make_unique(); + session->engine = static_cast(handle); + session->width = width; + session->bytecode.assign(data, data + size); + check(iree_vm_bytecode_module_create(session->engine->instance, + IREE_VM_BYTECODE_MODULE_FLAG_NONE, + iree_make_const_byte_span(session->bytecode.data(), size), + iree_allocator_null(), allocator(), &session->model)); + check(iree_vm_module_lookup_function_by_name(session->model, + IREE_VM_FUNCTION_LINKAGE_EXPORT, IREE_SV("recognition"), &session->function)); + iree_vm_module_t* modules[] = {session->engine->hal, session->engine->parameters, session->model}; + check(iree_vm_context_create_with_modules(session->engine->instance, + IREE_VM_CONTEXT_FLAG_NONE, 3, modules, allocator(), &session->context)); + *output = session.release(); + }); +} + +int run(void* handle, const float* input, size_t input_count, float* output, + size_t output_count, char* error, size_t capacity) { + return boundary(error, capacity, [&] { + auto* session = static_cast(handle); + if (!session || !input || !output || input_count != size_t(3*48)*session->width || + output_count != size_t(session->width/8)*kClasses) + throw std::invalid_argument("Recognition tensor dimensions differ from the bucket contract"); + iree_hal_dim_t shape[] = {1,3,48,session->width}; + iree_hal_buffer_params_t params = {}; + params.type = IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE; + params.usage = IREE_HAL_BUFFER_USAGE_DEFAULT; + iree_hal_buffer_view_t* input_view = nullptr; + iree_vm_list_t *inputs = nullptr, *outputs = nullptr; + auto status = iree_hal_buffer_view_allocate_buffer_copy( + session->engine->npu, iree_hal_device_allocator(session->engine->npu), + 4, shape, IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + params, iree_make_const_byte_span(input, input_count*sizeof(float)), &input_view); + if (iree_status_is_ok(status)) status = iree_vm_list_create( + iree_vm_make_undefined_type_def(), 1, allocator(), &inputs); + if (iree_status_is_ok(status)) status = iree_vm_list_create( + iree_vm_make_undefined_type_def(), 1, allocator(), &outputs); + if (iree_status_is_ok(status)) { + auto ref = iree_hal_buffer_view_move_ref(input_view); + input_view = nullptr; + status = iree_vm_list_push_ref_move(inputs, &ref); + iree_vm_ref_release(&ref); + } + if (iree_status_is_ok(status)) status = iree_vm_invoke(session->context, + session->function, IREE_VM_INVOCATION_FLAG_NONE, nullptr, inputs, outputs, allocator()); + if (iree_status_is_ok(status)) { + auto* view = static_cast(iree_vm_list_get_ref_deref( + outputs, 0, iree_hal_buffer_view_type())); + if (!view || iree_vm_list_size(outputs) != 1 || + iree_hal_buffer_view_element_type(view) != IREE_HAL_ELEMENT_TYPE_FLOAT_32 || + iree_hal_buffer_view_shape_rank(view) != 3 || + iree_hal_buffer_view_shape_dim(view,0) != 1 || + iree_hal_buffer_view_shape_dim(view,1) != session->width/8 || + iree_hal_buffer_view_shape_dim(view,2) != kClasses) { + status = iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "Invalid recognition output ABI"); + } else { + iree_hal_buffer_mapping_t mapping = {}; + status = iree_hal_buffer_map_range(iree_hal_buffer_view_buffer(view), + IREE_HAL_MAPPING_MODE_SCOPED, IREE_HAL_MEMORY_ACCESS_READ, + 0, output_count*sizeof(float), &mapping); + if (iree_status_is_ok(status)) { + std::memcpy(output,mapping.contents.data,output_count*sizeof(float)); + status = iree_hal_buffer_unmap_range(&mapping); + } + } + } + iree_hal_buffer_view_release(input_view); + iree_vm_list_release(outputs); + iree_vm_list_release(inputs); + check(status); + }); +} + +void destroy_session(void* handle) { delete static_cast(handle); } +void destroy_engine(void* handle) { delete static_cast(handle); } +const LightOcrAieApiV1 api = {1,sizeof(LightOcrAieApiV1),create,load,run,destroy_session,destroy_engine}; +} // namespace + +extern "C" __attribute__((visibility("default"))) +const LightOcrAieApiV1* LightOcrAieGetApi(uint32_t version) { + return version == 1 ? &api : nullptr; +} diff --git a/tools/npu/sdk.py b/tools/npu/sdk.py index 54ee341..4f1aed1 100644 --- a/tools/npu/sdk.py +++ b/tools/npu/sdk.py @@ -79,6 +79,26 @@ def artifact_set(items: list[dict]) -> str: separators=(",", ":")).encode()).hexdigest() +def validate_aie_configuration(compiler: dict, configuration: dict, artifacts: list[dict], root: Path) -> None: + if (compiler.get("runtimeAbi") != 1 or compiler.get("device") != "npu4" or + compiler.get("sourceModelSha256") != configuration["sourceModelSha256"] or + compiler.get("parameterScope") != "recognition" or + compiler.get("precision") != "bf16-bfp16ebs8" or + compiler.get("cpuPartitionRequired") is not True or + compiler.get("widths") != recognition_widths() or + not isinstance(compiler.get("runtimeVersion"), str) or not compiler["runtimeVersion"]): + raise ValueError("invalid AMD AIE deployment configuration") + parameters = compiler.get("parameters", {}) + relative_path(parameters.get("path")) + prefix = PurePosixPath(configuration["compilerConfiguration"]["path"]).parent + item = {**parameters, "path": (prefix / parameters["path"]).as_posix()} + verify_record(item, root) + if item not in artifacts: + raise ValueError("AMD shared parameters are outside the runtime inventory") + if any(not item["artifact"]["path"].endswith(".vmfb") for item in configuration["recognitionModels"]): + raise ValueError("AMD AIE requires compiled VMFB recognition modules") + + def validate_sdk(root: Path, provider: str | None = None) -> dict: root = root.absolute() if (root / "sdk-manifest.json").is_symlink(): @@ -142,7 +162,11 @@ def validate_sdk(root: Path, provider: str | None = None) -> dict: any(item["artifact"] not in artifacts for item in config["recognitionModels"])): raise ValueError("AMD NPU recognition models are outside the SDK inventory") compiler = json.loads(verify_record(config["compilerConfiguration"], root).read_text("utf-8")) - if (compiler.get("target") != "VAIML" or + if compiler.get("target") == "IREEAMDAIE": + validate_aie_configuration(compiler, config, artifacts, root) + if Path(value["runtimeLibrary"]).name != "liblight_ocr_amdaie.so.1": + raise ValueError("invalid AMD AIE runtime library") + elif (compiler.get("target") != "VAIML" or not any(item.get("name") == "vaiml_partition" for item in compiler.get("passes", []))): raise ValueError("AMD NPU SDK must use a VAIML BF16 configuration") # A production SDK must carry reviewed evidence bound to the exact bytes. @@ -174,8 +198,8 @@ def validate_descriptor_npu(descriptor: dict, root: Path) -> set[str]: if set(provider) != {"runtimeProvider", "providerVersion", "qualificationId", "providerLibrary", "configuration", "artifacts"}: raise ValueError("invalid NPU provider descriptor fields") - expected = "OpenVINO" if name == "openvino" else "VitisAIExecutionProvider" - if (provider["runtimeProvider"] != expected or not provider["providerVersion"] or + expected = {"OpenVINO"} if name == "openvino" else {"VitisAIExecutionProvider", "IREEAMDAIE"} + if (provider["runtimeProvider"] not in expected or not provider["providerVersion"] or not provider["qualificationId"] or not provider["artifacts"] or provider["providerLibrary"] not in provider["artifacts"]): raise ValueError("invalid NPU provider descriptor identity") @@ -216,7 +240,12 @@ def validate_descriptor_npu(descriptor: dict, root: Path) -> set[str]: any(set(item) != {"width", "artifact"} or item["artifact"] not in provider["artifacts"] for item in models)): raise ValueError("incomplete AMD NPU recognition buckets") - if not Path(provider["providerLibrary"]["path"]).name.startswith("libonnxruntime.so"): + if provider["runtimeProvider"] == "IREEAMDAIE": + compiler = json.loads(verify_record(config["compilerConfiguration"], root).read_text("utf-8")) + if compiler.get("target") != "IREEAMDAIE" or Path(provider["providerLibrary"]["path"]).name != "liblight_ocr_amdaie.so.1": + raise ValueError("invalid AMD AIE runtime identity") + validate_aie_configuration(compiler, config, provider["artifacts"], root) + elif not Path(provider["providerLibrary"]["path"]).name.startswith("libonnxruntime.so"): raise ValueError("invalid AMD ORT runtime library") return paths @@ -243,8 +272,9 @@ def stage_sdk(root: Path, stage: Path, descriptor: dict) -> dict: configuration["recognitionModels"] = [ {"width": item["width"], "artifact": by_path[item["artifact"]["path"]]} for item in configuration["recognitionModels"]] + amd_aie = provider == "amdnpu" and Path(value["runtimeLibrary"]).name == "liblight_ocr_amdaie.so.1" descriptor["providers"][provider] = { - "runtimeProvider": "OpenVINO" if provider == "openvino" else "VitisAIExecutionProvider", + "runtimeProvider": "OpenVINO" if provider == "openvino" else ("IREEAMDAIE" if amd_aie else "VitisAIExecutionProvider"), "providerVersion": value["providerVersion"], "qualificationId": value["qualificationId"], "providerLibrary": by_path[value["runtimeLibrary"]], From ae89d786d2e9ee60d6a952c817f3a389dcface1f Mon Sep 17 00:00:00 2001 From: eric8810 Date: Thu, 8 Oct 2026 18:21:21 +0800 Subject: [PATCH 3/3] =?UTF-8?q?test(amdnpu):=20=E6=97=A0=E8=8A=AF=E9=AA=8C?= =?UTF-8?q?=E5=8D=B7=EF=BC=8C=E9=94=99=E5=BE=84=E5=BD=92=E6=98=8E=20=C2=B7?= =?UTF-8?q?=20validate=20AMD=20software=20paths=20without=20hardware?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/npu-support-release.yml | 9 + CHANGELOG.md | 3 + docs/amd-aie-deployment-results.json | 3 +- docs/amd-aie-development.md | 38 ++- docs/amd-aie-software-validation-results.json | 287 ++++++++++++++++++ docs/amd-npu-acceleration.md | 8 +- docs/amd-npu-lightweight-plan.md | 6 +- docs/releases/npm-0.5.9.en.md | 4 +- docs/releases/npm-0.5.9.md | 4 +- packages/runtime/src/npu-descriptor.cjs | 3 +- packages/runtime/test/amdnpu-package.test.cjs | 71 +++++ packages/runtime/test/npu-descriptor.test.cjs | 94 ++++++ tests/python/test_amdaie_runtime.py | 67 ++++ tools/npu/check_cpu_recognition.py | 100 ++++++ 14 files changed, 683 insertions(+), 14 deletions(-) create mode 100644 docs/amd-aie-software-validation-results.json create mode 100644 packages/runtime/test/amdnpu-package.test.cjs create mode 100644 packages/runtime/test/npu-descriptor.test.cjs create mode 100644 tests/python/test_amdaie_runtime.py create mode 100644 tools/npu/check_cpu_recognition.py diff --git a/.github/workflows/npu-support-release.yml b/.github/workflows/npu-support-release.yml index af052ff..993be33 100644 --- a/.github/workflows/npu-support-release.yml +++ b/.github/workflows/npu-support-release.yml @@ -81,6 +81,15 @@ jobs: npm pack ./dist/package --ignore-scripts --json --pack-destination dist/tarballs > dist/tarballs/pack-manifest.json python tools/npu/record_support_pack.py --package-dir dist/package \ --pack-manifest dist/tarballs/pack-manifest.json --output-file dist/tarballs/support-size-report.json + - name: Test AMD package without NPU hardware + if: matrix.provider == 'amdnpu' + env: + LIGHT_OCR_AMD_TEST_PACKAGE: ${{ github.workspace }}/dist/package + LIGHT_OCR_AMD_TEST_BUNDLE: ${{ github.workspace }}/models/generated/ppocrv6-small-onnx-20260714.2 + LIGHT_OCR_AMD_TEST_ARCHIVE: ${{ github.workspace }}/dist/tarballs/arcships-light-ocr-amdnpu-linux-x64-gnu-${{ inputs.version }}.tgz + run: | + node --test packages/runtime/test/*.test.cjs + python -m unittest tests.python.test_amdaie_runtime - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 with: name: npu-support-${{ matrix.provider }}-${{ inputs.version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 55a1f21..bc6c582 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,9 @@ Release preparation; npm publication and GitHub Release are pending. numerical parity and hardware execution remain unverified. Deployment artifacts are bound to Small 0.3.4, with no vendor SDK, compiler, Python or XRT in the AMD payload. Legacy vendor deployments retain their separate ORT API/namespace path. + Added hardware-free package/C ABI regression checks and sampled CPU transform + comparisons; fixed null deployment JSON leaking an unclassified TypeError. + These checks do not qualify the AMD BFP16ebs8 kernel or device execution. Neither support package is automatically installed or selected by Auto. The existing Small model bundle remains unchanged. See [AMD NPU 加速技术方案](docs/amd-npu-acceleration.md) and [NPU SDK 构建与发布](docs/npu-runtime-release.md). diff --git a/docs/amd-aie-deployment-results.json b/docs/amd-aie-deployment-results.json index a8338ae..ebcbe95 100644 --- a/docs/amd-aie-deployment-results.json +++ b/docs/amd-aie-deployment-results.json @@ -566,5 +566,6 @@ "SDK assembly", "npm packaging and size/hash inventory" ] - } + }, + "subsequentSoftwareValidation": "amd-aie-software-validation-results.json" } diff --git a/docs/amd-aie-development.md b/docs/amd-aie-development.md index c4b2bef..181198a 100644 --- a/docs/amd-aie-development.md +++ b/docs/amd-aie-development.md @@ -59,7 +59,43 @@ npm pack ./dist/amdaie-package --ignore-scripts --pack-destination dist AMD 不自动安装、不参与 Auto。显式配置 `execution.provider="amdnpu"` 后才加载；`cpuPartition=forbid` 拒绝分区模型。 `npu-support-release.yml` 和 `npu-native.yml` 默认使用轻量源码构建入口，保留 Intel 路线。 -按维护者要求，本轮只执行生成部署产物所需的构建与编译，没有运行测试、数值比较或设备推理。 +初次开发仅执行构建与封装。2026-10-08 补充了下述无设备软件检查；未执行 AMD 设备推理。 + +## 无设备软件检查 + +2026-10-08 本地通过 40 项检查：运行时/配置/真实包集成 19 项、C ABI 4 项、发布工具与 workflow 合约 17 项。修复了部署 JSON 为 `null` 时泄漏普通 TypeError 的问题。 + +真实 tarball 离线安装、正式包解析与 addon 加载通过。没有设备时显式 AMD 返回 `unsupported_capability`；Auto 选择 CPU 并完成空白图 OCR。当前 `sessionFallback="cpu"` 被明确拒绝，不宣称自动回退。 + +```sh +export LIGHT_OCR_AMD_TEST_PACKAGE="$PWD/.cache/amd-aie/support-package" +export LIGHT_OCR_AMD_TEST_BUNDLE="$PWD/models/generated/ppocrv6-small-onnx-20260714.2" +export LIGHT_OCR_AMD_TEST_ARCHIVE="$PWD/.cache/amd-aie/tarballs/arcships-light-ocr-amdnpu-linux-x64-gnu-0.5.9.tgz" +node --test packages/runtime/test/*.test.cjs +python -m unittest tests.python.test_amdaie_runtime +python -m unittest tests.python.test_npm_release tests.python.test_ci_workflow_contracts +``` + +支持包 workflow 已加入运行时/真实包及 C ABI 检查；未设置这些环境变量时，依赖本地产物的集成检查跳过。 + +CPU 比较脚本使用独立前端与参考 Python 环境。默认比较 320/384/3200 三个宽桶的零值、固定种子随机值和渐变输入；可选文字 fixture 需要参考环境安装 OpenCV： + +```sh +python tools/npu/check_cpu_recognition.py \ + --source-model models/generated/ppocrv6-small-onnx-20260714.2/rec/inference.onnx \ + --frontend-environment .cache/amd-aie/host-compiler-venv \ + --reference-python .cache/ryzen-ai/venv/bin/python \ + --toolchain-dir .cache/amd-aie/toolchain-build \ + --runtime-tool .cache/amd-aie/runtime-build/tools/iree-run-module \ + --output-dir .cache/amd-aie/cpu-comparison-new \ + --text-image docs/assets/benchmark-generated-hello-123.png +``` + +上述 IREE CLI 是开发工具，可通过 `build_iree.py --runtime-only` 构建到单独目录，不进入 npm 包。 + +本地 9 组合成输入加一个 320 桶文字裁剪样本通过 FP32 `rtol=1e-3, atol=1e-4` 比较，最大绝对误差约 `5.79e-4`；混合相对/绝对容差并非最大绝对误差上限。BF16 CPU 模拟最大绝对误差约 `0.0374`，这些样本的 CTC 序列均与参考一致，文字样本为 `HELLO 123`。合成输入均解码为空文本，不能据此给出数据集准确率结论。 + +该流程在 CPU 上运行 BF16 操作数矩阵乘，未模拟 AMD 的 BFP16ebs8 内核、DMA 或设备同步；也未数值检查全部 20 桶。AMD `numericsValidated=false`、`deviceValidated=false` 保持不变。[完整结果记录](amd-aie-software-validation-results.json)。 ## 早期编译调查（历史记录） diff --git a/docs/amd-aie-software-validation-results.json b/docs/amd-aie-software-validation-results.json new file mode 100644 index 0000000..0d73a6a --- /dev/null +++ b/docs/amd-aie-software-validation-results.json @@ -0,0 +1,287 @@ +{ + "schemaVersion": "1.0", + "date": "2026-10-08", + "status": "software-checks-passed-npu-unqualified", + "sourceModelSha256": "5435fd747c9e0efe15a96d0b378d5bd157e9492ed8fd80edf08f30d02fa24634", + "sourceLockSha256": "a58136fc60e8eb5023798c3e8d3a7014abbeb2f3107f6a727c5b8f1f0cd0bf4a", + "npm": { + "package": "@arcships/light-ocr-amdnpu-linux-x64-gnu", + "version": "0.5.9", + "archive": { + "path": "arcships-light-ocr-amdnpu-linux-x64-gnu-0.5.9.tgz", + "bytes": 27869026, + "sha256": "2bee7052a4dbfcfea8338e1fa06e42061fc77b9d4aee21d52794f9f28cc4e8b8" + }, + "unpackedBytes": 59690023, + "fileCount": 50, + "lightweightAmd": true, + "deviceValidated": false, + "numericsValidated": false, + "budget": { + "compressedBytes": 67108864, + "unpackedBytes": 134217728 + } + }, + "softwareTests": { + "runtimeUnitAndIntegration": { + "passed": 19, + "failed": 0, + "skipped": 0 + }, + "runtimeCAbi": { + "passed": 4, + "failed": 0, + "skipped": 0 + }, + "releaseAndWorkflowTests": { + "passed": 17, + "failed": 0 + }, + "totalPassed": 40, + "coverage": [ + "20-bucket configuration", + "shared weight inventory and hashes", + "corrupt/missing artifacts", + "malformed deployment JSON", + "offline npm installation", + "production addon resolution", + "missing AMD NPU error", + "Auto selects CPU", + "CPU blank-image OCR", + "CPU partition forbid", + "unsupported CPU session fallback rejected", + "C ABI version/null arguments/error truncation/repeated no-device cleanup" + ] + }, + "cpuComparison": { + "widths": [ + 320, + 384, + 3200 + ], + "uniqueCases": 10, + "fp32Passed": true, + "fp32Tolerance": { + "rtol": 0.001, + "atol": 0.0001 + }, + "cases": [ + { + "width": 320, + "case": "zero", + "fp32": { + "maxAbsoluteError": 5.960464477539062e-07, + "meanAbsoluteError": 3.823016556103731e-12, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 5.960464477539062e-07, + "meanAbsoluteError": 4.307446760387634e-12, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 320, + "case": "random", + "fp32": { + "maxAbsoluteError": 0.00029271841049194336, + "meanAbsoluteError": 3.6059339958427472e-09, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.026643872261047363, + "meanAbsoluteError": 1.4035597359907115e-06, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 320, + "case": "gradient", + "fp32": { + "maxAbsoluteError": 0.0002880692481994629, + "meanAbsoluteError": 4.8882133896199775e-09, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.0052182674407958984, + "meanAbsoluteError": 3.335440510454646e-07, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 384, + "case": "zero", + "fp32": { + "maxAbsoluteError": 5.960464477539062e-07, + "meanAbsoluteError": 3.05321263063818e-12, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 5.960464477539062e-07, + "meanAbsoluteError": 3.5757265204328093e-12, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 384, + "case": "random", + "fp32": { + "maxAbsoluteError": 9.077787399291992e-05, + "meanAbsoluteError": 3.4979239504906445e-09, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.018251627683639526, + "meanAbsoluteError": 1.7710544852889143e-06, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 384, + "case": "gradient", + "fp32": { + "maxAbsoluteError": 0.0003910660743713379, + "meanAbsoluteError": 6.79352352150886e-09, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.005341827869415283, + "meanAbsoluteError": 2.1960502749607258e-07, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 3200, + "case": "zero", + "fp32": { + "maxAbsoluteError": 7.987022399902344e-06, + "meanAbsoluteError": 1.1363214189041848e-11, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 5.3048133850097656e-05, + "meanAbsoluteError": 3.410989199426062e-11, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 3200, + "case": "random", + "fp32": { + "maxAbsoluteError": 0.0005788207054138184, + "meanAbsoluteError": 2.732191184406929e-08, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.03741657733917236, + "meanAbsoluteError": 1.5979701402102364e-06, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 3200, + "case": "gradient", + "fp32": { + "maxAbsoluteError": 0.00017505884170532227, + "meanAbsoluteError": 3.630233669227323e-09, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.0013424158096313477, + "meanAbsoluteError": 1.5656084428883332e-08, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true, + "referenceCtcClassIds": [] + }, + { + "width": 320, + "case": "text", + "referenceCtcClassIds": [ + 50, + 47, + 54, + 54, + 57, + 18709, + 34, + 35, + 36 + ], + "fp32": { + "maxAbsoluteError": 5.424022674560547e-06, + "meanAbsoluteError": 2.7728121881898105e-11, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "bf16CpuEmulation": { + "maxAbsoluteError": 0.005755722522735596, + "meanAbsoluteError": 2.604011939411066e-08, + "argmaxAgreement": 1.0, + "ctcClassSequenceMatches": true + }, + "fp32WithinTolerance": true + } + ], + "allFp32AndBf16CpuCtcMatch": true, + "maximumFp32AbsoluteError": 0.0005788207054138184, + "maximumBf16CpuAbsoluteError": 0.03741657733917236, + "referenceRuntime": "ONNX Runtime CPU 1.27.0; packaged CPU addon uses 1.22.0", + "textFixture": { + "path": "docs/assets/benchmark-generated-hello-123.png", + "sha256": "3fc41659014523aea79a480143b6f76ae1951d27eee4192c7511ad84d3c5b8fc", + "preprocessing": "foreground crop, BGR OpenCV linear resize to height 48, aspect preserved, FP32 NCHW /127.5-1, zero padding", + "referenceText": "HELLO 123" + }, + "limitations": [ + "Only three representative widths, not all 20 widths numerically tested.", + "Synthetic inputs decode to empty text; the single text fixture does not establish OCR dataset accuracy.", + "CPU BF16 matmul does not emulate the AMD BFP16ebs8 kernel, DMA or device scheduling." + ] + }, + "regressionFix": "Null or array deployment JSON is rejected with package_load_failed instead of a raw TypeError.", + "ci": "npu-support-release.yml now runs software/package and C ABI checks for AMD builds; CPU numerical comparison remains an explicit host-tool command.", + "validation": { + "softwareValidated": true, + "cpuTransformationSampled": true, + "amdMicrokernelExecuted": false, + "deviceValidated": false, + "numericsValidated": false, + "performanceMeasured": false + }, + "scriptSha256": "5a704f263d727d114af4ba6addeac1c11e4eb037105c2459528965d9f7de5d7d" +} diff --git a/docs/amd-npu-acceleration.md b/docs/amd-npu-acceleration.md index 5ec7083..0e24f83 100644 --- a/docs/amd-npu-acceleration.md +++ b/docs/amd-npu-acceleration.md @@ -1,6 +1,6 @@ # AMD NPU 加速技术方案 -更新时间：2026-10-07。完整 CPU/NPU 分区后端、20 个识别宽度桶、共享权重和独立支持包构建已实现。AMD 默认关闭，用户通过配置显式开启；未运行测试、数值比较或 AMD 设备推理。 +更新时间：2026-10-08。完整 CPU/NPU 分区后端、20 个识别宽度桶、共享权重和独立支持包构建已实现。AMD 默认关闭，用户通过配置显式开启；无设备软件测试和 CPU 抽样比较已完成，AMD 设备推理仍未验证。 ## 使用方式 @@ -28,7 +28,7 @@ AMD 不参与 Auto，支持包不会自动安装。系统需可读写的 STX/KRK 矩阵 M/N/K 按 128 补齐并裁切；单次 dispatch 最多 3840 行，较宽桶分段执行，规避固定编译器的宽矩阵 shim DMA 维度 lowering 越界。 BF16 操作数在 Peano 微内核内转换为 BFP16ebs8，FP32 累加；精度诊断为 `bf16-bfp16ebs8`。 -这是完整识别分区图，没有纯 NPU、数值正确或性能提升的实测结论。 +这是完整识别分区图。CPU 转换抽样结果见 [软件验证记录](amd-aie-software-validation-results.json)；尚无 AMD 内核数值与性能结论。 `precision` 请求仍只接受 `auto`；`cpuPartition=forbid` 会拒绝此路线。 ## 独立包内容 @@ -44,11 +44,11 @@ BF16 操作数在 Peano 微内核内转换为 BFP16ebs8，FP32 累加；精度 ## 原生行为与诊断 库、配置、共享权重和桶模型均按字节数/SHA256 校验；哈希错误直接报告，不能静默回退。 -没有可访问 NPU 时返回 `adapter_unavailable`。后续桶按需加载；推理和创建错误保持原有错误/回退策略。 +没有可访问 NPU 时，选择原因是 `adapter_unavailable`，公开 API 返回 `unsupported_capability`，不会静默回退。后续桶按需加载。当前 `sessionFallback="cpu"` 不受支持；使用 Auto 或显式 CPU 可走 CPU 路线。 内部配置为 `target=IREEAMDAIE`、`runtimeAbi=1`；实际识别 provider 链为 `IREEAMDAIE → IREECPU`，检测为 `CPUExecutionProvider`。 会话关闭先释放模型 context，再释放参数模块与设备；动态库保持映射。 -`deviceValidated=false`，且没有数值验证。按维护者决定，验证不阻塞此次开发交付与显式启用支持的发布准备。 +`deviceValidated=false`，AMD 数值与性能未验证。40 项无设备软件测试通过；三个代表宽桶及文字样本的 CPU 转换比较通过，BF16 CPU 模拟不能代替 BFP16ebs8 内核验证。详见 [结果](amd-aie-software-validation-results.json) 与 [运行命令](amd-aie-development.md#无设备软件检查)。 ## 构建与兼容 diff --git a/docs/amd-npu-lightweight-plan.md b/docs/amd-npu-lightweight-plan.md index 7b00932..4b43a7d 100644 --- a/docs/amd-npu-lightweight-plan.md +++ b/docs/amd-npu-lightweight-plan.md @@ -1,6 +1,6 @@ # AMD NPU 轻量部署改造方案 -日期：2026-10-07。状态：完整 CPU/NPU 分区后端、20 桶、共享权重及独立支持包流程已实现；不做推理验证。 +日期：2026-10-08。状态：完整 CPU/NPU 分区后端、20 桶、共享权重及独立支持包流程已实现；无设备软件测试和 CPU 抽样比较通过，NPU 未验证。 ## 1. 目标与结论 @@ -147,7 +147,7 @@ P2 是主要选型决策点：如果关键算子无法编译、需要大量模 ## 8. 验收计划 -维护者已明确要求本轮不做验证。仅生成构建产物、库存、身份及体积记录；以下数值、设备和行为检查均为可选后续工作，不阻塞此次交付。 +初次开发按维护者要求仅完成产物生成。2026-10-08 经维护者同意补充了无设备软件测试及 CPU 转换抽样比较，见 [软件验证记录](amd-aie-software-validation-results.json)。设备验收仍不作为此次交付前置条件。 ### 无真机可完成 @@ -181,4 +181,4 @@ P0–P4 的构建、后端、完整桶和独立封装代码均已实现。IREE 原生后端和 descriptor 已识别新 ABI、配置和权重身份；CI 默认构建轻量 AMD SDK，独立包带许可证、SBOM 和哈希库存。 实际产物大小见 [部署结果清单](amd-aie-deployment-results.json)，构建步骤见 [完整部署开发说明](amd-aie-development.md)。 -没有执行测试、CPU 数值比较或 NPU 推理，没有精度和性能结论。`deviceValidated=false`，不作为合入和显式启用发布的阻塞项。 +后续 40 项无设备软件测试通过，三个代表宽桶及文字样本完成 CPU 数值比较。没有执行 NPU 推理，仍无 AMD 精度和性能结论。`deviceValidated=false`，不作为合入和显式启用发布的阻塞项。 diff --git a/docs/releases/npm-0.5.9.en.md b/docs/releases/npm-0.5.9.en.md index c025d56..9469652 100644 --- a/docs/releases/npm-0.5.9.en.md +++ b/docs/releases/npm-0.5.9.en.md @@ -36,11 +36,11 @@ NPU implementation: [PR #67](https://github.com/arcships/light-ocr/pull/67), mer ## Lightweight AMD preparation -[PR #71](https://github.com/arcships/light-ocr/pull/71) implements the complete IREE CPU/NPU recognition backend, all 20 buckets and shared weights. Nineteen dense pointwise convolution matrix cores run on the NPU; other recognition operators and detection remain on CPU. BF16 operands use BFP16ebs8 microkernels, without numerical or performance evidence. +[PR #71](https://github.com/arcships/light-ocr/pull/71) implements the complete IREE CPU/NPU recognition backend, all 20 buckets and shared weights. Nineteen dense pointwise convolution matrix cores run on the NPU; other recognition operators and detection remain on CPU. BF16 operands use BFP16ebs8 microkernels, without AMD kernel numerical or performance evidence. `npu-support-release.yml` builds pinned sources through `build_amdaie.py`. The optional package contains a small C ABI library, 20 VMFBs and one shared IRPA, with licenses, SBOM and hashes. Compiler tools, Peano, original ONNX, the vendor SDK, Python and XRT are excluded. AMD remains explicitly installed and selected, outside Auto. -The actual npm archive has a 64 MiB download / 128 MiB unpacked budget. CI records `support-size-report.json`; local bytes and SHA256 are in [the deployment report](../amd-aie-deployment-results.json). Version 0.5.9 remains unreleased. Only builds and packaging were performed; tests, numerical comparisons and hardware acceptance were omitted at the maintainer's request. +The actual npm archive has a 64 MiB download / 128 MiB unpacked budget. CI records `support-size-report.json`; local bytes and SHA256 are in [the deployment report](../amd-aie-deployment-results.json). Version 0.5.9 remains unreleased. The October 8 follow-up passed 40 software checks without NPU hardware and compared CPU transforms for three representative widths plus a text fixture. See [the software report](../amd-aie-software-validation-results.json). AMD kernel correctness, device compatibility and performance remain unqualified. Local lightweight candidate, including the addon, base dependencies, 20 buckets and shared weights: diff --git a/docs/releases/npm-0.5.9.md b/docs/releases/npm-0.5.9.md index 53555f6..6e3f32a 100644 --- a/docs/releases/npm-0.5.9.md +++ b/docs/releases/npm-0.5.9.md @@ -40,7 +40,7 @@ ## 轻量 AMD 构建与发布准备 -[PR #71](https://github.com/arcships/light-ocr/pull/71) 接入完整 IREE CPU/NPU 分区后端：19 个较大 pointwise 卷积矩阵核心由 NPU 执行，其余识别算子和检测保留 CPU。BF16 操作数使用 BFP16ebs8 微内核，没有数值和性能实测结论。 +[PR #71](https://github.com/arcships/light-ocr/pull/71) 接入完整 IREE CPU/NPU 分区后端：19 个较大 pointwise 卷积矩阵核心由 NPU 执行，其余识别算子和检测保留 CPU。BF16 操作数使用 BFP16ebs8 微内核，尚无 AMD 内核数值和性能实测结论。 `npu-support-release.yml` 使用 `build_amdaie.py` 从固定源码生成小型 C ABI 库、20 个 VMFB 和一份共享 IRPA；编译器、Peano、原始 ONNX、完整 vendor SDK、Python 与 XRT 均不进入支持包。运行时库、配置、权重及全部桶模型继续校验字节数和 SHA256。 @@ -55,7 +55,7 @@ npm SHA256：`2bee7052a4dbfcfea8338e1fa06e42061fc77b9d4aee21d52794f9f28cc4e8b8`。相比历史 vendor 包，下载体积减少约 94.9%。运行库 1,255,312 bytes，单份共享权重 17,145,856 bytes，全部 20 个 VMFB 合计 7,590,556 bytes。该包为本地候选，正式工作流会重新记录其制品身份。 -当前版本仍为补丁发布 `0.5.9`。本轮仅完成代码、构建和封装，没有 npm 发布或公开 Release；按维护者要求未运行测试、数值比较或真机验收。AMD 继续单独安装、显式启用、排除在 Auto 之外。 +当前版本仍为补丁发布 `0.5.9`。已经完成代码、构建、封装及 40 项无设备软件测试，另有三个代表宽桶和文字样本的 CPU 转换比较；详见 [软件验证记录](../amd-aie-software-validation-results.json)。没有 npm 发布、公开 Release 或 AMD 真机验收。AMD 继续单独安装、显式启用、排除在 Auto 之外。 ## 当前证据与废弃候选 diff --git a/packages/runtime/src/npu-descriptor.cjs b/packages/runtime/src/npu-descriptor.cjs index 4392be5..9aaa284 100644 --- a/packages/runtime/src/npu-descriptor.cjs +++ b/packages/runtime/src/npu-descriptor.cjs @@ -89,7 +89,8 @@ function validateNpuProviders(descriptor, root, { exactKeys, verifyArtifact, sam let compiler; try { compiler = JSON.parse(fs.readFileSync(configPath, 'utf8')); } catch { fail('Invalid AMD AIE deployment configuration'); } - if (compiler.target !== 'IREEAMDAIE' || compiler.runtimeAbi !== 1 || + if (!compiler || typeof compiler !== 'object' || Array.isArray(compiler) || + compiler.target !== 'IREEAMDAIE' || compiler.runtimeAbi !== 1 || compiler.device !== 'npu4' || compiler.sourceModelSha256 !== configuration.sourceModelSha256 || compiler.parameterScope !== 'recognition' || compiler.precision !== 'bf16-bfp16ebs8' || compiler.cpuPartitionRequired !== true || typeof compiler.runtimeVersion !== 'string' || diff --git a/packages/runtime/test/amdnpu-package.test.cjs b/packages/runtime/test/amdnpu-package.test.cjs new file mode 100644 index 0000000..012fe44 --- /dev/null +++ b/packages/runtime/test/amdnpu-package.test.cjs @@ -0,0 +1,71 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { spawnSync } = require('node:child_process'); +const packageRoot = process.env.LIGHT_OCR_AMD_TEST_PACKAGE; +const bundlePath = process.env.LIGHT_OCR_AMD_TEST_BUNDLE; +const archive = process.env.LIGHT_OCR_AMD_TEST_ARCHIVE; +const unavailable = !packageRoot || !bundlePath; + +// Run against generated artifacts on a host without an AMD NPU. Ordinary +// runtime tests remain dependency-free and skip these integration cases. +test('actual AMD addon loads, Auto uses CPU, and no-device policy is explicit', { skip: unavailable }, async t => { + const previous = [process.env.LIGHT_OCR_NODE_BINARY, process.env.LIGHT_OCR_RUNTIME_DESCRIPTOR]; + t.after(() => { + for (const [index, key] of ['LIGHT_OCR_NODE_BINARY', 'LIGHT_OCR_RUNTIME_DESCRIPTOR'].entries()) { + if (previous[index] === undefined) delete process.env[key]; + else process.env[key] = previous[index]; + } + }); + process.env.LIGHT_OCR_NODE_BINARY = path.resolve(packageRoot, 'native/light_ocr_node.node'); + process.env.LIGHT_OCR_RUNTIME_DESCRIPTOR = path.resolve(packageRoot, 'native/runtime-descriptor.json'); + const { createEngine } = require('../src/index.cjs'); + await assert.rejects(createEngine({ bundlePath, execution: { provider: 'amdnpu' } }), + e => e.code === 'unsupported_capability' && /No accessible AMD/.test(e.message)); + await assert.rejects(createEngine({ bundlePath, execution: { provider: 'amdnpu', cpuPartition: 'forbid' } }), + e => e.code === 'unsupported_capability' && /CPU detector partition/.test(e.message)); + await assert.rejects(createEngine({ bundlePath, execution: { provider: 'amdnpu', sessionFallback: 'cpu' } }), + e => e.code === 'invalid_argument'); + const engine = await createEngine({ bundlePath, execution: { provider: 'auto' } }); + try { + assert.equal(engine.info.execution.selectionTrace.selectedProvider, 'cpu'); + assert.deepEqual(engine.info.execution.selectionTrace.orderedCandidates, ['cpu']); + assert.deepEqual(engine.info.execution.sessions.recognition.actualProviderChain, ['CPUExecutionProvider']); + const result = await engine.recognize({ width: 64, height: 48, stride: 64, + pixelFormat: 'gray8', data: new Uint8Array(64 * 48).fill(255) }); + assert.ok(result); + } finally { await engine.close(); } +}); + +test('AMD tarball installs offline and production resolution reaches no-device error', + { skip: unavailable || !archive }, () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'light-ocr-amd-install-')); + try { + fs.writeFileSync(path.join(root, 'package.json'), JSON.stringify({ name: 'amd-offline-test', version: '1.0.0', private: true })); + const installed = spawnSync('npm', ['install', '--offline', '--ignore-scripts', '--no-audit', '--no-fund', + '--cache', path.join(root, 'cache'), '--prefix', root, path.resolve(archive)], { encoding: 'utf8' }); + assert.equal(installed.status, 0, installed.stderr); + const runtime = path.join(root, 'node_modules/@arcships/light-ocr-runtime'); + fs.mkdirSync(runtime, { recursive: true }); + fs.cpSync(path.resolve(__dirname, '../src'), path.join(runtime, 'src'), { recursive: true }); + fs.copyFileSync(path.resolve(__dirname, '../package.json'), path.join(runtime, 'package.json')); + // npm_release.stage replaces the workspace VERSION lookup with this + // literal metadata in published runtime packages. + const coreVersion = require('../src/metadata.cjs').coreVersion; + fs.writeFileSync(path.join(runtime, 'src/metadata.cjs'), + `'use strict';\nmodule.exports = Object.freeze({ coreVersion: ${JSON.stringify(coreVersion)} });\n`); + const env = { ...process.env }; + delete env.LIGHT_OCR_NODE_BINARY; delete env.LIGHT_OCR_RUNTIME_DESCRIPTOR; + const probe = spawnSync(process.execPath, ['-e', ` + const assert=require('node:assert/strict'); + const {createEngine}=require(${JSON.stringify(runtime)}); + createEngine({bundlePath:${JSON.stringify(path.resolve(bundlePath))},execution:{provider:'amdnpu'}}) + .then(async engine=>{await engine.close();throw new Error('Unexpected NPU device');}) + .catch(error=>{assert.equal(error.code,'unsupported_capability');assert.match(error.message,/No accessible AMD/);}); + `], { env, encoding: 'utf8', timeout: 30000 }); + assert.equal(probe.status, 0, probe.stderr); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }); diff --git a/packages/runtime/test/npu-descriptor.test.cjs b/packages/runtime/test/npu-descriptor.test.cjs new file mode 100644 index 0000000..7748c8b --- /dev/null +++ b/packages/runtime/test/npu-descriptor.test.cjs @@ -0,0 +1,94 @@ +'use strict'; +const assert = require('node:assert/strict'); +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const crypto = require('node:crypto'); +const { validateNpuProviders, WIDTHS } = require('../src/npu-descriptor.cjs'); + +function fixture(t) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'light-ocr-amd-')); + t.after(() => fs.rmSync(root, { recursive: true, force: true })); + const artifacts = []; + const write = (relative, bytes) => { + const destination = path.join(root, relative); + fs.mkdirSync(path.dirname(destination), { recursive: true }); + fs.writeFileSync(destination, bytes); + const record = { path: relative, bytes: Buffer.byteLength(bytes), + sha256: crypto.createHash('sha256').update(bytes).digest('hex') }; + artifacts.push(record); + return record; + }; + const library = write('native/amdnpu/lib/liblight_ocr_amdaie.so.1', 'test library'); + const parameters = write('native/amdnpu/models/recognition.irpa', 'test shared weights'); + const compiler = { target: 'IREEAMDAIE', runtimeAbi: 1, device: 'npu4', + sourceModelSha256: 'a'.repeat(64), parameterScope: 'recognition', + precision: 'bf16-bfp16ebs8', cpuPartitionRequired: true, runtimeVersion: 'pinned-test', + widths: [...WIDTHS], parameters: { ...parameters, path: 'models/recognition.irpa' } }; + const configuration = write('native/amdnpu/deployment.json', JSON.stringify(compiler)); + const provider = { runtimeProvider: 'IREEAMDAIE', providerVersion: 'pinned-test', + qualificationId: 'opt-in-unqualified', providerLibrary: library, + configuration: { sourceModelSha256: compiler.sourceModelSha256, compilerConfiguration: configuration, + recognitionModels: WIDTHS.map(width => ({ width, + artifact: write(`native/amdnpu/models/recognition-${width}.vmfb`, `test bucket ${width}`) })) }, artifacts }; + const descriptor = { schemaVersion: '2.1', platform: { id: 'linux-x64' }, + runtime: { artifacts }, providers: { amdnpu: provider } }; + const error = (code, message) => Object.assign(new Error(message), { code }); + const callbacks = { + adapterError: error, + exactKeys(value, expected) { + if (!value || typeof value !== 'object' || Array.isArray(value) || + Object.keys(value).sort().join() !== [...expected].sort().join()) { + throw error('package_load_failed', 'Unexpected keys'); + } + }, + sameArtifact(a, b) { return a.path === b.path && a.bytes === b.bytes && a.sha256 === b.sha256; }, + verifyArtifact(base, artifact) { + if (!artifact || typeof artifact.path !== 'string' || artifact.path.split('/').includes('..')) { + throw error('package_load_failed', 'Invalid path'); + } + let bytes; + try { bytes = fs.readFileSync(path.join(base, artifact.path)); } + catch { throw error('package_load_failed', 'Missing artifact'); } + if (bytes.length !== artifact.bytes || crypto.createHash('sha256').update(bytes).digest('hex') !== artifact.sha256) { + throw error('package_load_failed', 'Artifact identity differs'); + } + return path.join(base, artifact.path); + }, + }; + return { root, provider, descriptor, compiler, + validate: () => validateNpuProviders(descriptor, root, callbacks), + replaceCompiler(value) { + const bytes = JSON.stringify(value); + fs.writeFileSync(path.join(root, configuration.path), bytes); + configuration.bytes = Buffer.byteLength(bytes); + configuration.sha256 = crypto.createHash('sha256').update(bytes).digest('hex'); + } }; +} +const rejected = f => assert.throws(f.validate, e => e.code === 'package_load_failed'); + +test('AMD AIE accepts 20 buckets and shared weights with the unchanged native configuration shape', t => { + const f = fixture(t); const result = f.validate(); + assert.deepEqual(result.names, ['amdnpu']); + assert.equal(result.providers.amdnpu.configuration.recognitionModels.length, 20); + assert.deepEqual(Object.keys(result.providers.amdnpu.configuration).sort(), + ['compilerConfiguration', 'recognitionModels', 'sourceModelSha256']); + assert.ok(Object.isFrozen(result.providers.amdnpu.configuration)); +}); +for (const [name, change] of [ + ['missing bucket', f => f.provider.configuration.recognitionModels.pop()], + ['reordered bucket', f => f.provider.configuration.recognitionModels.reverse()], + ['duplicate bucket path', f => { f.provider.configuration.recognitionModels[1].artifact = f.provider.configuration.recognitionModels[0].artifact; }], + ['wrong runtime library', f => { f.provider.runtimeProvider = 'VitisAIExecutionProvider'; }], + ['wrong ABI', f => { f.compiler.runtimeAbi = 2; f.replaceCompiler(f.compiler); }], + ['wrong source identity', f => { f.compiler.sourceModelSha256 = 'b'.repeat(64); f.replaceCompiler(f.compiler); }], + ['CPU partition disabled', f => { f.compiler.cpuPartitionRequired = false; f.replaceCompiler(f.compiler); }], + ['traversal in shared weight path', f => { f.compiler.parameters.path = '../outside.irpa'; f.replaceCompiler(f.compiler); }], + ['absolute shared weight path', f => { f.compiler.parameters.path = '/tmp/outside.irpa'; f.replaceCompiler(f.compiler); }], + ['shared weights missing from inventory', f => { f.provider.artifacts.splice(f.provider.artifacts.findIndex(a => a.path.endsWith('.irpa')), 1); }], + ['corrupt shared weights', f => fs.appendFileSync(path.join(f.root, 'native/amdnpu/models/recognition.irpa'), 'corrupt')], + ['missing shared weights', f => fs.unlinkSync(path.join(f.root, 'native/amdnpu/models/recognition.irpa'))], + ['null deployment JSON', f => f.replaceCompiler(null)], + ['array deployment JSON', f => f.replaceCompiler([])], +]) test(`AMD AIE rejects ${name}`, t => { const f = fixture(t); change(f); rejected(f); }); diff --git a/tests/python/test_amdaie_runtime.py b/tests/python/test_amdaie_runtime.py new file mode 100644 index 0000000..b6abe41 --- /dev/null +++ b/tests/python/test_amdaie_runtime.py @@ -0,0 +1,67 @@ +"""Exercise the shipped C ABI on a host without an AMD NPU.""" +import ctypes as c +import os +from pathlib import Path +import unittest + +PACKAGE = os.environ.get('LIGHT_OCR_AMD_TEST_PACKAGE') +Create = c.CFUNCTYPE(c.c_int, c.c_void_p, c.c_size_t, c.POINTER(c.c_void_p), c.c_void_p, c.c_size_t) +Load = c.CFUNCTYPE(c.c_int, c.c_void_p, c.c_void_p, c.c_size_t, c.c_uint32, c.POINTER(c.c_void_p), c.c_void_p, c.c_size_t) +Run = c.CFUNCTYPE(c.c_int, c.c_void_p, c.c_void_p, c.c_size_t, c.c_void_p, c.c_size_t, c.c_void_p, c.c_size_t) +Destroy = c.CFUNCTYPE(None, c.c_void_p) +class Api(c.Structure): + _fields_ = [('version', c.c_uint32), ('size', c.c_uint32), ('create', Create), + ('load', Load), ('run', Run), ('destroy_session', Destroy), ('destroy_engine', Destroy)] + +@unittest.skipUnless(PACKAGE, 'requires generated AMD package, not an AMD device') +class AmdAieRuntimeTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.root = Path(PACKAGE) + cls.library = c.CDLL(str(cls.root/'native/amdnpu/lib/liblight_ocr_amdaie.so.1')) + cls.library.LightOcrAieGetApi.argtypes = [c.c_uint32] + cls.library.LightOcrAieGetApi.restype = c.POINTER(Api) + cls.api = cls.library.LightOcrAieGetApi(1).contents + + def test_versioned_abi(self): + self.assertFalse(self.library.LightOcrAieGetApi(0)) + self.assertFalse(self.library.LightOcrAieGetApi(2)) + self.assertEqual(self.api.version, 1) + self.assertEqual(self.api.size, c.sizeof(Api)) + + def test_null_input_errors_and_null_destruction(self): + error = c.create_string_buffer(256) + handle = c.c_void_p(123) + self.assertNotEqual(self.api.create(None, 0, c.byref(handle), error, len(error)), 0) + self.assertIsNone(handle.value) + self.assertIn(b'Missing parameter archive', error.value) + handle.value = 123 + self.assertNotEqual(self.api.load(None, None, 0, 320, c.byref(handle), error, len(error)), 0) + self.assertIsNone(handle.value) + self.assertIn(b'Invalid recognition module', error.value) + self.assertNotEqual(self.api.run(None, None, 0, None, 0, error, len(error)), 0) + self.assertIn(b'tensor dimensions', error.value) + self.api.destroy_session(None) + self.api.destroy_engine(None) + + def test_error_buffer_truncation(self): + handle = c.c_void_p() + error = c.create_string_buffer(1) + self.assertNotEqual(self.api.create(None, 0, c.byref(handle), error, 1), 0) + self.assertEqual(error.raw, b'\0') + self.assertNotEqual(self.api.create(None, 0, c.byref(handle), None, 0), 0) + + def test_real_weights_without_device_fail_cleanly(self): + weights = c.create_string_buffer((self.root/'native/amdnpu/models/recognition.irpa').read_bytes()) + for _ in range(3): + handle = c.c_void_p() + error = c.create_string_buffer(2048) + status = self.api.create(weights, len(weights)-1, c.byref(handle), error, len(error)) + if not status: + self.api.destroy_engine(handle) + self.fail('this test requires a host without AMD NPU') + self.assertIsNone(handle.value) + self.assertTrue(error.value) + self.assertRegex(error.value.decode(), r'(?i)(amdxdna|device|accel|driver)') + +if __name__ == '__main__': unittest.main() diff --git a/tools/npu/check_cpu_recognition.py b/tools/npu/check_cpu_recognition.py new file mode 100644 index 0000000..e48ca1a --- /dev/null +++ b/tools/npu/check_cpu_recognition.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Compare the original ONNX and IREE recognition transforms on CPU only. + +The partitioned CPU model emulates BF16 operands with CPU matmul; it does not +execute the AMD BFP16ebs8 microkernel or qualify the packaged NPU binaries. +""" +from __future__ import annotations +import argparse +import json +from pathlib import Path +import subprocess +import sys + +REFERENCE = '''import sys,numpy as np,onnxruntime as ort +options=ort.SessionOptions();options.intra_op_num_threads=2 +session=ort.InferenceSession(sys.argv[1],options,providers=["CPUExecutionProvider"]) +for source,destination in zip(sys.argv[2::2],sys.argv[3::2]): + np.save(destination,session.run(None,{session.get_inputs()[0].name:np.load(source)})[0]) +''' +COMPARE = '''import sys,json,numpy as np +records=[] +def decode(array): + result=[];previous=-1 + for value in array.argmax(-1).reshape(-1).tolist(): + if value!=previous and value!=0:result.append(value) + previous=value + return result +for record in json.load(open(sys.argv[1])): + reference=np.load(record.pop("reference"));fp32=np.load(record.pop("fp32"));bf16=np.load(record.pop("bf16")) + record["referenceCtcClassIds"]=decode(reference) + assert reference.shape==fp32.shape==bf16.shape==(1,record["width"]//8,18710) + assert all(np.isfinite(value).all() for value in (reference,fp32,bf16)) + for label,value in [("fp32",fp32),("bf16CpuEmulation",bf16)]: + delta=np.abs(value-reference) + record[label]={"maxAbsoluteError":float(delta.max()),"meanAbsoluteError":float(delta.mean()), + "argmaxAgreement":float(np.mean(reference.argmax(-1)==value.argmax(-1))), + "ctcClassSequenceMatches":decode(reference)==decode(value)} + record["fp32WithinTolerance"]=bool(np.allclose(reference,fp32,rtol=1e-3,atol=1e-4)) + records.append(record) +json.dump({"cases":records,"fp32Passed":all(r["fp32WithinTolerance"] for r in records), + "bf16CpuEmulationOnly":True,"amdMicrokernelExecuted":False,"deviceValidated":False},open(sys.argv[2],"w"),indent=2) +assert all(r["fp32WithinTolerance"] for r in records),"FP32 conversion differs beyond rtol=1e-3/atol=1e-4" +''' + +def main(): + p=argparse.ArgumentParser(description=__doc__) + for name in ('source-model','frontend-environment','reference-python','toolchain-dir','runtime-tool','output-dir'): + p.add_argument('--'+name,type=Path,required=True) + p.add_argument('--width',type=int,action='append') + p.add_argument('--text-image',type=Path,help='Optional text fixture; reference Python must provide OpenCV') + a=p.parse_args();out=a.output_dir.resolve() + if out.exists():p.error('select a fresh output directory') + out.mkdir(parents=True) + tool=Path(__file__).parent.resolve();frontend=a.frontend_environment.resolve()/'bin/python' + compiler=a.toolchain_dir.resolve()/'tools/iree-compile';runtime=a.runtime_tool.resolve() + records=[] + def run(command,log): + with log.open('w') as stream:subprocess.run(list(map(str,command)),stdout=stream,stderr=subprocess.STDOUT,check=True,timeout=900) + for width in a.width or [320,384,3200]: + work=out/str(width);work.mkdir();print(f'CPU comparison width={width}',flush=True) + run([frontend,tool/'export_iree_model.py','--source-model',a.source_model.resolve(),'--output-dir',work/'model','--width',width],work/'export.log') + run([sys.executable,tool/'import_iree_model.py','--model-dir',work/'model','--frontend-environment',a.frontend_environment.resolve(),'--output-dir',work/'imported'],work/'import.log') + source=work/'imported/core-linalg.mlir' + fp32=work/'fp32.mlir';fp32.write_text(source.read_text().replace('PaddlePaddle Graph in PIR mode','recognition')) + partitioned=work/'partitioned.mlir' + run([frontend,tool/'partition_iree_model.py','--input',source,'--output',partitioned,'--report',work/'partition.json'],work/'partition.log') + run([frontend,'-c', '''import numpy as np,sys +from pathlib import Path +p=Path(sys.argv[1]);w=int(sys.argv[2]);shape=(1,3,48,w) +np.save(p/'zero.npy',np.zeros(shape,np.float32)) +np.save(p/'random.npy',np.random.default_rng(20261008).uniform(-1,1,shape).astype(np.float32)) +np.save(p/'gradient.npy',np.broadcast_to(np.linspace(-1,1,w,dtype=np.float32),shape).copy()) +''',work,width],work/'inputs.log') + cases=['zero','random','gradient'] + if a.text_image: + run([a.reference_python.absolute(),'-c', '''import cv2,numpy as np,sys,math +image=cv2.imread(sys.argv[1]);assert image is not None +rows,cols=np.where(cv2.cvtColor(image,cv2.COLOR_BGR2GRAY)<200) +assert len(rows),"text fixture has no foreground" +image=image[max(0,rows.min()-2):rows.max()+3,max(0,cols.min()-2):cols.max()+3] +w=int(sys.argv[3]);content=min(w,math.ceil(48*image.shape[1]/image.shape[0])) +resized=cv2.resize(image,(content,48),interpolation=cv2.INTER_LINEAR) +tensor=np.zeros((1,3,48,w),np.float32) +tensor[0,:,:,:content]=(resized.astype(np.float32)/127.5-1).transpose(2,0,1) +np.save(sys.argv[2],tensor) +''',a.text_image.resolve(),work/'text.npy',width],work/'text-input.log') + cases.append('text') + pairs=[value for case in cases for value in (work/f'{case}.npy',work/f'{case}-reference.npy')] + run([a.reference_python.absolute(),'-c',REFERENCE,a.source_model.resolve(),*pairs],work/'reference.log') + for label,ir in [('fp32',fp32),('bf16',partitioned)]: + binary=work/f'{label}.vmfb' + run([compiler,ir,'--iree-hal-target-device=local','--iree-hal-local-target-device-backends=llvm-cpu','--iree-llvmcpu-target-cpu=generic','-o',binary],work/f'{label}-compile.log') + for case in cases: + run([runtime,'--device=local-sync',f'--module={binary}','--function=recognition',f'--input=@{work}/{case}.npy',f'--output=@{work}/{case}-{label}.npy'],work/f'{case}-{label}-run.log') + for case in cases:records.append({'width':width,'case':case,'reference':str(work/f'{case}-reference.npy'),'fp32':str(work/f'{case}-fp32.npy'),'bf16':str(work/f'{case}-bf16.npy')}) + paths=out/'comparison-inputs.json';paths.write_text(json.dumps(records)) + run([frontend,'-c',COMPARE,paths,out/'cpu-comparison.json'],out/'comparison.log') + print(out/'cpu-comparison.json',flush=True) + +if __name__=='__main__':main()