From 5017661dc980c5b11aeeadc8124dd9a0971b29c4 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Thu, 3 Sep 2026 09:23:14 +0700 Subject: [PATCH 1/3] init: report of llama.cpp commits working on supporting hexagon --- docs/backend/hexagon.md | 328 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 328 insertions(+) create mode 100644 docs/backend/hexagon.md diff --git a/docs/backend/hexagon.md b/docs/backend/hexagon.md new file mode 100644 index 0000000..3e9acad --- /dev/null +++ b/docs/backend/hexagon.md @@ -0,0 +1,328 @@ +# `vla.cpp` on Snapdragon (Hexagon NPU) + +Notes on what it would take to run `vla.cpp` on Qualcomm's Hexagon NPU, read out +of llama.cpp's own Hexagon history up to `b10729` - the tag this tree pins. + +> **Status: read and planned, never run.** There is no Hexagon rung in +> `src/backend.h`, no `GGML_HEXAGON` in `CMakeLists.txt`, and no IQ-9 or IQ-10 +> board on any desk here. Every claim below is either checked against llama.cpp +> at `b10729` (op tables, env vars, commit hashes - go verify them) or is +> explicitly a **projection**. Nothing in this file is a measurement on Qualcomm +> hardware, and the projected latencies are arithmetic, not results. +> +> Two things upstream did in 2026 make this worth writing down now: the NPU +> backend learned `IM2COL` for patch-embedding convolutions - which is exactly +> and only what a VLA's vision tower needs - and it learned multi-NPU. Both are +> already inside the tag we pin. + +The previous version of this file was a verbatim copy of upstream's +[`docs/backend/snapdragon/linux.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md). +That build recipe is still the right starting point and is kept below, but it +builds *llama.cpp*, not this engine; the gap between the two is the subject here. + +## What "IQ9" and "IQ10" refer to + +Two commits, both inside `b10729` (`458681e1`, 2026-09-01), and no Hexagon commit +lands after it as of this writing: + +| Commit | Date | What it actually did | +|---|---|---| +| `d81e63dc` `CI : support IOT device (IQ9) (#22987)` | 2026-05-14 | Added a third device to the Snapdragon CI matrix: `QCS9075M`, alongside the `SM8750` / `SM8850` phone parts. It is the only entry that is **Linux**, not Android - a separate build job (`linux-iot-snapdragon`, with `-DGGML_OPENCL=ON` on top of the preset), a BASH test framework instead of Appium/pytest, and `--retries 2 --retry-delay 300` because the IoT device in Qualcomm's Device Cloud is scarce. | +| `192067b7` `hexagon: support for multi-NPU devices (IQ9, IQ10) and fully asynchronous backend (#26501)` | 2026-08-26 | 5,314 insertions over 44 files, by Qualcomm. Physical vs virtual NPU sessions, an `ALLREDUCE` HTP kernel and sync tokens so a tensor split can span NPUs, a fully async backend with events, `GET_ROWS`/`SET_ROWS`, Q8_0 flash attention, op fusion, and the `build.py` / `run.py` scripts that replaced the old `adb/*.sh` and `windows/*.ps1` wrappers. | + +What the names do **not** tell you: neither `IQ9` nor `IQ10` appears anywhere in +the source, the docs, or the CI config as an identifier. The only board named in +code is `QCS9075M`, which places IQ-9 as the Dragonwing-class industrial/robotics +part rather than a phone SoC. IQ-10 exists in that commit title and nowhere else. +So "IQ9/IQ10" is upstream shorthand; what is real is the *capability* the commit +added, which is what the rest of this document reads against. + +The generational split visible in the code, rather than in the marketing: + +- **One physical NPU, N virtual sessions** (`HTP0:0`, `HTP0:1`, ...) - the IQ-9 + shape, and what every phone part does today. +- **N physical NPUs** (`HTP0:0`, `HTP1:0`) - discovered via `FASTRPC_GET_DOMAINS`, + with a static fallback that hardcodes physical 0 → CDSP domain 3 (`cdsp`) and + physical 1 → domain 4 (`cdsp1`). Ask for a third physical core without dynamic + discovery and the backend refuses by name: *"physical CDSP core %d not + supported without dynamic discovery"*. Two is what the fallback knows about; + the discovery path is open-ended. + +## What the backend gives you at `b10729` + +- **Two libraries.** `libggml-hexagon.so` on the CPU side, `libggml-htp-vNN.so` + on the NPU side. Skels are built for v68, v69, v73, v75, v79 and v81 and the + right one is picked at runtime from `htpdrv_get_arch`; a failed query falls + back to v73. `GGML_HEXAGON_ARCH` overrides. +- **Sessions are devices.** Each Hexagon process domain shows up to ggml as one + device and behaves like a GPU for offload and model splitting. + `GGML_HEXAGON_DEVICES` takes either a count or an explicit + `HTP:` list. `GGML_HEXAGON_NDEV` is the deprecated spelling. +- **~3.5 GB per session.** The backend now maps and unmaps execution buffers + during graph execution to fit larger models into one session, and layer- or + tensor-splitting across sessions is the alternative. +- **Repack buffers.** Q4_0, Q4_1, Q8_0, IQ4_NL and MXFP4 weights are repacked + into non-host buffers; since #26501 non-host is the default and + `GGML_HEXAGON_HOSTBUF=1` is the opt-out (needed to exercise `MUL_MAT` in + `test-backend-ops`). +- **VTCM is the real budget.** `supports_op` precomputes kernel params for + `MUL_MAT`, `FLASH_ATTN_EXT` and friends and returns false when the tile does + not fit VTCM. An op is not rejected by shape rules so much as by whether it + fits - which means coverage is a function of your tensor sizes, and has to be + measured on the board, not predicted from a table. +- **Fusion**, controlled by `GGML_HEXAGON_OPFUSION`: `RMS_NORM+MUL`, + `MUL_MAT+ADD`, N-way `MUL_MAT`, `ALLREDUCE+ADD`. + +The knobs worth knowing on day one: + +| Variable | Use | +|---|---| +| `GGML_HEXAGON_DEVICES=HTP0:0,HTP1:0` | which NPUs/sessions to open | +| `GGML_HEXAGON_VERBOSE=1` | log every op the NPU accepted, with dtypes and buffers | +| `GGML_HEXAGON_OPFILTER=` | force matching ops off the NPU - the bisection tool | +| `GGML_HEXAGON_PROFILE=1\|2` | per-op usecs/cycles (+PMU), pipe into `scripts/snapdragon/ggml-hexagon-profile.py -` | +| `GGML_HEXAGON_OPTRACE` | fine-grained HVX/HMX/DMA event trace | +| `GGML_HEXAGON_OPPOLL=1` | poll for op completion instead of sleeping - matters for many small graphs | +| `GGML_HEXAGON_HOSTBUF=1` | disable repack buffers (op testing) | + +## Op coverage against what `vla.cpp` actually builds + +This is the part that decides whether the port is a weekend or a month. The +column on the right is `ggml_backend_hexagon_device_supports_op` at `b10729`; +the middle column is a grep of `src/`. + +| ggml op | Where we build it | Hexagon at `b10729` | +|---|---|---| +| `MUL_MAT` / `MUL_MAT_ID` | 154 sites, everywhere | yes - Q4_0/Q4_1/Q8_0/IQ4_NL/MXFP4 (repacked), or F16/F32 direct. **No BF16.** Refuses `src0->ne[1] > 32768` (lm-heads), and anything whose tile misses VTCM | +| `IM2COL` | 6 `ggml_conv_2d`, every SigLIP/DINOv2 patch embed | yes, **partial** - added by `355303ed` (#26007) "targeting only patch-embedding convolutions": 2D only, F32 source, contiguous, **zero padding**. Our conv is kernel = stride = patch, pad 0, dilation 1. On paper it is precisely the supported case | +| `FLASH_ATTN_EXT` | 8 sites | yes - F16 or Q8_0 K/V, F16 mask, F32 sinks | +| `ROPE` / `ROPE_MULTI` | 14 sites | yes, including MROPE/IMROPE (`17d22a35`) and vision RoPE (`f2d1c2f3`) | +| `NORM`, `RMS_NORM`, `L2_NORM`, `SOFT_MAX`, `SCALE`, `CONCAT`, `SQR`, `GET_ROWS`, `ADD/SUB/MUL/DIV` | all over | yes | +| `CPY` / `CONT` (`ggml_cast`, 14 sites) | dtype hops between stages | yes, but **F32↔F16 only, and never a reshape and a conversion in the same op** | +| `SILU`, `GELU` (tanh), `GELU_QUICK`, `TANH`, `SIGMOID`, `NEG`, `ABS`, `EXP`, `SOFTPLUS` | 22 `ggml_silu`, 10 `ggml_gelu` | yes | +| **`RELU`** | **18 sites** - `action_expert.cpp` state encoder and action decoder, plus Evo-1, VLA-Adapter, VLA-JEPA, GR00T N1.7, OpenVLA-OFT, BitVLA | **not implemented.** `grep -i relu` over `ggml-hexagon/` returns nothing | +| **`GELU_ERF`** | **13 sites** - `ffn_gelu_erf` in `src/layers/ffn.h` and `dual_tower.h`, i.e. every tower trained with the exact erf form (DINOv2, SigLIP-so400m) | **not implemented.** `ggml_gelu` (tanh) is a *different op*, and substituting it silently changes numerics | +| `ggml_map_custom1` | BitVLA only | no, and BitVLA pins itself to CPU by design anyway | + +Two missing unary kernels, and one of them is in the vision tower of half the +model zoo. That is the whole gap. Note the shape of the mistake it invites: on +OpenVINO, "GELU is GELU" cost us a wrong arch and a day of bisection +([ov.md](ov.md)); here the op is simply absent, which is the *better* failure - +`supports_op` returns false and you find out at load time. + +**Except that in this engine, absent is fatal.** `src/backend.h` brings up one +backend and drives it through `gallocr` - there is no scheduler and no per-op CPU +fallback, by design, so an unsupported op asserts at predict time instead of +quietly limping. Every other backend we support (CUDA, Metal, SYCL, OpenVINO) +covers our op set completely, so this constraint has never bitten. Hexagon would +be the first backend where it does. + +So the port is gated on one of: + +1. **Two HVX kernels upstream.** `e70802a0` (#27786) added `ABS` and `LOG` as HTP + unary ops; `RELU` and `GELU_ERF` are the same shape of change, and `RELU` in + particular is a `vmax` against zero. This is the honest path and it is small. +2. **A scheduler in `vla.cpp`.** `ggml_backend_sched` with a CPU fallback, which + is an engine change with consequences for every backend, not a Hexagon one. +3. **Graph surgery** - swapping `gelu_erf` for `gelu` and re-checking fidelity. + Cheap, wrong, and it will cost more than it saves. Do not. + +## What would have to change in `vla.cpp` + +Nothing here has been written; this is the shape of the diff. + +**1. Build flags.** `GGML_HEXAGON` joins the mutually-exclusive accelerator list +at [CMakeLists.txt:15](../../CMakeLists.txt#L15) and defines `GGML_USE_HEXAGON` +the way the other four do. + +**2. A rung in the ladder** at [src/backend.h:176](../../src/backend.h#L176). +The Hexagon init API takes no ordinal - `ggml_backend_hexagon_init(void)` - so +device selection is environment-driven, and `VLA_DEVICE` (an integer ordinal for +CUDA and SYCL) does not map onto it. The right binding is `VLA_DEVICE=HTP0:0` +passed through to `GGML_HEXAGON_DEVICES`, which means the rung needs its own +parse rather than `backend_device_index()`. + +**3. Weights.** Our default resident dtype is BF16, which the NPU does not +accept in any op. Every Hexagon run needs a requantized checkpoint - +`python scripts/quantize_gguf.py --in model-bf16.gguf --out model-q8_0.gguf +--type Q8_0` (or `Q4_0`) - and the fidelity of *that* against the CPU reference +has to be established before any NPU number means anything. `--weight-dtype f32` +is the other option and doubles resident memory. + +**4. The build that is actually the most work: `vla-server`'s dependencies.** +`find_package(Protobuf REQUIRED)` and `pkg_check_modules(libzmq REQUIRED)` are +unconditional in our `CMakeLists.txt`, and the Snapdragon toolchain image carries +the Hexagon SDK and the OpenCL SDK but not protobuf, libzmq or cppzmq for arm64. +Configure fails before a single object compiles - including for `vla-cli` and +`vla-bench`, which link neither. Options, cheapest first: + +- Make the serving deps conditional so `vla-cli` / `vla-bench` configure without + them. Worth doing regardless of Hexagon; it is a dozen lines. +- Add the three to the cross sysroot. +- Build natively on the board, which for an IQ-9-class Linux part is plausible + and is how the first bring-up will probably actually happen. + +**5. Presets.** We have no `CMakeUserPresets.json`; upstream's +`arm64-linux-snapdragon-release` is the base to copy, with `GGML_HEXAGON=ON` and +`-march=armv8.2a+fp16+dotprod`. Note it sets `GGML_OPENCL=OFF` - upstream CI adds +`-DGGML_OPENCL=ON` on the command line for the IoT job. + +## Building llama.cpp for the board + +The upstream path, unchanged, and the thing to get working first - if +`llama-bench` will not run on the device, nothing downstream matters. Requires +only Docker on the host; `build.py` pulls +`ghcr.io/snapdragon-toolchain/arm64-linux:v0.7` itself. + +```bash +# cross-compile and deploy over SSH in one step +./scripts/snapdragon/build.py --target lnx:user@host --push + +# or by hand, inside the container +docker run -it --rm -u $(id -u):$(id -g) --volume $(pwd):/workspace \ + --platform linux/amd64 ghcr.io/snapdragon-toolchain/arm64-linux:v0.7 +[d]/workspace> cp docs/backend/snapdragon/CMakeUserPresets.json . +[d]/workspace> cmake --preset arm64-linux-snapdragon-release -B build-snapdragon +[d]/workspace> cmake --build build-snapdragon -j $(nproc) +[d]/workspace> cmake --install build-snapdragon --prefix pkg-linux +``` + +On the device, both library paths must be set - `ADSP_LIBRARY_PATH` is how the +FastRPC loader finds the HTP skel: + +```bash +export LD_LIBRARY_PATH=./lib +export ADSP_LIBRARY_PATH=./lib +./bin/llama-cli -m Llama-3.2-3B-Instruct-Q4_0.gguf --device HTP0 -ngl 99 -p "..." +``` + +`run.py` maps flags to those environment variables and can drive the device over +SSH or ADB from the host: + +```bash +./scripts/snapdragon/run.py --target lnx:user@host --devices HTP0 -- \ + llama-bench -m Llama-3.2-1B-Instruct-Q4_0.gguf -p 128 -n 64 + +# multi-NPU tensor split - the IQ-10 shape +./scripts/snapdragon/run.py --target lnx:user@host --devices HTP0:0,HTP1:0 -- \ + llama-completion -m gemma-2b-it-Q4_0.gguf -f prompt.txt --split-mode tensor +``` + +## Bring-up checklist, for the day a board arrives + +In order. Each step is cheap and each one has failed for somebody upstream. + +1. **Confirm the arch and session lines.** `ggml-hex: Hexagon Arch version v79` + and `allocating new session: HTP0:0`. A v73 line on a part you believe is + newer means `htpdrv_get_arch` failed and you are on the fallback skel. +2. **`test-backend-ops` before anything of ours.** + `run.py --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT`, + then `-o IM2COL`, `-o FLASH_ATTN_EXT`, `-o ROPE`. Filter to the dtypes we + would actually ship (`q8_0`, `q4_0`, `f16`). +3. **`llama-bench` on a 1B Q4_0** to get a throughput anchor on *this* board that + is comparable with upstream's published numbers, before any VLA is involved. +4. **Load a requantized VLA with `GGML_HEXAGON_VERBOSE=1`** and read which ops the + NPU accepted. This is where the two missing unaries will announce themselves. +5. **Fidelity before latency.** max|delta| against a CPU-backend reference on the + same checkpoint, the same 2.9e-3 bar SYCL and OpenVINO are held to + ([ov-progress.md](ov-progress.md)). A partially-failing run still prints a + wall-clock time; do not read a latency off a run whose actions did not come out. +6. **Then profile.** `GGML_HEXAGON_PROFILE=1 ... |& scripts/snapdragon/ggml-hexagon-profile.py -`, + and `GGML_HEXAGON_OPFILTER` to bisect anything that looks wrong. + +## Projections + +Everything in this section is arithmetic over numbers measured elsewhere. It is +here to set expectations and to be *falsified* by the first real run, not to be +quoted. + +Measured reference points, all ours, all `server_total_ms` per action chunk +(`ci/baselines/`, [sycl.md](sycl.md)): + +| Device | SmolVLA | π0 | GR00T N1.5 | Evo-1 | +|---|---:|---:|---:|---:| +| Ryzen 5 5500, CPU backend, 8 threads | 1,920 | - | - | 7,695 | +| Jetson Orin Nano | 510 | 1,485 | 1,183 | 3,552 | +| Apple M4 (Metal) | 324 | 1,129 | - | - | +| Arc A380 (SYCL) | 630 | - | - | 1,176 | +| RTX 3090 | 99 | 262 | 204 | 487 | + +The only Hexagon anchor that exists is upstream's own, on a v79 phone part: +Llama-3.2-1B Q4_0, **pp128 = 169 t/s**, **tg64 = 51.5 t/s** (single `HTP0` +session). Read as work rather than tokens: prefill at 169 t/s over 1.24 B params +is ≈ 0.42 TFLOP/s effective; decode at 51.5 t/s over 730 MiB of weights is +≈ 37 GB/s of weight traffic, which is a believable LPDDR5 read rate and a good +sign the published number is not cherry-picked. + +A VLA action chunk is **all prefill and no decode**: fixed shapes, a tiny KV +cache, no sampling loop. That is the regime the 0.42 TFLOP/s figure comes from, +which makes the extrapolation less dishonest than usual. SmolVLA is 1.07 GiB of +BF16 weights (≈ 0.54 B params), spent on three stages per prediction - a vision +tower over 2-3 views, a ~113-token prefix through the VLM, and 10 denoise steps +over a ~50-token suffix through the much smaller action expert. Multiplying that +out lands in the region of 300-400 GFLOP per chunk, so at the anchor rate: + +**SmolVLA, single v79-class HTP, Q4_0: 0.8-2.0 s per action chunk.** The band is +wide on purpose. The bottom needs the vision tower to land on the NPU via the new +`IM2COL` and the denoise loop to stay fed; the top is what you get if the loop is +dispatch-bound and the towers fall back. Either way that is somewhere between the +Orin Nano (510 ms) and a desktop CPU (1,920 ms) - useful for a 5-10 Hz policy on +a part that draws a few watts, not competitive with a discrete GPU. Expect the +first honest number to be nearer the top of the band and to improve with repack, +fusion and `OPPOLL`, exactly as upstream's own numbers did over the 107 commits +that have touched `ggml/src/ggml-hexagon/` since the backend landed in October +2025 (`63d2fc46`, "experimental"). + +Where the interesting risk is: + +- **The vision tower is the prize.** It is 58% of SmolVLA's CPU-backend time + (1,119 ms of 1,920 ms) and 7.1x faster on an A380. `IM2COL` landing for + patch-embed convolutions specifically is the single most encouraging thing in + this entire history, and it landed in July 2026 for reasons that had nothing to + do with us. +- **The denoise loop is the risk.** Ten sequential steps of small GEMMs is the + worst case for any offload engine - it is why SmolVLA gains least on SYCL + (3.0x when its own vision tower gains 7.1x). The fully-async backend, op + batching and graph reuse from #26501 are aimed at exactly this, which is + encouraging, but "many small ops over FastRPC" is a dispatch-latency problem + and NPUs do not usually win those. +- **The lm-head refusal does not touch us.** Hexagon bails on `src0->ne[1] > + 32768`; VLA action heads are small and no in-tree arch decodes a vocabulary on + the action path. + +### IQ-10, imagined + +Nothing about a second-generation part is knowable from this repository, so what +follows is a bet, not a forecast. The code in #26501 tells you what Qualcomm +built *for*: two or more physical NPUs, an `ALLREDUCE` kernel with a DMA solver +and fused `ALLREDUCE+ADD`, sync tokens to order work across devices, and a static +domain table that stops at two. + +If a dual-NPU IQ-10 lands, the obvious use - `--split-mode tensor` across +`HTP0:0,HTP1:0`, paying an allreduce per layer - is the *worse* fit for a VLA. +An action chunk has three stages that already hand off through host memory: the +vision tower, the VLM prefix, and the action expert. Putting the tower on one NPU +and the expert on the other pipelines across timesteps with **one** handoff per +stage instead of one collective per layer, and a policy server at 10 Hz has a +steady stream of chunks to pipeline. + +`vla.cpp` cannot do that today, for the same reason it cannot split across an +Intel iGPU and NPU ([ov.md](ov.md) reaches this conclusion from the other +direction): the core drives one backend for a whole prediction. It would need a +per-*stage* backend, not a per-op scheduler. The seam is already in the right +place. That is the one engine change this whole document argues for, and it pays +off on more than Hexagon. + +## What would falsify all of this + +- `RELU` and `GELU_ERF` turn out to be a 200-line HVX patch each → the port is a + weekend once the board exists, and everything above about schedulers is moot. +- `IM2COL`'s "partial" support turns out not to cover our patch shapes after all + (multi-view batching, a non-contiguous pixel buffer) → the vision tower stays + on CPU, the projection's lower bound disappears, and the whole exercise gets + much less interesting. +- Q8_0 requantization moves any arch past 2.9e-3 → the dtype story needs work + before the backend story does. +- VTCM budgets reject our GEMM shapes (which are wider and shorter than an LLM's) + → `supports_op` starts returning false for reasons no table here predicts, and + the answer is measurement, not more reading. From a587dec93fae75cc47fe46606ce90fb9427a9338 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Sun, 27 Sep 2026 10:18:31 +0700 Subject: [PATCH 2/3] Add Snapdragon X (Windows on Arm) support: Hexagon NPU and Adreno OpenCL backends with CPU fallback --- CHANGELOG.md | 18 + CMakeLists.txt | 132 ++++- README.md | 50 +- cmake/arm64-windows-llvm.cmake | 20 + cmake/copy_htp_skels.cmake | 10 + cmake/patch_sentencepiece_fpic.cmake | 8 + .../arm64-windows-clangcl.cmake | 22 + cmake/vcpkg-triplets/clangcl-toolchain.cmake | 26 + docs/backend/hexagon-windows.md | 305 ++++++++++++ docs/backend/hexagon.md | 309 +++++------- scripts/build_windows_snapdragon.ps1 | 92 ++++ src/backend.h | 140 +++++- src/backend_fallback.cpp | 456 ++++++++++++++++++ src/backend_fallback.h | 62 +++ src/gguf_reader.h | 26 +- src/layers/ffn.h | 3 +- src/model.cpp | 3 +- src/models/evo1.cpp | 6 +- src/models/gr00tn1d5.cpp | 2 +- src/models/gr00tn1d6.cpp | 2 +- src/models/gr00tn1d7.cpp | 2 +- src/models/openvla_oft.cpp | 2 +- src/models/pi0.cpp | 13 +- src/models/pi05.cpp | 8 +- src/models/smolvla.cpp | 90 +++- src/models/vla_adapter.cpp | 2 +- src/models/vla_jepa.cpp | 2 +- src/modules/qwen3vl_vit.h | 5 +- src/options.cpp | 14 +- src/serving/server.cpp | 7 + src/serving/vla-cli.cpp | 30 +- src/serving/vlm-server.cpp | 7 + tests/CMakeLists.txt | 5 + tests/predict_check.cpp | 9 +- 34 files changed, 1612 insertions(+), 276 deletions(-) create mode 100644 cmake/arm64-windows-llvm.cmake create mode 100644 cmake/copy_htp_skels.cmake create mode 100644 cmake/patch_sentencepiece_fpic.cmake create mode 100644 cmake/vcpkg-triplets/arm64-windows-clangcl.cmake create mode 100644 cmake/vcpkg-triplets/clangcl-toolchain.cmake create mode 100644 docs/backend/hexagon-windows.md create mode 100644 scripts/build_windows_snapdragon.ps1 create mode 100644 src/backend_fallback.cpp create mode 100644 src/backend_fallback.h diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a423c6..f98d4c6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,20 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke ### Added +- **Snapdragon X on Windows on Arm: Hexagon NPU, Adreno GPU and CPU.** + - `-DGGML_HEXAGON=ON` and `-DGGML_OPENCL=ON` build natively with Visual + Studio's Clang; `scripts/build_windows_snapdragon.ps1` drives the build, + including skel signing. + - Ops either accelerator rejects run on the CPU through a wrapper backend + (`src/backend_fallback.cpp`), with no change to any arch. + - SmolVLA runs in 1.23 s on the NPU (2.57 s CPU), within 1.5e-3 of the CPU + reference; eleven checkpoints run on both accelerators. + - Five ggml-hexagon kernels that give wrong answers for VLA shapes are routed + around. See `docs/backend/hexagon-windows.md`. +- `--weight-dtype f16`. It is the default on Hexagon and OpenCL, and 2.5-3.5x + faster than BF16 on CPUs without BF16 matmul. +- `VLA_BUILD_SERVER=OFF` builds `vla-cli` and `vla-bench` without protobuf or + ZeroMQ. - **OpenVINO backend.** `-DGGML_OPENVINO=ON` runs the archs on Intel CPUs, iGPUs and NPUs through ggml's OpenVINO backend. SmolVLA, π0.5, Evo-1 and VLA-Adapter match an F32 CPU reference to 1e-3; on an Arc B390 iGPU that is 3.0x to 9.6x @@ -31,6 +45,10 @@ Notable changes to vla.cpp. Format loosely follows [Keep a Changelog](https://ke ### Fixed +- A `scripts/quantize_gguf.py` file did not load for SmolVLA on any platform: + its loader read every weight as float and refused the packed connector. It now + keeps packed GEMM weights packed, like the other archs, and dequantizes the + rest. - Two elementwise adds stacked on a GEMM came out wrong on the Intel iGPU. The GPU plugin folds elementwise ops into the preceding GEMM as post-ops, and given `ADD(ADD(residual, GEMM), graph_input)` it folds both and silently drops the diff --git a/CMakeLists.txt b/CMakeLists.txt index 11fece1..5af93a5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ if(NOT CMAKE_BUILD_TYPE) endif() set(_vla_accel "") -foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL GGML_OPENVINO) +foreach(_flag GGML_CUDA GGML_SYCL GGML_METAL GGML_OPENVINO GGML_HEXAGON GGML_OPENCL) if(${_flag}) list(APPEND _vla_accel ${_flag}) endif() @@ -111,19 +111,55 @@ if(VLA_OCTO) set(SPM_BUILD_TEST OFF CACHE BOOL "" FORCE) set(SPM_ENABLE_TCMALLOC OFF CACHE BOOL "" FORCE) set(SPM_PROTOBUF_PROVIDER "package" CACHE STRING "" FORCE) + set(_vla_spm_patch "") + if(WIN32 AND NOT MSVC) + set(_vla_spm_patch PATCH_COMMAND ${CMAKE_COMMAND} -P + ${CMAKE_CURRENT_SOURCE_DIR}/cmake/patch_sentencepiece_fpic.cmake) + endif() FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG v0.2.0 GIT_SHALLOW TRUE + ${_vla_spm_patch} ) # sentencepiece v0.2.0 still asks for cmake_minimum_required(VERSION 3.1), # which CMake 4 refuses outright. set(CMAKE_POLICY_VERSION_MINIMUM 3.5) FetchContent_MakeAvailable(sentencepiece) unset(CMAKE_POLICY_VERSION_MINIMUM) + # Only the static library is linked; its CLI tools are dead weight. + vla_exclude_fetched_targets(${sentencepiece_SOURCE_DIR}) + # vcpkg's FindProtobuf shim reports the DLL, not its import library, as + # PROTOBUF_LITE_LIBRARY, and sentencepiece links that path verbatim. Link the + # imported target instead, which carries the import library and protobuf's + # abseil deps. The full runtime, not lite: vla-server links it too, and one + # protobuf runtime per binary is the rule above. + # Imported targets are directory-scoped, so find it again up here. + if(WIN32) + find_package(Protobuf CONFIG QUIET) + endif() + # The link goes on vla_core below rather than on sentencepiece-static: + # names in a target's link interface resolve in the directory that made the + # target, and sentencepiece's directory has its own copies of abseil's + # imported targets, which still carry the MSVC flag stripped further down. + if(WIN32 AND TARGET protobuf::libprotobuf) + set_property(TARGET sentencepiece-static PROPERTY + INTERFACE_LINK_LIBRARIES Threads::Threads) + set(VLA_SPM_PROTOBUF protobuf::libprotobuf) + endif() +endif() + +# llama.cpp turns BUILD_SHARED_LIBS on, so these follow it. That is harmless on +# ELF, which exports everything, but a Windows DLL exports nothing unannotated: +# the import library comes out empty and every consumer fails to link. Nothing +# outside this tree links the cores, so on Windows they are static. +if(WIN32) + set(VLA_CORE_LIB_TYPE STATIC) +else() + set(VLA_CORE_LIB_TYPE "") endif() -add_library(vla_core +add_library(vla_core ${VLA_CORE_LIB_TYPE} src/model.cpp src/loader.cpp src/options.cpp @@ -156,7 +192,7 @@ target_link_libraries(vla_core PUBLIC ggml) if(VLA_OCTO) target_sources(vla_core PRIVATE src/models/octo.cpp) target_include_directories(vla_core PRIVATE ${sentencepiece_SOURCE_DIR}/src) - target_link_libraries(vla_core PRIVATE sentencepiece-static) + target_link_libraries(vla_core PRIVATE sentencepiece-static ${VLA_SPM_PROTOBUF}) target_compile_definitions(vla_core PUBLIC VLA_USE_OCTO) endif() @@ -248,7 +284,20 @@ if(GGML_OPENVINO AND NOT GGML_CUDA AND NOT GGML_SYCL AND NOT GGML_METAL) target_compile_definitions(vla_core PUBLIC GGML_USE_OPENVINO) endif() -add_library(vlm_core +# Hexagon and OpenCL reject some ops the archs build; the wrapper in +# backend_fallback.cpp runs those on the CPU. It reaches into ggml's internal +# headers for the backend vtable, hence the private include. +if(GGML_HEXAGON OR GGML_OPENCL) + target_sources(vla_core PRIVATE src/backend_fallback.cpp) + target_include_directories(vla_core PRIVATE ${llama_SOURCE_DIR}/ggml/src) + if(GGML_HEXAGON) + target_compile_definitions(vla_core PUBLIC GGML_USE_HEXAGON) + else() + target_compile_definitions(vla_core PUBLIC GGML_USE_OPENCL) + endif() +endif() + +add_library(vlm_core ${VLA_CORE_LIB_TYPE} src/vlm/engine.cpp ) target_include_directories(vlm_core @@ -261,9 +310,23 @@ target_include_directories(vlm_core ) target_link_libraries(vlm_core PUBLIC llama ggml mtmd llama-common) +# vla-cli and vla-bench link neither protobuf nor ZeroMQ; turning this off lets +# them configure on a machine that has not installed the serving stack. +option(VLA_BUILD_SERVER "Build vla-server and vlm-server (needs protobuf, libzmq, cppzmq)" ON) +if(VLA_BUILD_SERVER) find_package(Protobuf REQUIRED) -find_package(PkgConfig REQUIRED) -pkg_check_modules(ZeroMQ REQUIRED IMPORTED_TARGET libzmq) +# vcpkg (the Windows route) ships a CMake package and no pkg-config binary; +# Linux distros ship the .pc file. Either yields one target to link. +find_package(ZeroMQ CONFIG QUIET) +if(TARGET libzmq) + set(VLA_ZMQ_TARGET libzmq) +elseif(TARGET libzmq-static) + set(VLA_ZMQ_TARGET libzmq-static) +else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(ZeroMQ REQUIRED IMPORTED_TARGET libzmq) + set(VLA_ZMQ_TARGET PkgConfig::ZeroMQ) +endif() find_path(CPPZMQ_INCLUDE_DIR NAMES zmq.hpp) if(NOT CPPZMQ_INCLUDE_DIR) @@ -299,7 +362,7 @@ target_include_directories(vla-server PRIVATE target_link_libraries(vla-server PRIVATE vla_core protobuf::libprotobuf - PkgConfig::ZeroMQ + ${VLA_ZMQ_TARGET} ) set(VLM_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/serving/vlm.proto) @@ -329,9 +392,11 @@ target_include_directories(vlm-server PRIVATE target_link_libraries(vlm-server PRIVATE vlm_core protobuf::libprotobuf - PkgConfig::ZeroMQ + ${VLA_ZMQ_TARGET} ) +endif() # VLA_BUILD_SERVER + add_library(vla SHARED src/vla_c_api.cpp) target_include_directories(vla PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include @@ -360,7 +425,10 @@ add_executable(vla-bench target_link_libraries(vla-bench PRIVATE vla_core) # Warnings and LTO for our own targets only, never the llama.cpp subtree. -set(VLA_FIRST_PARTY_TARGETS vla_core vlm_core vla vla-server vlm-server vla-cli vla-bench) +set(VLA_FIRST_PARTY_TARGETS vla_core vlm_core vla vla-cli vla-bench) +if(VLA_BUILD_SERVER) + list(APPEND VLA_FIRST_PARTY_TARGETS vla-server vlm-server) +endif() # The CUDA targets were the only first-party code compiled without warnings. if(GGML_CUDA) list(APPEND VLA_FIRST_PARTY_TARGETS bitvla_cuda_kernels vla_cuda_ops) @@ -370,6 +438,38 @@ foreach(tgt IN LISTS VLA_FIRST_PARTY_TARGETS) target_compile_options(${tgt} PRIVATE $<$:-Wall -Wextra>) endforeach() +# llama.cpp sends its DLLs to /bin, and Windows has no rpath: a binary +# finds a DLL beside itself or on PATH. Put ours next to them. +if(WIN32) + foreach(tgt IN LISTS VLA_FIRST_PARTY_TARGETS) + set_target_properties(${tgt} PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) + endforeach() + # Same problem for the servers' protobuf / abseil / libzmq DLLs, which live + # in the package manager's tree. + if(VLA_BUILD_SERVER) + foreach(tgt vla-server vlm-server) + add_custom_command(TARGET ${tgt} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + $ $ + COMMAND_EXPAND_LISTS) + endforeach() + endif() +endif() + +# The NPU-side HTP skels are built (and, with HEXAGON_HTP_CERT, signed) inside +# the fetched tree. FastRPC finds them through ADSP_LIBRARY_PATH, which the +# Hexagon rung defaults to the binary's own directory, so copy them there. +if(GGML_HEXAGON AND TARGET ggml-hexagon) + set(_vla_htp_dir ${CMAKE_BINARY_DIR}/_deps/llama-build/ggml/src/ggml-hexagon) + add_custom_target(vla-htp-skels ALL + COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_BINARY_DIR}/bin + COMMAND ${CMAKE_COMMAND} -DSRC=${_vla_htp_dir} -DDST=${CMAKE_BINARY_DIR}/bin + -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/copy_htp_skels.cmake + DEPENDS ggml-hexagon + COMMENT "copying HTP skels next to the binaries" + VERBATIM) +endif() + include(CheckIPOSupported) check_ipo_supported(RESULT VLA_IPO_OK OUTPUT VLA_IPO_MSG) if(VLA_IPO_OK AND CMAKE_BUILD_TYPE STREQUAL "Release") @@ -380,6 +480,20 @@ else() message(STATUS "vla: IPO/LTO disabled (${VLA_IPO_MSG})") endif() +# vcpkg's abseil hands every consumer "-ignore:4221", an MSVC link.exe flag, +# through INTERFACE_LINK_LIBRARIES. clang++ driving lld-link rejects it as an +# unknown argument. The warning it silences is harmless, so drop the flag. +if(WIN32 AND NOT MSVC) + get_directory_property(_vla_imported IMPORTED_TARGETS) + foreach(tgt IN LISTS _vla_imported) + get_target_property(_libs ${tgt} INTERFACE_LINK_LIBRARIES) + if(_libs AND "-ignore:4221" IN_LIST _libs) + list(REMOVE_ITEM _libs "-ignore:4221") + set_property(TARGET ${tgt} PROPERTY INTERFACE_LINK_LIBRARIES "${_libs}") + endif() + endforeach() +endif() + option(VLA_BUILD_TESTS "Build vla.cpp test harnesses and unit tests" OFF) if(VLA_BUILD_TESTS) enable_testing() diff --git a/README.md b/README.md index 69a9434..679e8d5 100644 --- a/README.md +++ b/README.md @@ -12,8 +12,8 @@ A C++ inference engine for **Vision-Language-Action (VLA) models**, built on [`l It runs the open VLA policies - SmolVLA, π0, BitVLA, Evo-1, GR00T N1.5/1.6/1.7 and more - under one runtime, each packaged as a single self-contained GGUF that needs no Python or PyTorch at inference time. The binaries drive robots on **CPU**, **Apple Silicon**, **CUDA** - -from consumer GPUs down to Jetson-class boards - or **Intel GPUs and NPUs** via -SYCL and OpenVINO. +from consumer GPUs down to Jetson-class boards - **Intel GPUs and NPUs** via +SYCL and OpenVINO, or the **Snapdragon X** Hexagon NPU and Adreno GPU on Windows on Arm. [**Learn vla.cpp**](https://fai-modelopt-tech.github.io/learn-vla-cpp/) walks through the engine design and how each policy is implemented on ggml. @@ -72,7 +72,8 @@ export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH ``` Check [docs/backend](docs/backend) for compiling `vla.cpp` on other platforms. -WSL2, Apple Silicon, and Intel GPU are all tested. +WSL2, Apple Silicon, Intel GPU and Snapdragon X on Windows on Arm +([docs/backend/hexagon-windows.md](docs/backend/hexagon-windows.md)) are all tested. To build and run in containers instead, see [docs/DOCKER.md](docs/DOCKER.md). --- @@ -161,7 +162,9 @@ Precision flags (`vla-server --help` for the full list). The fastest configuration per model, with measured latency and success rate, is in [`CHANGELOG.md`](CHANGELOG.md): -- `--weight-dtype f32|bf16` - resident dtype for GEMM weights. +- `--weight-dtype f32|bf16|f16` - resident dtype for GEMM weights. `f16` is the + default on Hexagon and OpenCL, which have no BF16 kernels. It is also 2.5-3.5x + faster than the BF16 default on CPUs without BF16 matmul (e.g. Snapdragon X). - `--act-dtype f32|bf16` - activation dtype; needs CUDA and bf16 weights. - `--flash-attn` - faster on the larger towers, but changes numerics. - `--mm-prec default|f32` - matmul accumulation precision. @@ -170,7 +173,9 @@ in [`CHANGELOG.md`](CHANGELOG.md): Environment knobs that apply to every arch: - `VLA_N_THREADS` - CPU backend thread count, default core count capped at 16. -- `VLA_DEVICE` - GPU ordinal for CUDA and SYCL builds, default 0. +- `VLA_DEVICE` - GPU ordinal for CUDA and SYCL builds, default 0. In Hexagon and + OpenCL builds it indexes the NPU session or GPU, and `VLA_DEVICE=cpu` skips the + accelerator. - `VLA_CACHE` - where `-hf` stores checkpoints, default `~/.cache/vla`. --- @@ -312,21 +317,28 @@ Experimental results on other platforms can be found in Support matrix of models (rows) against platforms (columns). Legend: `Y` = supported (released and benchmarked), `~` = in progress, `-` = planned. -| Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | Hexagon | +| Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | [Hexagon](docs/backend/hexagon-windows.md) | |---|:--:|:--:|:--:|:--:|:--:|:--:| -| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | - | -| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | - | -| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | - | -| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | - | -| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | - | -| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | - | -| [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | - | -| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | - | -| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | - | -| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | Y | - | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | - | -| [Octo-Small](https://hf.co/vrfai/octo-small-libero-gguf) | Y | Y | Y | Y | - | - | -| [TurboVLA](https://hf.co/vrfai/turbovla-libero-gguf) | Y | Y | Y | Y | Y | - | +| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | Y | +| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | ~ | +| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | Y | +| [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | - | +| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | Y | +| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | Y | +| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | Y | - | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | ~ | +| [Octo-Small](https://hf.co/vrfai/octo-small-libero-gguf) | Y | Y | Y | Y | - | Y | +| [TurboVLA](https://hf.co/vrfai/turbovla-libero-gguf) | Y | Y | Y | Y | Y | Y | + +Hexagon is measured on a Snapdragon X laptop (Windows on Arm, v73 NPU). `~` there +means the model runs but lands outside the 2.9e-3 fidelity bar: π0 and VLA-JEPA +are precision-sensitive even on the CPU at F16. The same laptop's **Adreno GPU** +(`GGML_OPENCL=ON`) runs the eleven models in the Hexagon column's `Y`/`~` rows +within 1.4e-3 of an F32 reference; see +[hexagon-windows.md](docs/backend/hexagon-windows.md). --- diff --git a/cmake/arm64-windows-llvm.cmake b/cmake/arm64-windows-llvm.cmake new file mode 100644 index 0000000..f54172a --- /dev/null +++ b/cmake/arm64-windows-llvm.cmake @@ -0,0 +1,20 @@ +# Copied from llama.cpp (MIT), cmake/arm64-windows-llvm.cmake: clang targeting +# the MSVC ABI on Windows on Arm. Needed before the fetch, so it lives here too. + +set( CMAKE_SYSTEM_NAME Windows ) +set( CMAKE_SYSTEM_PROCESSOR arm64 ) + +set( target arm64-pc-windows-msvc ) + +set( CMAKE_C_COMPILER clang ) +set( CMAKE_CXX_COMPILER clang++ ) + +set( CMAKE_C_COMPILER_TARGET ${target} ) +set( CMAKE_CXX_COMPILER_TARGET ${target} ) +set( CMAKE_ASM_COMPILER_TARGET ${target} ) + +set( arch_c_flags "-march=armv8.7-a -fvectorize -ffp-model=fast -fno-finite-math-only" ) +set( warn_c_flags "-Wno-format -Wno-unused-variable -Wno-unused-function -Wno-gnu-zero-variadic-macro-arguments" ) + +set( CMAKE_C_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) +set( CMAKE_CXX_FLAGS_INIT "${arch_c_flags} ${warn_c_flags}" ) diff --git a/cmake/copy_htp_skels.cmake b/cmake/copy_htp_skels.cmake new file mode 100644 index 0000000..4d090bb --- /dev/null +++ b/cmake/copy_htp_skels.cmake @@ -0,0 +1,10 @@ +# Copy libggml-htp-v*.so and, when the build signed them, libggml-htp.cat from +# SRC to DST. A script rather than a copy command because the skel list and the +# catalog's presence are only known after the build ran. +file(GLOB _skels "${SRC}/libggml-htp-v*.so" "${SRC}/libggml-htp.cat") +if(NOT _skels) + message(WARNING "no HTP skels under ${SRC}") +endif() +foreach(f IN LISTS _skels) + file(COPY "${f}" DESTINATION "${DST}") +endforeach() diff --git a/cmake/patch_sentencepiece_fpic.cmake b/cmake/patch_sentencepiece_fpic.cmake new file mode 100644 index 0000000..3e9eba6 --- /dev/null +++ b/cmake/patch_sentencepiece_fpic.cmake @@ -0,0 +1,8 @@ +# sentencepiece v0.2.0 adds -fPIC for every compiler that is not MSVC, and +# clang targeting arm64-pc-windows-msvc rejects the flag as a hard error. +# Windows has no PIC to ask for, so drop it. Run as the FetchContent patch step +# with the working directory at the sentencepiece source root. +set(_f "src/CMakeLists.txt") +file(READ "${_f}" _s) +string(REPLACE " -fPIC" "" _s "${_s}") +file(WRITE "${_f}" "${_s}") diff --git a/cmake/vcpkg-triplets/arm64-windows-clangcl.cmake b/cmake/vcpkg-triplets/arm64-windows-clangcl.cmake new file mode 100644 index 0000000..b85b6bb --- /dev/null +++ b/cmake/vcpkg-triplets/arm64-windows-clangcl.cmake @@ -0,0 +1,22 @@ +# vcpkg triplet: arm64 Windows, DLLs, built with Visual Studio's clang-cl. +# +# vla.cpp builds with clang, and protobuf's headers switch on __clang__. The one +# seen to matter: PROTOBUF_RESTRICT is __restrict under clang and empty under +# MSVC, the two mangle it differently, and clang-compiled generated code then +# fails to link against an MSVC-built protobuf (RepeatedPtrFieldBase:: +# InternalSwap). Building protobuf with the same compiler removes the mismatch +# without redefining keywords. +# +# Only protobuf (and abseil, which it pulls in) need it. ZeroMQ is a C API, so +# the stock MSVC triplet serves. +# +# vcpkg install protobuf --triplet arm64-windows-clangcl ^ +# --overlay-triplets=\cmake\vcpkg-triplets +# vcpkg install zeromq cppzmq --triplet arm64-windows +set(VCPKG_TARGET_ARCHITECTURE arm64) +set(VCPKG_CRT_LINKAGE dynamic) +set(VCPKG_LIBRARY_LINKAGE dynamic) +set(VCPKG_CHAINLOAD_TOOLCHAIN_FILE "${CMAKE_CURRENT_LIST_DIR}/clangcl-toolchain.cmake") +# A chainloaded toolchain makes vcpkg skip vcvars by default, and the link step +# still needs rc, mt and the SDK library paths from it. +set(VCPKG_LOAD_VCVARS_ENV ON) diff --git a/cmake/vcpkg-triplets/clangcl-toolchain.cmake b/cmake/vcpkg-triplets/clangcl-toolchain.cmake new file mode 100644 index 0000000..4fc9f29 --- /dev/null +++ b/cmake/vcpkg-triplets/clangcl-toolchain.cmake @@ -0,0 +1,26 @@ +# Chainloaded by arm64-windows-clangcl.cmake: vcpkg's own Windows toolchain, with +# Visual Studio's bundled clang-cl as the compiler. clang-cl takes the MSVC flags +# that toolchain sets. +# +# vcpkg scrubs PATH and VCPKG_ROOT, so clang-cl is looked up in the standard +# Visual Studio install locations. vcpkg's toolchains directory is found from +# CMAKE_TOOLCHAIN_FILE (scripts/buildsystems/vcpkg.cmake) and handed on to +# try_compile, which sees neither of those. +if(NOT _VLA_VCPKG_TOOLCHAINS) + get_filename_component(_VLA_VCPKG_TOOLCHAINS "${CMAKE_TOOLCHAIN_FILE}" DIRECTORY) + set(_VLA_VCPKG_TOOLCHAINS "${_VLA_VCPKG_TOOLCHAINS}/../toolchains") +endif() +list(APPEND CMAKE_TRY_COMPILE_PLATFORM_VARIABLES _VLA_VCPKG_TOOLCHAINS) + +file(GLOB _vla_llvm_dirs + "$ENV{ProgramFiles}/Microsoft Visual Studio/*/*/VC/Tools/Llvm/ARM64/bin" + "C:/Program Files/Microsoft Visual Studio/*/*/VC/Tools/Llvm/ARM64/bin") +find_program(_vla_clang_cl clang-cl REQUIRED + HINTS "$ENV{VCINSTALLDIR}/Tools/Llvm/ARM64/bin" ${_vla_llvm_dirs}) +set(CMAKE_C_COMPILER "${_vla_clang_cl}") +set(CMAKE_CXX_COMPILER "${_vla_clang_cl}") +include("${_VLA_VCPKG_TOOLCHAINS}/windows.cmake") +# vcpkg sets CMAKE_RC_FLAGS to "/c65001 /DWIN32" unconditionally, and CMake's +# Ninja generator runs the C compiler over .rc files for dependencies: clang-cl +# takes /c65001 for a file name. rc.exe defaults to the right code page anyway. +set(CMAKE_RC_FLAGS "/DWIN32" CACHE STRING "" FORCE) diff --git a/docs/backend/hexagon-windows.md b/docs/backend/hexagon-windows.md new file mode 100644 index 0000000..37a870c --- /dev/null +++ b/docs/backend/hexagon-windows.md @@ -0,0 +1,305 @@ +# vla.cpp on Snapdragon X (Windows on Arm): Hexagon NPU, Adreno GPU and CPU + +Date: 2026-09-27 +vla.cpp: `3f0a38a` plus the changes described here (uncommitted) +llama.cpp: local checkout, build 11201, commit `2145525a4` (not the `b10729` pin) + +## Summary + +vla.cpp now builds natively on a Snapdragon X laptop in three flavours: + +- the Oryon CPU; +- the Hexagon NPU (HTP), with ops the NPU rejects running on the CPU; +- the Adreno GPU (OpenCL), with the same CPU fallback. + +`vla-server`, `vla-cli`, `vla-bench` and the tests all build. + +Eleven of the twelve published checkpoints run on all three. BitVLA does not: its only published GGUF is int2-packed, which only CUDA builds load. OpenVLA-OFT was not attempted, because at F16 it needs about 14 GB (more than the 8 GB budget). + +Every accelerator result below was checked against a CPU-backend reference on identical inputs before its latency was recorded. + +- **SmolVLA runs fastest on the NPU:** 1.23 s per action chunk against 2.57 s on the CPU and 3.05 s on the GPU. It is within 1.5e-3 of the CPU reference. +- **The NPU also wins on π0 (5.6 s vs 10.4 s CPU) and VLA-JEPA (0.85 s vs 1.6 s).** Both models are sensitive to precision, and both land 4.8e-3 to 5.6e-3 from an F32 reference, above the 2.9e-3 bar the other backends are held to. Running the same F16 weights on the CPU moves them further (8.5e-3 and 1.2e-2). +- **On the other eight models the CPU is fastest**, once it runs F16 weights instead of the BF16 it defaults to. Oryon has no BF16 matmul, so the shipped default is 3-4x slower than it needs to be. Both accelerators still agree with the reference on these models, most to within 1e-3. +- **Five ggml-hexagon kernels give wrong answers for shapes vla.cpp uses.** The first three can flip the gripper channel (max |Δ| 0.7 to 1.9): + - GELU runs as GELU_QUICK. + - A broadcast with a gap is mis-indexed. + - IM2COL is wrong unless the conv is a patch embedding. + - F16 copies read back wrong on the host. + - Op fusion costs precision. + + The CPU fallback now routes around each of them, and each is described [below](#hexagon-issues-found-and-worked-around). + +The NPU path needs the same one-off setup as llama.cpp: a newer NPU driver, test-signing, and a self-signed certificate. That setup is described in `../llama.cpp/snapdragon-x-windows-report.md` and was reused unchanged. + +## Device + +Same laptop as the llama.cpp report: + +| Item | Value | +|---|---| +| Laptop | ASUS Vivobook 14 X1407QA | +| SoC | Snapdragon X X1-26-100, 8 Oryon cores | +| GPU | Adreno X1-45, driver 31.0.112.0 | +| NPU | Hexagon v73, driver 30.0.220.3000; 4 HVX threads, 1 HMX unit, 8 MB VTCM | +| RAM | 16 GB | +| OS | Windows 11 Home, build 26200 | + +## Toolchain + +Everything llama.cpp needed, plus the following: + +| Component | Version / location | Used for | +|---|---|---| +| vcpkg | `C:\vcpkg` | protobuf 6.33.4 (triplet `arm64-windows-clangcl`, in this repo), zeromq 4.3.5 and cppzmq (stock `arm64-windows`) for `vla-server` | +| Python | 3.14.7 ARM64, venv in `.venv` | `transformers` 5.17, `tokenizers`, `gguf`, `numpy`: tokenizing prompts and `scripts/quantize_gguf.py` | + +vcpkg needs no admin rights. protobuf has to be compiled by clang, like +vla.cpp. Otherwise clang's and MSVC's different mangling of `__restrict` leaves +the servers with unresolved protobuf symbols. The overlay triplet in +`cmake/vcpkg-triplets` builds it with Visual Studio's clang-cl. ZeroMQ is a C +API, so the stock MSVC build serves. Run these from a Visual Studio developer +shell: + +``` +git clone --depth 1 https://github.com/microsoft/vcpkg C:\vcpkg +C:\vcpkg\bootstrap-vcpkg.bat -disableMetrics +C:\vcpkg\vcpkg install zeromq cppzmq --triplet arm64-windows +C:\vcpkg\vcpkg install protobuf --triplet arm64-windows-clangcl --overlay-triplets=\cmake\vcpkg-triplets +``` + +## Building + +The build script sets up the Visual Studio shell, the compiler flags llama.cpp's Snapdragon preset uses, vcpkg, and (for the NPU) skel signing: + +``` +.\scripts\build_windows_snapdragon.ps1 -Backend htp -LlamaDir C:\Users\khanh\git\llama.cpp -HtpCert C:\Users\khanh\Certs\ggml-htp-v1.pfx +.\scripts\build_windows_snapdragon.ps1 -Backend opencl -LlamaDir C:\Users\khanh\git\llama.cpp +.\scripts\build_windows_snapdragon.ps1 -Backend cpu -LlamaDir C:\Users\khanh\git\llama.cpp +``` + +Each build goes into `build-wos-`, with every binary and DLL in `build-wos-\bin`. `-NoServer` skips `vla-server`, Octo and their protobuf and ZeroMQ dependencies. + +`-LlamaDir` points the build at an existing llama.cpp checkout through `FETCHCONTENT_SOURCE_DIR_LLAMA`. This build uses the local checkout, which already builds and signs the HTP skels on this machine. The `b10729` pin in `CMakeLists.txt` still applies when `-LlamaDir` is not given. That pin was not tested here. + +The HTP build also signs `libggml-htp-v*.so` with the certificate and copies the skels and their catalog next to the binaries. At startup the Hexagon backend points `ADSP_LIBRARY_PATH` at the executable's own folder, but only if the variable is unset. The llama.cpp setup set it for the user to `llama.cpp\pkg-wos\lib`. Those skels come from the same llama.cpp commit, so they work too. + +### What had to change for Windows + +| Problem | Fix | +|---|---| +| `fseeko` and `popen` are POSIX; `clock_gettime` in the test harness | `_fseeki64`, `_popen` with cmd.exe quoting, `std::chrono` | +| `vla_core` became a DLL (llama.cpp turns on `BUILD_SHARED_LIBS`) that exports nothing, so every consumer failed to link | The two internal libraries are static on Windows | +| DLLs land in `\bin` and Windows has no rpath | Executables go to `bin` too; the servers' vcpkg DLLs are copied next to them | +| protobuf and ZeroMQ were required even for `vla-cli` | New option `VLA_BUILD_SERVER` (default ON); ZeroMQ is also found through its CMake package, since vcpkg ships no pkg-config | +| SentencePiece adds `-fPIC`, which clang rejects when targeting Windows | Patched out at fetch time, on Windows only | +| vcpkg's protobuf shim hands SentencePiece the DLL instead of its import library | SentencePiece's protobuf dependency is linked as the `protobuf::libprotobuf` target, from the top-level directory | +| clang and MSVC mangle `__restrict` parameters differently, so clang code cannot link against an MSVC-built protobuf | protobuf built with clang-cl (the overlay triplet above) | +| `setvbuf(stdout, NULL, _IOLBF, 0)` in both servers: the Windows CRT has no line buffering and treats size 0 as invalid, so it fails fast (`0xC0000409`) before printing anything | `_IONBF` on Windows | +| vcpkg's abseil passes the MSVC-only flag `-ignore:4221` to its consumers | Stripped from the imported targets | +| ggml-hexagon's catalog-signing step can run before the skels exist in a parallel build | The script builds the four skels first | + +## Running + +``` +$env:VLA_PYTHON = "$PWD\.venv\Scripts\python.exe" # tokenizer for --text +.\build-wos-htp\bin\vla-cli.exe --ckpt models\smolvla\smolvla-libero.gguf --image assets\front.jpg --text "pick up the black bowl" --pretty +.\build-wos-htp\bin\vla-server.exe models\smolvla\smolvla-libero.gguf +``` + +`vla-server` on the NPU answered a two-view SmolVLA request over ZeroMQ in +1.25 s server-side (1.67 s for the first, cold request), matching +`vla_predict_check`. The LIBERO client could not be run, because it needs +PyTorch and LIBERO; the smoke test used a 20-line pyzmq client instead. + +| Variable / flag | Effect in the Hexagon and OpenCL builds | +|---|---| +| `VLA_DEVICE=cpu` | Skip the accelerator. The same binary then produces the CPU reference. | +| `VLA_DEVICE=` | Registry device index: an HTP session, or an OpenCL GPU | +| `--weight-dtype f16\|f32\|bf16` | Resident dtype for GEMM weights. New: `f16`, which is also the default on both accelerators. | +| `--flash-attn on\|off` | On by default in Hexagon builds, off elsewhere | +| `VLA_FALLBACK_STATS=1` | Print how each graph was split between the accelerator and the CPU | +| `GGML_HEXAGON_OPFUSION` | Defaults to `0` here (see below); set to `1` to trade precision for speed | +| `GGML_OPENCL_ADRENO_XMEM_GEMM` | Defaults to `0` here (see below); set to `1` to trade precision for speed | + +Each process prints which ops the accelerator gave to the CPU, once per op type, and a summary when it exits: + +``` +vla: HTP0+CPU rejects GET_ROWS (f32 <- bf16, [960,48,1,1], e.g. 'node_15'); running it on CPU +vla: HTP0+CPU: 7 of 35 graphs split; 7 nodes in 7 runs went to CPU, 90.2 MiB copied in, 1.2 MiB out, 90.2 MiB of weights mirrored on host +``` + +## How it works + +### The CPU fallback + +The engine drives one backend per model through `gallocr`, with no scheduler. That works for CUDA, Metal, SYCL and OpenVINO, which run every op the archs build. Hexagon and OpenCL do not. An NPU rejects an op when it has no kernel for it (GELU_ERF), or when the tile does not fit VTCM (softmax rows that are not a multiple of 32). Neither can be known until the graph exists. + +`src/backend_fallback.cpp` wraps the accelerator in a backend that looks like it to the arch: same device, same buffer type, so weights and activations still live on the accelerator. Its `graph_compute` splits each graph: + +- Runs of ops the accelerator accepts go to it unchanged. +- Runs it rejects are copied to host memory, computed on a CPU backend, and copied back. +- Weights that a CPU op reads are copied once and kept. + +No arch's call sites changed, and CUDA, Metal, SYCL and OpenVINO builds do not compile the wrapper at all. The Hexagon- and OpenCL-specific graph changes (`fa_kv`, `gelu`, the default weight dtype and flash attention) sit behind the same build flags and reduce to the old calls elsewhere. SmolVLA's CPU BF16 output was checked bit-identical before and after them. The other archs were not re-checked against a pre-change build, because the unmodified tree does not build on Windows. + +The wrapper is also where Hexagon's wrong kernels are refused (next section). Forcing every op through the CPU path (`GGML_HEXAGON_OPFILTER=.*`) reproduces a pure-CPU run bit for bit. + +### Hexagon issues found and worked around + +Each issue below was found by moving op families to the CPU with `GGML_HEXAGON_OPFILTER` until the action chunk matched the reference. None are vla.cpp bugs; all should go upstream. + +| Issue | Seen on | Error on the NPU | After the fix | Where fixed | +|---|---|---:|---:|---| +| Unary `GELU` runs `x*sigmoid(1.702x)`, which is GELU_QUICK. Upstream maps both ops to one kernel. | SmolVLA (SigLIP) | 1.9 | 1.5e-3 | `vla::gelu()` builds the tanh form exactly as `x*sigmoid(2c(x+0.044715x³))` from ops the NPU gets right. The wrapper refuses any remaining `GGML_UNARY_OP_GELU`. | +| Binary op with `src1` broadcast over one dimension but not a higher one, e.g. `[64,12,261,2] * [64,1,261,1]` | TurboVLA (RoPE), Octo | 0.72 | 6.0e-4 | Wrapper sends these to the CPU | +| `IM2COL` accepts any geometry, but is right only for patch embeddings (kernel = stride > 1, no padding) | Octo (stride-2 pad-1 stem, 1x1 projection) | 1.05 | 5.2e-4 | Wrapper sends other geometries to the CPU; SigLIP's 16x16/16 stays on the NPU | +| An F32→F16 `CPY` computed on the NPU reads back wrong on the host, though the NPU's own consumers see it correctly | SmolVLA with FA forced to CPU | 1.2 | 1.7e-3 | A copy that feeds a CPU op runs on the CPU too | +| Op fusion (MUL_MAT+ADD, RMS_NORM+MUL) | SmolVLA | 1.2e-2 | 1.6e-3 | `GGML_HEXAGON_OPFUSION` defaults to 0 | + +Two further changes keep work on the NPU: + +- **F16 K/V for flash attention.** The HTP kernel takes F16 or Q8_0 K/V only. The archs pass F32, which sent every SmolVLA attention through the CPU: 352 runs and 348 MiB of copies per chunk. `vla::fa_kv()` casts K/V before the permute in Hexagon builds, and is a no-op elsewhere. Evo-1 keeps F32 K/V (its attention runs on the CPU), because the cast measured 3.9e-2. +- **Flash attention on by default.** In the towers, the explicit path's 1024x1024 F32 score matmuls took SmolVLA's vision stage to 3.8 s. Flash attention does it in 0.44 s. + +Not fixed, left on the CPU: + +- `GELU_ERF` has no HTP kernel (DINOv2 and SigLIP-so400m towers). +- `SOFT_MAX` needs rows that are a multiple of 32. +- `GROUP_NORM`, `SIN` and `COS` are also unsupported. + +Evo-1 is where this costs the most: 3,900 CPU runs and 11 GB of copies per chunk, which is why the NPU is slower than the CPU there. + +### Adreno (OpenCL) issues + +| Issue | Error | After the fix | Fix | +|---|---:|---:|---| +| The "xmem" F16xF32 GEMM (weights prepacked into images) rounds activations | SmolVLA 5.0e-3 | 2.9e-4 | `GGML_OPENCL_ADRENO_XMEM_GEMM` defaults to 0. It is about 1.4x faster on SmolVLA where it works. | +| The same GEMM aborts with `CL_OUT_OF_RESOURCES` on GR00T N1.7's larger weights | crash | 2.9e-4 | same | +| MUL_MAT has no BF16 kernel | every GEMM on CPU | – | F16 resident weights by default | + +### Weights: F16, not BF16 + +The shipped GGUFs are BF16, and most archs keep them BF16 in memory. HTP has no BF16 kernel for any op, and Adreno none for MUL_MAT, so both accelerator builds default to F16 (`vla::default_weight_dtype`). F16 holds every normal-range BF16 value exactly, and `--weight-dtype` still overrides. + +SmolVLA used to convert every weight to its resident dtype, so a `quantize_gguf.py` file would not load for it on any platform. It now keeps a GEMM weight the file stores packed, as the other archs already do, and it dequantizes any packed tensor it keeps as float. + +## Benchmark + +### Setup + +| Item | Value | +|---|---| +| Tool | `vla_predict_check`: fixed images, tokens, state and noise; `VLA_BENCH_ITERS=3` after 3 warm-ups; minimum reported | +| Inputs | Each model at its native size and view count (the README's table); VLA-JEPA with its 32 `` tokens; Octo's prompt padded to 16 tokens, un-normalized against `libero_object` | +| Checkpoints | The published `vrfai/*` GGUFs (GR00T N1.7 and VLA-Adapter: `libero_object`) | +| Fidelity | max \|Δ\| of the action chunk against the CPU backend with `--weight-dtype f32`. A second number against the CPU BF16 default is given where it is tighter, following [ov.md](ov.md#picking-the-right-baseline). The bar the other backends are held to is 2.9e-3. | +| Memory | Peak working set of the process, from `GetProcessMemoryInfo` | +| Threads | 8 (all cores); power plan and temperature not controlled | + +### Latency (ms per action chunk) + +CPU BF16 is what a stock build does today. CPU F16 is the fair CPU baseline. HTP and OpenCL use their defaults (F16 weights). + +| Model | Views × size | CPU BF16 | CPU F16 | NPU (HTP) | GPU (OpenCL) | Fastest | +|---|---|--:|--:|--:|--:|---| +| SmolVLA | 2 × 512 | 7,609 | 2,570 | **1,227** | 3,048 | NPU | +| VLA-JEPA | 1 × 256 | 5,557 | 1,617 | **845** | 2,402 | NPU | +| π0 | 2 × 224 | 30,937 | 10,387 | **5,563** | 13,644 | NPU | +| π0.5 | 2 × 224 | 29,957 | 9,987 | **9,280** | 13,358 | NPU (7% faster) | +| GR00T N1.7 | 1 × 256 | 6,700 | **1,983** | 3,068 | 3,535 | CPU | +| GR00T N1.6 | 1 × 224 | 7,627 | **2,232** | 5,019 | 4,032 | CPU | +| GR00T N1.5 | 1 × 224 | 9,299 | **3,186** | 5,215 | 4,572 | CPU | +| VLA-Adapter | 1 × 224 | 7,281 | **2,140** | 4,127 | 3,655 | CPU | +| Evo-1 | 1 × 448 | 16,451 | **6,451** | 7,297 | 9,321 | CPU | +| TurboVLA | 2 × 256 | 598 | **388** | 2,428 | 717 | CPU | +| Octo-Small | 1 × 256 | 104 | **88** | 196 | 148 | CPU | + +TurboVLA and Octo ship F32 GGUFs, so their "CPU BF16" column is the F32 default. + +### Fidelity (max |Δ| against CPU F32) + +| Model | CPU F16 | NPU (HTP) | GPU (OpenCL) | +|---|--:|--:|--:| +| SmolVLA | 6.7e-4 | 2.0e-3 (1.5e-3 vs BF16) | 2.9e-4 | +| VLA-JEPA | 8.5e-3 | **4.8e-3** | 1.7e-4 | +| π0 | 1.2e-2 | **5.6e-3** | 1.4e-3 | +| π0.5 † | 6.5e-4 | 7.1e-4 | 6.3e-4 | +| GR00T N1.7 | 2.0e-2 | 1.6e-3 | 2.9e-4 | +| GR00T N1.6 | 1.0e-3 | 4.1e-4 | 3.1e-4 | +| GR00T N1.5 | 9.4e-4 | 1.5e-3 | 8.9e-4 | +| VLA-Adapter | 1.5e-3 | 7.0e-4 | 3.0e-5 | +| Evo-1 | 1.6e-3 | 4.1e-4 | 2.1e-4 | +| TurboVLA | 7.1e-3 | 6.0e-4 | 1.1e-6 | +| Octo-Small | 0 | 5.2e-4 | 8.2e-4 | + +† Against the CPU BF16 run. An F32 reference for π0.5 needs more memory than the 8 GB budget. + +The accelerators are often closer to F32 than the CPU running the same F16 weights. ggml's CPU matmul converts activations to the weight's type (F16 here), while HTP and Adreno keep them in F32. + +Bold marks the two results above the 2.9e-3 bar. Both models are precision-sensitive: their own CPU F16 run is worse. The fix, F32 weights, would need 12 GB (π0) and 8.6 GB (VLA-JEPA). Turning flash attention off only brings them to 4.9e-3 and 3.4e-3, at 1.6-2x the latency. + +### Peak memory (GB) + +Peak working set of the whole process: + +| Model | CPU BF16 | CPU F16 | NPU | GPU | +|---|--:|--:|--:|--:| +| SmolVLA | 1.15 | 1.15 | 1.27 | 1.85 | +| VLA-JEPA | 3.75 | 3.75 | 3.82 | 4.20 | +| π0 | 5.20 | 5.20 | 5.25 | 5.67 | +| π0.5 | 5.63 | 5.63 | 5.71 | 6.10 | +| GR00T N1.7 | 6.00 | 6.00 | 6.07 | 7.44 | +| GR00T N1.6 | 5.62 | 5.62 | 5.70 | 7.09 | +| GR00T N1.5 | 4.57 | 4.57 | 4.66 | 6.02 | +| VLA-Adapter | 2.64 | 2.64 | 2.73 | 3.77 | +| Evo-1 | 1.40 | 1.40 | 1.47 | 1.84 | +| TurboVLA | 0.83 | 0.48 | 0.93 | 1.40 | +| Octo-Small | 0.61 | 0.61 | 0.66 | 1.11 | + +Every configuration stays under 8 GB. The NPU's weights live in FastRPC shared +memory: they count toward the working set, but the process's private commit stays +under 1.4 GB. The GPU build is the largest, because the OpenCL driver's buffers +count on top of the host copy made during loading. + +### Q8_0 + +`scripts/quantize_gguf.py --type Q8_0` packs the LM backbones. It saves memory; it does not preserve fidelity. The CPU itself moves π0 by 4.9e-2 and the GR00Ts by 7e-3 to 4e-2 on the Q8_0 files. Latency and fidelity on the Q8_0 files (fidelity against the CPU F32 run of the original): + +| Model | Weights BF16 → Q8_0 | NPU | GPU | CPU (Q8_0) | +|---|---|--:|--:|--:| +| SmolVLA | 1.13 → 0.72 GB | 1,140 ms, 1.2e-2 | 3,015 ms, 9.7e-3 | 2,512 ms, 1.0e-2 | +| π0 | 6.49 → 4.33 GB | 5,010 ms, 2.3e-2 | 7,336 ms, 2.7e-1 | 8,893 ms, 4.9e-2 | +| π0.5 † | 12.59 → 5.85 GB | 8,666 ms, 1.5e-3 | 7,009 ms, 4.4e-3 | 8,947 ms, 2.3e-3 | +| GR00T N1.6 | 9.16 → 7.99 GB | 4,930 ms, 7.9e-3 | 3,473 ms, 8.5e-3 | 6,509 ms, 6.6e-3 | +| VLA-JEPA | 4.57 → 3.24 GB | 745 ms, 2.5e-2 | 1,477 ms, 4.9e-2 | 2,979 ms, 5.2e-2 | + +SmolVLA's CPU Q8_0 run keeps its float weights at F16; the other rows keep the CPU default, BF16. The π0 Q8_0 result on the GPU (0.27) is far worse than the same file on the CPU, and was not investigated. Evo-1's Q8_0 file fails to load on every backend, including the CPU, with a `ggml_view` assertion; that is an arch bug, not a Snapdragon one. + +## Observations + +- **The NPU wins where the graph stays on it.** + - SmolVLA's whole graph runs on the NPU except 7 embedding row fetches, and its vision tower is 4x faster there than on the CPU. + - π0 and π0.5 lose only their odd-width softmaxes to the CPU. + - Where GELU_ERF or a 1025-wide softmax sits in every tower layer (Evo-1, TurboVLA, VLA-Adapter), the copies cost more than the NPU saves. +- **The CPU's shipped default is its slow path.** BF16 → F16 alone makes every BF16 model 2.5-3.5x faster on Oryon, with fidelity inside or near the bar. That is a one-flag win (`--weight-dtype f16`) for any ARM CPU without BF16 matmul. +- **The Adreno GPU is never the fastest here**, but it is the most faithful accelerator: every model is within 1.4e-3 of F32. +- **The NPU's vision is slow on SigLIP-so400m.** π0's tower takes 3.5 s on the NPU against 1.9 s on the CPU, with nothing falling back. So400m's 4304-wide MLP is not a multiple of 32, which likely keeps it off the matrix unit; this was not profiled. + +## Limitations + +- **No task-success numbers.** LIBERO does not run on Windows on Arm here, so fidelity is the only check that actions are right. +- **One device, one driver, one llama.cpp commit.** ggml-hexagon changes fast, and several of the bugs above may already be fixed upstream. +- **Few repetitions, no power or thermal control.** The minimum of 3 runs is reported, and the laptop was on its default power plan. +- **BitVLA and OpenVLA-OFT were not run.** BitVLA's published GGUF is int2 for CUDA; a BF16 GGUF would have to be converted from the PyTorch checkpoint. OpenVLA-OFT exceeds the 8 GB budget at F16 and was not downloaded. +- **Q8_0 on the NPU uses dynamically quantized activations**, which is part of its error; not separated here. + +## Known issues + +- The NPU prints `ggml-hex: FASTRPC_GET_DOMAINS query failed (0x6c), using static CDSP domains` on every run. It works regardless, as in the llama.cpp report. +- The Adreno driver refuses `GGML_OPENCL_ADRENO_USE_LARGE_BUFFER` ("not supported by driver"), so no single allocation can exceed about 1 GB. F16 weights keep every model here under that. +- `Launch-VsDevShell.ps1` prints `'vswhere.exe' is not recognized`; the build is not affected. + +## Security state + +Unchanged from the llama.cpp report: Secure Boot off, test-signing on, and a self-signed certificate trusted as a root authority. This build signs its skels with the same certificate. Rollback is described in that report. diff --git a/docs/backend/hexagon.md b/docs/backend/hexagon.md index 3e9acad..ec1e123 100644 --- a/docs/backend/hexagon.md +++ b/docs/backend/hexagon.md @@ -1,24 +1,21 @@ # `vla.cpp` on Snapdragon (Hexagon NPU) -Notes on what it would take to run `vla.cpp` on Qualcomm's Hexagon NPU, read out -of llama.cpp's own Hexagon history up to `b10729` - the tag this tree pins. - -> **Status: read and planned, never run.** There is no Hexagon rung in -> `src/backend.h`, no `GGML_HEXAGON` in `CMakeLists.txt`, and no IQ-9 or IQ-10 -> board on any desk here. Every claim below is either checked against llama.cpp -> at `b10729` (op tables, env vars, commit hashes - go verify them) or is -> explicitly a **projection**. Nothing in this file is a measurement on Qualcomm -> hardware, and the projected latencies are arithmetic, not results. +What running `vla.cpp` on Qualcomm's Hexagon NPU takes, and what it turned out +to take once it ran. + +> **Status: running on Windows on Arm, not yet on a Linux board.** vla.cpp builds +> with `GGML_HEXAGON=ON` and runs eleven checkpoints on a Snapdragon X laptop's +> v73 NPU, each checked against a CPU reference. The build, the measurements and +> the upstream bugs found along the way are in +> [hexagon-windows.md](hexagon-windows.md). What follows is the backend in +> general: the parts that hold on any Snapdragon part, and the Linux / IQ-9 notes +> from before anything ran, which are still unverified. > -> Two things upstream did in 2026 make this worth writing down now: the NPU -> backend learned `IM2COL` for patch-embedding convolutions - which is exactly -> and only what a VLA's vision tower needs - and it learned multi-NPU. Both are -> already inside the tag we pin. - -The previous version of this file was a verbatim copy of upstream's -[`docs/backend/snapdragon/linux.md`](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/linux.md). -That build recipe is still the right starting point and is kept below, but it -builds *llama.cpp*, not this engine; the gap between the two is the subject here. +> The earlier version of this file was written without hardware and projected +> 0.8-2.0 s per SmolVLA action chunk on a v79 part. On a v73 part it measured +> **1.23 s**, inside the band. Its op-gap prediction was half right: `GELU_ERF` +> is still missing, and `RELU` has landed upstream. What it did not foresee was +> ops the NPU claims and then computes wrongly. ## What "IQ9" and "IQ10" refer to @@ -85,87 +82,92 @@ The knobs worth knowing on day one: | `GGML_HEXAGON_OPPOLL=1` | poll for op completion instead of sleeping - matters for many small graphs | | `GGML_HEXAGON_HOSTBUF=1` | disable repack buffers (op testing) | -## Op coverage against what `vla.cpp` actually builds +## Op coverage, measured + +The previous version read `supports_op` at `b10729` and found two gaps, `RELU` +and `GELU_ERF`, and warned that in this engine an absent op is fatal. Running it +changed all three conclusions: + +- **`RELU` landed upstream**, and the local checkout used here (build 11201) has + it. +- **`GELU_ERF` is still missing.** It now runs on the CPU (next point) in every + DINOv2 and SigLIP-so400m tower, and costs what that implies: Evo-1 spends more + on copies than the NPU saves. +- **Absent is no longer fatal.** `src/backend_fallback.cpp` wraps the NPU in a + backend that runs whatever it rejects on the CPU, without touching an arch. + That also covers ops refused for shape rather than type: softmax rows that are + not a multiple of 32, `GROUP_NORM`, `SIN` and `COS`. -This is the part that decides whether the port is a weekend or a month. The -column on the right is `ggml_backend_hexagon_device_supports_op` at `b10729`; -the middle column is a grep of `src/`. +The larger finding was not a missing op but **wrong ones**. `supports_op` claims +these, and they compute something else: -| ggml op | Where we build it | Hexagon at `b10729` | +| Op | What goes wrong | Handled by | |---|---|---| -| `MUL_MAT` / `MUL_MAT_ID` | 154 sites, everywhere | yes - Q4_0/Q4_1/Q8_0/IQ4_NL/MXFP4 (repacked), or F16/F32 direct. **No BF16.** Refuses `src0->ne[1] > 32768` (lm-heads), and anything whose tile misses VTCM | -| `IM2COL` | 6 `ggml_conv_2d`, every SigLIP/DINOv2 patch embed | yes, **partial** - added by `355303ed` (#26007) "targeting only patch-embedding convolutions": 2D only, F32 source, contiguous, **zero padding**. Our conv is kernel = stride = patch, pad 0, dilation 1. On paper it is precisely the supported case | -| `FLASH_ATTN_EXT` | 8 sites | yes - F16 or Q8_0 K/V, F16 mask, F32 sinks | -| `ROPE` / `ROPE_MULTI` | 14 sites | yes, including MROPE/IMROPE (`17d22a35`) and vision RoPE (`f2d1c2f3`) | -| `NORM`, `RMS_NORM`, `L2_NORM`, `SOFT_MAX`, `SCALE`, `CONCAT`, `SQR`, `GET_ROWS`, `ADD/SUB/MUL/DIV` | all over | yes | -| `CPY` / `CONT` (`ggml_cast`, 14 sites) | dtype hops between stages | yes, but **F32↔F16 only, and never a reshape and a conversion in the same op** | -| `SILU`, `GELU` (tanh), `GELU_QUICK`, `TANH`, `SIGMOID`, `NEG`, `ABS`, `EXP`, `SOFTPLUS` | 22 `ggml_silu`, 10 `ggml_gelu` | yes | -| **`RELU`** | **18 sites** - `action_expert.cpp` state encoder and action decoder, plus Evo-1, VLA-Adapter, VLA-JEPA, GR00T N1.7, OpenVLA-OFT, BitVLA | **not implemented.** `grep -i relu` over `ggml-hexagon/` returns nothing | -| **`GELU_ERF`** | **13 sites** - `ffn_gelu_erf` in `src/layers/ffn.h` and `dual_tower.h`, i.e. every tower trained with the exact erf form (DINOv2, SigLIP-so400m) | **not implemented.** `ggml_gelu` (tanh) is a *different op*, and substituting it silently changes numerics | -| `ggml_map_custom1` | BitVLA only | no, and BitVLA pins itself to CPU by design anyway | - -Two missing unary kernels, and one of them is in the vision tower of half the -model zoo. That is the whole gap. Note the shape of the mistake it invites: on -OpenVINO, "GELU is GELU" cost us a wrong arch and a day of bisection -([ov.md](ov.md)); here the op is simply absent, which is the *better* failure - -`supports_op` returns false and you find out at load time. - -**Except that in this engine, absent is fatal.** `src/backend.h` brings up one -backend and drives it through `gallocr` - there is no scheduler and no per-op CPU -fallback, by design, so an unsupported op asserts at predict time instead of -quietly limping. Every other backend we support (CUDA, Metal, SYCL, OpenVINO) -covers our op set completely, so this constraint has never bitten. Hexagon would -be the first backend where it does. - -So the port is gated on one of: - -1. **Two HVX kernels upstream.** `e70802a0` (#27786) added `ABS` and `LOG` as HTP - unary ops; `RELU` and `GELU_ERF` are the same shape of change, and `RELU` in - particular is a `vmax` against zero. This is the honest path and it is small. -2. **A scheduler in `vla.cpp`.** `ggml_backend_sched` with a CPU fallback, which - is an engine change with consequences for every backend, not a Hexagon one. -3. **Graph surgery** - swapping `gelu_erf` for `gelu` and re-checking fidelity. - Cheap, wrong, and it will cost more than it saves. Do not. - -## What would have to change in `vla.cpp` - -Nothing here has been written; this is the shape of the diff. - -**1. Build flags.** `GGML_HEXAGON` joins the mutually-exclusive accelerator list -at [CMakeLists.txt:15](../../CMakeLists.txt#L15) and defines `GGML_USE_HEXAGON` -the way the other four do. - -**2. A rung in the ladder** at [src/backend.h:176](../../src/backend.h#L176). -The Hexagon init API takes no ordinal - `ggml_backend_hexagon_init(void)` - so -device selection is environment-driven, and `VLA_DEVICE` (an integer ordinal for -CUDA and SYCL) does not map onto it. The right binding is `VLA_DEVICE=HTP0:0` -passed through to `GGML_HEXAGON_DEVICES`, which means the rung needs its own -parse rather than `backend_device_index()`. - -**3. Weights.** Our default resident dtype is BF16, which the NPU does not -accept in any op. Every Hexagon run needs a requantized checkpoint - -`python scripts/quantize_gguf.py --in model-bf16.gguf --out model-q8_0.gguf ---type Q8_0` (or `Q4_0`) - and the fidelity of *that* against the CPU reference -has to be established before any NPU number means anything. `--weight-dtype f32` -is the other option and doubles resident memory. - -**4. The build that is actually the most work: `vla-server`'s dependencies.** -`find_package(Protobuf REQUIRED)` and `pkg_check_modules(libzmq REQUIRED)` are -unconditional in our `CMakeLists.txt`, and the Snapdragon toolchain image carries -the Hexagon SDK and the OpenCL SDK but not protobuf, libzmq or cppzmq for arm64. -Configure fails before a single object compiles - including for `vla-cli` and -`vla-bench`, which link neither. Options, cheapest first: - -- Make the serving deps conditional so `vla-cli` / `vla-bench` configure without - them. Worth doing regardless of Hexagon; it is a dozen lines. -- Add the three to the cross sysroot. -- Build natively on the board, which for an IQ-9-class Linux part is plausible - and is how the first bring-up will probably actually happen. - -**5. Presets.** We have no `CMakeUserPresets.json`; upstream's -`arm64-linux-snapdragon-release` is the base to copy, with `GGML_HEXAGON=ON` and -`-march=armv8.2a+fp16+dotprod`. Note it sets `GGML_OPENCL=OFF` - upstream CI adds -`-DGGML_OPENCL=ON` on the command line for the IoT job. +| `GELU` | Runs `x*sigmoid(1.702x)` (GELU_QUICK) | `vla::gelu()` builds the tanh form from correct ops | +| `ADD`/`MUL`/... | `src1` broadcast over one dim but not a higher one | Refused, runs on the CPU | +| `IM2COL` | Wrong unless kernel = stride > 1 and no padding | Refused, runs on the CPU | +| `CPY` F32→F16 | Reads back wrong on the host | Copies feeding a CPU op run on the CPU | +| fused `MUL_MAT+ADD`, `RMS_NORM+MUL` | ~10x the error of the unfused ops | `GGML_HEXAGON_OPFUSION=0` by default | + +Each one flips the gripper channel, or comes close, on some checkpoint. The +earlier warning that a wrong op would be the worse failure than a missing one +was right: a missing op fails at load, and these only show in the actions. +[hexagon-windows.md](hexagon-windows.md#hexagon-issues-found-and-worked-around) +has the numbers and how each was bisected. They belong upstream. + +## What changed in `vla.cpp` + +The shape of the diff was predicted correctly, apart from the scheduler: + +1. **Build flags.** `GGML_HEXAGON` (and `GGML_OPENCL`) joined the accelerator + list and define `GGML_USE_HEXAGON` / `GGML_USE_OPENCL`. +2. **A rung in [src/backend.h](../../src/backend.h).** + - It goes through the registry, because `ggml_backend_hexagon_init` is + declared but not defined upstream. + - `VLA_DEVICE` indexes HTP sessions, and `VLA_DEVICE=cpu` gives the reference. + - It sets the defaults above and points `ADSP_LIBRARY_PATH` at the + executable's folder. +3. **Weights.** + - The NPU takes no BF16. Both accelerator builds default to F16 resident + weights, which hold BF16 exactly. + - Q8_0 works, but moves actions by 1e-2 or more on most checkpoints, and does + so on the CPU too. + - SmolVLA's loader learned to keep packed weights. +4. **Serving dependencies** are optional (`VLA_BUILD_SERVER`), and on Windows + come from vcpkg. +5. **No scheduler.** + - The fallback wraps one backend instead, so the ~84 gallocr/compute sites + across 13 archs stay as they were. + - CUDA, Metal, SYCL and OpenVINO do not compile it. + - K/V for flash attention are cast to F16 on Hexagon only (`vla::fa_kv`), + because the NPU kernel takes nothing else. + +## Bring-up checklist, for a new part + +In order. Each step is cheap, and steps 4 and 5 are where every bug above was +found. + +1. **Confirm the arch and session lines.** + - Expect `ggml-hex: Hexagon Arch version v73` (or v79/v81) and + `allocating new session`. + - A v73 line on a part you believe is newer means `htpdrv_get_arch` failed, + and you are on the fallback skel. +2. **`llama-bench` on a small model**, to prove the driver, signing and + `ADSP_LIBRARY_PATH` before any VLA is involved. +3. **Run a VLA and read the fallback lines.** Each process prints which ops went + to the CPU and how much was copied (`VLA_FALLBACK_STATS=1` for per-graph + detail). +4. **Fidelity before latency.** + - Compare `vla_predict_check` output against the same binary with + `VLA_DEVICE=cpu`, and against `--weight-dtype f32`. + - The bar is 2.9e-3. + - A run with a wrong kernel still prints a latency, and it is usually a good + one. +5. **Bisect anything off the bar** with `GGML_HEXAGON_OPFILTER=`, which + moves matching ops to the CPU fallback. Families first, then single ops. + `GGML_HEXAGON_OPFILTER=.*` must reproduce the CPU run exactly; if it does + not, the fallback is at fault, not the NPU. +6. **Then profile.** Use `GGML_HEXAGON_PROFILE=1 ... |& scripts/snapdragon/ggml-hexagon-profile.py -`. ## Building llama.cpp for the board @@ -208,89 +210,7 @@ SSH or ADB from the host: llama-completion -m gemma-2b-it-Q4_0.gguf -f prompt.txt --split-mode tensor ``` -## Bring-up checklist, for the day a board arrives - -In order. Each step is cheap and each one has failed for somebody upstream. - -1. **Confirm the arch and session lines.** `ggml-hex: Hexagon Arch version v79` - and `allocating new session: HTP0:0`. A v73 line on a part you believe is - newer means `htpdrv_get_arch` failed and you are on the fallback skel. -2. **`test-backend-ops` before anything of ours.** - `run.py --hex-hostbuf 0 --devices HTP0:0 -- test-backend-ops -b HTP0:0 -o MUL_MAT`, - then `-o IM2COL`, `-o FLASH_ATTN_EXT`, `-o ROPE`. Filter to the dtypes we - would actually ship (`q8_0`, `q4_0`, `f16`). -3. **`llama-bench` on a 1B Q4_0** to get a throughput anchor on *this* board that - is comparable with upstream's published numbers, before any VLA is involved. -4. **Load a requantized VLA with `GGML_HEXAGON_VERBOSE=1`** and read which ops the - NPU accepted. This is where the two missing unaries will announce themselves. -5. **Fidelity before latency.** max|delta| against a CPU-backend reference on the - same checkpoint, the same 2.9e-3 bar SYCL and OpenVINO are held to - ([ov-progress.md](ov-progress.md)). A partially-failing run still prints a - wall-clock time; do not read a latency off a run whose actions did not come out. -6. **Then profile.** `GGML_HEXAGON_PROFILE=1 ... |& scripts/snapdragon/ggml-hexagon-profile.py -`, - and `GGML_HEXAGON_OPFILTER` to bisect anything that looks wrong. - -## Projections - -Everything in this section is arithmetic over numbers measured elsewhere. It is -here to set expectations and to be *falsified* by the first real run, not to be -quoted. - -Measured reference points, all ours, all `server_total_ms` per action chunk -(`ci/baselines/`, [sycl.md](sycl.md)): - -| Device | SmolVLA | π0 | GR00T N1.5 | Evo-1 | -|---|---:|---:|---:|---:| -| Ryzen 5 5500, CPU backend, 8 threads | 1,920 | - | - | 7,695 | -| Jetson Orin Nano | 510 | 1,485 | 1,183 | 3,552 | -| Apple M4 (Metal) | 324 | 1,129 | - | - | -| Arc A380 (SYCL) | 630 | - | - | 1,176 | -| RTX 3090 | 99 | 262 | 204 | 487 | - -The only Hexagon anchor that exists is upstream's own, on a v79 phone part: -Llama-3.2-1B Q4_0, **pp128 = 169 t/s**, **tg64 = 51.5 t/s** (single `HTP0` -session). Read as work rather than tokens: prefill at 169 t/s over 1.24 B params -is ≈ 0.42 TFLOP/s effective; decode at 51.5 t/s over 730 MiB of weights is -≈ 37 GB/s of weight traffic, which is a believable LPDDR5 read rate and a good -sign the published number is not cherry-picked. - -A VLA action chunk is **all prefill and no decode**: fixed shapes, a tiny KV -cache, no sampling loop. That is the regime the 0.42 TFLOP/s figure comes from, -which makes the extrapolation less dishonest than usual. SmolVLA is 1.07 GiB of -BF16 weights (≈ 0.54 B params), spent on three stages per prediction - a vision -tower over 2-3 views, a ~113-token prefix through the VLM, and 10 denoise steps -over a ~50-token suffix through the much smaller action expert. Multiplying that -out lands in the region of 300-400 GFLOP per chunk, so at the anchor rate: - -**SmolVLA, single v79-class HTP, Q4_0: 0.8-2.0 s per action chunk.** The band is -wide on purpose. The bottom needs the vision tower to land on the NPU via the new -`IM2COL` and the denoise loop to stay fed; the top is what you get if the loop is -dispatch-bound and the towers fall back. Either way that is somewhere between the -Orin Nano (510 ms) and a desktop CPU (1,920 ms) - useful for a 5-10 Hz policy on -a part that draws a few watts, not competitive with a discrete GPU. Expect the -first honest number to be nearer the top of the band and to improve with repack, -fusion and `OPPOLL`, exactly as upstream's own numbers did over the 107 commits -that have touched `ggml/src/ggml-hexagon/` since the backend landed in October -2025 (`63d2fc46`, "experimental"). - -Where the interesting risk is: - -- **The vision tower is the prize.** It is 58% of SmolVLA's CPU-backend time - (1,119 ms of 1,920 ms) and 7.1x faster on an A380. `IM2COL` landing for - patch-embed convolutions specifically is the single most encouraging thing in - this entire history, and it landed in July 2026 for reasons that had nothing to - do with us. -- **The denoise loop is the risk.** Ten sequential steps of small GEMMs is the - worst case for any offload engine - it is why SmolVLA gains least on SYCL - (3.0x when its own vision tower gains 7.1x). The fully-async backend, op - batching and graph reuse from #26501 are aimed at exactly this, which is - encouraging, but "many small ops over FastRPC" is a dispatch-latency problem - and NPUs do not usually win those. -- **The lm-head refusal does not touch us.** Hexagon bails on `src0->ne[1] > - 32768`; VLA action heads are small and no in-tree arch decodes a vocabulary on - the action path. - -### IQ-10, imagined +## IQ-10, imagined Nothing about a second-generation part is knowable from this repository, so what follows is a bet, not a forecast. The code in #26501 tells you what Qualcomm @@ -308,21 +228,24 @@ steady stream of chunks to pipeline. `vla.cpp` cannot do that today, for the same reason it cannot split across an Intel iGPU and NPU ([ov.md](ov.md) reaches this conclusion from the other -direction): the core drives one backend for a whole prediction. It would need a +direction): the core drives one backend for a whole prediction. (The CPU fallback +splits a graph by op, not by stage, so it does not change this.) It would need a per-*stage* backend, not a per-op scheduler. The seam is already in the right place. That is the one engine change this whole document argues for, and it pays off on more than Hexagon. -## What would falsify all of this - -- `RELU` and `GELU_ERF` turn out to be a 200-line HVX patch each → the port is a - weekend once the board exists, and everything above about schedulers is moot. -- `IM2COL`'s "partial" support turns out not to cover our patch shapes after all - (multi-view batching, a non-contiguous pixel buffer) → the vision tower stays - on CPU, the projection's lower bound disappears, and the whole exercise gets - much less interesting. -- Q8_0 requantization moves any arch past 2.9e-3 → the dtype story needs work - before the backend story does. -- VTCM budgets reject our GEMM shapes (which are wider and shorter than an LLM's) - → `supports_op` starts returning false for reasons no table here predicts, and - the answer is measurement, not more reading. +## What the measurements settled + +The previous version listed four ways it could be wrong. Here is how each came out: + +- **"`RELU` and `GELU_ERF` are a weekend each."** `RELU` came from upstream; + `GELU_ERF` has not. The CPU fallback made both moot for correctness. +- **"`IM2COL`'s partial support might not cover our patch shapes."** It covers + SigLIP's 16x16/16 patch embed, and SmolVLA's vision tower runs 4x faster than + on the CPU. It gets every other geometry wrong while claiming it. +- **"Q8_0 might move an arch past 2.9e-3."** It does, for most archs and on the + CPU as much as the NPU, so F16 is the default. +- **"VTCM might reject our GEMM shapes."** Nothing was rejected for VTCM. The + NPU loses where the fallback copies dominate (GELU_ERF towers, 1025-wide + softmaxes), and on SigLIP-so400m, whose tower is slower on the NPU than on the + CPU with nothing falling back. diff --git a/scripts/build_windows_snapdragon.ps1 b/scripts/build_windows_snapdragon.ps1 new file mode 100644 index 0000000..7e5da9e --- /dev/null +++ b/scripts/build_windows_snapdragon.ps1 @@ -0,0 +1,92 @@ +# Build vla.cpp natively on a Snapdragon X laptop (Windows on Arm). +# +# .\scripts\build_windows_snapdragon.ps1 -Backend htp # Hexagon NPU (+ CPU fallback) +# .\scripts\build_windows_snapdragon.ps1 -Backend opencl # Adreno GPU (+ CPU fallback) +# .\scripts\build_windows_snapdragon.ps1 -Backend cpu # Oryon CPU only +# +# Binaries land in build-wos-\bin. See docs/backend/hexagon-windows.md +# for the one-off setup this assumes: Visual Studio with its bundled Clang, +# CMake and Ninja; the Hexagon and OpenCL SDKs (HEXAGON_SDK_ROOT, +# HEXAGON_TOOLS_ROOT, OPENCL_SDK_ROOT); vcpkg for the server's dependencies; and, +# for the NPU, a code-signing certificate for the HTP skels. +param( + [ValidateSet("htp", "opencl", "cpu")][string]$Backend = "htp", + # A llama.cpp checkout to build against. Empty fetches the tag CMakeLists.txt pins. + [string]$LlamaDir = "", + [string]$VsRoot = "C:\Program Files\Microsoft Visual Studio\18\Insiders", + [string]$VcpkgRoot = $(if ($env:VCPKG_ROOT) { $env:VCPKG_ROOT } else { "C:\vcpkg" }), + # Signs the HTP skels. Without it the NPU refuses to load them. + [string]$HtpCert = $env:HEXAGON_HTP_CERT, + [string]$WindowsSdkBin = "C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0", + [switch]$NoServer, + [switch]$Reconfigure +) +# Not "Stop": Windows PowerShell turns any stderr line from a native tool (a CMake +# warning) into a terminating error. Exit codes are checked after each step. +$ErrorActionPreference = "Continue" +$root = Split-Path -Parent $PSScriptRoot +Set-Location $root + +& "$VsRoot\Common7\Tools\Launch-VsDevShell.ps1" -Arch arm64 -HostArch arm64 -SkipAutomaticLocation | Out-Null +$env:PATH = "$VsRoot\VC\Tools\Llvm\ARM64\bin;$env:PATH" +Set-Location $root + +$dir = "build-wos-$Backend" +$flags = "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -D_GNU_SOURCE" +# protobuf and abseil must be built by clang too, with the triplet in +# cmake/vcpkg-triplets: clang code does not link against an MSVC-built protobuf. +# ZeroMQ is a C API and comes from the stock triplet, searched second. +$triplet = "$VcpkgRoot\installed\arm64-windows-clangcl" +$tripletC = "$VcpkgRoot\installed\arm64-windows" + +# llama.cpp's toolchain file (vendored: a fetch has it only after configure) +# drives clang against the MSVC ABI. +$toolchain = "$root\cmake\arm64-windows-llvm.cmake" + +$args_ = @( + "-S", ".", "-B", $dir, "-G", "Ninja", + "-DCMAKE_BUILD_TYPE=Release", + "-DCMAKE_TOOLCHAIN_FILE=$($toolchain -replace '\\','/')", + "-DCMAKE_C_FLAGS=$flags", "-DCMAKE_CXX_FLAGS=$flags", + "-DGGML_OPENMP=OFF", "-DGGML_LLAMAFILE=OFF", "-DLLAMA_OPENSSL=OFF", + "-DVLA_BUILD_TESTS=ON" +) +if ($LlamaDir) { $args_ += "-DFETCHCONTENT_SOURCE_DIR_LLAMA=$($LlamaDir -replace '\\','/')" } +if ($NoServer) { + # Octo tokenizes in-process through SentencePiece, which needs protobuf too. + $args_ += @("-DVLA_BUILD_SERVER=OFF", "-DVLA_OCTO=OFF") +} else { + $args_ += @( + "-DCMAKE_PREFIX_PATH=$($triplet -replace '\\','/');$($tripletC -replace '\\','/');$($env:OPENCL_SDK_ROOT -replace '\\','/')", + # vcpkg's protobuf is found through its CMake package, which carries the + # abseil dependencies the old FindProtobuf module does not. + "-DCMAKE_FIND_PACKAGE_PREFER_CONFIG=ON", "-Dprotobuf_MODULE_COMPATIBLE=ON", + "-DProtobuf_PROTOC_EXECUTABLE=$($triplet -replace '\\','/')/tools/protobuf/protoc.exe" + ) +} +switch ($Backend) { + "htp" { + if (-not $HtpCert) { Write-Warning "no -HtpCert / HEXAGON_HTP_CERT: the HTP skels will be unsigned and the NPU will not load them" } + $env:HEXAGON_HTP_CERT = $HtpCert + $env:WINDOWS_SDK_BIN = $WindowsSdkBin + $args_ += @("-DGGML_HEXAGON=ON", "-DGGML_OPENCL=OFF", "-DPREBUILT_LIB_DIR=windows_aarch64", + "-DHEXAGON_SDK_ROOT=$env:HEXAGON_SDK_ROOT", "-DHEXAGON_TOOLS_ROOT=$env:HEXAGON_TOOLS_ROOT") + } + "opencl" { $args_ += @("-DGGML_OPENCL=ON", "-DGGML_HEXAGON=OFF") } + "cpu" { $args_ += @("-DGGML_OPENCL=OFF", "-DGGML_HEXAGON=OFF") } +} + +if ($Reconfigure -or -not (Test-Path "$dir\CMakeCache.txt")) { + cmake @args_ + if ($LASTEXITCODE -ne 0) { throw "configure failed" } +} +if ($Backend -eq "htp") { + # ggml-hexagon signs the skel catalog in a step that depends on the .so + # files but not on the targets that build them, so a parallel build can + # reach inf2cat first. Build the skels on their own before everything else. + cmake --build $dir -j 8 --target htp-v73 htp-v75 htp-v79 htp-v81 + if ($LASTEXITCODE -ne 0) { throw "HTP skel build failed" } +} +cmake --build $dir -j 8 +if ($LASTEXITCODE -ne 0) { throw "build failed" } +Write-Host "built $dir\bin" diff --git a/src/backend.h b/src/backend.h index 9223fc5..5af6acb 100644 --- a/src/backend.h +++ b/src/backend.h @@ -21,10 +21,12 @@ * instead; the ladder lives here once. * * Exactly one accelerator is compiled in, picked by the CMake flag that was - * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL` / `GGML_OPENVINO`). There is no - * per-op CPU fallback: the core drives a single backend through `gallocr` rather - * than a scheduler, so an arch that hits an op the backend does not implement - * asserts at predict time instead of silently limping. + * used (`GGML_CUDA` / `GGML_SYCL` / `GGML_METAL` / `GGML_OPENVINO` / + * `GGML_HEXAGON` / `GGML_OPENCL`). The core drives a single backend through + * `gallocr` rather than a scheduler. The first four run every op the archs + * build, so there is no per-op CPU fallback for them. Hexagon and OpenCL do + * not, and they are wrapped by @ref fallback_backend_new, which runs the ops + * they reject on the CPU. */ #pragma once @@ -45,6 +47,15 @@ #ifdef GGML_USE_OPENVINO #include "ggml-openvino.h" #endif +#ifdef GGML_USE_HEXAGON +#include "ggml-hexagon.h" +#endif +#ifdef GGML_USE_OPENCL +#include "ggml-opencl.h" +#endif +#if defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) +#include "backend_fallback.h" +#endif #include #include @@ -53,14 +64,14 @@ #include #include #endif -#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) #include // setenv / _putenv_s #include #endif namespace vla { -#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) // setenv is POSIX. _putenv_s has no "do not overwrite" mode, so check first. // Empty counts as unset; an empty KEY= in a compose file is not a choice. inline void setenv_default(const char * key, const char * val) { @@ -151,6 +162,77 @@ inline ggml_backend_buffer_t alloc_weights(ggml_context * ctx, ggml_backend_t ba return buf; } +/** + * @brief K or V as the compiled-in backend's flash-attention kernel takes it. + * + * The archs hand `ggml_flash_attn_ext` F32 K/V. Hexagon's kernel accepts only + * F16 or Q8_0 K/V, so F32 sends every attention to the CPU fallback, and a + * 10-step denoise loop then copies its activations back and forth once per + * layer per step. Cast before any permute: the NPU's copy kernel converts + * contiguous tensors only. Everywhere else this returns @p t untouched, so the + * graph and its numerics are unchanged. + */ +inline ggml_tensor * fa_kv([[maybe_unused]] ggml_context * ctx, ggml_tensor * t) { +#ifdef GGML_USE_HEXAGON + return t->type == GGML_TYPE_F16 ? t : ggml_cast(ctx, t, GGML_TYPE_F16); +#else + return t; +#endif +} + +/** + * @brief `ggml_gelu` (the tanh form), in ops the compiled-in backend gets right. + * + * ggml-hexagon runs GELU as x*sigmoid(1.702x), which is GELU_QUICK, so the + * fallback wrapper refuses it and every tower layer would round-trip its MLP + * activations through the CPU. The tanh form is exactly + * x*sigmoid(2*sqrt(2/pi)*(x + 0.044715x^3)), since (1+tanh z)/2 = sigmoid(2z), + * and HTP implements each of those ops. Elsewhere this is plain ggml_gelu. + */ +inline ggml_tensor * gelu(ggml_context * ctx, ggml_tensor * x) { +#ifdef GGML_USE_HEXAGON + const float c = 0.7978845608028654f; // sqrt(2/pi) + ggml_tensor * x3 = ggml_mul(ctx, ggml_mul(ctx, x, x), x); + ggml_tensor * z = ggml_add(ctx, x, ggml_scale(ctx, x3, 0.044715f)); + return ggml_mul(ctx, x, ggml_sigmoid(ctx, ggml_scale(ctx, z, 2.0f * c))); +#else + return ggml_gelu(ctx, x); +#endif +} + +/** + * @brief An arch's default resident type for GEMM weights, on this backend. + * + * Most archs keep BF16 weights resident. Hexagon has no BF16 kernel for any op + * and Adreno's OpenCL backend none for MUL_MAT, so BF16 there means every GEMM + * runs on the CPU fallback. F16 holds every BF16 value in the normal range + * exactly (it has more mantissa bits), so the swap costs nothing. (On Adreno + * that holds only with the xmem GEMM off; see the OpenCL rung.) + * `--weight-dtype` still overrides. + */ +inline ggml_type default_weight_dtype(ggml_type arch_default) { +#if defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) + return arch_default == GGML_TYPE_BF16 ? GGML_TYPE_F16 : arch_default; +#else + return arch_default; +#endif +} + +/** + * @brief Whether towers use flash attention when `--flash-attn` is not given. + * + * On Hexagon the explicit path's 1024x1024 F32 score matmuls are what make + * SmolVLA's vision stage take 3.8 s; flash attention runs it in 0.44 s within + * the fidelity bar. Everywhere else the default stays off. + */ +constexpr bool default_flash_attn() { +#ifdef GGML_USE_HEXAGON + return true; +#else + return false; +#endif +} + /// GPU ordinal for CUDA and SYCL; `VLA_DEVICE` overrides. Junk is rejected, not /// silently read as device 0. inline int backend_device_index() { @@ -293,6 +375,52 @@ inline Backend backend_init(const char * tag, int n_threads) { std::fprintf(stderr, "%s: ggml_backend_openvino_init failed; falling back to CPU\n", tag); } } +#elif defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) + { + // Both go through the registry: ggml_backend_hexagon_init is declared + // but has no definition upstream. VLA_DEVICE indexes the registry's + // devices (HTP sessions, or OpenCL GPUs), and VLA_DEVICE=cpu skips the + // accelerator so the same build yields the CPU reference an + // accelerator run is checked against. + const char * want = std::getenv("VLA_DEVICE"); + if (want && (std::strcmp(want, "cpu") == 0 || std::strcmp(want, "CPU") == 0)) { + std::printf("%s: VLA_DEVICE=%s, not starting the accelerator\n", tag, want); + } else { +#ifdef GGML_USE_HEXAGON + // Op fusion (MUL_MAT+ADD, RMS_NORM+MUL) takes SmolVLA from 1.6e-3 to + // 1.2e-2 max|delta| against the CPU reference, past the 2.9e-3 bar + // the other backends are held to. Only a default: an explicit + // GGML_HEXAGON_OPFUSION wins. Read at registry init, so set it first. + static std::once_flag fuse_once; + std::call_once(fuse_once, [] { setenv_default("GGML_HEXAGON_OPFUSION", "0"); }); + hexagon_default_skel_path(); + ggml_backend_reg_t reg = ggml_backend_hexagon_reg(); + const char * what = "Hexagon"; +#else + // Adreno's "xmem" F16xF32 GEMM prepacks weights into images: it + // rounds activations (SmolVLA 5.0e-3 max|delta| against an F32 + // reference, 2.9e-4 without it) and aborts with CL_OUT_OF_RESOURCES + // on GR00T N1.7's larger weights. It is ~1.4x faster where it works, + // so only a default: GGML_OPENCL_ADRENO_XMEM_GEMM=1 turns it back on. + static std::once_flag xmem_once; + std::call_once(xmem_once, [] { setenv_default("GGML_OPENCL_ADRENO_XMEM_GEMM", "0"); }); + ggml_backend_reg_t reg = ggml_backend_opencl_reg(); + const char * what = "OpenCL"; +#endif + const int dev = backend_device_index(); + const size_t n_dev = reg ? ggml_backend_reg_dev_count(reg) : 0; + ggml_backend_t accel = nullptr; + if ((size_t) dev >= n_dev) { + std::fprintf(stderr, "%s: %s device %d out of range (%zu visible); falling back to CPU\n", + tag, what, dev, n_dev); + } else if ((accel = ggml_backend_dev_init(ggml_backend_reg_dev_get(reg, dev), nullptr)) == nullptr) { + std::fprintf(stderr, "%s: %s init failed; falling back to CPU\n", tag, what); + } else if ((b.handle = fallback_backend_new(accel, n_threads)) != nullptr) { + std::printf("%s: backend = %s (%s), rejected ops on CPU (%d threads)\n", + tag, what, ggml_backend_name(b.handle), n_threads); + } + } + } #endif if (!b.handle) { diff --git a/src/backend_fallback.cpp b/src/backend_fallback.cpp new file mode 100644 index 0000000..21d7b9a --- /dev/null +++ b/src/backend_fallback.cpp @@ -0,0 +1,456 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "backend_fallback.h" + +#include "ggml.h" +#include "ggml-backend.h" +#include "ggml-cpu.h" +// Internal: the backend vtable, and struct ggml_cgraph for slicing a graph. +// ggml_graph_view itself is not exported from ggml-base.dll, so the slice is +// built here the same way. +#include "ggml-backend-impl.h" +#include "ggml-impl.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#define NOMINMAX +#include +#endif + +namespace vla { +namespace { + +enum class Where : uint8_t { Accel, Cpu, Either }; + +struct FallbackCtx { + ggml_backend_t accel = nullptr; + ggml_backend_t cpu = nullptr; + std::string name; + + // Host copies of weights some CPU-side op reads, made once. Keyed by tensor: + // the wrapper lives exactly as long as the model that owns the weights. + std::unordered_map> weights; + + // Host storage for one CPU run; the inner vectors keep their capacity across + // runs, so a steady-state predict does not allocate. + std::vector> pool; + size_t pool_used = 0; + + std::vector where; + std::set reported; + bool stats = false; + bool hexagon = false; // accel is an HTP session, see accel_runs() + + uint64_t n_graphs = 0, n_split_graphs = 0, n_cpu_nodes = 0, n_cpu_runs = 0; + uint64_t bytes_in = 0, bytes_out = 0; +}; + +uint8_t * take(FallbackCtx * fc, size_t n) { + if (fc->pool_used == fc->pool.size()) + fc->pool.emplace_back(); + std::vector & v = fc->pool[fc->pool_used++]; + if (v.size() < n) + v.resize(n); + return v.data(); +} + +// Ops that only reinterpret memory. They compute nothing, so they go wherever +// their neighbours go instead of splitting a run. +bool is_view_op(const ggml_tensor * t) { + switch (t->op) { + case GGML_OP_NONE: + case GGML_OP_VIEW: + case GGML_OP_RESHAPE: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + return true; + default: + return false; + } +} + +ggml_backend_buffer_t buffer_of(const ggml_tensor * t) { + return t->view_src ? t->view_src->buffer : t->buffer; +} + +bool is_weight(const ggml_tensor * t) { + ggml_backend_buffer_t b = buffer_of(t); + return b && ggml_backend_buffer_get_usage(b) == GGML_BACKEND_BUFFER_USAGE_WEIGHTS; +} + +bool is_host(const ggml_tensor * t) { + ggml_backend_buffer_t b = buffer_of(t); + return b && ggml_backend_buffer_is_host(b); +} + +void report(FallbackCtx * fc, const ggml_tensor * t, const char * why) { + std::string key = std::string(ggml_op_desc(t)) + "/" + ggml_type_name(t->type); + if (t->src[0]) + key += std::string("/") + ggml_type_name(t->src[0]->type); + if (!fc->reported.insert(key).second) + return; + std::fprintf(stderr, + "vla: %s %s %s (%s <- %s, [%lld,%lld,%lld,%lld], e.g. '%s'); running it on CPU\n", + fc->name.c_str(), why, ggml_op_desc(t), ggml_type_name(t->type), + t->src[0] ? ggml_type_name(t->src[0]->type) : "-", + (long long) t->ne[0], (long long) t->ne[1], (long long) t->ne[2], (long long) t->ne[3], + t->name); +} + +// A host tensor holding what `t` holds on the device, for the CPU to read. +ggml_tensor * host_input(FallbackCtx * fc, ggml_context * meta, + std::unordered_map & shadow, + ggml_tensor * t) { + auto it = shadow.find(t); + if (it != shadow.end()) + return it->second; + + ggml_tensor * s = ggml_new_tensor(meta, t->type, GGML_MAX_DIMS, t->ne); + std::memcpy(s->nb, t->nb, sizeof(s->nb)); + ggml_set_name(s, t->name); + + // ggml_nbytes follows the strides, so a permuted or strided view copies the + // span it covers, and the copy keeps the same strides. + const size_t n = ggml_nbytes(t); + if (is_host(t)) { + s->data = t->data; + } else if (is_weight(t)) { + std::vector & w = fc->weights[t]; + if (w.empty()) { + // get_tensor undoes the accelerator's repacking, so this is plain ggml layout. + w.resize(n); + ggml_backend_tensor_get(t, w.data(), 0, n); + fc->bytes_in += n; + } + s->data = w.data(); + } else { + s->data = take(fc, n); + ggml_backend_tensor_get(t, s->data, 0, n); + fc->bytes_in += n; + } + shadow[t] = s; + return s; +} + +enum ggml_status run_on_cpu(FallbackCtx * fc, ggml_cgraph * g, int i0, int i1) { + // The accelerator may still be writing what this run reads. + ggml_backend_synchronize(fc->accel); + + const int n_run = i1 - i0; + const size_t max_objs = (size_t) n_run * (GGML_MAX_SRC + 2); + const size_t gsize = std::max(GGML_DEFAULT_GRAPH_SIZE, max_objs); + ggml_init_params p = { + ggml_tensor_overhead() * max_objs + ggml_graph_overhead_custom(gsize, false), + nullptr, + true, + }; + ggml_context * meta = ggml_init(p); + if (!meta) + return GGML_STATUS_ALLOC_FAILED; + ggml_cgraph * sg = ggml_new_graph_custom(meta, gsize, false); + + fc->pool_used = 0; + std::unordered_map shadow; + shadow.reserve(max_objs); + + for (int k = i0; k < i1; ++k) { + ggml_tensor * node = g->nodes[k]; + ggml_tensor * s = ggml_new_tensor(meta, node->type, GGML_MAX_DIMS, node->ne); + s->op = node->op; + s->flags = node->flags; + std::memcpy(s->op_params, node->op_params, sizeof(s->op_params)); + std::memcpy(s->nb, node->nb, sizeof(s->nb)); + for (int j = 0; j < GGML_MAX_SRC; ++j) { + if (node->src[j]) + s->src[j] = host_input(fc, meta, shadow, node->src[j]); + } + if (node->view_src) { + // In-place ops and views write through their source, so they must + // land in the same host copy the source got. + ggml_tensor * vs = host_input(fc, meta, shadow, node->view_src); + s->view_src = vs; + s->view_offs = node->view_offs; + s->data = (char *) vs->data + node->view_offs; + } else { + s->data = take(fc, ggml_nbytes(node)); + } + ggml_set_name(s, node->name); + shadow[node] = s; + ggml_build_forward_expand(sg, s); + } + + const enum ggml_status st = ggml_backend_graph_compute(fc->cpu, sg); + + if (st == GGML_STATUS_SUCCESS) { + // Pure views wrote nothing; every other node's result goes back, since + // later accelerator runs (or the arch's tensor_get) read it there. + for (int k = i0; k < i1; ++k) { + ggml_tensor * node = g->nodes[k]; + if (is_view_op(node) || !node->data || !buffer_of(node)) + continue; + const size_t n = ggml_nbytes(node); + ggml_backend_tensor_set(node, shadow[node]->data, 0, n); + fc->bytes_out += n; + } + } + fc->n_cpu_nodes += (uint64_t) n_run; + fc->n_cpu_runs += 1; + ggml_free(meta); + return st; +} + +enum ggml_status run_on_accel(FallbackCtx * fc, ggml_cgraph * g, int i0, int i1) { + // Same slice ggml_graph_view makes: the node range, no leafs, and the parent's + // hash set so use counts (which fusion reads) still resolve. + ggml_cgraph v = *g; + v.size = 0; + v.n_nodes = i1 - i0; + v.n_leafs = 0; + v.nodes = g->nodes + i0; + v.grads = nullptr; + v.grad_accs = nullptr; + v.leafs = nullptr; + v.uid = 0; + return ggml_backend_graph_compute_async(fc->accel, &v); +} + +const char * fb_get_name(ggml_backend_t be) { + return static_cast(be->context)->name.c_str(); +} + +void fb_free(ggml_backend_t be) { + auto * fc = static_cast(be->context); + if (fc->n_split_graphs > 0) { + std::fprintf(stderr, + "vla: %s: %llu of %llu graphs split; %llu nodes in %llu runs went to CPU, " + "%.1f MiB copied in, %.1f MiB out, %.1f MiB of weights mirrored on host\n", + fc->name.c_str(), + (unsigned long long) fc->n_split_graphs, (unsigned long long) fc->n_graphs, + (unsigned long long) fc->n_cpu_nodes, (unsigned long long) fc->n_cpu_runs, + fc->bytes_in / 1048576.0, fc->bytes_out / 1048576.0, + [fc] { + size_t s = 0; + for (auto & kv : fc->weights) s += kv.second.size(); + return s / 1048576.0; + }()); + } + ggml_backend_free(fc->accel); + ggml_backend_free(fc->cpu); + delete fc; + delete be; +} + +void fb_synchronize(ggml_backend_t be) { + ggml_backend_synchronize(static_cast(be->context)->accel); +} + +// Whether b broadcasts over some dim d (b->ne[d] == 1 < a->ne[d]) while +// spanning a higher one. Row vectors and trailing broadcasts have no gap. +bool broadcast_has_gap(const ggml_tensor * a, const ggml_tensor * b) { + bool broadcast_below = false; + for (int d = 1; d < GGML_MAX_DIMS; ++d) { + if (b->ne[d] == 1 && a->ne[d] > 1) + broadcast_below = true; + else if (b->ne[d] > 1 && broadcast_below) + return true; + } + return false; +} + +// Whether the accelerator should run t. Normally its own supports_op, but an +// op it claims and computes differently from the CPU reference is refused here. +bool accel_runs(FallbackCtx * fc, const ggml_tensor * t, const char ** why) { + *why = "rejects"; + if (fc->hexagon && t->op == GGML_OP_UNARY && ggml_get_unary_op(t) == GGML_UNARY_OP_GELU) { + // ggml-hexagon maps GELU and GELU_QUICK to one kernel, x*sigmoid(1.702x), + // which is GELU_QUICK. ggml_gelu is the tanh form; on SmolVLA's SigLIP + // tower the difference flips the gripper channel of the action chunk. + *why = "computes GELU as GELU_QUICK for"; + return false; + } + if (fc->hexagon && + (t->op == GGML_OP_ADD || t->op == GGML_OP_SUB || t->op == GGML_OP_MUL || t->op == GGML_OP_DIV) && + broadcast_has_gap(t->src[0], t->src[1])) { + // Claimed, but a src1 that broadcasts over one dim while spanning a + // higher one comes out wrong. TurboVLA's RoPE multiplies + // [64,12,261,2] by a cos table [64,1,261,1]: max|delta| 0.72 on the + // action chunk, 1.2e-3 with these on the CPU. + *why = "miscomputes a gapped broadcast in"; + return false; + } + if (fc->hexagon && t->op == GGML_OP_IM2COL) { + // Written for patch embeddings (kernel == stride > 1, no padding, no + // dilation) but claimed for any geometry. Octo's stride-2, pad-1 stem + // convs and its 1x1 projection come out wrong: max|delta| 1.05, 5.2e-4 + // with them on the CPU. SigLIP's 16x16/16 patch embed is fine. + const int32_t * p = (const int32_t *) t->op_params; // s0 s1 p0 p1 d0 d1 is_2D + const int64_t kw = t->src[0]->ne[0]; + const int64_t kh = p[6] ? t->src[0]->ne[1] : 1; + const bool patch = p[2] == 0 && p[3] == 0 && p[4] == 1 && p[5] == 1 && kw > 1 && + p[0] == kw && (!p[6] || p[1] == kh); + if (!patch) { + *why = "miscomputes an overlapping or padded"; + return false; + } + } + return ggml_backend_supports_op(fc->accel, t); +} + +// The node whose memory t reads, looking through pure views. +const ggml_tensor * producer(const ggml_tensor * t) { + while (t && t->op != GGML_OP_NONE && is_view_op(t)) + t = t->src[0]; + return t; +} + +enum ggml_status fb_graph_compute(ggml_backend_t be, ggml_cgraph * g) { + auto * fc = static_cast(be->context); + const int n = g->n_nodes; + fc->n_graphs++; + + fc->where.resize((size_t) n); + bool any_cpu = false; + for (int i = 0; i < n; ++i) { + ggml_tensor * t = g->nodes[i]; + if (is_view_op(t) || ggml_is_empty(t)) { + fc->where[i] = Where::Either; + } else if (const char * why; accel_runs(fc, t, &why)) { + fc->where[i] = Where::Accel; + } else { + fc->where[i] = Where::Cpu; + any_cpu = true; + report(fc, t, why); + } + } + if (!any_cpu) + return ggml_backend_graph_compute_async(fc->accel, g); + fc->n_split_graphs++; + + if (fc->hexagon) { + // An F32->F16 CPY computed on the NPU reads back wrong on the host, while + // the NPU's own consumers see it right (SmolVLA: flash attention moved to + // the CPU gets max|delta| 1.2 on its F16 K/V, 1.7e-3 once the cast moves + // with it). So a copy feeding the CPU is done on the CPU too. Walking + // backwards lets a pulled copy pull the copy that feeds it. + std::unordered_map idx; + idx.reserve((size_t) n); + for (int i = 0; i < n; ++i) + idx[g->nodes[i]] = i; + for (int i = n - 1; i >= 0; --i) { + if (fc->where[i] != Where::Cpu) + continue; + for (int s = 0; s < GGML_MAX_SRC; ++s) { + const ggml_tensor * p = producer(g->nodes[i]->src[s]); + if (!p) + continue; + auto it = idx.find(p); + if (it == idx.end() || fc->where[it->second] != Where::Accel) + continue; + if (p->op == GGML_OP_CPY || p->op == GGML_OP_CONT || p->op == GGML_OP_DUP) + fc->where[it->second] = Where::Cpu; + } + } + } + + Where prev = Where::Accel; + for (int i = 0; i < n; ++i) { + if (fc->where[i] == Where::Either) + fc->where[i] = prev; + prev = fc->where[i]; + } + + int n_runs = 0; + for (int i = 0; i < n;) { + int j = i + 1; + while (j < n && fc->where[j] == fc->where[i]) + ++j; + const enum ggml_status st = fc->where[i] == Where::Cpu ? run_on_cpu(fc, g, i, j) + : run_on_accel(fc, g, i, j); + if (st != GGML_STATUS_SUCCESS) + return st; + ++n_runs; + i = j; + } + if (fc->stats) { + std::fprintf(stderr, "vla: %s: graph of %d nodes ran as %d runs\n", fc->name.c_str(), n, n_runs); + } + return GGML_STATUS_SUCCESS; +} + +ggml_guid_t fb_guid() { + static ggml_guid guid = { 0x76, 0x6c, 0x61, 0x2d, 0x66, 0x61, 0x6c, 0x6c, + 0x62, 0x61, 0x63, 0x6b, 0x2d, 0x63, 0x70, 0x75 }; + return &guid; +} + +} // namespace + +ggml_backend_t fallback_backend_new(ggml_backend_t accel, int n_threads) { + ggml_backend_t cpu = ggml_backend_cpu_init(); + if (!cpu) { + ggml_backend_free(accel); + return nullptr; + } + ggml_backend_cpu_set_n_threads(cpu, n_threads); + + auto * fc = new FallbackCtx; + fc->accel = accel; + fc->cpu = cpu; + fc->name = std::string(ggml_backend_name(accel)) + "+CPU"; + const char * st = std::getenv("VLA_FALLBACK_STATS"); + fc->stats = st && st[0] == '1'; + fc->hexagon = std::strncmp(ggml_backend_name(accel), "HTP", 3) == 0; + + ggml_backend_i iface = {}; + iface.get_name = fb_get_name; + iface.free = fb_free; + iface.synchronize = fb_synchronize; + iface.graph_compute = fb_graph_compute; + // Everything else stays null: tensor_set/get then go through the buffer, + // which is the accelerator's, and nobody here plans graphs or uses events. + + return new ggml_backend{ fb_guid(), iface, accel->device, fc }; +} + +void hexagon_default_skel_path() { +#ifdef _WIN32 + size_t len = 0; + if (getenv_s(&len, nullptr, 0, "ADSP_LIBRARY_PATH") == 0 && len > 1) + return; + char path[MAX_PATH]; + const DWORD n = GetModuleFileNameA(nullptr, path, MAX_PATH); + if (n == 0 || n >= MAX_PATH) + return; + char * slash = std::strrchr(path, '\\'); + if (!slash) + return; + *slash = '\0'; + // _putenv_s updates the process environment too, which is where the FastRPC + // loader in libcdsprpc.dll reads it from. + _putenv_s("ADSP_LIBRARY_PATH", path); +#endif +} + +} // namespace vla diff --git a/src/backend_fallback.h b/src/backend_fallback.h new file mode 100644 index 0000000..e42c4a2 --- /dev/null +++ b/src/backend_fallback.h @@ -0,0 +1,62 @@ +// Copyright 2026 VinRobotics +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + * @file backend_fallback.h + * @brief An accelerator backend that hands the ops it rejects to the CPU. + * + * The core drives one backend through `gallocr`, with no scheduler, and that is + * fine for CUDA, Metal, SYCL and OpenVINO because each of them runs every op the + * archs build. Hexagon and OpenCL do not: an NPU rejects an op it has no kernel + * for (GELU_ERF) or whose tile does not fit its VTCM, and the answer depends on + * tensor shapes, so it cannot be known before the graph exists. + * + * Instead of a `ggml_backend_sched` at every call site, the arch gets one + * backend that looks like the accelerator -- same device, same buffer type, so + * weights and activations still live on it -- but whose graph_compute walks the + * graph and splits it: runs of ops the accelerator accepts go to it unchanged, + * and runs it rejects are copied to host memory, computed on a CPU backend, and + * copied back. Weights read on the CPU side are copied once and kept. + * + * Only built with GGML_HEXAGON or GGML_OPENCL; nothing else links it. + */ + +#pragma once + +#include "ggml-backend.h" + +namespace vla { + +/** + * @brief Wrap @p accel so that ops it cannot run execute on the CPU. + * + * @param accel Accelerator backend; ownership passes to the wrapper, which + * frees it. Must not be null. + * @param n_threads Thread count for the CPU backend that runs rejected ops. + * @return The wrapper, or null if the CPU backend could not be created (in + * which case @p accel has been freed). + */ +ggml_backend_t fallback_backend_new(ggml_backend_t accel, int n_threads); + +/** + * @brief Point the Hexagon FastRPC loader at this executable's directory. + * + * The HTP skels (`libggml-htp-vNN.so` and their signed catalog) are copied + * beside the binaries at build time, and FastRPC looks for them only on + * `ADSP_LIBRARY_PATH`. Sets it to the executable's directory unless the + * environment already names one. No-op off Windows. + */ +void hexagon_default_skel_path(); + +} // namespace vla diff --git a/src/gguf_reader.h b/src/gguf_reader.h index be675c2..532d161 100644 --- a/src/gguf_reader.h +++ b/src/gguf_reader.h @@ -28,6 +28,16 @@ #include #include +// fseeko is POSIX; the Windows CRT spells the 64-bit seek _fseeki64. A plain +// fseek takes a 32-bit long there and cannot reach past 2 GiB into a GGUF. +inline int vla_fseek64(FILE * fp, uint64_t off) { +#ifdef _WIN32 + return _fseeki64(fp, (long long) off, SEEK_SET); +#else + return fseeko(fp, (off_t) off, SEEK_SET); +#endif +} + namespace vla { struct gguf_reader { @@ -129,7 +139,7 @@ struct gguf_reader { arch, name, nb, cap); return false; } - if (fseeko(fp, (off_t) off, SEEK_SET) != 0) + if (vla_fseek64(fp, off) != 0) return false; return std::fread(buf, 1, nb, fp) == nb; } @@ -141,15 +151,16 @@ struct gguf_reader { std::vector out(n); if (t->type == GGML_TYPE_F32) { if (!read_raw(name, out.data(), out.size()*sizeof(float))) return {}; } else if (t->type == GGML_TYPE_BF16) { std::vector tmp(n); if (!read_raw(name, tmp.data(), tmp.size()*sizeof(ggml_bf16_t))) return {}; ggml_bf16_to_fp32_row(tmp.data(), out.data(), n); } + else if (t->type == GGML_TYPE_F16) { std::vector tmp(n); if (!read_raw(name, tmp.data(), tmp.size()*sizeof(ggml_fp16_t))) return {}; ggml_fp16_to_fp32_row(tmp.data(), out.data(), n); } else { std::fprintf(stderr, "vla(%s): tensor %s unsupported type %d\n", arch, name, (int) t->type); return {}; } return out; } - // F32/BF16 targets dequantize to that resident type. Any other target (I8, - // Q8_0, Q4_0, ...) is stored raw so ggml_mul_mat dequantizes at compute. + // F32/BF16/F16 targets dequantize to that resident type. Any other target + // (I8, Q8_0, Q4_0, ...) is stored raw so ggml_mul_mat dequantizes at compute. // gemma_norm adds 1.0 per weight. std::vector read_convert(const char * name, ggml_type target, bool gemma_norm = false) { - if (target != GGML_TYPE_F32 && target != GGML_TYPE_BF16) { + if (target != GGML_TYPE_F32 && target != GGML_TYPE_BF16 && target != GGML_TYPE_F16) { if (gemma_norm) { // The +1 needs unpacked floats; skipping it would silently give wrong // norm weights. @@ -177,6 +188,11 @@ struct gguf_reader { ggml_fp32_to_bf16_row(f.data(), reinterpret_cast(o.data()), n); return o; } + if (target == GGML_TYPE_F16) { + std::vector o(n * sizeof(ggml_fp16_t)); + ggml_fp32_to_fp16_row(f.data(), reinterpret_cast(o.data()), n); + return o; + } std::fprintf(stderr, "vla(%s): unsupported resident type %d for %s\n", arch, (int) target, name); return {}; } @@ -202,7 +218,7 @@ struct gguf_reader { std::fprintf(stderr, "vla(%s): row %d out of range for %s\n", arch, r, name); return false; } - if (fseeko(fp, (off_t) (base+(size_t) r * rb), SEEK_SET) != 0) + if (vla_fseek64(fp, base+(size_t) r * rb) != 0) return false; if (std::fread(row.data(), 1, rb, fp) != rb) return false; diff --git a/src/layers/ffn.h b/src/layers/ffn.h index 2e26690..7b2b623 100644 --- a/src/layers/ffn.h +++ b/src/layers/ffn.h @@ -15,6 +15,7 @@ #pragma once +#include "backend.h" #include "layers/linear.h" #include "ggml.h" @@ -23,7 +24,7 @@ namespace vla { inline ggml_tensor * ffn_gelu(ggml_context * C, ggml_tensor * W1, ggml_tensor * b1, ggml_tensor * W2, ggml_tensor * b2, ggml_tensor * x) { - return linear(C, W2, b2, ggml_gelu(C, linear(C, W1, b1, x))); + return linear(C, W2, b2, gelu(C, linear(C, W1, b1, x))); } inline ggml_tensor * ffn_relu(ggml_context * C, ggml_tensor * W1, ggml_tensor * b1, diff --git a/src/model.cpp b/src/model.cpp index 1b14754..6bf4f55 100644 --- a/src/model.cpp +++ b/src/model.cpp @@ -13,6 +13,7 @@ // limitations under the License. #include "arch.h" +#include "backend.h" #include "model.h" #include "options.h" @@ -231,7 +232,7 @@ Model* model_load(const std::string& mmproj_path, const std::string& ckpt_path, } } - set_flash_attn(opts.flash_attn.value_or(false)); + set_flash_attn(opts.flash_attn.value_or(default_flash_attn())); set_mm_prec_f32(opts.mm_prec_f32.value_or(true)); switch (arch) { diff --git a/src/models/evo1.cpp b/src/models/evo1.cpp index 1721fb6..bdeb9a9 100644 --- a/src/models/evo1.cpp +++ b/src/models/evo1.cpp @@ -210,7 +210,9 @@ ggml_tensor * evo1_flash_attn(ggml_context * C, ggml_tensor * q, ggml_tensor * k // K/V stay F32. Casting them to F16 (as some in-tree FA helpers do) costs // real precision: over 24 ViT layers it moved evo1's actions by ~1e-2, which // is enough to change a LIBERO episode's outcome. smolvla's expert passes - // F32 K/V to the same op, so the backend handles it. + // F32 K/V to the same op, so the backend handles it. Hexagon's kernel takes + // F16 K/V only, and casting there measured 3.9e-2 against the CPU reference, + // so on Hexagon this attention runs on the CPU fallback instead. ggml_tensor * o = ggml_flash_attn_ext(C, q, k, v, nullptr, scale, 0.0f, 0.0f); // F32 accumulation keeps the softmax/AV reduction at the precision the // explicit path used, so switching kernels does not move the actions. @@ -366,7 +368,7 @@ std::unique_ptr evo1_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); if (!m->io.open(ckpt_path)) return nullptr; diff --git a/src/models/gr00tn1d5.cpp b/src/models/gr00tn1d5.cpp index bd3cac8..03a70ea 100644 --- a/src/models/gr00tn1d5.cpp +++ b/src/models/gr00tn1d5.cpp @@ -210,7 +210,7 @@ std::unique_ptr gr00t_n1_5_create(const std::string& mmproj_path, std::printf("vla(gr00tn1d5): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) diff --git a/src/models/gr00tn1d6.cpp b/src/models/gr00tn1d6.cpp index d2e49aa..c017f36 100644 --- a/src/models/gr00tn1d6.cpp +++ b/src/models/gr00tn1d6.cpp @@ -244,7 +244,7 @@ std::unique_ptr gr00t_n1_6_create(const std::string& mmproj_path, std::printf("vla(gr00tn1d6): note - mmproj '%s' is ignored (the vision tower is bundled in the combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); m->lm.cfg.rope.freq_base = 1000000.0f; if (!m->io.open(ckpt_path)) diff --git a/src/models/gr00tn1d7.cpp b/src/models/gr00tn1d7.cpp index a64cf7e..0e959b3 100644 --- a/src/models/gr00tn1d7.cpp +++ b/src/models/gr00tn1d7.cpp @@ -245,7 +245,7 @@ std::unique_ptr gr00t_n1_7_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); gguf_reader g("gr00tn1d7"); if (!g.open(ckpt_path)) diff --git a/src/models/openvla_oft.cpp b/src/models/openvla_oft.cpp index c355b43..735b1fe 100644 --- a/src/models/openvla_oft.cpp +++ b/src/models/openvla_oft.cpp @@ -159,7 +159,7 @@ std::unique_ptr openvla_oft_create(const std::string& mmproj_path if (!mmproj_path.empty()) std::printf("vla(openvla_oft): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->mt = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); gguf_reader g("openvla_oft"); if (!g.open(ckpt_path)) diff --git a/src/models/pi0.cpp b/src/models/pi0.cpp index 43c8076..27f25ff 100644 --- a/src/models/pi0.cpp +++ b/src/models/pi0.cpp @@ -137,9 +137,10 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_ten ggml_tensor * att; if (vla::flash_attn_enabled()) { // Avoids materialising the per-head score matrix; K/V stay F32 so the - // numerics track the explicit path below. + // numerics track the explicit path below (except on Hexagon, whose + // kernel takes F16 K/V only; see fa_kv). ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * fa = ggml_flash_attn_ext(C, Q, K, V, nullptr, scale, 0.0f, 0.0f); + ggml_tensor * fa = ggml_flash_attn_ext(C, Q, vla::fa_kv(C, K), vla::fa_kv(C, V), nullptr, scale, 0.0f, 0.0f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); att = ggml_reshape_2d(C, fa, hidden, seq); } else { @@ -150,7 +151,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_ten } ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, mm_act(C, w.Wo, as_type(C, att, at), at), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, mm_act(C, w.Wfc2, ggml_gelu(C, ggml_add(C, mm_act(C, w.Wfc1, n2, at), w.bfc1)), at), w.bfc2); + ggml_tensor * ff = ggml_add(C, mm_act(C, w.Wfc2, vla::gelu(C, ggml_add(C, mm_act(C, w.Wfc1, n2, at), w.bfc1)), at), w.bfc2); return ggml_add(C, h1, ff); } @@ -209,7 +210,7 @@ ggml_tensor * build_gemma_layer( // ggml_flash_attn_ext asserts an F16 mask. The mask holds only 0 and // -inf, both exactly representable in F16, so the cast is lossless. ggml_tensor * mask_f16 = mask ? ggml_cast(ctx, mask, GGML_TYPE_F16) : nullptr; - ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, K, V, mask_f16, scale, 0.0f, 0.0f); + ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, vla::fa_kv(ctx, K), vla::fa_kv(ctx, V), mask_f16, scale, 0.0f, 0.0f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); att_pre = ggml_reshape_2d(ctx, fa, qf, seq); } else { @@ -227,7 +228,7 @@ ggml_tensor * build_gemma_layer( ggml_tensor * x_norm_mlp = ggml_mul(ctx, ggml_rms_norm(ctx, h1, cfg.rms_eps), w.ln_post); ggml_tensor * gate = mm_act(ctx, w.Wgate, x_norm_mlp, at); ggml_tensor * up = mm_act(ctx, w.Wup, x_norm_mlp, at); - ggml_tensor * inter_t = ggml_mul(ctx, ggml_gelu(ctx, gate), up); + ggml_tensor * inter_t = ggml_mul(ctx, vla::gelu(ctx, gate), up); ggml_tensor * mlp_out = mm_act(ctx, w.Wdown, inter_t, at); return ggml_add(ctx, h1, mlp_out); } @@ -359,7 +360,7 @@ std::unique_ptr pi0_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); if (!m->io.open(ckpt_path)) return nullptr; diff --git a/src/models/pi05.cpp b/src/models/pi05.cpp index 6a0079b..02a5cce 100644 --- a/src/models/pi05.cpp +++ b/src/models/pi05.cpp @@ -143,7 +143,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_ten ggml_tensor * att = ggml_reshape_2d(C, ggml_cont(C, ggml_permute(C, ggml_mul_mat(C, V, aw), 0, 2, 1, 3)), hidden, seq); ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); + ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, vla::gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); return ggml_add(C, h1, ff); } @@ -200,7 +200,7 @@ ggml_tensor * build_vlm_layer( ggml_tensor * x_norm_mlp = ggml_mul(ctx, ggml_rms_norm(ctx, h1, cfg.rms_eps), w.ln_post); ggml_tensor * gate = ggml_mul_mat(ctx, w.Wgate, x_norm_mlp); ggml_tensor * up = ggml_mul_mat(ctx, w.Wup, x_norm_mlp); - ggml_tensor * inter_t = ggml_mul(ctx, ggml_gelu(ctx, gate), up); + ggml_tensor * inter_t = ggml_mul(ctx, vla::gelu(ctx, gate), up); ggml_tensor * mlp_out = ggml_mul_mat(ctx, w.Wdown, inter_t); return ggml_add(ctx, h1, mlp_out); } @@ -275,7 +275,7 @@ ggml_tensor * build_expert_layer( ggml_tensor * x_norm_mlp = build_adarms(ctx, h1, w.ada_post_w, w.ada_post_b, cond, h, cfg.rms_eps, &gate_ffn); ggml_tensor * gate = ggml_mul_mat(ctx, w.Wgate, x_norm_mlp); ggml_tensor * up = ggml_mul_mat(ctx, w.Wup, x_norm_mlp); - ggml_tensor * inter_t = ggml_mul(ctx, ggml_gelu(ctx, gate), up); + ggml_tensor * inter_t = ggml_mul(ctx, vla::gelu(ctx, gate), up); ggml_tensor * mlp_out = ggml_mul_mat(ctx, w.Wdown, inter_t); return ggml_add(ctx, h1, ggml_mul(ctx, mlp_out, gate_ffn)); @@ -399,7 +399,7 @@ std::unique_ptr pi05_create(const std::string& mmproj_path, auto m = std::make_unique(); m->ckpt_path_ = ckpt_path; - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); if (!m->io.open(ckpt_path)) return nullptr; diff --git a/src/models/smolvla.cpp b/src/models/smolvla.cpp index 4e522c8..d9fd0fc 100644 --- a/src/models/smolvla.cpp +++ b/src/models/smolvla.cpp @@ -213,7 +213,7 @@ struct gguf_source { const int64_t id = gguf_find_tensor(gctx, name.c_str()); const size_t offset = data_off+gguf_get_tensor_offset(gctx, id); const size_t bytes = gguf_get_tensor_size(gctx, id); - if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) { + if (vla_fseek64(fp, offset) != 0) { std::fprintf(stderr, "vla: fseek failed for %s\n", name.c_str()); return false; } @@ -225,6 +225,19 @@ struct gguf_source { if (std::fread(tmp.data(), 1, bytes, fp) != bytes) return false; ggml_bf16_to_fp32_row(tmp.data(), dst, tmp.size()); + } else if (t->type == GGML_TYPE_F16 || ggml_is_quantized(t->type)) { + // A requantized file (scripts/quantize_gguf.py) may pack a tensor + // this model keeps float; unpack it rather than refuse the file. + const ggml_type_traits * tt = ggml_get_type_traits(t->type); + if (!tt->to_float) { + std::fprintf(stderr, "vla: gguf cannot dequantize %s for %s\n", + ggml_type_name(t->type), name.c_str()); + return false; + } + std::vector tmp(bytes); + if (std::fread(tmp.data(), 1, bytes, fp) != bytes) + return false; + tt->to_float(tmp.data(), dst, ggml_nelements(t)); } else { std::fprintf(stderr, "vla: gguf unsupported dtype %d for %s\n", (int) t->type, name.c_str()); @@ -233,6 +246,24 @@ struct gguf_source { return true; } + /// Type of a tensor in the file, or GGML_TYPE_COUNT if it is absent. + ggml_type file_type(const std::string & name) const { + const ggml_tensor * t = ggml_get_tensor(meta_ctx, name.c_str()); + return t ? t->type : GGML_TYPE_COUNT; + } + + /// Packed bytes, stored as they are: the caller made the tensor that type. + bool read_packed(const std::string & name, void * dst, ggml_type want, size_t expected_bytes) { + const int64_t id = gguf_find_tensor(gctx, name.c_str()); + if (id < 0 || file_type(name) != want || gguf_get_tensor_size(gctx, id) != expected_bytes) { + std::fprintf(stderr, "vla: gguf bad packed read for %s\n", name.c_str()); + return false; + } + if (vla_fseek64(fp, data_off+gguf_get_tensor_offset(gctx, id)) != 0) + return false; + return std::fread(dst, 1, expected_bytes, fp) == expected_bytes; + } + bool read_raw(const std::string & name, void * dst, size_t expected_bytes, const char * expected_dtype) { ggml_tensor * t = ggml_get_tensor(meta_ctx, name.c_str()); @@ -249,7 +280,7 @@ struct gguf_source { return false; } const size_t offset = data_off+gguf_get_tensor_offset(gctx, id); - if (fseeko(fp, (off_t) offset, SEEK_SET) != 0) + if (vla_fseek64(fp, offset) != 0) return false; return std::fread(dst, 1, bytes, fp) == bytes; } @@ -410,7 +441,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_ten // numerics match the explicit path; the expert layers below already call // this op the same way. ggml_tensor * V = ggml_cont(C, ggml_permute(C, ggml_reshape_3d(C, v, head_dim, heads, seq), 0, 2, 1, 3)); - ggml_tensor * fa = ggml_flash_attn_ext(C, Q, K, V, nullptr, scale, 0.0f, 0.0f); + ggml_tensor * fa = ggml_flash_attn_ext(C, Q, vla::fa_kv(C, K), vla::fa_kv(C, V), nullptr, scale, 0.0f, 0.0f); ggml_flash_attn_ext_set_prec(fa, GGML_PREC_F32); att = ggml_reshape_2d(C, fa, hidden, seq); } else { @@ -421,7 +452,7 @@ ggml_tensor * build_siglip_layer(ggml_context * C, const EncBlockW & w, ggml_ten } ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); + ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, vla::gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); return ggml_add(C, h1, ff); } @@ -794,8 +825,8 @@ ggml_tensor * build_vlm_layer(ggml_context * ctx, const VlmLayerW & w, *v_out = v_h; ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); - ggml_tensor * K = ggml_permute(ctx, k_rope, 0, 2, 1, 3); - ggml_tensor * V = ggml_permute(ctx, v_h, 0, 2, 1, 3); + ggml_tensor * K = ggml_permute(ctx, vla::fa_kv(ctx, k_rope), 0, 2, 1, 3); + ggml_tensor * V = ggml_permute(ctx, vla::fa_kv(ctx, v_h), 0, 2, 1, 3); const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, K, V, mask, scale, 0.f, 0.f); @@ -833,8 +864,8 @@ ggml_tensor * build_expert_self_attn_layer( ggml_tensor * V_full = ggml_concat(ctx, cached_V, v_h, 2); ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); - ggml_tensor * Kp = ggml_permute(ctx, K_full, 0, 2, 1, 3); - ggml_tensor * Vp = ggml_permute(ctx, V_full, 0, 2, 1, 3); + ggml_tensor * Kp = ggml_permute(ctx, vla::fa_kv(ctx, K_full), 0, 2, 1, 3); + ggml_tensor * Vp = ggml_permute(ctx, vla::fa_kv(ctx, V_full), 0, 2, 1, 3); const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, Kp, Vp, mask_full, scale, 0.f, 0.f); @@ -872,8 +903,8 @@ ggml_tensor * build_expert_cross_attn_layer( ggml_tensor * q_rope = rope_q_or_k(ctx, q_h, positions_rebased, cfg); ggml_tensor * Q = ggml_permute(ctx, q_rope, 0, 2, 1, 3); - ggml_tensor * Kp = ggml_permute(ctx, K_repro, 0, 2, 1, 3); - ggml_tensor * Vp = ggml_permute(ctx, V_repro, 0, 2, 1, 3); + ggml_tensor * Kp = ggml_permute(ctx, vla::fa_kv(ctx, K_repro), 0, 2, 1, 3); + ggml_tensor * Vp = ggml_permute(ctx, vla::fa_kv(ctx, V_repro), 0, 2, 1, 3); const float scale = 1.f/std::sqrt(static_cast(cfg.head_dim)); ggml_tensor * fa = ggml_flash_attn_ext(ctx, Q, Kp, Vp, mask_prefix_only, scale, 0.f, 0.f); @@ -1241,6 +1272,35 @@ SmolVLAModelArch* smolvla_load_impl(ggml_type weight_dtype, cfg.expert_h, cfg.n_suffix); } + // A GEMM weight the file stores packed (Q8_0, Q4_0, ...) stays packed and + // ggml_mul_mat dequantizes at compute, as in the shared WeightLoader. Every + // tensor above that uses wdt is a mul_mat operand, so those are the ones + // eligible. Retyping is safe here: nothing is allocated yet. + struct PendingPacked { std::string name; ggml_tensor * t; }; + std::vector pending_packed; + if (use_gguf) { + std::vector keep; + keep.reserve(pending_f32.size()); + for (auto & p : pending_f32) { + const ggml_type ft = gst.file_type(hf_to_gguf(p.name)); + if (p.t->type == wdt && ft != GGML_TYPE_COUNT && ggml_is_quantized(ft) && + p.t->ne[0] % ggml_blck_size(ft) == 0) { + p.t->type = ft; + p.t->nb[0] = ggml_type_size(ft); + p.t->nb[1] = p.t->nb[0] * (p.t->ne[0] / ggml_blck_size(ft)); + for (int d = 2; d < GGML_MAX_DIMS; ++d) + p.t->nb[d] = p.t->nb[d-1] * p.t->ne[d-1]; + pending_packed.push_back({p.name, p.t}); + } else { + keep.push_back(std::move(p)); + } + } + pending_f32.swap(keep); + if (!pending_packed.empty()) + std::printf("vla: %zu GEMM weights kept packed as in the file (%s)\n", + pending_packed.size(), ggml_type_name(pending_packed[0].t->type)); + } + m->weight_buf = alloc_weights(m->ctx_weights, m->backend); if (!m->weight_buf) { std::fprintf(stderr, "vla: alloc_weights failed\n"); @@ -1289,6 +1349,14 @@ SmolVLAModelArch* smolvla_load_impl(ggml_type weight_dtype, return nullptr; } } + for (auto & p : pending_packed) { + std::vector hbuf(ggml_nbytes(p.t)); + if (!gst.read_packed(hf_to_gguf(p.name), hbuf.data(), p.t->type, hbuf.size())) { + delete m; + return nullptr; + } + ggml_backend_tensor_set(p.t, hbuf.data(), 0, hbuf.size()); + } { const float dt = -1.f/static_cast(cfg.num_steps); @@ -1972,7 +2040,7 @@ std::unique_ptr smolvla_create(const std::string& mmproj_path, const std::string& ckpt_path, const std::string& config_path, const Options& opts) { - SmolVLAModelArch* raw = smolvla_load_impl(opts.weight_dtype.value_or(GGML_TYPE_BF16), + SmolVLAModelArch* raw = smolvla_load_impl(opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)), mmproj_path, ckpt_path, config_path); if (!raw) return nullptr; diff --git a/src/models/vla_adapter.cpp b/src/models/vla_adapter.cpp index 12821e0..3a308da 100644 --- a/src/models/vla_adapter.cpp +++ b/src/models/vla_adapter.cpp @@ -185,7 +185,7 @@ std::unique_ptr vla_adapter_create(const std::string& mmproj_path if (!mmproj_path.empty()) std::printf("vla(vla_adapter): note - mmproj '%s' ignored (vision baked into combined GGUF)\n", mmproj_path.c_str()); auto m = std::make_unique(); - m->mt = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->mt = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); gguf_reader g("vla_adapter"); if (!g.open(ckpt_path)) diff --git a/src/models/vla_jepa.cpp b/src/models/vla_jepa.cpp index d297b91..00667ec 100644 --- a/src/models/vla_jepa.cpp +++ b/src/models/vla_jepa.cpp @@ -221,7 +221,7 @@ std::unique_ptr vla_jepa_create(const std::string& mmproj_path, auto m = std::make_unique(); m->gguf_path = ckpt_path; - m->matmul_type = opts.weight_dtype.value_or(GGML_TYPE_BF16); + m->matmul_type = opts.weight_dtype.value_or(vla::default_weight_dtype(GGML_TYPE_BF16)); gguf_reader g("vla_jepa"); if (!g.open(ckpt_path)) diff --git a/src/modules/qwen3vl_vit.h b/src/modules/qwen3vl_vit.h index 851f471..e7a0c5f 100644 --- a/src/modules/qwen3vl_vit.h +++ b/src/modules/qwen3vl_vit.h @@ -16,6 +16,7 @@ #pragma once +#include "backend.h" #include "layers/attn.h" #include "loader.h" #include "layers/rope.h" @@ -106,7 +107,7 @@ inline ggml_tensor * build_vit_layer(ggml_context * C, const VitLayerW & w, ggml att = attention (C, Q, K, to_heads_v(C, v, hd, heads, seq), nullptr, scale, hidden, seq); ggml_tensor * h1 = ggml_add(C, x, ggml_add(C, ggml_mul_mat(C, w.Wo, att), w.bo)); ggml_tensor * n2 = ggml_add(C, ggml_mul(C, ggml_norm(C, h1, ln_eps), w.ln2w), w.ln2b); - ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, ggml_gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); + ggml_tensor * ff = ggml_add(C, ggml_mul_mat(C, w.Wfc2, vla::gelu(C, ggml_add(C, ggml_mul_mat(C, w.Wfc1, n2), w.bfc1))), w.bfc2); return ggml_add(C, h1, ff); } @@ -123,7 +124,7 @@ inline ggml_tensor * build_merger(ggml_context * C, const MergerW & w, ggml_tens m = ggml_add(C, ggml_mul(C, ggml_norm(C, mr, ln_eps), w.nw), w.nb); } ggml_tensor * z1 = ggml_add(C, ggml_mul_mat(C, w.fc1w, m), w.fc1b); - return ggml_add(C, ggml_mul_mat(C, w.fc2w, ggml_gelu(C, z1)), w.fc2b); + return ggml_add(C, ggml_mul_mat(C, w.fc2w, vla::gelu(C, z1)), w.fc2b); } // Patch row/col after the spatial merge. diff --git a/src/options.cpp b/src/options.cpp index abf6319..f3614c4 100644 --- a/src/options.cpp +++ b/src/options.cpp @@ -34,6 +34,10 @@ bool parse_dtype(const std::string & v, ggml_type & out) { out = GGML_TYPE_BF16; return true; } + if (v == "f16" || v == "fp16") { + out = GGML_TYPE_F16; + return true; + } return false; } @@ -53,7 +57,7 @@ bool parse_bool(const std::string & v, bool & out) { } const char * dtype_name(ggml_type t) { - return t == GGML_TYPE_BF16 ? "bf16" : "f32"; + return t == GGML_TYPE_BF16 ? "bf16" : t == GGML_TYPE_F16 ? "f16" : "f32"; } namespace { @@ -76,7 +80,7 @@ bool mm_prec_f32_enabled() { } const char * Options::usage() { - return " --weight-dtype f32|bf16 resident dtype for GEMM weights\n" + return " --weight-dtype f32|bf16|f16 resident dtype for GEMM weights\n" " --act-dtype f32|bf16 activation dtype (needs CUDA and bf16 weights)\n" " --flash-attn [0|1] flash attention; faster, changes numerics\n" " --mm-prec default|f32 matmul accumulation precision\n"; @@ -101,6 +105,12 @@ bool Options::parse_arg(int argc, char ** argv, int & i, std::string & err) { ggml_type t; if (!parse_dtype(v, t)) { + err = a+": expected f32, bf16 or f16, got '"+v+"'"; + return false; + } + // F16 weights are for backends without BF16 kernels (Hexagon, OpenCL); + // the BF16 activation path is CUDA's and has no F16 twin. + if (a == "--act-dtype" && t == GGML_TYPE_F16) { err = a+": expected f32 or bf16, got '"+v+"'"; return false; } diff --git a/src/serving/server.cpp b/src/serving/server.cpp index d01d9df..3c853dd 100644 --- a/src/serving/server.cpp +++ b/src/serving/server.cpp @@ -207,7 +207,14 @@ void usage(const char * prog) { int main(int argc, char ** argv) { GOOGLE_PROTOBUF_VERIFY_VERSION; +#ifdef _WIN32 + // The Windows CRT has no line buffering (_IOLBF means full there) and treats + // a zero size as an invalid argument, which fails fast before main prints a + // word. Unbuffered is the nearest match. + std::setvbuf(stdout, nullptr, _IONBF, 0); +#else std::setvbuf(stdout, nullptr, _IOLBF, 0); +#endif std::string bind_addr = "tcp://*:5555"; std::string mmproj_path; diff --git a/src/serving/vla-cli.cpp b/src/serving/vla-cli.cpp index ec48e13..95bf099 100644 --- a/src/serving/vla-cli.cpp +++ b/src/serving/vla-cli.cpp @@ -186,6 +186,12 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { std::fprintf(stderr, "vla-cli: --text takes plain prose (letters, digits, space . , - _ ')\n"); return ""; } +#ifdef _WIN32 + // cmd.exe has no single quotes. text_ok already rules out '"', so the text + // needs no escaping inside double quotes. + const std::string esc = text; + const std::string q = "\""; +#else std::string esc; for (const char c : text) { if (c == '\'') @@ -193,16 +199,28 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { else esc += c; } + const std::string q = "'"; +#endif // Env first so a packaged binary can point at its own copy of the script. const char * env = std::getenv("VLA_TOKENIZE_SCRIPT"); const std::string script = (env && *env) ? std::string(env) : std::string(VLA_SOURCE_DIR) + "/scripts/tokenize_prompt.py"; const char * py = std::getenv("VLA_PYTHON"); - const std::string interp = (py && *py) ? std::string(py) : std::string("python3"); - const std::string cmd = "'" + interp + "' '" + script + "' --arch " + arch_slug(arch) + - " --text '" + esc + "'"; - +#ifdef _WIN32 + const char * def_py = "python"; // python3.exe is the Store stub on Windows +#else + const char * def_py = "python3"; +#endif + const std::string interp = (py && *py) ? std::string(py) : std::string(def_py); + std::string cmd = q + interp + q + " " + q + script + q + " --arch " + arch_slug(arch) + + " --text " + q + esc + q; +#ifdef _WIN32 + // cmd /c strips the first and last quote of the line; give it a pair to eat. + cmd = "\"" + cmd + "\""; + FILE * fp = _popen(cmd.c_str(), "r"); +#else FILE * fp = popen(cmd.c_str(), "r"); +#endif if (!fp) { std::fprintf(stderr, "vla-cli: cannot run %s\n", cmd.c_str()); return ""; @@ -211,7 +229,11 @@ std::string tokenize_text(const std::string & ckpt, const std::string & text) { char buf[4096]; while (std::fgets(buf, sizeof(buf), fp)) out += buf; +#ifdef _WIN32 + if (_pclose(fp) != 0) { +#else if (pclose(fp) != 0) { +#endif std::fprintf(stderr, "vla-cli: tokenizing failed. Install the client extras with\n" " pip install -e \".[client]\"\n" diff --git a/src/serving/vlm-server.cpp b/src/serving/vlm-server.cpp index e960611..7a948f4 100644 --- a/src/serving/vlm-server.cpp +++ b/src/serving/vlm-server.cpp @@ -69,7 +69,14 @@ void usage(const char * prog) { int main(int argc, char ** argv) { GOOGLE_PROTOBUF_VERIFY_VERSION; +#ifdef _WIN32 + // The Windows CRT has no line buffering (_IOLBF means full there) and treats + // a zero size as an invalid argument, which fails fast before main prints a + // word. Unbuffered is the nearest match. + std::setvbuf(stdout, nullptr, _IONBF, 0); +#else std::setvbuf(stdout, nullptr, _IOLBF, 0); +#endif std::string bind_addr = "tcp://*:5567"; vlm::LoadParams lp; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e3175f7..ddffcfd 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,6 +1,11 @@ # vla.cpp tests, enabled with -DVLA_BUILD_TESTS=ON. predict_check needs a real # GGUF so it is built but not run by ctest; the unit tests below are pure. +# Beside the ggml DLLs in /bin; Windows has no rpath (see ../CMakeLists.txt). +if(WIN32) + set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin) +endif() + add_executable(vla_predict_check predict_check.cpp) target_link_libraries(vla_predict_check PRIVATE vla_core) target_compile_options(vla_predict_check PRIVATE -Wall -Wextra) diff --git a/tests/predict_check.cpp b/tests/predict_check.cpp index e7c425b..831ef0a 100644 --- a/tests/predict_check.cpp +++ b/tests/predict_check.cpp @@ -28,7 +28,7 @@ #include #include #include -#include +#include #include using namespace vla; @@ -113,11 +113,10 @@ int main(int argc, char** argv) { for (int w = 0; w < 3; ++w) (void)predict(m, in); double best = 1e30, sum = 0.0; for (int i = 0; i < iters; ++i) { - struct timespec t0, t1; - clock_gettime(CLOCK_MONOTONIC, &t0); + const auto t0 = std::chrono::steady_clock::now(); std::vector a = predict(m, in); - clock_gettime(CLOCK_MONOTONIC, &t1); - double ms = (t1.tv_sec - t0.tv_sec) * 1e3 + (t1.tv_nsec - t0.tv_nsec) / 1e6; + const auto t1 = std::chrono::steady_clock::now(); + double ms = std::chrono::duration(t1 - t0).count(); best = ms < best ? ms : best; sum += ms; } From 136170d1f2575ae1219ab605165ca408dd83b619 Mon Sep 17 00:00:00 2001 From: Khanh Nguyen Date: Sun, 27 Sep 2026 18:43:48 +0700 Subject: [PATCH 3/3] Clean up Snapdragon X support for merge: generic paths in docs, trimmed README, script autodetection, style fixes --- CMakeLists.txt | 150 +++++++++++++-------------- README.md | 47 ++++----- docs/backend/hexagon-windows.md | 54 +++++++--- scripts/build_windows_snapdragon.ps1 | 50 +++++++-- src/backend.h | 15 +-- 5 files changed, 183 insertions(+), 133 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5af93a5..1066aea 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -314,88 +314,84 @@ target_link_libraries(vlm_core PUBLIC llama ggml mtmd llama-common) # them configure on a machine that has not installed the serving stack. option(VLA_BUILD_SERVER "Build vla-server and vlm-server (needs protobuf, libzmq, cppzmq)" ON) if(VLA_BUILD_SERVER) -find_package(Protobuf REQUIRED) -# vcpkg (the Windows route) ships a CMake package and no pkg-config binary; -# Linux distros ship the .pc file. Either yields one target to link. -find_package(ZeroMQ CONFIG QUIET) -if(TARGET libzmq) - set(VLA_ZMQ_TARGET libzmq) -elseif(TARGET libzmq-static) - set(VLA_ZMQ_TARGET libzmq-static) -else() - find_package(PkgConfig REQUIRED) - pkg_check_modules(ZeroMQ REQUIRED IMPORTED_TARGET libzmq) - set(VLA_ZMQ_TARGET PkgConfig::ZeroMQ) -endif() - -find_path(CPPZMQ_INCLUDE_DIR NAMES zmq.hpp) -if(NOT CPPZMQ_INCLUDE_DIR) - message(FATAL_ERROR - "cppzmq (zmq.hpp) not found. Install 'cppzmq-dev' (Debian/Ubuntu) or set " - "CMAKE_PREFIX_PATH to your cppzmq install prefix.") -endif() + find_package(Protobuf REQUIRED) + if(WIN32) + # vcpkg ships ZeroMQ's CMake package and no pkg-config. + find_package(ZeroMQ CONFIG REQUIRED) + set(VLA_ZMQ_TARGET libzmq) + else() + find_package(PkgConfig REQUIRED) + pkg_check_modules(ZeroMQ REQUIRED IMPORTED_TARGET libzmq) + set(VLA_ZMQ_TARGET PkgConfig::ZeroMQ) + endif() -set(VLA_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/serving/vla.proto) -set(VLA_PROTO_GEN_DIR ${CMAKE_CURRENT_BINARY_DIR}/proto-gen/serving) -file(MAKE_DIRECTORY ${VLA_PROTO_GEN_DIR}) - -add_custom_command( - OUTPUT ${VLA_PROTO_GEN_DIR}/vla.pb.h - ${VLA_PROTO_GEN_DIR}/vla.pb.cc - COMMAND ${Protobuf_PROTOC_EXECUTABLE} - --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/src/serving - --cpp_out=${VLA_PROTO_GEN_DIR} - ${VLA_PROTO_SRC} - DEPENDS ${VLA_PROTO_SRC} - VERBATIM -) + find_path(CPPZMQ_INCLUDE_DIR NAMES zmq.hpp) + if(NOT CPPZMQ_INCLUDE_DIR) + message(FATAL_ERROR + "cppzmq (zmq.hpp) not found. Install 'cppzmq-dev' (Debian/Ubuntu) or set " + "CMAKE_PREFIX_PATH to your cppzmq install prefix.") + endif() -add_executable(vla-server - src/serving/server.cpp - ${VLA_PROTO_GEN_DIR}/vla.pb.cc -) -target_include_directories(vla-server PRIVATE - ${CMAKE_CURRENT_BINARY_DIR}/proto-gen - ${llama_SOURCE_DIR}/vendor/stb - ${CPPZMQ_INCLUDE_DIR} -) -target_link_libraries(vla-server PRIVATE - vla_core - protobuf::libprotobuf - ${VLA_ZMQ_TARGET} -) + set(VLA_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/serving/vla.proto) + set(VLA_PROTO_GEN_DIR ${CMAKE_CURRENT_BINARY_DIR}/proto-gen/serving) + file(MAKE_DIRECTORY ${VLA_PROTO_GEN_DIR}) + + add_custom_command( + OUTPUT ${VLA_PROTO_GEN_DIR}/vla.pb.h + ${VLA_PROTO_GEN_DIR}/vla.pb.cc + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/src/serving + --cpp_out=${VLA_PROTO_GEN_DIR} + ${VLA_PROTO_SRC} + DEPENDS ${VLA_PROTO_SRC} + VERBATIM + ) -set(VLM_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/serving/vlm.proto) -set(VLM_PROTO_GEN_DIR ${CMAKE_CURRENT_BINARY_DIR}/proto-gen/serving) -file(MAKE_DIRECTORY ${VLM_PROTO_GEN_DIR}) - -add_custom_command( - OUTPUT ${VLM_PROTO_GEN_DIR}/vlm.pb.h - ${VLM_PROTO_GEN_DIR}/vlm.pb.cc - COMMAND ${Protobuf_PROTOC_EXECUTABLE} - --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/src/serving - --cpp_out=${VLM_PROTO_GEN_DIR} - ${VLM_PROTO_SRC} - DEPENDS ${VLM_PROTO_SRC} - VERBATIM -) + add_executable(vla-server + src/serving/server.cpp + ${VLA_PROTO_GEN_DIR}/vla.pb.cc + ) + target_include_directories(vla-server PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/proto-gen + ${llama_SOURCE_DIR}/vendor/stb + ${CPPZMQ_INCLUDE_DIR} + ) + target_link_libraries(vla-server PRIVATE + vla_core + protobuf::libprotobuf + ${VLA_ZMQ_TARGET} + ) -add_executable(vlm-server - src/serving/vlm-server.cpp - ${VLM_PROTO_GEN_DIR}/vlm.pb.cc -) -target_include_directories(vlm-server PRIVATE - ${CMAKE_CURRENT_BINARY_DIR}/proto-gen - ${llama_SOURCE_DIR}/vendor/stb - ${CPPZMQ_INCLUDE_DIR} -) -target_link_libraries(vlm-server PRIVATE - vlm_core - protobuf::libprotobuf - ${VLA_ZMQ_TARGET} -) + set(VLM_PROTO_SRC ${CMAKE_CURRENT_SOURCE_DIR}/src/serving/vlm.proto) + set(VLM_PROTO_GEN_DIR ${CMAKE_CURRENT_BINARY_DIR}/proto-gen/serving) + file(MAKE_DIRECTORY ${VLM_PROTO_GEN_DIR}) + + add_custom_command( + OUTPUT ${VLM_PROTO_GEN_DIR}/vlm.pb.h + ${VLM_PROTO_GEN_DIR}/vlm.pb.cc + COMMAND ${Protobuf_PROTOC_EXECUTABLE} + --proto_path=${CMAKE_CURRENT_SOURCE_DIR}/src/serving + --cpp_out=${VLM_PROTO_GEN_DIR} + ${VLM_PROTO_SRC} + DEPENDS ${VLM_PROTO_SRC} + VERBATIM + ) -endif() # VLA_BUILD_SERVER + add_executable(vlm-server + src/serving/vlm-server.cpp + ${VLM_PROTO_GEN_DIR}/vlm.pb.cc + ) + target_include_directories(vlm-server PRIVATE + ${CMAKE_CURRENT_BINARY_DIR}/proto-gen + ${llama_SOURCE_DIR}/vendor/stb + ${CPPZMQ_INCLUDE_DIR} + ) + target_link_libraries(vlm-server PRIVATE + vlm_core + protobuf::libprotobuf + ${VLA_ZMQ_TARGET} + ) +endif() add_library(vla SHARED src/vla_c_api.cpp) target_include_directories(vla diff --git a/README.md b/README.md index 679e8d5..59a06af 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,8 @@ It runs the open VLA policies - SmolVLA, π0, BitVLA, Evo-1, GR00T N1.5/1.6/1.7 under one runtime, each packaged as a single self-contained GGUF that needs no Python or PyTorch at inference time. The binaries drive robots on **CPU**, **Apple Silicon**, **CUDA** - from consumer GPUs down to Jetson-class boards - **Intel GPUs and NPUs** via -SYCL and OpenVINO, or the **Snapdragon X** Hexagon NPU and Adreno GPU on Windows on Arm. +SYCL and OpenVINO, or **Qualcomm Snapdragon CPUs, Adreno GPUs and Hexagon NPUs** +via OpenCL and the Hexagon backend. [**Learn vla.cpp**](https://fai-modelopt-tech.github.io/learn-vla-cpp/) walks through the engine design and how each policy is implemented on ggml. @@ -72,8 +73,7 @@ export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH ``` Check [docs/backend](docs/backend) for compiling `vla.cpp` on other platforms. -WSL2, Apple Silicon, Intel GPU and Snapdragon X on Windows on Arm -([docs/backend/hexagon-windows.md](docs/backend/hexagon-windows.md)) are all tested. +WSL2, Apple Silicon, and Intel GPU are all tested. To build and run in containers instead, see [docs/DOCKER.md](docs/DOCKER.md). --- @@ -162,9 +162,7 @@ Precision flags (`vla-server --help` for the full list). The fastest configuration per model, with measured latency and success rate, is in [`CHANGELOG.md`](CHANGELOG.md): -- `--weight-dtype f32|bf16|f16` - resident dtype for GEMM weights. `f16` is the - default on Hexagon and OpenCL, which have no BF16 kernels. It is also 2.5-3.5x - faster than the BF16 default on CPUs without BF16 matmul (e.g. Snapdragon X). +- `--weight-dtype f32|bf16` - resident dtype for GEMM weights. - `--act-dtype f32|bf16` - activation dtype; needs CUDA and bf16 weights. - `--flash-attn` - faster on the larger towers, but changes numerics. - `--mm-prec default|f32` - matmul accumulation precision. @@ -173,9 +171,7 @@ in [`CHANGELOG.md`](CHANGELOG.md): Environment knobs that apply to every arch: - `VLA_N_THREADS` - CPU backend thread count, default core count capped at 16. -- `VLA_DEVICE` - GPU ordinal for CUDA and SYCL builds, default 0. In Hexagon and - OpenCL builds it indexes the NPU session or GPU, and `VLA_DEVICE=cpu` skips the - accelerator. +- `VLA_DEVICE` - GPU ordinal for CUDA and SYCL builds, default 0. - `VLA_CACHE` - where `-hf` stores checkpoints, default `~/.cache/vla`. --- @@ -319,26 +315,19 @@ supported (released and benchmarked), `~` = in progress, `-` = planned. | Model | CPU (x86-64 / ARM) | CUDA | [SYCL (Intel)](docs/backend/sycl.md) | [Metal](docs/backend/metal.md) | [OpenVINO](docs/backend/ov.md) | [Hexagon](docs/backend/hexagon-windows.md) | |---|:--:|:--:|:--:|:--:|:--:|:--:| -| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | Y | -| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | ~ | -| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | Y | -| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | Y | -| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | Y | -| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | Y | -| [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | - | -| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | Y | -| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | Y | -| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | Y | - | -| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | ~ | -| [Octo-Small](https://hf.co/vrfai/octo-small-libero-gguf) | Y | Y | Y | Y | - | Y | -| [TurboVLA](https://hf.co/vrfai/turbovla-libero-gguf) | Y | Y | Y | Y | Y | Y | - -Hexagon is measured on a Snapdragon X laptop (Windows on Arm, v73 NPU). `~` there -means the model runs but lands outside the 2.9e-3 fidelity bar: π0 and VLA-JEPA -are precision-sensitive even on the CPU at F16. The same laptop's **Adreno GPU** -(`GGML_OPENCL=ON`) runs the eleven models in the Hexagon column's `Y`/`~` rows -within 1.4e-3 of an F32 reference; see -[hexagon-windows.md](docs/backend/hexagon-windows.md). +| [SmolVLA](https://hf.co/vrfai/smolvla-libero-gguf) | Y | Y | Y | Y | Y | Y | +| [π0](https://hf.co/vrfai/pi0-libero-finetuned-v044-gguf) | Y | Y | - | Y | Y | ~ | +| [π0.5](https://hf.co/vrfai/pi05-libero-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.5](https://hf.co/vrfai/gr00tn1d5-libero-object-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.6](https://hf.co/vrfai/gr00tn1d6-libero-gguf) | Y | Y | - | Y | Y | Y | +| [GR00T N1.7](https://hf.co/vrfai/gr00tn1d7-libero-gguf) | Y | Y | - | Y | Y | Y | +| [BitVLA](https://hf.co/vrfai/bitvla-libero-gguf) | Y | Y | - | ~ | - | - | +| [Evo-1](https://hf.co/vrfai/evo1-libero-gguf) | Y | Y | Y | Y | Y | Y | +| [VLA-Adapter](https://hf.co/vrfai/vla-adapter-libero-gguf) | Y | Y | ~ | Y | Y | Y | +| [OpenVLA-OFT](https://hf.co/vrfai/openvla-oft-libero-gguf) | Y | Y | - | Y | Y | - | +| [VLA-JEPA](https://hf.co/vrfai/vla-jepa-libero) | Y | Y | - | Y | Y | ~ | +| [Octo-Small](https://hf.co/vrfai/octo-small-libero-gguf) | Y | Y | Y | Y | - | Y | +| [TurboVLA](https://hf.co/vrfai/turbovla-libero-gguf) | Y | Y | Y | Y | Y | Y | --- diff --git a/docs/backend/hexagon-windows.md b/docs/backend/hexagon-windows.md index 37a870c..686f2dc 100644 --- a/docs/backend/hexagon-windows.md +++ b/docs/backend/hexagon-windows.md @@ -1,8 +1,7 @@ # vla.cpp on Snapdragon X (Windows on Arm): Hexagon NPU, Adreno GPU and CPU -Date: 2026-09-27 -vla.cpp: `3f0a38a` plus the changes described here (uncommitted) -llama.cpp: local checkout, build 11201, commit `2145525a4` (not the `b10729` pin) +Measured 2026-09 against llama.cpp build 11201 (`2145525a4`), passed in with +`-LlamaDir`. The `b10729` tag that `CMakeLists.txt` pins was not tested. ## Summary @@ -30,12 +29,10 @@ Every accelerator result below was checked against a CPU-backend reference on id The CPU fallback now routes around each of them, and each is described [below](#hexagon-issues-found-and-worked-around). -The NPU path needs the same one-off setup as llama.cpp: a newer NPU driver, test-signing, and a self-signed certificate. That setup is described in `../llama.cpp/snapdragon-x-windows-report.md` and was reused unchanged. +The NPU path needs the same one-off setup as llama.cpp's Hexagon backend: a newer NPU driver, test-signing, and a self-signed certificate. See [NPU prerequisites](#npu-prerequisites). ## Device -Same laptop as the llama.cpp report: - | Item | Value | |---|---| | Laptop | ASUS Vivobook 14 X1407QA | @@ -47,10 +44,12 @@ Same laptop as the llama.cpp report: ## Toolchain -Everything llama.cpp needed, plus the following: - | Component | Version / location | Used for | |---|---|---| +| Visual Studio | 2026 (18), MSVC 14.51, ARM64 libraries | Its bundled Clang 22.1.3, CMake and Ninja build everything | +| Windows SDK / WDK | 10.0.26100 | `inf2cat` and `signtool`, to sign the HTP skels | +| Hexagon SDK | 6.6.0.0, tools 19.0.07 (`HEXAGON_SDK_ROOT`, `HEXAGON_TOOLS_ROOT`) | Building the HTP skels | +| OpenCL SDK | 2.3.2 (`OPENCL_SDK_ROOT`) | The Adreno backend | | vcpkg | `C:\vcpkg` | protobuf 6.33.4 (triplet `arm64-windows-clangcl`, in this repo), zeromq 4.3.5 and cppzmq (stock `arm64-windows`) for `vla-server` | | Python | 3.14.7 ARM64, venv in `.venv` | `transformers` 5.17, `tokenizers`, `gguf`, `numpy`: tokenizing prompts and `scripts/quantize_gguf.py` | @@ -73,16 +72,33 @@ C:\vcpkg\vcpkg install protobuf --triplet arm64-windows-clangcl --overlay-triple The build script sets up the Visual Studio shell, the compiler flags llama.cpp's Snapdragon preset uses, vcpkg, and (for the NPU) skel signing: ``` -.\scripts\build_windows_snapdragon.ps1 -Backend htp -LlamaDir C:\Users\khanh\git\llama.cpp -HtpCert C:\Users\khanh\Certs\ggml-htp-v1.pfx -.\scripts\build_windows_snapdragon.ps1 -Backend opencl -LlamaDir C:\Users\khanh\git\llama.cpp -.\scripts\build_windows_snapdragon.ps1 -Backend cpu -LlamaDir C:\Users\khanh\git\llama.cpp +.\scripts\build_windows_snapdragon.ps1 -Backend htp -LlamaDir -HtpCert +.\scripts\build_windows_snapdragon.ps1 -Backend opencl -LlamaDir +.\scripts\build_windows_snapdragon.ps1 -Backend cpu -LlamaDir ``` Each build goes into `build-wos-`, with every binary and DLL in `build-wos-\bin`. `-NoServer` skips `vla-server`, Octo and their protobuf and ZeroMQ dependencies. -`-LlamaDir` points the build at an existing llama.cpp checkout through `FETCHCONTENT_SOURCE_DIR_LLAMA`. This build uses the local checkout, which already builds and signs the HTP skels on this machine. The `b10729` pin in `CMakeLists.txt` still applies when `-LlamaDir` is not given. That pin was not tested here. +`-LlamaDir` points the build at an existing llama.cpp checkout through `FETCHCONTENT_SOURCE_DIR_LLAMA`. Without it, the `b10729` pin in `CMakeLists.txt` applies, which was not tested here. + +The HTP build also signs `libggml-htp-v*.so` with the certificate and copies the skels and their catalog next to the binaries. At startup the Hexagon backend points `ADSP_LIBRARY_PATH` at the executable's own folder, but only if the variable is unset. If it is already set, for example by a llama.cpp install, the skels it names must come from the same llama.cpp commit. + +### NPU prerequisites + +These follow llama.cpp's [Windows guide](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/snapdragon/windows.md). They are one-off per machine, and the steps that need admin rights are marked. + +1. **Update the NPU driver.** Install Qualcomm's HND package from the Qualcomm Software Center (driver 30.0.220 or later; admin and a reboot). The driver shipped with the laptop (30.0.143) lacks the `dspqueue_*` functions, and the backend fails with `failed to dlsym dspqueue_create`. +2. **Enable test-signing** (admin and a reboot). Secure Boot must be off first, in the firmware setup. Then run `bcdedit /set TESTSIGNING ON`. +3. **Create and trust a code-signing certificate** (admin, for the trust step): -The HTP build also signs `libggml-htp-v*.so` with the certificate and copies the skels and their catalog next to the binaries. At startup the Hexagon backend points `ADSP_LIBRARY_PATH` at the executable's own folder, but only if the variable is unset. The llama.cpp setup set it for the user to `llama.cpp\pkg-wos\lib`. Those skels come from the same llama.cpp commit, so they work too. + ``` + New-SelfSignedCertificate -Subject "CN=GGML.HTP.v1" -Type CodeSigningCert -CertStoreLocation Cert:\CurrentUser\My + # export it to and , then: + certutil -addstore Root + certutil -addstore TrustedPublisher + ``` + +4. **Pass the `.pfx` to the build** with `-HtpCert`. Without it the skels are unsigned, and opening a session fails with error `0x80000406`. ### What had to change for Windows @@ -296,10 +312,18 @@ SmolVLA's CPU Q8_0 run keeps its float weights at F16; the other rows keep the C ## Known issues -- The NPU prints `ggml-hex: FASTRPC_GET_DOMAINS query failed (0x6c), using static CDSP domains` on every run. It works regardless, as in the llama.cpp report. +- The NPU prints `ggml-hex: FASTRPC_GET_DOMAINS query failed (0x6c), using static CDSP domains` on every run. It works regardless. - The Adreno driver refuses `GGML_OPENCL_ADRENO_USE_LARGE_BUFFER` ("not supported by driver"), so no single allocation can exceed about 1 GB. F16 weights keep every model here under that. - `Launch-VsDevShell.ps1` prints `'vswhere.exe' is not recognized`; the build is not affected. ## Security state -Unchanged from the llama.cpp report: Secure Boot off, test-signing on, and a self-signed certificate trusted as a root authority. This build signs its skels with the same certificate. Rollback is described in that report. +Using the NPU leaves the machine with Secure Boot off, test-signing on, and a self-signed certificate trusted as a root authority. Anyone who can read the `.pfx` can sign code that machine will trust, so keep it private. To undo, run the following; the NPU backend then stops loading, while the CPU and GPU are unaffected: + +``` +certutil -delstore Root +certutil -delstore TrustedPublisher +bcdedit /set TESTSIGNING OFF +``` + +Then turn Secure Boot back on in the firmware setup. diff --git a/scripts/build_windows_snapdragon.ps1 b/scripts/build_windows_snapdragon.ps1 index 7e5da9e..ed2de01 100644 --- a/scripts/build_windows_snapdragon.ps1 +++ b/scripts/build_windows_snapdragon.ps1 @@ -1,3 +1,17 @@ +# Copyright 2026 VinRobotics +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + # Build vla.cpp natively on a Snapdragon X laptop (Windows on Arm). # # .\scripts\build_windows_snapdragon.ps1 -Backend htp # Hexagon NPU (+ CPU fallback) @@ -13,11 +27,15 @@ param( [ValidateSet("htp", "opencl", "cpu")][string]$Backend = "htp", # A llama.cpp checkout to build against. Empty fetches the tag CMakeLists.txt pins. [string]$LlamaDir = "", - [string]$VsRoot = "C:\Program Files\Microsoft Visual Studio\18\Insiders", - [string]$VcpkgRoot = $(if ($env:VCPKG_ROOT) { $env:VCPKG_ROOT } else { "C:\vcpkg" }), + # Empty picks the newest Visual Studio (prereleases included) through vswhere. + [string]$VsRoot = "", + # Empty picks VCPKG_ROOT or C:\vcpkg, whichever holds the clang-cl triplet. + # Visual Studio's dev shell points VCPKG_ROOT at its own bundled vcpkg. + [string]$VcpkgRoot = "", # Signs the HTP skels. Without it the NPU refuses to load them. [string]$HtpCert = $env:HEXAGON_HTP_CERT, - [string]$WindowsSdkBin = "C:\Program Files (x86)\Windows Kits\10\bin\10.0.26100.0", + # Empty picks the newest SDK under Windows Kits\10\bin (inf2cat, signtool). + [string]$WindowsSdkBin = "", [switch]$NoServer, [switch]$Reconfigure ) @@ -25,13 +43,33 @@ param( # warning) into a terminating error. Exit codes are checked after each step. $ErrorActionPreference = "Continue" $root = Split-Path -Parent $PSScriptRoot -Set-Location $root + +if (-not $VsRoot) { + $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" + if (Test-Path $vswhere) { + $VsRoot = & $vswhere -latest -prerelease -products * -property installationPath + } + if (-not $VsRoot) { throw "Visual Studio not found; pass -VsRoot" } +} +if (-not $VcpkgRoot) { + $VcpkgRoot = @($env:VCPKG_ROOT, "C:\vcpkg") | Where-Object { + $_ -and (Test-Path "$_\installed\arm64-windows-clangcl") + } | Select-Object -First 1 + if (-not $VcpkgRoot -and -not $NoServer) { + throw "vcpkg with the arm64-windows-clangcl triplet not found; pass -VcpkgRoot" + } +} +if (-not $WindowsSdkBin) { + $WindowsSdkBin = Get-ChildItem "${env:ProgramFiles(x86)}\Windows Kits\10\bin" -Directory -Filter "10.*" | + Sort-Object { [version] $_.Name } | Select-Object -Last 1 -ExpandProperty FullName +} & "$VsRoot\Common7\Tools\Launch-VsDevShell.ps1" -Arch arm64 -HostArch arm64 -SkipAutomaticLocation | Out-Null $env:PATH = "$VsRoot\VC\Tools\Llvm\ARM64\bin;$env:PATH" Set-Location $root $dir = "build-wos-$Backend" +$jobs = $env:NUMBER_OF_PROCESSORS $flags = "-march=armv8.7a+fp16+dotprod+i8mm -fvectorize -ffp-model=fast -D_GNU_SOURCE" # protobuf and abseil must be built by clang too, with the triplet in # cmake/vcpkg-triplets: clang code does not link against an MSVC-built protobuf. @@ -84,9 +122,9 @@ if ($Backend -eq "htp") { # ggml-hexagon signs the skel catalog in a step that depends on the .so # files but not on the targets that build them, so a parallel build can # reach inf2cat first. Build the skels on their own before everything else. - cmake --build $dir -j 8 --target htp-v73 htp-v75 htp-v79 htp-v81 + cmake --build $dir -j $jobs --target htp-v73 htp-v75 htp-v79 htp-v81 if ($LASTEXITCODE -ne 0) { throw "HTP skel build failed" } } -cmake --build $dir -j 8 +cmake --build $dir -j $jobs if ($LASTEXITCODE -ne 0) { throw "build failed" } Write-Host "built $dir\bin" diff --git a/src/backend.h b/src/backend.h index 5af6acb..087638e 100644 --- a/src/backend.h +++ b/src/backend.h @@ -64,14 +64,16 @@ #include #include #endif -#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || \ + defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) #include // setenv / _putenv_s #include #endif namespace vla { -#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) +#if defined(GGML_USE_SYCL) || defined(GGML_USE_OPENVINO) || \ + defined(GGML_USE_HEXAGON) || defined(GGML_USE_OPENCL) // setenv is POSIX. _putenv_s has no "do not overwrite" mode, so check first. // Empty counts as unset; an empty KEY= in a compose file is not a choice. inline void setenv_default(const char * key, const char * val) { @@ -407,13 +409,14 @@ inline Backend backend_init(const char * tag, int n_threads) { ggml_backend_reg_t reg = ggml_backend_opencl_reg(); const char * what = "OpenCL"; #endif - const int dev = backend_device_index(); - const size_t n_dev = reg ? ggml_backend_reg_dev_count(reg) : 0; + const int dev = backend_device_index(); + const size_t n_dev = reg ? ggml_backend_reg_dev_count(reg) : 0; ggml_backend_t accel = nullptr; if ((size_t) dev >= n_dev) { - std::fprintf(stderr, "%s: %s device %d out of range (%zu visible); falling back to CPU\n", + std::fprintf(stderr, + "%s: %s device %d out of range (%zu visible); falling back to CPU\n", tag, what, dev, n_dev); - } else if ((accel = ggml_backend_dev_init(ggml_backend_reg_dev_get(reg, dev), nullptr)) == nullptr) { + } else if (!(accel = ggml_backend_dev_init(ggml_backend_reg_dev_get(reg, dev), nullptr))) { std::fprintf(stderr, "%s: %s init failed; falling back to CPU\n", tag, what); } else if ((b.handle = fallback_backend_new(accel, n_threads)) != nullptr) { std::printf("%s: backend = %s (%s), rejected ops on CPU (%d threads)\n",