From 2b9f12edfab4d0a174dfba5a53485c8e131bc351 Mon Sep 17 00:00:00 2001 From: sunwen Date: Mon, 17 Aug 2026 03:05:04 +0800 Subject: [PATCH] feat: add QCS8550 QNN deployment for LIBERO Object --- README.md | 1 + deployment/qcs8550/.gitignore | 4 + deployment/qcs8550/AGENT_DEPLOYMENT.md | 211 +++++++ deployment/qcs8550/README.md | 131 +++++ .../qcs8550/native/turbovla_qnn_server.cpp | 539 ++++++++++++++++++ deployment/qcs8550/qnn_policy.py | 201 +++++++ .../qcs8550/tools/build_native_server.py | 93 +++ deployment/qcs8550/tools/download_contexts.py | 75 +++ .../qcs8550/tools/export_static_onnx.py | 267 +++++++++ .../tools/extract_checkpoint_contract.py | 74 +++ deployment/qcs8550/tools/native_client.py | 132 +++++ deployment/qcs8550/tools/qcs8550_reference.py | 220 +++++++ .../qcs8550/tools/run_libero_rollout.py | 30 + .../qcs8550/tools/submit_qai_hub_compile.py | 89 +++ 14 files changed, 2067 insertions(+) create mode 100644 deployment/qcs8550/.gitignore create mode 100644 deployment/qcs8550/AGENT_DEPLOYMENT.md create mode 100644 deployment/qcs8550/README.md create mode 100644 deployment/qcs8550/native/turbovla_qnn_server.cpp create mode 100644 deployment/qcs8550/qnn_policy.py create mode 100644 deployment/qcs8550/tools/build_native_server.py create mode 100644 deployment/qcs8550/tools/download_contexts.py create mode 100644 deployment/qcs8550/tools/export_static_onnx.py create mode 100644 deployment/qcs8550/tools/extract_checkpoint_contract.py create mode 100644 deployment/qcs8550/tools/native_client.py create mode 100644 deployment/qcs8550/tools/qcs8550_reference.py create mode 100644 deployment/qcs8550/tools/run_libero_rollout.py create mode 100644 deployment/qcs8550/tools/submit_qai_hub_compile.py diff --git a/README.md b/README.md index 4567b29..99dcb75 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ This repository contains the official implementation of **TurboVLA** for the pap ## 📅 TODO * [ ] Support Huawei Ascend NPUs +* [x] Experimental Qualcomm QCS8550 QNN deployment for LIBERO Object: see [deployment/qcs8550](deployment/qcs8550/README.md) --- ## 📄 Abstract diff --git a/deployment/qcs8550/.gitignore b/deployment/qcs8550/.gitignore new file mode 100644 index 0000000..e53a5fd --- /dev/null +++ b/deployment/qcs8550/.gitignore @@ -0,0 +1,4 @@ +# All files produced by export, AI Hub compilation, replay, and rollout are local deployment assets. +artifacts/ +__pycache__/ +tools/__pycache__/ diff --git a/deployment/qcs8550/AGENT_DEPLOYMENT.md b/deployment/qcs8550/AGENT_DEPLOYMENT.md new file mode 100644 index 0000000..11927cf --- /dev/null +++ b/deployment/qcs8550/AGENT_DEPLOYMENT.md @@ -0,0 +1,211 @@ +# QCS8550 Deployment Instructions For AI Agents + +Use this runbook when the task is to deploy the released TurboVLA **LIBERO +Object** checkpoint on a Qualcomm QCS8550 board. Follow the phases in order. +Do not substitute model outputs, change graph topology, or claim FP32-equivalent +results: this target is an experimental QAIRT 2.48 QNN deployment. + +## Safety And Scope + +- Work from the TurboVLA repository root and keep generated files below + `deployment/qcs8550/artifacts/`; this directory is intentionally ignored. +- Do not commit, print, copy into source files, or request any API token. Run + `qai-hub configure` only in an interactive user terminal. It stores credentials + in the user's local configuration and is not part of this repository. +- Use an isolated board root such as `/opt/turbovla-qcs8550` and port `10092`. + Do not stop, overwrite, or reconfigure an unrelated board service. The build + helper refuses to start if the selected port is already occupied. +- Never use `last_hidden_state` for DINO features. Upstream issue + [#4](https://github.com/H-EmbodVis/TurboVLA/issues/4) requires + `outputs.hidden_states[-1]`. Pin `transformers==4.56.*` for reference/export. +- Stop and report the blocker rather than guessing if the checkpoint, QAIRT + version, QCS8550 target, QNN headers, runtime, board access, or AI Hub access + is unavailable. + +## Inputs To Obtain From The User + +Ask only for values that cannot be discovered locally: + +| Variable | Meaning | +| --- | --- | +| `BOARD_HOST` | SSH host/IP for the QCS8550 board; key-based SSH must work. | +| `QAIRT_INCLUDE` | Host QAIRT 2.48 SDK `include` directory (or its `include/QNN` child). | +| `QAIRT_RUNTIME` | Board QAIRT 2.48 runtime root. | +| `LIBERO_ROOT` | Local LIBERO installation, required for rollout. | + +The agent must verify these local assets before doing expensive work: + +```bash +test -f pretrained/TurboVLA/checkpoints/libero/object.pth +test -d pretrained/bert-base-uncased +test -d "$QAIRT_INCLUDE" +ssh -o BatchMode=yes "$BOARD_HOST" true +ssh -o BatchMode=yes "$BOARD_HOST" "test -d '$QAIRT_RUNTIME'" +python -c 'import transformers; assert transformers.__version__.startswith("4.56.")' +``` + +The release checkpoint supplies the DINO weights used by static export. The +shared LIBERO command line still accepts a local DINO path for its upstream +configuration; retain a licensed local DINOv3 ViT-B asset for rollout. + +## Phase 1: Prepare The Host + +Use a Python 3.10 environment with TurboVLA's LIBERO dependencies, then install +the export and cloud client dependencies: + +```bash +pip install -e '.[libero]' +pip install 'transformers==4.56.*' onnx onnxruntime qai-hub +``` + +Do not proceed if installing `transformers` changes the pin. Review the source +tree before continuing; do not stage local model or deployment assets: + +```bash +git status --short +python -m py_compile deployment/qcs8550/tools/*.py deployment/qcs8550/qnn_policy.py +``` + +## Phase 2: Export Static ONNX + +Export the exact fixed-shape graph set. This command also checks that the +eager-attention split reference preserves the original normalized action: + +```bash +python deployment/qcs8550/tools/extract_checkpoint_contract.py +python deployment/qcs8550/tools/export_static_onnx.py +``` + +Require all of the following before going on: + +```text +deployment/qcs8550/artifacts/object/onnx/dinov3_one_view_fp32.onnx +deployment/qcs8550/artifacts/object/onnx/bert_l11_fp32.onnx +deployment/qcs8550/artifacts/object/onnx/bert_l14_fp32.onnx +deployment/qcs8550/artifacts/object/onnx/bert_l21_fp32.onnx +deployment/qcs8550/artifacts/object/onnx/policy_core_l21_fp32.onnx +deployment/qcs8550/artifacts/object/export_report.json +deployment/qcs8550/artifacts/object/reference_bundle_l11.npz +deployment/qcs8550/artifacts/checkpoint_contract.json +``` + +Read `export_report.json`; preserve it locally with the generated artifacts. +Any failed exact split-reference check is a hard stop. Do not work around it by +changing DINO feature selection, operators, shapes, tokenizer behavior, or +normalization. + +## Phase 3: Compile And Download Contexts + +This phase needs an authorized Qualcomm AI Hub account. The user must complete +interactive login; an agent must never handle the token. + +```bash +qai-hub configure +python deployment/qcs8550/tools/submit_qai_hub_compile.py --qairt-version 2.48 +python deployment/qcs8550/tools/download_contexts.py +``` + +Do not target another SoC or silently use a different QAIRT version. Verify the +following six files exist after every job reaches `SUCCESS`: + +```text +deployment/qcs8550/artifacts/object/qcs8550_contexts/dinov3.bin +deployment/qcs8550/artifacts/object/qcs8550_contexts/bert_l11.bin +deployment/qcs8550/artifacts/object/qcs8550_contexts/bert_l14.bin +deployment/qcs8550/artifacts/object/qcs8550_contexts/bert_l21.bin +deployment/qcs8550/artifacts/object/qcs8550_contexts/policy_core.bin +deployment/qcs8550/artifacts/object/qcs8550_contexts/manifest.json +``` + +Keep the manifest with the `.bin` files. They are deployment assets, not Git +assets: do not add them to a commit or upload them without confirming QAIRT and +model-license redistribution terms. + +## Phase 4: Install And Start The Board Service + +Set the discovered values and copy only the generated context directory to the +isolated board root. The `rsync --delete` below is safe only because its target +is the dedicated `contexts/` directory; do not point it at a shared location. + +```bash +export BOARD_HOST= +export QAIRT_INCLUDE= +export QAIRT_RUNTIME= +export REMOTE_ROOT=/opt/turbovla-qcs8550 + +ssh -o BatchMode=yes "$BOARD_HOST" "mkdir -p '$REMOTE_ROOT/contexts'" +rsync -a --delete deployment/qcs8550/artifacts/object/qcs8550_contexts/ \ + "$BOARD_HOST:$REMOTE_ROOT/contexts/" + +python deployment/qcs8550/tools/build_native_server.py \ + --host "$BOARD_HOST" \ + --remote-root "$REMOTE_ROOT" \ + --qairt-include "$QAIRT_INCLUDE" \ + --runtime "$QAIRT_RUNTIME" \ + --port 10092 \ + --start +``` + +The helper compiles the native service against the copied QAIRT headers and +starts it with the board runtime's aarch64 and Hexagon v73 library paths. It +does not create, alter, or stop any other service. If port `10092` is already +occupied, stop here and ask the user whether it belongs to this deployment or +choose a different unused port explicitly. + +Check the board process and log without exposing secrets: + +```bash +ssh -o BatchMode=yes "$BOARD_HOST" \ + "ss -ltn | grep ':10092 ' && tail -n 80 '$REMOTE_ROOT/logs/turbovla_qnn_server.log'" +``` + +## Phase 5: Native Replay And LIBERO Smoke Test + +First send the frozen reference input repeatedly. The output must be stable +across warm requests. This checks persistent context loading and the wire +protocol; it does not imply FP32 equivalence. + +```bash +python deployment/qcs8550/tools/native_client.py \ + --bundle deployment/qcs8550/artifacts/object/reference_bundle_l11.npz \ + --host "$BOARD_HOST" --port 10092 --requests 20 +``` + +Then perform a one-trial-per-task LIBERO Object smoke test using the board +service. Set the environment variables so the host adapter connects to the +same service: + +```bash +export TURBOVLA_QNN_HOST="$BOARD_HOST" +export TURBOVLA_QNN_PORT=10092 +export MUJOCO_GL=egl +export PYOPENGL_PLATFORM=egl +export PYTHONPATH="$PWD/deployment/qcs8550:$PWD:$PWD/third_party/vla_adapter" + +python deployment/qcs8550/tools/run_libero_rollout.py \ + --ckpt_path pretrained/TurboVLA/checkpoints/libero/object.pth \ + --dinov3_path pretrained/dinov3-vitb16 \ + --bert_path pretrained/bert-base-uncased \ + --stats_path experiments/libero/configs/libero_all4_stats.json \ + --stats_key libero_all4_no_noops \ + --libero_root "$LIBERO_ROOT" \ + --task_suite_name libero_object \ + --num_trials_per_task 1 +``` + +Report the service median/p95 latency, native replay stability, task-level +results, exact QAIRT version, and context manifest hashes. Do not describe a +one-trial smoke test as an upstream 50-trial evaluation. On the original +bring-up setup, board service P50/P95 was 60.1/67.6 ms and one episode for each +of the ten LIBERO Object tasks succeeded; those numbers are reference evidence, +not guaranteed results on another board or build. + +## Failure Triage + +| Symptom | Required action | +| --- | --- | +| Export split-reference check fails | Check the Transformers 4.56 pin and DINO `hidden_states[-1]`; do not modify the graph to force a pass. | +| AI Hub compile fails | Save the job URL/status and stop; do not compile with a different target or QAIRT version without approval. | +| Native service cannot load a context | Confirm context manifest target/version, board runtime, and Hexagon v73 libraries match. | +| Port is in use | Do not kill its owner. Ask the user or select an approved unused port. | +| Replay is unstable or rollout fails | Preserve local logs/results, report the first failing phase and manifest hashes, and do not claim deployment success. | diff --git a/deployment/qcs8550/README.md b/deployment/qcs8550/README.md new file mode 100644 index 0000000..ae4eeb3 --- /dev/null +++ b/deployment/qcs8550/README.md @@ -0,0 +1,131 @@ +# QCS8550 QNN Deployment + +This experimental deployment targets the released **LIBERO Object** checkpoint +on Qualcomm QCS8550 HTP v73 with QAIRT 2.48. It keeps TurboVLA's two 256x256 +DINOv3 views, 8-D normalized state, static BERT lengths (`11`, `14`, `21`), +and `[1,12,7]` normalized action chunk. + +The graph partition is: + +```text +host normalization + tokenizer/masks + -> DINOv3 one-view context, executed twice + -> BERT context selected by instruction length + -> host zero-pad BERT hidden state to 21 tokens + -> policy-core context + -> host action denormalization +``` + +The board-native server loads these five contexts once and uses direct QNN API +execution, avoiding per-request process launch, context creation, and raw-file +I/O. + +For an AI agent that must carry out the complete deployment, including explicit +validation and safety stop conditions, read [AGENT_DEPLOYMENT.md](AGENT_DEPLOYMENT.md). + +## Important Compatibility Rule + +The DINO wrapper deliberately uses `outputs.hidden_states[-1]`, not +`last_hidden_state`. This preserves the behavior documented in upstream issue +[#4](https://github.com/H-EmbodVis/TurboVLA/issues/4). Export and numerical +validation must use `transformers==4.56.*`; loading the checkpoint alone is +not sufficient to establish equivalent actions. + +## Requirements + +- Python 3.10 with the TurboVLA dependencies, `onnx`, `onnxruntime`, and + `qai-hub` for export and cloud compilation. +- A QCS8550 board with a QAIRT 2.48 runtime and Hexagon v73 skeletons. +- The QAIRT 2.48 Linux development SDK on the host for QNN headers. +- The released Object checkpoint under `pretrained/TurboVLA` and local BERT + assets under `pretrained/bert-base-uncased`. + +Model weights and context binaries are intentionally not committed. + +## Export And Compile + +Export the fixed-shape ONNX graphs and validate their CPU ONNX Runtime outputs +against the pinned PyTorch reference: + +```bash +python deployment/qcs8550/tools/export_static_onnx.py +``` + +Authenticate with Qualcomm AI Hub, compile for the target, then download the +context binaries: + +```bash +qai-hub configure +python deployment/qcs8550/tools/submit_qai_hub_compile.py --qairt-version 2.48 +python deployment/qcs8550/tools/download_contexts.py +``` + +The compile command uses `qnn_context_binary`, QAIRT 2.48, and native 64-bit +I/O. The resulting `artifacts/object/qcs8550_contexts` directory must contain +`dinov3.bin`, `bert_l11.bin`, `bert_l14.bin`, `bert_l21.bin`, `policy_core.bin`, +and `manifest.json`. + +## Board Service + +Copy the contexts to an isolated board root, then compile and start the native +server. The commands below use an SSH host alias and deliberately do not +manage any unrelated service or port. + +```bash +export QAIRT_INCLUDE=/path/to/qairt-2.48/include +export QAIRT_RUNTIME=/path/on/board/qairt-2.48.0.260626-linux + +rsync -a deployment/qcs8550/artifacts/object/qcs8550_contexts/ \ + qcs8550:/opt/turbovla-qcs8550/contexts/ + +python deployment/qcs8550/tools/build_native_server.py \ + --host qcs8550 \ + --remote-root /opt/turbovla-qcs8550 \ + --qairt-include "$QAIRT_INCLUDE" \ + --runtime "$QAIRT_RUNTIME" \ + --start +``` + +The server listens on port `10092` by default. It accepts a framed named-array +request containing two normalized pixel tensors, BERT inputs/masks, and state; +it returns one normalized action chunk plus timing JSON. `native_client.py` +contains the reference client implementation. + +## LIBERO Rollout + +Extract the instruction-length contract once from the released checkpoint: + +```bash +python deployment/qcs8550/tools/extract_checkpoint_contract.py +``` + +The adapter performs no PyTorch model forward pass. It reproduces image +preprocessing, tokenizer special-token masks, state normalization, and action +decoding on the host, then sends the request to the board service. + +```bash +export TURBOVLA_QNN_HOST= +export TURBOVLA_QNN_PORT=10092 +export MUJOCO_GL=egl +export PYOPENGL_PLATFORM=egl +export PYTHONPATH="$PWD/deployment/qcs8550:$PWD:$PWD/third_party/vla_adapter" + +python deployment/qcs8550/tools/run_libero_rollout.py \ + --ckpt_path pretrained/TurboVLA/checkpoints/libero/object.pth \ + --dinov3_path pretrained/dinov3-vitb16 \ + --bert_path pretrained/bert-base-uncased \ + --stats_path experiments/libero/configs/libero_all4_stats.json \ + --stats_key libero_all4_no_noops \ + --libero_root /path/to/LIBERO \ + --task_suite_name libero_object --num_trials_per_task 1 +``` + +## Numerical Validation + +Do not assume HTP outputs are exactly FP32-equivalent. The native server must +first match the same compiled QNN context through `qnn-net-run`; then compare +full normalized actions and rollout success against the pinned reference. +On the contributor's QCS8550/QAIRT 2.48 setup, the HTP service matched board +replay exactly across 20 warm requests and completed a one-episode smoke for +each LIBERO Object task. This is a bring-up result, not a replacement for the +upstream 50-trial-per-task evaluation. diff --git a/deployment/qcs8550/native/turbovla_qnn_server.cpp b/deployment/qcs8550/native/turbovla_qnn_server.cpp new file mode 100644 index 0000000..4d65138 --- /dev/null +++ b/deployment/qcs8550/native/turbovla_qnn_server.cpp @@ -0,0 +1,539 @@ +// Persistent QNN runner for the fixed TurboVLA LIBERO Object graph split. +// The wire protocol is intentionally the same framed named-array format used +// by the earlier XR runners: host preprocessing stays explicit and testable. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "QnnInterface.h" +#include "System/QnnSystemInterface.h" + +using Bytes = std::vector; +using GetProvidersFn = Qnn_ErrorHandle_t (*)(const QnnInterface_t ***, uint32_t *); +using GetSystemProvidersFn = Qnn_ErrorHandle_t (*)(const QnnSystemInterface_t ***, uint32_t *); + +constexpr uint32_t kMaxFrameBytes = 16u * 1024u * 1024u; +constexpr size_t kActionBytes = 12u * 7u * sizeof(float); +constexpr size_t kTextHiddenBytes = 21u * 768u * sizeof(float); + +uint32_t read_u32_be(const uint8_t *data) { + return (static_cast(data[0]) << 24) | + (static_cast(data[1]) << 16) | + (static_cast(data[2]) << 8) | static_cast(data[3]); +} + +uint64_t read_u64_be(const uint8_t *data) { + uint64_t value = 0; + for (int index = 0; index < 8; ++index) value = (value << 8) | data[index]; + return value; +} + +void append_u32_be(std::string *out, uint32_t value) { + out->push_back(static_cast((value >> 24) & 0xff)); + out->push_back(static_cast((value >> 16) & 0xff)); + out->push_back(static_cast((value >> 8) & 0xff)); + out->push_back(static_cast(value & 0xff)); +} + +bool recv_all(int fd, void *buffer, size_t size) { + auto *cursor = static_cast(buffer); + while (size != 0) { + const ssize_t received = recv(fd, cursor, size, MSG_WAITALL); + if (received <= 0) return false; + cursor += received; + size -= static_cast(received); + } + return true; +} + +bool send_all(int fd, const void *buffer, size_t size) { + const auto *cursor = static_cast(buffer); + while (size != 0) { + const ssize_t sent = send(fd, cursor, size, MSG_NOSIGNAL); + if (sent <= 0) return false; + cursor += sent; + size -= static_cast(sent); + } + return true; +} + +uint64_t now_us() { + return static_cast(std::chrono::duration_cast( + std::chrono::steady_clock::now().time_since_epoch()) + .count()); +} + +size_t dtype_bytes(Qnn_DataType_t dtype) { + switch (dtype) { + case QNN_DATATYPE_FLOAT_32: + case QNN_DATATYPE_INT_32: + case QNN_DATATYPE_UINT_32: + case QNN_DATATYPE_SFIXED_POINT_32: + case QNN_DATATYPE_UFIXED_POINT_32: + return 4; + case QNN_DATATYPE_FLOAT_16: + case QNN_DATATYPE_BFLOAT_16: + case QNN_DATATYPE_INT_16: + case QNN_DATATYPE_UINT_16: + case QNN_DATATYPE_SFIXED_POINT_16: + case QNN_DATATYPE_UFIXED_POINT_16: + return 2; + case QNN_DATATYPE_INT_8: + case QNN_DATATYPE_UINT_8: + case QNN_DATATYPE_SFIXED_POINT_8: + case QNN_DATATYPE_UFIXED_POINT_8: + case QNN_DATATYPE_BOOL_8: + return 1; + default: + throw std::runtime_error("unsupported QNN tensor dtype"); + } +} + +uint64_t tensor_elements(const Qnn_Tensor_t &tensor) { + const uint32_t rank = tensor.version == QNN_TENSOR_VERSION_2 ? tensor.v2.rank : tensor.v1.rank; + const uint32_t *dims = + tensor.version == QNN_TENSOR_VERSION_2 ? tensor.v2.dimensions : tensor.v1.dimensions; + uint64_t count = 1; + for (uint32_t index = 0; index < rank; ++index) count *= dims[index]; + return count; +} + +const char *tensor_name(const Qnn_Tensor_t &tensor) { + return tensor.version == QNN_TENSOR_VERSION_2 ? tensor.v2.name : tensor.v1.name; +} + +Qnn_DataType_t tensor_dtype(const Qnn_Tensor_t &tensor) { + return tensor.version == QNN_TENSOR_VERSION_2 ? tensor.v2.dataType : tensor.v1.dataType; +} + +size_t tensor_bytes(const Qnn_Tensor_t &tensor) { + return static_cast(tensor_elements(tensor)) * dtype_bytes(tensor_dtype(tensor)); +} + +void bind_tensor(Qnn_Tensor_t *tensor, Qnn_TensorType_t type, void *data, size_t size) { + if (tensor->version == QNN_TENSOR_VERSION_2) { + tensor->v2.type = type; + tensor->v2.memType = QNN_TENSORMEMTYPE_RAW; + tensor->v2.clientBuf.data = data; + tensor->v2.clientBuf.dataSize = static_cast(size); + } else { + tensor->v1.type = type; + tensor->v1.memType = QNN_TENSORMEMTYPE_RAW; + tensor->v1.clientBuf.data = data; + tensor->v1.clientBuf.dataSize = static_cast(size); + } +} + +struct TensorSpec { + Qnn_Tensor_t tensor{}; + std::string name; + std::vector dimensions; + std::vector dynamic_dimensions; + + void copy_from(const Qnn_Tensor_t &source) { + tensor = source; + name = tensor_name(source) ? tensor_name(source) : ""; + const uint32_t rank = source.version == QNN_TENSOR_VERSION_2 ? source.v2.rank : source.v1.rank; + const uint32_t *dims = source.version == QNN_TENSOR_VERSION_2 ? source.v2.dimensions + : source.v1.dimensions; + dimensions.assign(dims, dims + rank); + if (source.version == QNN_TENSOR_VERSION_2 && source.v2.isDynamicDimensions) { + dynamic_dimensions.assign(source.v2.isDynamicDimensions, source.v2.isDynamicDimensions + rank); + } + if (tensor.version == QNN_TENSOR_VERSION_2) { + tensor.v2.name = name.c_str(); + tensor.v2.dimensions = dimensions.empty() ? nullptr : dimensions.data(); + tensor.v2.isDynamicDimensions = dynamic_dimensions.empty() ? nullptr : dynamic_dimensions.data(); + } else { + tensor.v1.name = name.c_str(); + tensor.v1.dimensions = dimensions.empty() ? nullptr : dimensions.data(); + } + } +}; + +struct Runtime { + void *backend_library = nullptr; + void *system_library = nullptr; + const QNN_INTERFACE_VER_TYPE *provider = nullptr; + const QNN_SYSTEM_INTERFACE_VER_TYPE *system = nullptr; + Qnn_DeviceHandle_t device = nullptr; + Qnn_BackendHandle_t backend = nullptr; + + ~Runtime() { + if (backend && provider && provider->backendFree) provider->backendFree(backend); + if (device && provider && provider->deviceFree) provider->deviceFree(device); + if (system_library) dlclose(system_library); + if (backend_library) dlclose(backend_library); + } + + static void check(Qnn_ErrorHandle_t status, const std::string &operation) { + if (status == QNN_SUCCESS) return; + std::ostringstream message; + message << operation << " failed: 0x" << std::hex << static_cast(status); + throw std::runtime_error(message.str()); + } + + void initialize(const std::string &library_root) { + backend_library = dlopen((library_root + "/libQnnHtp.so").c_str(), RTLD_NOW | RTLD_LOCAL); + if (!backend_library) throw std::runtime_error("could not open libQnnHtp.so: " + std::string(dlerror())); + const auto get_providers = reinterpret_cast( + dlsym(backend_library, "QnnInterface_getProviders")); + if (!get_providers) throw std::runtime_error("QnnInterface_getProviders is missing"); + const QnnInterface_t **providers = nullptr; + uint32_t provider_count = 0; + check(get_providers(&providers, &provider_count), "QnnInterface_getProviders"); + if (!providers || provider_count == 0) throw std::runtime_error("QNN provider list is empty"); + provider = &providers[0]->QNN_INTERFACE_VER_NAME; + + system_library = dlopen((library_root + "/libQnnSystem.so").c_str(), RTLD_NOW | RTLD_LOCAL); + if (!system_library) throw std::runtime_error("could not open libQnnSystem.so: " + std::string(dlerror())); + const auto get_system_providers = reinterpret_cast( + dlsym(system_library, "QnnSystemInterface_getProviders")); + if (!get_system_providers) throw std::runtime_error("QnnSystemInterface_getProviders is missing"); + const QnnSystemInterface_t **system_providers = nullptr; + uint32_t system_count = 0; + check(get_system_providers(&system_providers, &system_count), "QnnSystemInterface_getProviders"); + if (!system_providers || system_count == 0) throw std::runtime_error("QNN system provider list is empty"); + system = &system_providers[0]->QNN_SYSTEM_INTERFACE_VER_NAME; + check(provider->deviceCreate(nullptr, nullptr, &device), "deviceCreate"); + check(provider->backendCreate(nullptr, nullptr, &backend), "backendCreate"); + } +}; + +class Graph { + public: + Graph(Runtime *runtime, std::string label, std::string path) + : runtime_(runtime), label_(std::move(label)), path_(std::move(path)) {} + + ~Graph() { + if (context_) runtime_->provider->contextFree(context_, nullptr); + if (mapped_) munmap(mapped_, mapped_size_); + if (fd_ >= 0) close(fd_); + } + + void load() { + fd_ = open(path_.c_str(), O_RDONLY | O_CLOEXEC); + if (fd_ < 0) throw std::runtime_error("could not open context " + path_); + struct stat info {}; + if (fstat(fd_, &info) != 0 || info.st_size <= 0) throw std::runtime_error("invalid context " + path_); + mapped_size_ = static_cast(info.st_size); + mapped_ = mmap(nullptr, mapped_size_, PROT_READ, MAP_PRIVATE, fd_, 0); + if (mapped_ == MAP_FAILED) { + mapped_ = nullptr; + throw std::runtime_error("mmap failed for " + path_); + } + + QnnSystemContext_Handle_t system_context = nullptr; + Runtime::check(runtime_->system->systemContextCreate(&system_context), "systemContextCreate"); + const QnnSystemContext_BinaryInfo_t *binary = nullptr; + Runtime::check(runtime_->system->systemContextGetMetaData(system_context, mapped_, mapped_size_, &binary), + "systemContextGetMetaData"); + QnnSystemContext_GraphInfo_t *graphs = nullptr; + uint32_t graph_count = 0; + if (binary->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_3) { + graphs = binary->contextBinaryInfoV3.graphs; + graph_count = binary->contextBinaryInfoV3.numGraphs; + } else if (binary->version == QNN_SYSTEM_CONTEXT_BINARY_INFO_VERSION_2) { + graphs = binary->contextBinaryInfoV2.graphs; + graph_count = binary->contextBinaryInfoV2.numGraphs; + } else { + graphs = binary->contextBinaryInfoV1.graphs; + graph_count = binary->contextBinaryInfoV1.numGraphs; + } + if (!graphs || graph_count == 0) { + runtime_->system->systemContextFree(system_context); + throw std::runtime_error("context contains no graphs " + path_); + } + const auto &metadata = graphs[0].graphInfoV1; + graph_name_ = metadata.graphName ? metadata.graphName : ""; + if (graph_name_.empty()) { + runtime_->system->systemContextFree(system_context); + throw std::runtime_error("context graph has no name " + path_); + } + for (uint32_t index = 0; index < metadata.numGraphInputs; ++index) { + TensorSpec spec; + spec.copy_from(metadata.graphInputs[index]); + inputs_.push_back(std::move(spec)); + } + for (uint32_t index = 0; index < metadata.numGraphOutputs; ++index) { + TensorSpec spec; + spec.copy_from(metadata.graphOutputs[index]); + outputs_.push_back(std::move(spec)); + } + runtime_->system->systemContextFree(system_context); + Runtime::check(runtime_->provider->contextCreateFromBinary(runtime_->backend, runtime_->device, nullptr, + mapped_, mapped_size_, &context_, nullptr), + "contextCreateFromBinary " + label_); + Runtime::check(runtime_->provider->graphRetrieve(context_, graph_name_.c_str(), &graph_), + "graphRetrieve " + label_); + std::printf("loaded %-14s graph=%s inputs=%zu outputs=%zu\n", label_.c_str(), graph_name_.c_str(), + inputs_.size(), outputs_.size()); + } + + Bytes execute(const std::map &provided, uint64_t *elapsed_us) const { + std::vector inputs; + std::vector outputs; + std::vector output_buffers; + output_buffers.reserve(outputs_.size()); + inputs.reserve(inputs_.size()); + outputs.reserve(outputs_.size()); + for (const auto &spec : inputs_) { + const auto found = provided.find(spec.name); + if (found == provided.end()) throw std::runtime_error(label_ + " missing input " + spec.name); + if (found->second->size() != tensor_bytes(spec.tensor)) { + throw std::runtime_error(label_ + " input size mismatch for " + spec.name); + } + Qnn_Tensor_t tensor = spec.tensor; + bind_tensor(&tensor, QNN_TENSOR_TYPE_APP_WRITE, const_cast(found->second->data()), + found->second->size()); + inputs.push_back(tensor); + } + for (const auto &spec : outputs_) { + output_buffers.emplace_back(tensor_bytes(spec.tensor)); + Qnn_Tensor_t tensor = spec.tensor; + bind_tensor(&tensor, QNN_TENSOR_TYPE_APP_READ, output_buffers.back().data(), output_buffers.back().size()); + outputs.push_back(tensor); + } + if (outputs.size() != 1) throw std::runtime_error(label_ + " must expose one output"); + const uint64_t started = now_us(); + Runtime::check(runtime_->provider->graphExecute(graph_, inputs.data(), static_cast(inputs.size()), + outputs.data(), static_cast(outputs.size()), nullptr, + nullptr), + "graphExecute " + label_); + *elapsed_us += now_us() - started; + return std::move(output_buffers.front()); + } + + private: + Runtime *runtime_; + std::string label_; + std::string path_; + int fd_ = -1; + void *mapped_ = nullptr; + size_t mapped_size_ = 0; + std::string graph_name_; + Qnn_ContextHandle_t context_ = nullptr; + Qnn_GraphHandle_t graph_ = nullptr; + std::vector inputs_; + std::vector outputs_; +}; + +struct Request { + std::map arrays; +}; + +Request parse_request(const std::string &payload) { + if (payload.size() < 4) throw std::runtime_error("request is truncated"); + Request request; + const auto *data = reinterpret_cast(payload.data()); + size_t offset = 0; + const uint32_t count = read_u32_be(data + offset); + offset += 4; + if (count == 0 || count > 32) throw std::runtime_error("invalid request tensor count"); + for (uint32_t index = 0; index < count; ++index) { + if (offset + 2 > payload.size()) throw std::runtime_error("request name is truncated"); + const uint16_t name_size = static_cast((data[offset] << 8) | data[offset + 1]); + offset += 2; + if (name_size == 0 || offset + name_size + 8 > payload.size()) { + throw std::runtime_error("request tensor header is invalid"); + } + const std::string name(reinterpret_cast(data + offset), name_size); + offset += name_size; + const uint64_t size = read_u64_be(data + offset); + offset += 8; + if (size > kMaxFrameBytes || size > payload.size() - offset) { + throw std::runtime_error("request tensor size is invalid"); + } + Bytes value(static_cast(size)); + std::memcpy(value.data(), data + offset, value.size()); + offset += value.size(); + if (!request.arrays.emplace(name, std::move(value)).second) { + throw std::runtime_error("request contains a duplicate tensor name"); + } + } + if (offset != payload.size()) throw std::runtime_error("request has trailing bytes"); + return request; +} + +const Bytes &require(const Request &request, const std::string &name) { + const auto found = request.arrays.find(name); + if (found == request.arrays.end()) throw std::runtime_error("missing request tensor " + name); + return found->second; +} + +class TurboVlaServer { + public: + TurboVlaServer(std::string model_root, std::string runtime_root, uint16_t port) + : model_root_(std::move(model_root)), runtime_root_(std::move(runtime_root)), port_(port) {} + + void initialize() { + runtime_.initialize(runtime_root_ + "/lib/aarch64-oe-linux-gcc11.2"); + load_graph("dinov3", "contexts/dinov3.bin"); + load_graph("bert_l11", "contexts/bert_l11.bin"); + load_graph("bert_l14", "contexts/bert_l14.bin"); + load_graph("bert_l21", "contexts/bert_l21.bin"); + load_graph("policy_core", "contexts/policy_core.bin"); + std::printf("TurboVLA QNN service ready on 0.0.0.0:%u\n", port_); + std::fflush(stdout); + } + + void serve() { + const int listener = socket(AF_INET, SOCK_STREAM, 0); + if (listener < 0) throw std::runtime_error("socket failed"); + int reuse = 1; + setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); + sockaddr_in address{}; + address.sin_family = AF_INET; + address.sin_addr.s_addr = htonl(INADDR_ANY); + address.sin_port = htons(port_); + if (bind(listener, reinterpret_cast(&address), sizeof(address)) != 0 || + listen(listener, 1) != 0) { + close(listener); + throw std::runtime_error("bind/listen failed"); + } + while (true) { + const int client = accept(listener, nullptr, nullptr); + if (client >= 0) { + serve_client(client); + close(client); + } + } + } + + private: + void load_graph(const std::string &label, const std::string &relative_path) { + auto graph = std::make_unique(&runtime_, label, model_root_ + "/" + relative_path); + graph->load(); + graphs_.emplace(label, std::move(graph)); + } + + Bytes run(const std::string &label, const std::map &inputs, + uint64_t *elapsed_us) const { + return graphs_.at(label)->execute(inputs, elapsed_us); + } + + Bytes infer(const Request &request, std::string *metrics) const { + uint64_t dino_us = 0; + uint64_t bert_us = 0; + uint64_t core_us = 0; + const Bytes &view0 = require(request, "pixels_view0"); + const Bytes &view1 = require(request, "pixels_view1"); + const Bytes dino0 = run("dinov3", {{"pixel_values", &view0}}, &dino_us); + const Bytes dino1 = run("dinov3", {{"pixel_values", &view1}}, &dino_us); + + const Bytes &input_ids = require(request, "input_ids"); + const size_t text_length = input_ids.size() / sizeof(int32_t); + if (input_ids.size() != text_length * sizeof(int32_t) || + (text_length != 11 && text_length != 14 && text_length != 21)) { + throw std::runtime_error("input_ids must contain exactly 11, 14, or 21 int32 tokens"); + } + const std::string bert_name = "bert_l" + std::to_string(text_length); + const Bytes bert = run(bert_name, + {{"input_ids", &input_ids}, + {"token_type_ids", &require(request, "token_type_ids")}, + {"text_self_attention_mask", &require(request, "bert_attention_mask")}, + {"position_ids", &require(request, "position_ids")}}, + &bert_us); + if (bert.size() != text_length * 768u * sizeof(float)) { + throw std::runtime_error("BERT returned an unexpected hidden size"); + } + Bytes bert_padded(kTextHiddenBytes, 0); + std::memcpy(bert_padded.data(), bert.data(), bert.size()); + Bytes action = run("policy_core", + {{"vision_view0", &dino0}, + {"vision_view1", &dino1}, + {"bert_hidden_padded", &bert_padded}, + {"text_key_padding_mask", &require(request, "text_key_padding_mask")}, + {"text_self_attention_mask", &require(request, "text_self_attention_mask")}, + {"state", &require(request, "state")}}, + &core_us); + if (action.size() != kActionBytes) throw std::runtime_error("policy core returned an unexpected action size"); + std::ostringstream json; + json << "{\"dino_us\":" << dino_us << ",\"bert_us\":" << bert_us + << ",\"policy_core_us\":" << core_us << "}"; + *metrics = json.str(); + return action; + } + + static std::string response(uint32_t status, const Bytes &action, const std::string &metrics) { + std::string body; + append_u32_be(&body, status); + append_u32_be(&body, static_cast(action.size())); + body.append(reinterpret_cast(action.data()), action.size()); + append_u32_be(&body, static_cast(metrics.size())); + body.append(metrics); + std::string framed; + append_u32_be(&framed, static_cast(body.size())); + framed.append(body); + return framed; + } + + void serve_client(int client) const { + while (true) { + uint8_t size[4]; + if (!recv_all(client, size, sizeof(size))) return; + const uint32_t frame_size = read_u32_be(size); + if (frame_size == 0 || frame_size > kMaxFrameBytes) return; + std::string payload(frame_size, '\0'); + if (!recv_all(client, payload.data(), payload.size())) return; + const uint64_t started = now_us(); + Bytes action; + std::string metrics; + uint32_t status = 0; + try { + action = infer(parse_request(payload), &metrics); + metrics.insert(metrics.size() - 1, ",\"request_us\":" + std::to_string(now_us() - started)); + } catch (const std::exception &error) { + status = 1; + metrics = std::string("{\"error\":\"") + error.what() + "\"}"; + } + const std::string result = response(status, action, metrics); + if (!send_all(client, result.data(), result.size())) return; + } + } + + std::string model_root_; + std::string runtime_root_; + uint16_t port_; + Runtime runtime_; + std::map> graphs_; +}; + +int main(int argc, char **argv) { + if (argc < 3) { + std::fprintf(stderr, "usage: %s [port]\n", argv[0]); + return 2; + } + const std::string model_root = argv[1]; + const std::string runtime_root = argv[2]; + const uint16_t port = argc > 3 ? static_cast(std::strtoul(argv[3], nullptr, 10)) : 10092; + try { + TurboVlaServer server(model_root, runtime_root, port); + server.initialize(); + server.serve(); + } catch (const std::exception &error) { + std::fprintf(stderr, "fatal: %s\n", error.what()); + return 1; + } + return 0; +} diff --git a/deployment/qcs8550/qnn_policy.py b/deployment/qcs8550/qnn_policy.py new file mode 100644 index 0000000..2563a58 --- /dev/null +++ b/deployment/qcs8550/qnn_policy.py @@ -0,0 +1,201 @@ +"""Host-side preprocessing and action decoding for the persistent TurboVLA QNN service.""" + +from __future__ import annotations + +import json +import os +import sys +from pathlib import Path +from typing import Any + +import numpy as np +from transformers import AutoTokenizer + + +QCS_ROOT = Path(__file__).resolve().parent +REPO_ROOT = QCS_ROOT.parents[1] +if str(QCS_ROOT / "tools") not in sys.path: + sys.path.insert(0, str(QCS_ROOT / "tools")) + +from native_client import run_request + + +DEFAULT_CONTRACT = QCS_ROOT / "artifacts/checkpoint_contract.json" +DEFAULT_BERT = REPO_ROOT / "pretrained/bert-base-uncased" +ACTION_MIN = np.asarray( + (-0.9375, -0.9375, -0.9375, -0.23642857372760773, -0.3053571283817291, -0.3675000071525574), + dtype=np.float32, +) +ACTION_MAX = np.asarray( + (0.9375, 0.9375, 0.9375, 0.30000001192092896, 0.29357144236564636, 0.375), + dtype=np.float32, +) + + +def rotate_libero_image(image: np.ndarray) -> np.ndarray: + return np.ascontiguousarray(np.asarray(image)[::-1, ::-1]) + + +def quat2axisangle(quat: np.ndarray) -> np.ndarray: + quat = np.asarray(quat, dtype=np.float32).copy() + quat[3] = np.clip(quat[3], -1.0, 1.0) + denominator = np.sqrt(max(0.0, 1.0 - float(quat[3]) ** 2)) + if np.isclose(denominator, 0.0): + return np.zeros(3, dtype=np.float32) + return (quat[:3] * 2.0 * np.arccos(float(quat[3])) / denominator).astype(np.float32) + + +def _special_token_masks(input_ids: np.ndarray, special_ids: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + """Match TurboVLA generate_masks_with_special_tokens for batch size one.""" + ids = np.asarray(input_ids, dtype=np.int32) + if ids.ndim != 2 or ids.shape[0] != 1: + raise ValueError(f"expected one tokenized instruction, got {ids.shape}") + length = ids.shape[1] + attention = np.eye(length, dtype=np.bool_)[None] + position_ids = np.zeros((1, length), dtype=np.int32) + previous = 0 + for column in np.flatnonzero(np.isin(ids[0], special_ids)): + if column == 0 or column == length - 1: + attention[0, column, column] = True + position_ids[0, column] = 0 + else: + attention[0, previous + 1 : column + 1, previous + 1 : column + 1] = True + position_ids[0, previous + 1 : column + 1] = np.arange(0, column - previous, dtype=np.int32) + previous = int(column) + return attention, position_ids + + +class TurboVLAQnnPolicy: + """TurboVLA LIBERO policy with DINO/BERT/policy-core execution on QCS8550.""" + + def __init__( + self, + *_, + service_host: str = os.environ.get("TURBOVLA_QNN_HOST", "127.0.0.1"), + service_port: int = int(os.environ.get("TURBOVLA_QNN_PORT", "10092")), + contract_path: str | Path = DEFAULT_CONTRACT, + bert_path: str | Path = DEFAULT_BERT, + timeout: float = 120.0, + **__, + ) -> None: + contract = json.loads(Path(contract_path).read_text(encoding="utf-8")) + self.instruction_lengths = {str(key): int(value) for key, value in contract["instruction_lengths"].items()} + self.output_length = int(contract["text_padding_length"]) + self.mean = np.asarray(contract["image"]["mean"], dtype=np.float32)[:, None, None] + self.std = np.asarray(contract["image"]["std"], dtype=np.float32)[:, None, None] + self.proprio_mean = np.asarray(contract["state"]["mean"], dtype=np.float32) + self.proprio_std = np.asarray(contract["state"]["std"], dtype=np.float32) + self.tokenizer = AutoTokenizer.from_pretrained(str(bert_path), local_files_only=True, use_fast=True) + self.special_ids = np.asarray( + self.tokenizer.convert_tokens_to_ids(contract["special_token_strings"]), dtype=np.int32 + ) + self.service_host = service_host + self.service_port = int(service_port) + self.timeout = float(timeout) + self.last_metrics: dict[str, object] = {} + + def _text_inputs(self, instruction: str) -> dict[str, np.ndarray]: + if instruction not in self.instruction_lengths: + raise KeyError(f"instruction is not in the checkpoint static-length contract: {instruction!r}") + length = self.instruction_lengths[instruction] + tokens = self.tokenizer( + [instruction], padding="max_length", truncation=True, max_length=length, return_tensors="np" + ) + input_ids = np.ascontiguousarray(tokens["input_ids"], dtype=np.int32) + token_type_ids = np.ascontiguousarray(tokens["token_type_ids"], dtype=np.int32) + token_attention = np.asarray(tokens["attention_mask"], dtype=np.bool_) + bert_attention_mask, position_ids = _special_token_masks(input_ids, self.special_ids) + key_padding = np.ones((1, self.output_length), dtype=np.bool_) + key_padding[:, :length] = ~token_attention + policy_attention = np.eye(self.output_length, dtype=np.bool_)[None] + policy_attention[:, :length, :length] = bert_attention_mask + return { + "input_ids": input_ids, + "token_type_ids": token_type_ids, + "bert_attention_mask": bert_attention_mask, + "position_ids": position_ids, + "text_key_padding_mask": key_padding, + "text_self_attention_mask": policy_attention, + } + + def _state(self, state_or_obs: np.ndarray | dict[str, Any]) -> np.ndarray: + if isinstance(state_or_obs, dict): + state = np.concatenate( + ( + np.asarray(state_or_obs["robot0_eef_pos"], dtype=np.float32).reshape(-1), + quat2axisangle(state_or_obs["robot0_eef_quat"]), + np.asarray(state_or_obs["robot0_gripper_qpos"], dtype=np.float32).reshape(-1), + ) + ) + else: + state = np.asarray(state_or_obs, dtype=np.float32).reshape(-1) + if state.shape != (8,): + raise ValueError(f"TurboVLA requires an 8-D state, got {state.shape}") + return np.ascontiguousarray(((state - self.proprio_mean) / (self.proprio_std + 1e-6))[None], dtype=np.float32) + + def _pixels(self, primary: np.ndarray, wrist: np.ndarray) -> tuple[np.ndarray, np.ndarray]: + views = [] + for image in (primary, wrist): + rgb = np.asarray(image, dtype=np.float32) + if rgb.shape != (256, 256, 3): + raise ValueError(f"TurboVLA requires a 256x256 RGB image, got {rgb.shape}") + views.append(np.ascontiguousarray((np.transpose(rgb / 255.0, (2, 0, 1)) - self.mean) / self.std)) + return views[0][None], views[1][None] + + def request_arrays( + self, + primary: np.ndarray, + wrist: np.ndarray, + instruction: str, + state_or_obs: np.ndarray | dict[str, Any], + ) -> dict[str, np.ndarray]: + view0, view1 = self._pixels(primary, wrist) + return { + "pixels_view0": view0, + "pixels_view1": view1, + **self._text_inputs(instruction), + "state": self._state(state_or_obs), + } + + def predict_normalized_action_chunk( + self, + primary: np.ndarray, + wrist: np.ndarray, + instruction: str, + state_or_obs: np.ndarray | dict[str, Any], + ) -> np.ndarray: + action, self.last_metrics = run_request( + self.service_host, + self.service_port, + self.request_arrays(primary, wrist, instruction, state_or_obs), + self.timeout, + ) + return np.nan_to_num(action[0], nan=0.0, posinf=1.0, neginf=-1.0).clip(-1.0, 1.0) + + def predict_env_action_chunk( + self, + primary: np.ndarray, + wrist: np.ndarray, + instruction: str, + state_or_obs: np.ndarray | dict[str, Any], + execute_steps: int | None = None, + ) -> np.ndarray: + normalized = self.predict_normalized_action_chunk(primary, wrist, instruction, state_or_obs) + arm = 0.5 * (normalized[:, :6] + 1.0) * (ACTION_MAX - ACTION_MIN) + ACTION_MIN + gripper = np.where(normalized[:, 6:7] >= 0.0, 1.0, -1.0).astype(np.float32) + actions = np.concatenate((arm, gripper), axis=1).astype(np.float32) + return actions if execute_steps is None else actions[: int(execute_steps)] + + def predict_env_action_chunk_from_obs( + self, + obs: dict[str, Any], + instruction: str, + execute_steps: int | None = None, + ) -> np.ndarray: + return self.predict_env_action_chunk( + rotate_libero_image(obs["agentview_image"]), + rotate_libero_image(obs["robot0_eye_in_hand_image"]), + instruction, + obs, + execute_steps, + ) diff --git a/deployment/qcs8550/tools/build_native_server.py b/deployment/qcs8550/tools/build_native_server.py new file mode 100644 index 0000000..53b9ba6 --- /dev/null +++ b/deployment/qcs8550/tools/build_native_server.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Build the TurboVLA native QNN service on the isolated QCS8550 directory.""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SERVER_SOURCE = ROOT / "native/turbovla_qnn_server.cpp" + + +def run(command: list[str]) -> None: + print("+", shlex.join(command), flush=True) + subprocess.run(command, check=True) + + +def ssh(host: str, command: str) -> None: + run(["ssh", "-o", "BatchMode=yes", host, command]) + + +def rsync(host: str, source: Path, destination: str) -> None: + source_arg = f"{source}/" if source.is_dir() else str(source) + run(["rsync", "-a", "--delete", source_arg, f"{host}:{destination}"]) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--host", required=True) + parser.add_argument("--remote-root", default="/opt/turbovla-qcs8550") + parser.add_argument("--qairt-include", type=Path, default=os.environ.get("QAIRT_INCLUDE")) + parser.add_argument("--runtime", default=os.environ.get("QAIRT_RUNTIME")) + parser.add_argument("--port", type=int, default=10092) + parser.add_argument("--start", action="store_true") + args = parser.parse_args() + + qairt_include = args.qairt_include / "QNN" if args.qairt_include and args.qairt_include.name != "QNN" else args.qairt_include + if not SERVER_SOURCE.is_file() or not qairt_include or not qairt_include.is_dir(): + raise FileNotFoundError("native source or QAIRT 2.48 headers are missing") + remote_native = f"{args.remote_root}/native" + remote_include = f"{args.remote_root}/include" + ssh(args.host, f"mkdir -p {shlex.quote(remote_native)} {shlex.quote(remote_include)}") + rsync(args.host, SERVER_SOURCE, f"{remote_native}/turbovla_qnn_server.cpp") + rsync(args.host, qairt_include, f"{remote_include}/QNN/") + build = " ".join( + [ + "g++ -std=c++17 -O2 -pipe -static-libstdc++ -static-libgcc", + f"-I{shlex.quote(remote_include + '/QNN')}", + shlex.quote(remote_native + "/turbovla_qnn_server.cpp"), + "-ldl", + "-o", + shlex.quote(remote_native + "/turbovla_qnn_server"), + "&& file", + shlex.quote(remote_native + "/turbovla_qnn_server"), + "&& sha256sum", + shlex.quote(remote_native + "/turbovla_qnn_server"), + ] + ) + ssh(args.host, build) + if args.start: + if not args.runtime: + raise ValueError("--runtime or QAIRT_RUNTIME is required with --start") + runtime = args.runtime + arm_lib = f"{runtime}/lib/aarch64-oe-linux-gcc11.2" + dsp_lib = f"{runtime}/lib/hexagon-v73/unsigned" + launch = " ".join( + [ + f"if ss -ltn | grep -q {shlex.quote(':' + str(args.port) + ' ')}; then", + f"echo port {args.port} is already in use >&2; exit 3; fi;", + f"mkdir -p {shlex.quote(args.remote_root + '/logs')};", + "nohup env", + f"LD_LIBRARY_PATH={shlex.quote(arm_lib)}", + f"ADSP_LIBRARY_PATH={shlex.quote(dsp_lib)}", + f"DSP_LIBRARY_PATH={shlex.quote(dsp_lib)}", + shlex.quote(remote_native + "/turbovla_qnn_server"), + shlex.quote(args.remote_root), + shlex.quote(runtime), + str(args.port), + ">", + shlex.quote(args.remote_root + "/logs/turbovla_qnn_server.log"), + "2>&1 < /dev/null &", + ] + ) + ssh(args.host, launch) + ssh(args.host, f"sleep 1; cat {shlex.quote(args.remote_root + '/logs/turbovla_qnn_server.log')}") + + +if __name__ == "__main__": + main() diff --git a/deployment/qcs8550/tools/download_contexts.py b/deployment/qcs8550/tools/download_contexts.py new file mode 100644 index 0000000..5a6aed5 --- /dev/null +++ b/deployment/qcs8550/tools/download_contexts.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +"""Download the fixed TurboVLA QCS8550 contexts with a local manifest.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +from typing import Any + +import qai_hub as hub + + +PORT_ROOT = Path(__file__).resolve().parents[1] +ARTIFACT_DIR = PORT_ROOT / "artifacts" / "object" +DEFAULT_COMPILE_JOBS = ARTIFACT_DIR / "qcs8550_compile_jobs.json" +DEFAULT_OUTPUT_DIR = ARTIFACT_DIR / "qcs8550_contexts" + + +def sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--compile-jobs", type=Path, default=DEFAULT_COMPILE_JOBS) + parser.add_argument("--output-dir", type=Path, default=DEFAULT_OUTPUT_DIR) + args = parser.parse_args() + + jobs = json.loads(args.compile_jobs.read_text(encoding="utf-8")) + graphs: dict[str, Any] = jobs["graphs"] + args.output_dir.mkdir(parents=True, exist_ok=True) + manifest_path = args.output_dir / "manifest.json" + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) if manifest_path.is_file() else {} + manifest["target"] = jobs["target"] + manifest["graphs"] = {} + + for name, entry in graphs.items(): + compile_job = hub.get_job(entry["compile_job_id"]) + status = compile_job.get_status() + if status.code != "SUCCESS": + raise RuntimeError(f"{name} compile job {entry['compile_job_id']} is {status.code}: {status.message}") + target_model = compile_job.get_target_model() + if target_model is None: + raise RuntimeError(f"{name} compile job has no target model") + output = args.output_dir / f"{name}.bin" + previous = manifest["graphs"].get(name, {}) + if output.is_file() and previous.get("model_id") == target_model.model_id and previous.get("sha256") == sha256(output): + print(f"reuse {name}: {output}") + else: + print(f"download {name}: {target_model.model_id}") + downloaded = Path(target_model.download(str(output))) + if downloaded.resolve() != output.resolve(): + downloaded.replace(output) + manifest["graphs"][name] = { + "compile_job_id": entry["compile_job_id"], + "model_id": target_model.model_id, + "filename": output.name, + "bytes": output.stat().st_size, + "sha256": sha256(output), + "input_spec": str(target_model.input_spec), + "output_spec": str(target_model.output_spec), + } + manifest_path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + + print(json.dumps(manifest, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/deployment/qcs8550/tools/export_static_onnx.py b/deployment/qcs8550/tools/export_static_onnx.py new file mode 100644 index 0000000..a7f1e92 --- /dev/null +++ b/deployment/qcs8550/tools/export_static_onnx.py @@ -0,0 +1,267 @@ +#!/usr/bin/env python3 +"""Export and validate the first static TurboVLA QNN graph set.""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import sys +from pathlib import Path +from typing import Any + +import numpy as np +import onnx +import onnxruntime as ort +import torch + +PORT_ROOT = Path(__file__).resolve().parents[1] +if str(PORT_ROOT) not in sys.path: + sys.path.insert(0, str(PORT_ROOT)) + +from tools.qcs8550_reference import ( + OneViewDINOv3, + StaticBert, + StaticPolicyCore, + load_object_reference, + pad_bert_hidden, + prepare_static_text_inputs, +) + + +INSTRUCTIONS = { + 11: "put the bowl on the plate", + 14: "pick up the orange juice and place it in the basket", + 21: "put the white mug on the left plate and put the yellow and white mug on the right plate", +} + + +def numpy_value(value: torch.Tensor) -> np.ndarray: + return value.detach().cpu().contiguous().numpy() + + +def relative_l2(actual: np.ndarray, expected: np.ndarray) -> float: + numerator = np.linalg.norm(np.asarray(actual, dtype=np.float64) - np.asarray(expected, dtype=np.float64)) + denominator = np.linalg.norm(np.asarray(expected, dtype=np.float64)) + return float(numerator / max(denominator, 1e-12)) + + +def error_metrics(actual: torch.Tensor, expected: torch.Tensor) -> dict[str, float]: + difference = actual - expected + return { + "relative_l2": float(difference.norm() / expected.norm().clamp_min(1e-12)), + "max_abs": float(difference.abs().max()), + "rmse": float(difference.square().mean().sqrt()), + } + + +def export_model( + module: torch.nn.Module, + inputs: tuple[torch.Tensor, ...], + output: Path, + input_names: list[str], + output_name: str, + opset: int, +) -> None: + module.eval() + torch.onnx.export( + module, + inputs, + output, + export_params=True, + opset_version=opset, + do_constant_folding=True, + input_names=input_names, + output_names=[output_name], + dynamic_axes=None, + ) + onnx.checker.check_model(str(output)) + + +def run_ort(path: Path, feed: dict[str, np.ndarray]) -> np.ndarray: + session = ort.InferenceSession(str(path), providers=["CPUExecutionProvider"]) + outputs = session.run(None, feed) + if len(outputs) != 1: + raise RuntimeError(f"expected one output from {path.name}, got {len(outputs)}") + return np.asarray(outputs[0]) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--output-dir", type=Path, default=PORT_ROOT / "artifacts" / "object") + parser.add_argument("--opset", type=int, default=17) + parser.add_argument("--seed", type=int, default=20260817) + args = parser.parse_args() + + output_dir = args.output_dir.resolve() + onnx_dir = output_dir / "onnx" + onnx_dir.mkdir(parents=True, exist_ok=True) + + torch.manual_seed(args.seed) + sdpa_model = load_object_reference(device="cpu") + model = load_object_reference( + device="cpu", + dino_attention_implementation="eager", + bert_attention_implementation="eager", + ) + # TransformerDecoder's eval fastpath becomes aten::_native_multi_head_attention, + # which is not an ONNX operator. The portable path is action-identical here. + torch.backends.mha.set_fastpath_enabled(False) + dino = OneViewDINOv3(model).eval() + bert = StaticBert(model).eval() + core = StaticPolicyCore(model).eval() + + pixels = torch.randn((1, 2, 3, 256, 256), dtype=torch.float32) + state = torch.randn((1, 8), dtype=torch.float32) + core_instruction = INSTRUCTIONS[11] + + with torch.inference_mode(): + sdpa_action = sdpa_model([core_instruction], {"dinov3": pixels}, state) + full_action = model([core_instruction], {"dinov3": pixels}, state) + vision_view0 = dino(pixels[:, 0]) + vision_view1 = dino(pixels[:, 1]) + core_text = prepare_static_text_inputs(model, core_instruction, torch.device("cpu")) + core_bert_hidden = bert( + core_text["input_ids"], + core_text["token_type_ids"], + core_text["bert_attention_mask"], + core_text["position_ids"], + ) + core_bert_padded = pad_bert_hidden(core_bert_hidden) + split_action = core( + vision_view0, + vision_view1, + core_bert_padded, + core_text["text_key_padding_mask"], + core_text["text_self_attention_mask"], + state, + ) + + if not torch.equal(full_action, split_action): + max_abs = float((full_action - split_action).abs().max()) + raise RuntimeError(f"split reference changed full action, max_abs={max_abs}") + eager_vs_sdpa = error_metrics(full_action, sdpa_action) + if eager_vs_sdpa["relative_l2"] > 1e-5: + raise RuntimeError(f"eager lowering changed reference action too much: {eager_vs_sdpa}") + + dino_path = onnx_dir / "dinov3_one_view_fp32.onnx" + export_model(dino, (pixels[:, 0],), dino_path, ["pixel_values"], "patch_tokens", args.opset) + + bert_paths: dict[int, Path] = {} + bert_reference: dict[int, dict[str, torch.Tensor]] = {} + for length, instruction in INSTRUCTIONS.items(): + text = prepare_static_text_inputs(model, instruction, torch.device("cpu")) + bert_inputs = ( + text["input_ids"], + text["token_type_ids"], + text["bert_attention_mask"], + text["position_ids"], + ) + with torch.inference_mode(): + hidden = bert(*bert_inputs) + path = onnx_dir / f"bert_l{length}_fp32.onnx" + export_model( + bert, + bert_inputs, + path, + ["input_ids", "token_type_ids", "text_self_attention_mask", "position_ids"], + "bert_hidden", + args.opset, + ) + bert_paths[length] = path + bert_reference[length] = {**text, "bert_hidden": hidden} + + core_path = onnx_dir / "policy_core_l21_fp32.onnx" + core_inputs = ( + vision_view0, + vision_view1, + core_bert_padded, + core_text["text_key_padding_mask"], + core_text["text_self_attention_mask"], + state, + ) + core_input_names = [ + "vision_view0", + "vision_view1", + "bert_hidden_padded", + "text_key_padding_mask", + "text_self_attention_mask", + "state", + ] + export_model(core, core_inputs, core_path, core_input_names, "normalized_action", args.opset) + + precision: dict[str, Any] = {} + dino_input = {"pixel_values": numpy_value(pixels[:, 0])} + precision["dinov3_one_view"] = { + "relative_l2": relative_l2(run_ort(dino_path, dino_input), numpy_value(vision_view0)), + "shape": list(vision_view0.shape), + } + for length, text in bert_reference.items(): + feed = { + "input_ids": numpy_value(text["input_ids"]), + "token_type_ids": numpy_value(text["token_type_ids"]), + "text_self_attention_mask": numpy_value(text["bert_attention_mask"]), + "position_ids": numpy_value(text["position_ids"]), + } + precision[f"bert_l{length}"] = { + "relative_l2": relative_l2(run_ort(bert_paths[length], feed), numpy_value(text["bert_hidden"])), + "shape": list(text["bert_hidden"].shape), + } + core_feed = dict(zip(core_input_names, (numpy_value(value) for value in core_inputs))) + ort_action = run_ort(core_path, core_feed) + precision["policy_core_l21"] = { + "relative_l2": relative_l2(ort_action, numpy_value(full_action)), + "shape": list(full_action.shape), + } + + np.savez( + output_dir / "reference_bundle_l11.npz", + pixels=numpy_value(pixels), + state=numpy_value(state), + full_action=numpy_value(full_action), + vision_view0=numpy_value(vision_view0), + vision_view1=numpy_value(vision_view1), + input_ids=numpy_value(core_text["input_ids"]), + token_type_ids=numpy_value(core_text["token_type_ids"]), + bert_attention_mask=numpy_value(core_text["bert_attention_mask"]), + position_ids=numpy_value(core_text["position_ids"]), + bert_hidden=numpy_value(core_bert_hidden), + bert_hidden_padded=numpy_value(core_bert_padded), + text_key_padding_mask=numpy_value(core_text["text_key_padding_mask"]), + text_self_attention_mask=numpy_value(core_text["text_self_attention_mask"]), + ) + + report = { + "source": { + "checkpoint": str(model.config.name), + "transformers_version": __import__("transformers").__version__, + "torch_version": torch.__version__, + "python": platform.python_version(), + "dino_feature": "outputs.hidden_states[-1]", + "static_attention_lowering": "eager", + "mha_fastpath_enabled": False, + }, + "input_contract": { + "pixels": [1, 2, 3, 256, 256], + "state": [1, 8], + "action": [1, 12, 7], + "bert_lengths": sorted(INSTRUCTIONS), + }, + "instruction_by_length": {str(key): value for key, value in INSTRUCTIONS.items()}, + "onnx": { + "dinov3": str(dino_path), + "bert": {str(key): str(value) for key, value in bert_paths.items()}, + "policy_core": str(core_path), + }, + "precision": precision, + "eager_vs_sdpa_action": eager_vs_sdpa, + "reference_bundle": str(output_dir / "reference_bundle_l11.npz"), + } + (output_dir / "export_report.json").write_text(json.dumps(report, indent=2), encoding="utf-8") + print(json.dumps(report, indent=2)) + + +if __name__ == "__main__": + os.environ.setdefault("TRANSFORMERS_NO_ADVISORY_WARNINGS", "1") + main() diff --git a/deployment/qcs8550/tools/extract_checkpoint_contract.py b/deployment/qcs8550/tools/extract_checkpoint_contract.py new file mode 100644 index 0000000..19e04cc --- /dev/null +++ b/deployment/qcs8550/tools/extract_checkpoint_contract.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Extract the small runtime contract needed by the TurboVLA QNN host adapter.""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +import torch + + +ROOT = Path(__file__).resolve().parents[1] +CHECKPOINT_PATH = ROOT.parents[1] / "pretrained/TurboVLA/checkpoints/libero/object.pth" + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint", type=Path, default=CHECKPOINT_PATH) + parser.add_argument("--output", type=Path, default=ROOT / "artifacts/checkpoint_contract.json") + args = parser.parse_args() + + payload = torch.load(args.checkpoint, map_location="cpu", weights_only=False) + config = payload.get("model_config") + if not isinstance(config, dict): + raise RuntimeError("checkpoint has no model_config") + text = config.get("text") + if not isinstance(text, dict): + raise RuntimeError("checkpoint has no text configuration") + layout = text.get("padding_length_by_instruction", {}) + if not isinstance(layout, dict): + raise RuntimeError("padding_length_by_instruction is not a mapping") + contract = { + "checkpoint": str(args.checkpoint.resolve()), + "text_padding_length": int(text["padding_length"]), + "instruction_lengths": {str(key): int(value) for key, value in sorted(layout.items())}, + "special_token_strings": ["[CLS]", "[SEP]", ".", "?"], + "image": { + "height": 256, + "width": 256, + "mean": [0.485, 0.456, 0.406], + "std": [0.229, 0.224, 0.225], + "libero_rotation": "flip height and width", + }, + "state": { + "mean": [ + -0.04190646484494209, + 0.03539437800645828, + 0.8257066607475281, + 2.908315658569336, + -0.5562158823013306, + -0.16649103164672852, + 0.02831534668803215, + -0.028561558574438095, + ], + "std": [ + 0.10743443667888641, + 0.14424759149551392, + 0.25723373889923096, + 0.34413808584213257, + 1.234430193901062, + 0.35798805952072144, + 0.013308786787092686, + 0.013174591585993767, + ], + }, + } + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(contract, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps({"output": str(args.output), "instructions": len(layout)}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/deployment/qcs8550/tools/native_client.py b/deployment/qcs8550/tools/native_client.py new file mode 100644 index 0000000..4aee1fd --- /dev/null +++ b/deployment/qcs8550/tools/native_client.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Send a frozen TurboVLA request to the persistent QCS8550 QNN service.""" + +from __future__ import annotations + +import argparse +import json +import socket +import struct +from pathlib import Path + +import numpy as np + + +ROOT = Path(__file__).resolve().parents[1] + + +def recv_all(connection: socket.socket, size: int) -> bytes: + result = bytearray() + while len(result) < size: + chunk = connection.recv(size - len(result)) + if not chunk: + raise RuntimeError("native service closed the connection") + result.extend(chunk) + return bytes(result) + + +def request_arrays(bundle: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + return { + "pixels_view0": np.ascontiguousarray(bundle["pixels"][:, 0], dtype=np.float32), + "pixels_view1": np.ascontiguousarray(bundle["pixels"][:, 1], dtype=np.float32), + "input_ids": np.ascontiguousarray(bundle["input_ids"], dtype=np.int32), + "token_type_ids": np.ascontiguousarray(bundle["token_type_ids"], dtype=np.int32), + "bert_attention_mask": np.ascontiguousarray(bundle["bert_attention_mask"], dtype=np.bool_), + "position_ids": np.ascontiguousarray(bundle["position_ids"], dtype=np.int32), + "text_key_padding_mask": np.ascontiguousarray(bundle["text_key_padding_mask"], dtype=np.bool_), + "text_self_attention_mask": np.ascontiguousarray(bundle["text_self_attention_mask"], dtype=np.bool_), + "state": np.ascontiguousarray(bundle["state"], dtype=np.float32), + } + + +def run_request(host: str, port: int, arrays: dict[str, np.ndarray], timeout: float) -> tuple[np.ndarray, dict[str, object]]: + rows = [] + for name, array in arrays.items(): + encoded = name.encode("ascii") + value = np.ascontiguousarray(array) + rows.append(struct.pack(">H", len(encoded)) + encoded + struct.pack(">Q", value.nbytes) + value.tobytes()) + payload = struct.pack(">I", len(rows)) + b"".join(rows) + with socket.create_connection((host, port), timeout=timeout) as connection: + connection.sendall(struct.pack(">I", len(payload)) + payload) + response = recv_all(connection, struct.unpack(">I", recv_all(connection, 4))[0]) + if len(response) < 12: + raise RuntimeError("native service response is truncated") + status, action_size = struct.unpack(">II", response[:8]) + offset = 8 + if offset + action_size + 4 > len(response): + raise RuntimeError("native service action payload is malformed") + action = np.frombuffer(response[offset : offset + action_size], dtype=np.float32).copy() + offset += action_size + metrics_size = struct.unpack(">I", response[offset : offset + 4])[0] + metrics_raw = response[offset + 4 : offset + 4 + metrics_size] + try: + metrics: dict[str, object] = json.loads(metrics_raw) + except json.JSONDecodeError: + metrics = {"raw": metrics_raw.decode("utf-8", errors="replace")} + if status != 0: + raise RuntimeError(f"native service rejected request: {metrics}") + if action.size != 12 * 7: + raise RuntimeError(f"native service returned {action.size} floats, expected 84") + return action.reshape(1, 12, 7), metrics + + +def metric(actual: np.ndarray, expected: np.ndarray) -> dict[str, float]: + diff = np.asarray(actual, dtype=np.float64) - np.asarray(expected, dtype=np.float64) + return { + "relative_l2": float(np.linalg.norm(diff) / max(np.linalg.norm(expected), 1e-12)), + "max_abs": float(np.abs(diff).max()), + "rmse": float(np.sqrt(np.mean(np.square(diff)))), + } + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--bundle", type=Path, required=True) + parser.add_argument("--host", required=True) + parser.add_argument("--port", type=int, default=10092) + parser.add_argument("--timeout", type=float, default=120.0) + parser.add_argument("--requests", type=int, default=1) + parser.add_argument("--expected", type=Path) + parser.add_argument("--expected-key", default="board_action") + parser.add_argument("--output", type=Path, default=ROOT / "artifacts/native_client_output.npz") + args = parser.parse_args() + + with np.load(args.bundle, allow_pickle=False) as archive: + bundle = {name: archive[name] for name in archive.files} + if args.requests < 1: + raise ValueError("--requests must be positive") + actions = [] + service_metrics = [] + arrays = request_arrays(bundle) + for _ in range(args.requests): + action, metrics = run_request(args.host, args.port, arrays, args.timeout) + actions.append(action) + service_metrics.append(metrics) + action = actions[-1] + request_us = np.asarray([row.get("request_us", np.nan) for row in service_metrics], dtype=np.float64) + stability = np.asarray(actions, dtype=np.float32) + report: dict[str, object] = { + "service_last": service_metrics[-1], + "action_shape": list(action.shape), + "requests": args.requests, + "action_max_abs_across_requests": float(np.max(np.abs(stability - stability[0]))), + "service_request_us": { + "p50": float(np.nanpercentile(request_us, 50)), + "p95": float(np.nanpercentile(request_us, 95)), + "min": float(np.nanmin(request_us)), + "max": float(np.nanmax(request_us)), + }, + } + if args.expected: + with np.load(args.expected, allow_pickle=False) as archive: + report["vs_expected"] = metric(action, archive[args.expected_key]) + if "full_action" in bundle: + report["vs_reference"] = metric(action, bundle["full_action"]) + args.output.parent.mkdir(parents=True, exist_ok=True) + np.savez(args.output, action=action) + args.output.with_suffix(".json").write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps(report, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/deployment/qcs8550/tools/qcs8550_reference.py b/deployment/qcs8550/tools/qcs8550_reference.py new file mode 100644 index 0000000..9cd7aa9 --- /dev/null +++ b/deployment/qcs8550/tools/qcs8550_reference.py @@ -0,0 +1,220 @@ +"""Exact TurboVLA LIBERO reference under the upstream-supported runtime.""" + +from __future__ import annotations + +import sys +from contextlib import contextmanager +from pathlib import Path +from typing import Iterator + +import torch +from torch import nn +from transformers import DINOv3ViTConfig, DINOv3ViTModel + + +PORT_ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = PORT_ROOT.parents[1] +CHECKPOINT_PATH = SOURCE_ROOT / "pretrained" / "TurboVLA" / "checkpoints" / "libero" / "object.pth" +BERT_PATH = SOURCE_ROOT / "pretrained" / "bert-base-uncased" + +TEXT_OUTPUT_LENGTH = 21 +IMAGE_SIZE = 256 +PATCH_SIZE = 16 + + +def dinov3_vitb16_config() -> DINOv3ViTConfig: + """Return the fixed ViT-B/16 architecture used by the released checkpoint.""" + return DINOv3ViTConfig( + image_size=IMAGE_SIZE, + patch_size=PATCH_SIZE, + hidden_size=768, + intermediate_size=3072, + num_hidden_layers=12, + num_attention_heads=12, + num_register_tokens=4, + hidden_act="gelu", + attention_dropout=0.0, + layer_norm_eps=1e-5, + rope_theta=100.0, + query_bias=True, + key_bias=False, + value_bias=True, + proj_bias=True, + mlp_bias=True, + use_gated_mlp=False, + ) + + +def _source_modules(): + if str(SOURCE_ROOT) not in sys.path: + sys.path.insert(0, str(SOURCE_ROOT)) + from turbovla.models.configuration import TurboVLAConfig + from turbovla.models.turbovla import TurboVLA, build_turbovla + import turbovla.models.vision_encoder as vision_encoder + + return TurboVLAConfig, TurboVLA, build_turbovla, vision_encoder + + +@contextmanager +def _instantiate_dinov3_from_config(vision_encoder) -> Iterator[None]: + """Avoid an HF gated download: release checkpoint strictly supplies every tensor.""" + original_loader = vision_encoder._load_pretrained_model + + def build_backbone(_config): + return DINOv3ViTModel(dinov3_vitb16_config()) + + vision_encoder._load_pretrained_model = build_backbone + try: + yield + finally: + vision_encoder._load_pretrained_model = original_loader + + +def load_object_reference( + device: str | torch.device = "cpu", + checkpoint_path: Path = CHECKPOINT_PATH, + bert_path: Path = BERT_PATH, + dino_attention_implementation: str | None = None, + bert_attention_implementation: str | None = None, +) -> nn.Module: + """Strict-load the released LIBERO Object policy without version drift. + + TurboVLA's checkpoint includes all DINOv3 and BERT weights. The local + BERT directory supplies tokenizer/config construction; DINOv3 is created + from its fixed architecture and immediately fully overwritten by the + release state dict. This retains the upstream `hidden_states[-1]` path. + """ + checkpoint_path = Path(checkpoint_path) + bert_path = Path(bert_path) + if not checkpoint_path.is_file(): + raise FileNotFoundError(checkpoint_path) + if not bert_path.is_dir(): + raise FileNotFoundError(bert_path) + + payload = torch.load(checkpoint_path, map_location="cpu", weights_only=False) + config_payload = payload.get("model_config") + state_dict = payload.get("model_state_dict") + if not isinstance(config_payload, dict) or not isinstance(state_dict, dict): + raise RuntimeError("expected a released TurboVLA checkpoint with model_config and model_state_dict") + + TurboVLAConfig, _TurboVLA, build_turbovla, vision_encoder = _source_modules() + config = TurboVLAConfig.from_mapping(config_payload) + config.text.model_name_or_path = str(bert_path) + config.text.local_files_only = True + config.text.attention_implementation = bert_attention_implementation + config.vision.model_name_or_path = "local-dinov3-vitb16-from-release-state" + config.vision.local_files_only = True + config.vision.compute_precision = "fp32" + + with _instantiate_dinov3_from_config(vision_encoder): + model = build_turbovla(config) + model.load_state_dict(state_dict, strict=True) + if dino_attention_implementation is not None: + model.vision_encoder.backbone.config._attn_implementation = dino_attention_implementation + model.to(device=device, dtype=torch.float32) + model.eval() + model.requires_grad_(False) + return model + + +class OneViewDINOv3(nn.Module): + """One original TurboVLA DINOv3 view, retaining issue #4's feature choice.""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.backbone = model.vision_encoder.backbone + self.prefix_tokens = int(model.vision_encoder.prefix_tokens) + + def forward(self, pixel_values: torch.Tensor) -> torch.Tensor: + outputs = self.backbone(pixel_values=pixel_values, output_hidden_states=True) + # Issue #4: do not replace this with last_hidden_state. + patch_tokens = outputs.hidden_states[-1][:, self.prefix_tokens :, :] + return patch_tokens.reshape(1, 256, 768) + + +class StaticBert(nn.Module): + """BERT invocation with host-generated TurboVLA special-token masks.""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.bert = model.text_encoder.bert + + def forward( + self, + input_ids: torch.Tensor, + token_type_ids: torch.Tensor, + text_self_attention_mask: torch.Tensor, + position_ids: torch.Tensor, + ) -> torch.Tensor: + return self.bert( + input_ids=input_ids, + token_type_ids=token_type_ids, + attention_mask=text_self_attention_mask, + position_ids=position_ids, + ).last_hidden_state + + +class StaticPolicyCore(nn.Module): + """TurboVLA after DINO/BERT, with the original fixed 21-token policy layout.""" + + def __init__(self, model: nn.Module) -> None: + super().__init__() + self.vision_projection = model.vision_projection + self.view_embedding = model.view_embedding + self.vision_language_interaction = model.vision_language_interaction + self.text_projection = model.text_encoder.text_projection + self.action_head = model.action_head + + def forward( + self, + vision_view0: torch.Tensor, + vision_view1: torch.Tensor, + bert_hidden_padded: torch.Tensor, + text_key_padding_mask: torch.Tensor, + text_self_attention_mask: torch.Tensor, + state: torch.Tensor, + ) -> torch.Tensor: + vision = torch.stack((vision_view0, vision_view1), dim=1) + visual_tokens = self.vision_projection(vision) + visual_tokens = visual_tokens + self.view_embedding[:, :, None, :].to( + device=visual_tokens.device, dtype=visual_tokens.dtype + ) + visual_tokens = visual_tokens.flatten(1, 2) + text_tokens = self.text_projection(bert_hidden_padded) + visual_tokens, text_tokens = self.vision_language_interaction( + visual_tokens=visual_tokens, + text_tokens=text_tokens, + text_key_padding_mask=text_key_padding_mask, + text_self_attention_masks=text_self_attention_mask, + ) + return self.action_head(torch.cat((visual_tokens, text_tokens), dim=1), state) + + +def prepare_static_text_inputs(model: nn.Module, instruction: str, device: torch.device) -> dict[str, torch.Tensor]: + """Reproduce TurboVLA's per-instruction BERT length and 21-token padding.""" + text_encoder = model.text_encoder + configured_length = int(text_encoder.config.padding_length or TEXT_OUTPUT_LENGTH) + group_length = int(text_encoder.config.padding_length_by_instruction.get(instruction, configured_length)) + tokenized, group_self_mask, position_ids = text_encoder._tokenize_group([instruction], device, group_length) + + text_key_padding_mask = torch.ones((1, configured_length), dtype=torch.bool, device=device) + text_key_padding_mask[:, :group_length] = ~tokenized.attention_mask.bool() + text_self_attention_mask = torch.eye(configured_length, dtype=torch.bool, device=device).unsqueeze(0) + text_self_attention_mask[:, :group_length, :group_length] = group_self_mask + return { + "input_ids": tokenized.input_ids, + "token_type_ids": tokenized.token_type_ids, + "bert_attention_mask": group_self_mask, + "position_ids": position_ids, + "text_key_padding_mask": text_key_padding_mask, + "text_self_attention_mask": text_self_attention_mask, + "group_length": torch.tensor(group_length, device=device), + } + + +def pad_bert_hidden(bert_hidden: torch.Tensor, output_length: int = TEXT_OUTPUT_LENGTH) -> torch.Tensor: + if bert_hidden.ndim != 3 or bert_hidden.shape[0] != 1 or bert_hidden.shape[1] > output_length: + raise ValueError(f"expected BERT hidden [1,L,768] where L<={output_length}, got {tuple(bert_hidden.shape)}") + padded = bert_hidden.new_zeros((1, output_length, bert_hidden.shape[-1])) + padded[:, : bert_hidden.shape[1]] = bert_hidden + return padded diff --git a/deployment/qcs8550/tools/run_libero_rollout.py b/deployment/qcs8550/tools/run_libero_rollout.py new file mode 100644 index 0000000..b674b3b --- /dev/null +++ b/deployment/qcs8550/tools/run_libero_rollout.py @@ -0,0 +1,30 @@ +#!/usr/bin/env python3 +"""Run the upstream LIBERO rollout protocol with QCS8550 TurboVLA execution.""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_ROOT = ROOT.parents[1] +for path in (ROOT, SOURCE_ROOT, SOURCE_ROOT / "third_party/vla_adapter"): + if str(path) not in sys.path: + sys.path.insert(0, str(path)) + +from qnn_policy import TurboVLAQnnPolicy, rotate_libero_image +from vla_adapter import rollout + + +def qnn_policy_import(): + from turbovla.evaluation.policy import get_libero_dummy_action, set_seed_everywhere + + return TurboVLAQnnPolicy, get_libero_dummy_action, rotate_libero_image, set_seed_everywhere + + +if __name__ == "__main__": + os.environ.setdefault("TOKENIZERS_PARALLELISM", "false") + rollout._import_turbovla_adapter = qnn_policy_import + rollout.main() diff --git a/deployment/qcs8550/tools/submit_qai_hub_compile.py b/deployment/qcs8550/tools/submit_qai_hub_compile.py new file mode 100644 index 0000000..7581efe --- /dev/null +++ b/deployment/qcs8550/tools/submit_qai_hub_compile.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +"""Submit the static TurboVLA ONNX graph set to QCS8550 QAIRT 2.48.""" + +from __future__ import annotations + +import argparse +import json +from datetime import datetime, timezone +from pathlib import Path + +import qai_hub as hub + + +PORT_ROOT = Path(__file__).resolve().parents[1] +DEFAULT_ONNX_DIR = PORT_ROOT / "artifacts" / "object" / "onnx" +DEFAULT_JOBS = PORT_ROOT / "artifacts" / "object" / "qcs8550_compile_jobs.json" + + +def job_id(job) -> str: + value = getattr(job, "job_id", None) + if value: + return str(value) + return str(job.url).rstrip("/").split("/")[-1] + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--onnx-dir", type=Path, default=DEFAULT_ONNX_DIR) + parser.add_argument("--jobs", type=Path, default=DEFAULT_JOBS) + parser.add_argument("--device", default="QCS8550 (Proxy)") + parser.add_argument("--qairt-version", default="2.48") + parser.add_argument("--only", nargs="*", choices=["dinov3", "bert_l11", "bert_l14", "bert_l21", "policy_core"]) + args = parser.parse_args() + + graphs = { + "dinov3": "dinov3_one_view_fp32.onnx", + "bert_l11": "bert_l11_fp32.onnx", + "bert_l14": "bert_l14_fp32.onnx", + "bert_l21": "bert_l21_fp32.onnx", + "policy_core": "policy_core_l21_fp32.onnx", + } + selected = args.only or list(graphs) + args.jobs.parent.mkdir(parents=True, exist_ok=True) + saved = json.loads(args.jobs.read_text(encoding="utf-8")) if args.jobs.is_file() else {} + saved.setdefault("target", {"device": args.device, "qairt_version": args.qairt_version}) + saved.setdefault("graphs", {}) + + client = hub.Client() + device = hub.Device(args.device) + options = f"--target_runtime qnn_context_binary --qairt_version {args.qairt_version} --truncate_64bit_io" + for name in selected: + model = (args.onnx_dir / graphs[name]).resolve() + if not model.is_file(): + raise FileNotFoundError(model) + prior = saved["graphs"].get(name) + if prior and prior.get("model_sha256") == _sha256(model): + print(f"reuse {name}: {prior['compile_job_id']}") + continue + job = client.submit_compile_job( + model=str(model), + device=device, + name=f"turbovla_object_{name}_qcs8550_qairt{args.qairt_version.replace('.', '')}", + options=options, + ) + saved["graphs"][name] = { + "compile_job_id": job_id(job), + "url": job.url, + "model": str(model), + "model_sha256": _sha256(model), + "options": options, + "submitted_at": datetime.now(timezone.utc).isoformat(), + } + args.jobs.write_text(json.dumps(saved, indent=2), encoding="utf-8") + print(f"submitted {name}: {job.url}") + args.jobs.write_text(json.dumps(saved, indent=2), encoding="utf-8") + + +def _sha256(path: Path) -> str: + import hashlib + + digest = hashlib.sha256() + with path.open("rb") as handle: + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + return digest.hexdigest() + + +if __name__ == "__main__": + main()