diff --git a/.executorch-version b/.executorch-version new file mode 100644 index 0000000..8b20e48 --- /dev/null +++ b/.executorch-version @@ -0,0 +1 @@ +v0.7.0 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..64c13ed --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,128 @@ +name: CI + +on: + push: + branches: [master] + pull_request: + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # --------------------------------------------------------------------------- + # Cheap checks that don't need a native build, so obvious mistakes fail fast. + # --------------------------------------------------------------------------- + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: false + + - name: Check Ruby syntax + run: | + find lib test bench -name '*.rb' -print0 \ + | xargs -0 -n1 ruby -c > /dev/null + ruby -c Rakefile > /dev/null + ruby -c ext/executorch/extconf.rb > /dev/null + + - name: Check shell syntax + run: bash -n script/build-executorch.sh + + - name: Build the gem package + run: gem build executorch.gemspec + + - name: Version matches CHANGELOG + run: | + version=$(ruby -r./lib/executorch/version -e 'print Executorch::VERSION') + grep -q "## \[$version\]" CHANGELOG.md \ + || { echo "CHANGELOG.md has no entry for $version"; exit 1; } + + # --------------------------------------------------------------------------- + # ExecuTorch takes a long time to build, so build it once per OS and cache it. + # The test matrix restores that cache instead of rebuilding per Ruby version. + # --------------------------------------------------------------------------- + executorch: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + # Same expression as the test job below -- they must agree or the test + # job restores nothing. + - name: Restore ExecuTorch + id: cache + uses: actions/cache@v4 + with: + path: vendor/executorch + key: executorch-${{ matrix.os }}-${{ hashFiles('.executorch-version', 'script/build-executorch.sh') }} + + - name: Install build tools + if: steps.cache.outputs.cache-hit != 'true' + run: | + python3 -m pip install --upgrade "cmake>=3.29" zstd + if [ "$RUNNER_OS" = "Linux" ]; then + sudo apt-get update && sudo apt-get install -y ninja-build + else + brew install ninja + fi + + - name: Build ExecuTorch + if: steps.cache.outputs.cache-hit != 'true' + run: script/build-executorch.sh + env: + # Portable kernels only: correct, and enough to test the bindings. + # The XNNPACK delegate is what you want in production (see + # bench/FINDINGS.md) but roughly triples the build time. + EXECUTORCH_BACKENDS: none + + test: + needs: executorch + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest] + ruby: ["3.1", "3.2", "3.3", "3.4"] + include: + - os: macos-latest + ruby: "3.3" + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + + - name: Restore ExecuTorch + uses: actions/cache/restore@v4 + with: + path: vendor/executorch + key: executorch-${{ matrix.os }}-${{ hashFiles('.executorch-version', 'script/build-executorch.sh') }} + fail-on-cache-miss: true + + # A partial restore would be worse than none: an install built by an older + # script can be missing headers the extension needs. + - name: Verify the install is usable + run: | + test -f vendor/executorch/include/executorch/extension/module/module.h \ + || { echo "ExecuTorch install is incomplete"; exit 1; } + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: ${{ matrix.ruby }} + bundler-cache: true + + - name: Compile + run: bundle exec rake compile + env: + EXECUTORCH_DIR: ${{ github.workspace }}/vendor/executorch + + - name: Test + run: bundle exec rake test + env: + EXECUTORCH_DIR: ${{ github.workspace }}/vendor/executorch diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..28ac387 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,94 @@ +name: Release + +# Publishes to RubyGems when a v* tag is pushed: +# +# 1. bump Executorch::VERSION and add a CHANGELOG entry +# 2. git tag v0.2.0 && git push origin v0.2.0 +# +# Publishing uses RubyGems trusted publishing (OIDC), so there is no API key in +# repository secrets. Set it up once at +# https://rubygems.org/gems/executorch/trusted_publishers with: +# +# repository: benngarcia/executorch-ruby +# workflow: release.yml +# environment: release +# +# This ships a source gem -- the C++ extension is compiled on the user's +# machine at install time against their own ExecuTorch build, so no +# cross-compilation matrix is needed here. + +on: + push: + tags: ["v*"] + workflow_dispatch: + inputs: + dry_run: + description: "Build and verify without publishing" + type: boolean + default: true + +jobs: + release: + runs-on: ubuntu-latest + environment: release + + permissions: + contents: write # create the GitHub release + id-token: write # OIDC token for trusted publishing + + steps: + - uses: actions/checkout@v4 + + - uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.3" + bundler-cache: false + + - name: Check the tag matches Executorch::VERSION + if: startsWith(github.ref, 'refs/tags/') + run: | + version=$(ruby -r./lib/executorch/version -e 'print Executorch::VERSION') + tag="${GITHUB_REF#refs/tags/v}" + if [ "$version" != "$tag" ]; then + echo "Tag v$tag does not match Executorch::VERSION ($version)." >&2 + echo "Bump lib/executorch/version.rb, or retag." >&2 + exit 1 + fi + echo "version=$version" >> "$GITHUB_ENV" + + - name: Check the CHANGELOG has an entry + if: startsWith(github.ref, 'refs/tags/') + run: | + grep -q "## \[$version\]" CHANGELOG.md \ + || { echo "CHANGELOG.md has no entry for $version"; exit 1; } + + - name: Build the gem + run: gem build executorch.gemspec + + # Catch a packaging mistake before it reaches RubyGems: the extension + # sources must be in the gem or it cannot build on install. + - name: Verify the package contents + run: | + gem_file=$(ls executorch-*.gem) + for required in \ + ext/executorch/extconf.rb \ + ext/executorch/executorch.cpp \ + ext/executorch/utils.h \ + lib/executorch.rb \ + LICENSE.txt + do + tar -xOf "$gem_file" data.tar.gz | tar -tzf - | grep -qx "$required" \ + || { echo "Missing from gem package: $required"; exit 1; } + done + echo "Package contents OK" + + - name: Publish to RubyGems + if: startsWith(github.ref, 'refs/tags/') && !inputs.dry_run + uses: rubygems/release-gem@v1 + + - name: Create the GitHub release + if: startsWith(github.ref, 'refs/tags/') && !inputs.dry_run + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + files: executorch-*.gem diff --git a/.gitignore b/.gitignore index 9097d09..524d8ad 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,11 @@ test/support/models/*.pte # Coverage /coverage/ + +# Benchmark models (regenerate with bench/make_pt_models.py + bench/pt_to_pte.py). +# Results are kept in git -- they're the evidence behind bench/FINDINGS.md. +/bench/models/ + +# Python +__pycache__/ +*.pyc diff --git a/CHANGELOG.md b/CHANGELOG.md index 625ced1..c172af6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,41 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added + +- `Tensor.from_bytes(packed, shape:, dtype:)` and `Tensor#to_binary` - raw + binary tensor I/O that skips per-element conversion. Roughly 174x faster than + the Array constructor on a 150k-element input. +- `bench/` - eval and profiling harness. Converts `.pt` checkpoints to `.pte`, + checks output against PyTorch, and times each leg of an inference call. See + `bench/FINDINGS.md`. +- `script/build-executorch.sh` - reproducible ExecuTorch build used by both + humans and CI. +- CI (GitHub Actions) for lint and tests across Ruby 3.1-3.4 on Linux and + macOS, plus a tag-triggered RubyGems release workflow. + +### Changed + +- Large reduction in Ruby/C++ boundary cost. Tensor creation is ~13x faster, + `#to_a` up to ~65x faster, and building a tensor from a nested Array now + allocates 11 objects instead of 153,238 for a 150k-element input. +- `extconf.rb` now links optimized CPU kernels and the XNNPACK delegate when + they are present in the ExecuTorch install, whole-archiving them so their + self-registration is not dropped by the linker. + +### Fixed + +- The documented install path did not work: `cmake --install` does not install + `extension/module/module.h` or the `runtime/executor` headers, so + `rake compile` failed with "module.h header not found" on a correct build. + `script/build-executorch.sh` copies them. +- `extconf.rb` whole-archived two operator libraries when both were present, + which aborts the runtime at init on duplicate operator registration. +- Build failure against Rice 4.12, where `Rice::Array::Proxy` no longer + converts implicitly to `Rice::Object`. + ## [0.1.0] - 2024-12-27 ### Added diff --git a/README.md b/README.md index 5b9050d..c691b87 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # ExecuTorch Ruby +[![CI](https://github.com/benngarcia/executorch-ruby/actions/workflows/ci.yml/badge.svg)](https://github.com/benngarcia/executorch-ruby/actions/workflows/ci.yml) + Run PyTorch models in Ruby. [ExecuTorch](https://pytorch.org/executorch/) is Meta's lightweight runtime for deploying PyTorch models on edge devices. This gem provides Ruby bindings so you can run exported models (`.pte` files) directly in your Ruby applications. @@ -26,24 +28,28 @@ puts output.to_a # => [[3.0, 5.0, 7.0]] ### Step 1: Build ExecuTorch -ExecuTorch must be built from source. Follow the [official guide](https://pytorch.org/executorch/stable/getting-started-setup.html), or use these commands: +ExecuTorch must be built from source. This repo ships a script that does it: ```bash -git clone https://github.com/pytorch/executorch.git -cd executorch -./install_requirements.sh +script/build-executorch.sh +``` -cmake -B cmake-out \ - -DCMAKE_INSTALL_PREFIX=vendor/executorch \ - -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ - -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ - -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ - -DCMAKE_BUILD_TYPE=Release +It clones ExecuTorch at a pinned version, builds it, installs into +`vendor/executorch`, and copies the headers that `cmake --install` leaves +behind (`extension/module/module.h` and the `runtime/executor` tree are not +installed by ExecuTorch's own install step, and this gem includes them +directly). + +Add the XNNPACK delegate — strongly recommended, see [Performance](#performance) +— with: -cmake --build cmake-out -j4 -cmake --install cmake-out +```bash +EXECUTORCH_BACKENDS=xnnpack script/build-executorch.sh ``` +Needs CMake ≥ 3.29, Ninja, and a C++17 compiler. The first build takes a while; +the ExecuTorch source and submodules are several GB. + ### Step 2: Install the Gem Tell Bundler where ExecuTorch is installed (only needed once per project): @@ -94,6 +100,23 @@ tensor = Executorch::Tensor.new([1.0, 2.0, 3.0, 4.0], shape: [2, 2]) **Supported dtypes:** `:float` (default), `:double`, `:int`, `:long` +For large tensors, skip per-element conversion entirely and hand over packed +bytes — the runtime memcpys them straight into the tensor buffer: + +```ruby +# ~13x faster than the Array constructor on a 150k-element input +tensor = Executorch::Tensor.from_bytes(pixels.pack("f*"), shape: [1, 3, 224, 224]) + +# and back out +bytes = tensor.to_binary +values = bytes.unpack("f*") +``` + +This is the right path whenever your data is already bytes (an image decoded to +a string, a file, a socket) or when you can pack once and reuse. Data must be +native-endian and match the dtype's element width: `:float` → `"f*"`, +`:double` → `"d*"`, `:int` → `"l*"`, `:long` → `"q*"`. + ### Models ```ruby @@ -131,6 +154,43 @@ with open("model.pte", "wb") as f: et_program.write_to_file(f) ``` +## Performance + +The single biggest factor in inference speed is **which kernels your ExecuTorch +build links** — not the Ruby layer. + +A default build uses the portable kernels: reference implementations written for +correctness and portability, with no vectorization or threading. They work +everywhere and they are slow — resnet18 takes **8.3 s** per call. + +Build with the XNNPACK delegate instead: + +```bash +cmake -B cmake-out \ + -DEXECUTORCH_BUILD_XNNPACK=ON \ + ... # other flags as above +``` + +and export your model through the XNNPACK partitioner (see +`bench/pt_to_pte.py --xnnpack`). Both halves are required — the backend has to +be linked at build time *and* targeted at export time. + +That takes resnet18 from 8.3 s to **12.6 ms** — 658×, and about 2× faster than +PyTorch eager on the same machine. `extconf.rb` links the backend automatically +when it finds it in your ExecuTorch install. + +On the Ruby side, once the runtime is fast the boundary becomes the bottleneck: + +- Prefer `Tensor.from_bytes` over the Array constructor for large inputs. On + mobilenet_v2 that's 0.85 ms instead of 7.6 ms, against a 3.4 ms inference. +- Prefer `Tensor#flat_to_a` over `#to_a` when you don't need the nested shape. +- Prefer `Tensor.new(flat, shape: ...)` over a nested Array when you have the + choice — shape inference has to walk the nesting. + +See [`bench/`](bench/) for the eval + profiling harness, and +[`bench/FINDINGS.md`](bench/FINDINGS.md) for the full walkthrough — including +why fixing the kernels is what makes the binding work matter. + ## Troubleshooting
@@ -147,15 +207,20 @@ bundle config set --local build.executorch --with-executorch-dir=vendor/executor
"module.h header not found" -ExecuTorch was built without required extensions. Rebuild with: +Usually not a missing build flag — `cmake --install` doesn't install +`extension/module/module.h` at all, even on a correct build. If you built +ExecuTorch by hand, copy the remaining headers across: ```bash -cmake -B cmake-out \ - -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \ - -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \ - -DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \ - ... +cd /path/to/executorch +find runtime kernels extension -name '*.h' -not -path '*/test/*' \ + | while read -r f; do + mkdir -p "$PREFIX/include/executorch/$(dirname "$f")" + cp "$f" "$PREFIX/include/executorch/$f" + done ``` + +Or just use `script/build-executorch.sh`, which handles it.
diff --git a/Rakefile b/Rakefile index 7cd5c27..2041edf 100644 --- a/Rakefile +++ b/Rakefile @@ -17,30 +17,33 @@ task default: %i[compile test] namespace :executorch do desc 'Build ExecuTorch from source and install to vendor/executorch' task :build_deps do - source_dir = ENV.fetch('EXECUTORCH_SRC', File.expand_path('../executorch', __dir__)) - install_dir = File.expand_path('vendor/executorch', __dir__) - - unless File.directory?(source_dir) - abort "ExecuTorch source not found at #{source_dir}. Clone it or set EXECUTORCH_SRC." - end - - puts "Building ExecuTorch from #{source_dir}..." - puts "Installing to #{install_dir}..." - - Dir.chdir(source_dir) do - system('cmake', '-B', 'cmake-out', - "-DCMAKE_INSTALL_PREFIX=#{install_dir}", - '-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON', - '-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON', - '-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON', - '-DCMAKE_BUILD_TYPE=Release') || abort('CMake configure failed') - - system('cmake', '--build', 'cmake-out', "-j#{Etc.nprocessors}") || abort('CMake build failed') - system('cmake', '--install', 'cmake-out') || abort('CMake install failed') - end - - puts "ExecuTorch installed to #{install_dir}" - puts "Run: bundle config set --local build.executorch --with-executorch-dir=#{install_dir}" + # Delegates to the script so this task, the README, and CI all build + # ExecuTorch exactly the same way. Set EXECUTORCH_BACKENDS=xnnpack to + # include the delegate. + sh File.expand_path('script/build-executorch.sh', __dir__) + end +end + +namespace :bench do + models_dir = File.expand_path('bench/models', __dir__) + + desc 'Export the .pt benchmark checkpoints and convert them to .pte' + task :prepare do + sh 'python3', 'bench/make_pt_models.py', '--all' + sh "python3 bench/pt_to_pte.py #{models_dir}/*.pt" + end + + desc 'Run evals + profiling (LABEL=name to tag the results file)' + task run: :compile do + sh 'ruby', 'bench/run_bench.rb', '--label', ENV.fetch('LABEL', 'current') + end + + desc 'Compare two result files: rake bench:compare BASE=baseline CURRENT=mine' + task :compare do + base = ENV.fetch('BASE', 'baseline') + current = ENV.fetch('CURRENT', 'current') + sh 'ruby', 'bench/compare.rb', + "bench/results/#{base}.json", "bench/results/#{current}.json" end end diff --git a/bench/FINDINGS.md b/bench/FINDINGS.md new file mode 100644 index 0000000..d845552 --- /dev/null +++ b/bench/FINDINGS.md @@ -0,0 +1,315 @@ +# What the benchmarks found + +Measured on the machine that produced `bench/results/*.json` (4-core x86_64, +Ruby 3.3.6, Rice 4.12, ExecuTorch 0.7.0). Absolute numbers are +machine-specific; the ratios are the durable part. + +There are two separate performance stories here, and it's worth keeping them +apart: + +1. **The bindings** — the cost of getting data across the Ruby ↔ C++ boundary. + This is what the gem controls, and it was expensive. +2. **The kernels** — the cost of the math itself. The gem doesn't control this, + but it does control which kernels get linked, and the default is the slow + one. + +Short version: the kernels are worth **658×** and the bindings are worth up to +**20×**, and you need both — on a fast runtime, 96.6% of a mobilenet_v2 call +was binding overhead. Part 3 has the cross-product. + +--- + +## Part 1: the bindings + +### The finding + +Profiling a call by phase showed that on a small model, most of an "inference" +call wasn't inference: + +| model | e2e | of which `forward()` | binding overhead | +|---|---|---|---| +| `tiny_mlp` | 0.082 ms | 0.020 ms | **76%** | +| `mlp_512x2` | 4.22 ms | 3.24 ms | 23% | +| `resnet18` | 9313 ms | 8261 ms | 11% | + +Broken down per element, the boundary cost about **0.6 µs per element in** and +**1.1 µs per element out**. For a resnet18 input that's 92 ms to build a tensor +from an Array of 150,528 floats — before any math happens. + +For scale: 1.1 µs per element is roughly 3,000 CPU cycles to move one float +from a C array into a Ruby Array. + +### The cause + +Rice is a lovely API, but its default conversion path is built for safety, not +throughput. Every `Array` element access and every scalar conversion goes +through `Rice::detail::protect()`: + +```cpp +// rice.hpp +inline VALUE Array::Proxy::value() const { + return detail::protect(rb_ary_entry, array_.value(), index_); +} + +static T convert(VALUE value) { + return (T)protect(RubyType::fromRuby, value); // rb_num2dbl +} +``` + +and `protect()` is: + +```cpp +int state = (int)JumpException::RUBY_TAG_NONE; +rb_protect(trampoline, (VALUE)(&invoker), &state); +``` + +`rb_protect` pushes a VM tag and does a `setjmp` — it exists so a Ruby +exception can't longjmp past C++ destructors. Entirely correct for arbitrary +Ruby calls. But the original code read tensor data like this: + +```cpp +for (size_t i = 0; i < data.size(); i++) { + float_data.push_back(static_cast( + detail::From_Ruby().convert(data[i].value()))); +} +``` + +That's **two `rb_protect` calls per element** — one for `data[i].value()`, one +for the numeric conversion — to read a `Float` that's sitting right there in +the array. On the way out, `result.push(data[i])` cost another one each. + +The setjmp was the entire cost. The conversion itself is a pointer dereference. + +### The fixes + +**1. Inline the common conversions** (`ext/executorch/utils.h`). `Float` and +`Fixnum` cover essentially all real tensor data and can't raise, so they take a +direct path; anything else (Bignum, Rational, an object with `to_f`) still goes +through Rice's protected call, because those genuinely can run Ruby code: + +```cpp +inline double to_double_fast(VALUE v) { + if (RB_FLOAT_TYPE_P(v)) return RFLOAT_VALUE(v); + if (FIXNUM_P(v)) return static_cast(FIX2LONG(v)); + return Rice::detail::protect(rb_num2dbl, v); // rare, and really can raise +} +``` + +Building the output array uses `rb_ary_new_capa` + `rb_ary_push` — sized once, +and `rb_ary_push` on an array we just created can't raise, so no `protect` is +needed. Doubles mostly become flonums, so nothing is allocated per element. + +**2. Stop deep-copying every input.** `forward()` cloned each input tensor to +"own the data during forward". But the caller's Array holds a live reference to +every input for the whole call, so nothing can be collected underneath us — the +copy bought nothing and cost a full pass over the input. Outputs *are* still +cloned, and must be: they point into the method's planned memory arena, which +the next call overwrites. + +**3. Dispatch on type, not on exceptions.** Input type detection was: + +```cpp +try { RubyTensor& t = detail::From_Ruby().convert(...); } +catch (...) { try { /* RubyEValue */ } catch (...) { rb_raise(...); } } +``` + +Throwing a C++ exception to discover a type costs more than a small model's +entire inference. Replaced with `Data_Type::is_descendant(v)`. + +**4. Flatten nested input level-by-level in Ruby.** `Tensor.new([[1.0, 2.0]])` +went through a recursive `flat_map` that allocated an intermediate Array *per +leaf* — 153,238 objects for a resnet18 input, before a single number reached +C++. Walking one level at a time with `Array#concat` does the gathering in C. +Jagged-array detection is preserved (the level walk catches size mismatches +directly; a final flatten-size check catches uneven nesting depth). + +**5. A raw-binary escape hatch.** `Tensor.from_bytes` / `Tensor#to_binary` skip +per-element conversion entirely and memcpy the buffer. This is the right path +whenever the data is already bytes — an image decoded to a string, a file, a +socket — or when you can pack once and reuse. + +### The results + +`bundle exec ruby bench/compare.rb bench/results/baseline.json bench/results/optimized.json` + +| phase | model | before | after | speedup | +|---|---|---|---|---| +| `tensor_new_flat` | resnet18 | 92.20 ms | 7.00 ms | **13.2×** | +| `tensor_new_nested` | resnet18 | 135.22 ms | 13.27 ms | **10.2×** | +| `tensor_from_bytes` | resnet18 | — | 0.53 ms | **174×** vs baseline | +| `to_a_flat` | mobilenet_v2 | 1.105 ms | 0.017 ms | **64.6×** | +| `to_a_nested` | mlp_1024x4 | 1.174 ms | 0.038 ms | **31.3×** | +| `to_binary` | mobilenet_v2 | — | 0.004 ms | **291×** vs baseline | +| `forward` | add_mul | 0.019 ms | 0.014 ms | 1.33× | + +Allocations per call, resnet18 nested input: **153,238 → 11**. + +Binding overhead as a share of an end-to-end call: + +| model | before | after | +|---|---|---| +| `mnist_cnn` | 10.7% | 0.2% | +| `mlp_512x2` | 23.3% | 1.1% | +| `mlp_1024x4` | 17.3% | 2.5% | + +All 59 existing tests pass, and every model's output still matches PyTorch to +within 1e-4 (most to 1e-7). + +`tiny_mlp` still shows ~74% overhead, and that's honest: when `forward()` is +13 µs, the fixed per-call cost of crossing the boundary at all — allocating the +result Tensor object, wrapping outputs — is the floor. It's just a much lower +floor than before (0.082 ms → 0.050 ms e2e). + +### What's left on the table + +- **A zero-copy input view.** `from_bytes` still copies once into a tensor-owned + buffer. A tensor that borrows a frozen Ruby String's memory would remove even + that, at the cost of a lifetime rule users have to respect. +- **Reusing output tensors.** Every `forward()` allocates a fresh `Tensor` per + output and clones the data. An opt-in "write into this tensor I already have" + API would suit a hot inference loop. +- **`Tensor#to_a` reshaping** still happens in Ruby. It's now a small share of + the cost, but for a many-dimensional output it could move into C. +- **`shape()`** still uses `Rice::Array::push` (one `rb_protect` per dimension). + Irrelevant at 4 dimensions, noted for consistency. + +--- + +## Part 2: the kernels + +This one dwarfs everything above, and it isn't a binding problem at all. + +With the runtime built the way the README described, `forward()` on resnet18 +takes **8.26 seconds**. PyTorch eager, same checkpoint, same machine: **25 ms**. +That's ~330× slower, and no amount of binding work touches it. + +The reason is that a default ExecuTorch build links `portable_ops_lib` — the +reference kernel implementations. They're written for correctness and +portability, not speed: no vectorization, no blocking, no threading. They're the +right default for a runtime that has to build anywhere, and the wrong choice for +anything latency-sensitive. + +Two candidate fixes, and it's worth being precise about what each one bought, +because they were not equal: + +### Optimized CPU kernels — no measurable help + +`EXECUTORCH_BUILD_KERNELS_OPTIMIZED=ON` swaps in vectorized implementations and +works on existing `.pte` files with no re-export, which makes it sound like the +easy win. Measured (`results/optimized_kernels.json`): + +| model | portable | optimized kernels | +|---|---|---| +| resnet18 `forward` | 8262 ms | 8204 ms | +| mobilenet_v2 `forward` | 1618 ms | 1629 ms | +| mlp_512x2 `forward` | 2.95 ms | 3.26 ms | + +Noise. The optimized set covers elementwise and a few BLAS-backed ops; these +models spend all their time in convolution, which still falls back to the +portable implementation. Worth knowing before reaching for it as a fix. + +### The XNNPACK delegate — 658× + +`EXECUTORCH_BUILD_XNNPACK=ON` plus a re-export through `XnnpackPartitioner` +hands whole subgraphs to XNNPACK. This needs both a rebuild *and* a re-export — +the backend has to exist at build time and be targeted at export time, and +missing either half silently leaves you on portable kernels. + +| model | portable `forward` | XNNPACK `forward` | speedup | PyTorch eager | +|---|---|---|---|---| +| resnet18 | 8262 ms | **12.56 ms** | 658× | 25.3 ms | +| mobilenet_v2 | 1618 ms | **3.39 ms** | 477× | 16.7 ms | +| mlp_1024x4 | 48.6 ms | **0.27 ms** | 178× | 0.30 ms | +| mnist_cnn | 4.91 ms | **0.26 ms** | 18.6× | 0.29 ms | +| mlp_512x2 | 3.24 ms | **0.12 ms** | 26.6× | 0.055 ms | + +Delegated ExecuTorch is *faster than PyTorch eager* on both vision models — +2.0× on resnet18, 4.9× on mobilenet_v2 — which is the whole point of an +ahead-of-time-compiled edge runtime. + +`bench/pt_to_pte.py --xnnpack` produces the lowered variant; `run_bench.rb +--variant xnnpack` benchmarks it. + +### A linker trap worth knowing about + +Kernel libraries and backend delegates register themselves from global +constructors. A plain `-lxnnpack_backend` only pulls in object files that +resolve some undefined symbol — and a self-registering object resolves nothing, +so the linker discards it and the registration silently never happens. You find +out much later, when a model fails to load with a missing-operator or +missing-backend error. + +They have to be whole-archived: `-Wl,--whole-archive` on Linux, +`-Wl,-force_load` on macOS. `extconf.rb` already did this for +`portable_ops_lib`; it's now generalized to the optimized set and the XNNPACK +backend. + +One constraint that follows: **exactly one operator library may be +whole-archived.** Each registers the full op set into the same global table, so +linking two aborts the runtime at init on duplicate registration. `extconf.rb` +picks `optimized_native_cpu_ops_lib` when present and `portable_ops_lib` +otherwise; `EXECUTORCH_OPS_LIB` overrides. + +--- + +## Part 3: why the two halves need each other + +Neither piece of work looks impressive alone. Together they're the whole story. + +On portable kernels, the binding fixes barely move end-to-end time — `forward()` +is so slow that nothing else is visible. That's exactly why the original +bindings could carry a 0.6 µs/element conversion cost without anyone noticing. + +So the real test is the cross-product: **original bindings, XNNPACK kernels** +(`results/baseline_xnnpack.json`). Once the math is fast, the boundary is all +that's left: + +| model | e2e | `forward()` | overhead | +|---|---|---|---| +| mobilenet_v2 | 107.9 ms | 3.7 ms | **96.6%** | +| resnet18 | 116.4 ms | 11.7 ms | **90.0%** | +| mlp_1024x4 | 2.61 ms | 0.30 ms | 88.4% | +| mnist_cnn | 1.21 ms | 0.25 ms | 79.6% | + +96.6% of a mobilenet_v2 call spent not doing inference. Building the input +tensor alone (102.9 ms) cost **28× more than running the model** (3.7 ms). + +With the optimized bindings on the same XNNPACK runtime: + +| model | e2e before | e2e after | with `from_bytes` | best speedup | +|---|---|---|---|---| +| mobilenet_v2 | 107.9 ms | 12.97 ms | **5.26 ms** | **20.5×** | +| resnet18 | 116.4 ms | 19.67 ms | **13.69 ms** | **8.5×** | +| mlp_1024x4 | 2.61 ms | 0.47 ms | **0.38 ms** | **6.9×** | +| mnist_cnn | 1.21 ms | 0.47 ms | **0.29 ms** | **4.2×** | + +The takeaway: **fix the kernels first, because that's the 658×** — but the +moment you do, the bindings become the bottleneck, and on mobilenet_v2 they're +worth another 20×. + +It also changes which API matters. On a fast runtime, `Tensor.from_bytes` isn't +a micro-optimization: for mobilenet_v2 it's the difference between spending +7.6 ms or 0.85 ms getting the input across, against a 3.4 ms inference. + +--- + +## Notes for reproducing + +Four result files, the full cross-product: + +| file | bindings | kernels | +|---|---|---| +| `baseline.json` | original | portable | +| `optimized.json` | optimized | portable | +| `optimized_kernels.json` | optimized | optimized CPU kernels | +| `baseline_xnnpack.json` | original | XNNPACK | +| `xnnpack.json` | optimized | XNNPACK | + +```bash +bundle exec ruby bench/compare.rb bench/results/baseline_xnnpack.json bench/results/xnnpack.json +``` + +The eager PyTorch timings recorded in each `.meta.json` were measured on the +same machine but not under controlled conditions (some runs overlapped a +compile). Treat them as an anchor, not a benchmark; regenerate the models on an +idle machine if you want to lean on them. diff --git a/bench/README.md b/bench/README.md new file mode 100644 index 0000000..ec755fa --- /dev/null +++ b/bench/README.md @@ -0,0 +1,121 @@ +# executorch-ruby benchmarks + +An eval + profiling harness for the gem. It answers two questions: + +1. **Are we right?** Does Ruby produce the same numbers PyTorch does? +2. **Where does the time go?** How much of an inference call is actual + inference, and how much is the Ruby ↔ C++ boundary? + +## The pipeline + +``` +bench_models.py model definitions (shared by both scripts below) + │ + ▼ +make_pt_models.py → models/.pt eager module, torch.save + → models/.meta.json shapes, param count, eager latency + → models/.io.bin golden input + output, raw float32 + │ + ▼ +pt_to_pte.py → models/.pte torch.export → to_edge → to_executorch + │ + ▼ +run_bench.rb → results/