Add eval/benchmark suite, cut Ruby↔C++ overhead, fix the install path, add CI

.executorch-version

@@ -0,0 +1 @@
+v0.7.0

.github/workflows/ci.yml

@@ -0,0 +1,128 @@
+name: CI
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Cheap checks that don't need a native build, so obvious mistakes fail fast.
+  # ---------------------------------------------------------------------------
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.3"
+          bundler-cache: false
+
+      - name: Check Ruby syntax
+        run: |
+          find lib test bench -name '*.rb' -print0 \
+            | xargs -0 -n1 ruby -c > /dev/null
+          ruby -c Rakefile > /dev/null
+          ruby -c ext/executorch/extconf.rb > /dev/null
+
+      - name: Check shell syntax
+        run: bash -n script/build-executorch.sh
+
+      - name: Build the gem package
+        run: gem build executorch.gemspec
+
+      - name: Version matches CHANGELOG
+        run: |
+          version=$(ruby -r./lib/executorch/version -e 'print Executorch::VERSION')
+          grep -q "## \[$version\]" CHANGELOG.md \
+            || { echo "CHANGELOG.md has no entry for $version"; exit 1; }
+
+  # ---------------------------------------------------------------------------
+  # ExecuTorch takes a long time to build, so build it once per OS and cache it.
+  # The test matrix restores that cache instead of rebuilding per Ruby version.
+  # ---------------------------------------------------------------------------
+  executorch:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      # Same expression as the test job below -- they must agree or the test
+      # job restores nothing.
+      - name: Restore ExecuTorch
+        id: cache
+        uses: actions/cache@v4
+        with:
+          path: vendor/executorch
+          key: executorch-${{ matrix.os }}-${{ hashFiles('.executorch-version', 'script/build-executorch.sh') }}
+
+      - name: Install build tools
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: |
+          python3 -m pip install --upgrade "cmake>=3.29" zstd
+          if [ "$RUNNER_OS" = "Linux" ]; then
+            sudo apt-get update && sudo apt-get install -y ninja-build
+          else
+            brew install ninja
+          fi
+
+      - name: Build ExecuTorch
+        if: steps.cache.outputs.cache-hit != 'true'
+        run: script/build-executorch.sh
+        env:
+          # Portable kernels only: correct, and enough to test the bindings.
+          # The XNNPACK delegate is what you want in production (see
+          # bench/FINDINGS.md) but roughly triples the build time.
+          EXECUTORCH_BACKENDS: none
+
+  test:
+    needs: executorch
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        ruby: ["3.1", "3.2", "3.3", "3.4"]
+        include:
+          - os: macos-latest
+            ruby: "3.3"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Restore ExecuTorch
+        uses: actions/cache/restore@v4
+        with:
+          path: vendor/executorch
+          key: executorch-${{ matrix.os }}-${{ hashFiles('.executorch-version', 'script/build-executorch.sh') }}
+          fail-on-cache-miss: true
+
+      # A partial restore would be worse than none: an install built by an older
+      # script can be missing headers the extension needs.
+      - name: Verify the install is usable
+        run: |
+          test -f vendor/executorch/include/executorch/extension/module/module.h \
+            || { echo "ExecuTorch install is incomplete"; exit 1; }
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby }}
+          bundler-cache: true
+
+      - name: Compile
+        run: bundle exec rake compile
+        env:
+          EXECUTORCH_DIR: ${{ github.workspace }}/vendor/executorch
+
+      - name: Test
+        run: bundle exec rake test
+        env:
+          EXECUTORCH_DIR: ${{ github.workspace }}/vendor/executorch

.github/workflows/release.yml

@@ -0,0 +1,94 @@
+name: Release
+
+# Publishes to RubyGems when a v* tag is pushed:
+#
+#   1. bump Executorch::VERSION and add a CHANGELOG entry
+#   2. git tag v0.2.0 && git push origin v0.2.0
+#
+# Publishing uses RubyGems trusted publishing (OIDC), so there is no API key in
+# repository secrets. Set it up once at
+# https://rubygems.org/gems/executorch/trusted_publishers with:
+#
+#   repository:  benngarcia/executorch-ruby
+#   workflow:    release.yml
+#   environment: release
+#
+# This ships a source gem -- the C++ extension is compiled on the user's
+# machine at install time against their own ExecuTorch build, so no
+# cross-compilation matrix is needed here.
+
+on:
+  push:
+    tags: ["v*"]
+  workflow_dispatch:
+    inputs:
+      dry_run:
+        description: "Build and verify without publishing"
+        type: boolean
+        default: true
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    environment: release
+
+    permissions:
+      contents: write # create the GitHub release
+      id-token: write # OIDC token for trusted publishing
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: "3.3"
+          bundler-cache: false
+
+      - name: Check the tag matches Executorch::VERSION
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          version=$(ruby -r./lib/executorch/version -e 'print Executorch::VERSION')
+          tag="${GITHUB_REF#refs/tags/v}"
+          if [ "$version" != "$tag" ]; then
+            echo "Tag v$tag does not match Executorch::VERSION ($version)." >&2
+            echo "Bump lib/executorch/version.rb, or retag." >&2
+            exit 1
+          fi
+          echo "version=$version" >> "$GITHUB_ENV"
+
+      - name: Check the CHANGELOG has an entry
+        if: startsWith(github.ref, 'refs/tags/')
+        run: |
+          grep -q "## \[$version\]" CHANGELOG.md \
+            || { echo "CHANGELOG.md has no entry for $version"; exit 1; }
+
+      - name: Build the gem
+        run: gem build executorch.gemspec
+
+      # Catch a packaging mistake before it reaches RubyGems: the extension
+      # sources must be in the gem or it cannot build on install.
+      - name: Verify the package contents
+        run: |
+          gem_file=$(ls executorch-*.gem)
+          for required in \
+            ext/executorch/extconf.rb \
+            ext/executorch/executorch.cpp \
+            ext/executorch/utils.h \
+            lib/executorch.rb \
+            LICENSE.txt
+          do
+            tar -xOf "$gem_file" data.tar.gz | tar -tzf - | grep -qx "$required" \
+              || { echo "Missing from gem package: $required"; exit 1; }
+          done
+          echo "Package contents OK"
+
+      - name: Publish to RubyGems
+        if: startsWith(github.ref, 'refs/tags/') && !inputs.dry_run
+        uses: rubygems/release-gem@v1
+
+      - name: Create the GitHub release
+        if: startsWith(github.ref, 'refs/tags/') && !inputs.dry_run
+        uses: softprops/action-gh-release@v2
+        with:
+          generate_release_notes: true
+          files: executorch-*.gem

.gitignore

@@ -33,3 +33,11 @@ test/support/models/*.pte
 
 # Coverage
 /coverage/
+
+# Benchmark models (regenerate with bench/make_pt_models.py + bench/pt_to_pte.py).
+# Results are kept in git -- they're the evidence behind bench/FINDINGS.md.
+/bench/models/
+
+# Python
+__pycache__/
+*.pyc

CHANGELOG.md

@@ -5,6 +5,41 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- `Tensor.from_bytes(packed, shape:, dtype:)` and `Tensor#to_binary` - raw
+  binary tensor I/O that skips per-element conversion. Roughly 174x faster than
+  the Array constructor on a 150k-element input.
+- `bench/` - eval and profiling harness. Converts `.pt` checkpoints to `.pte`,
+  checks output against PyTorch, and times each leg of an inference call. See
+  `bench/FINDINGS.md`.
+- `script/build-executorch.sh` - reproducible ExecuTorch build used by both
+  humans and CI.
+- CI (GitHub Actions) for lint and tests across Ruby 3.1-3.4 on Linux and
+  macOS, plus a tag-triggered RubyGems release workflow.
+
+### Changed
+
+- Large reduction in Ruby/C++ boundary cost. Tensor creation is ~13x faster,
+  `#to_a` up to ~65x faster, and building a tensor from a nested Array now
+  allocates 11 objects instead of 153,238 for a 150k-element input.
+- `extconf.rb` now links optimized CPU kernels and the XNNPACK delegate when
+  they are present in the ExecuTorch install, whole-archiving them so their
+  self-registration is not dropped by the linker.
+
+### Fixed
+
+- The documented install path did not work: `cmake --install` does not install
+  `extension/module/module.h` or the `runtime/executor` headers, so
+  `rake compile` failed with "module.h header not found" on a correct build.
+  `script/build-executorch.sh` copies them.
+- `extconf.rb` whole-archived two operator libraries when both were present,
+  which aborts the runtime at init on duplicate operator registration.
+- Build failure against Rice 4.12, where `Rice::Array::Proxy` no longer
+  converts implicitly to `Rice::Object`.
+
 ## [0.1.0] - 2024-12-27
 
 ### Added