diff --git a/.github/assemble-docs.sh b/.github/assemble-docs.sh index cadd938dc..f9c47947e 100755 --- a/.github/assemble-docs.sh +++ b/.github/assemble-docs.sh @@ -13,7 +13,8 @@ fi rm -rf site_docs mkdir -p site_docs/docs site_docs/python site_docs/apple site_docs/android \ - site_docs/flutter site_docs/rust site_docs/blog site_docs/assets + site_docs/flutter site_docs/rust site_docs/swift site_docs/kotlin \ + site_docs/blog site_docs/assets cp -r assets/* site_docs/assets/ @@ -31,7 +32,9 @@ cp docs/*.md site_docs/docs/ cp python/README.md site_docs/python/README.md cp apple/README.md site_docs/apple/README.md cp android/README.md site_docs/android/README.md -cp flutter/README.md site_docs/flutter/README.md +cp bindings/flutter/README.md site_docs/flutter/README.md +cp bindings/swift/README.md site_docs/swift/README.md +cp bindings/kotlin/README.md site_docs/kotlin/README.md mkdir -p site_docs/react-native if curl -sfL "https://raw.githubusercontent.com/cactus-compute/cactus-react-native/main/README.md" -o site_docs/react-native/README.md; then @@ -54,9 +57,7 @@ else echo "Warning: Could not fetch React Native README, using fallback" fi -if [ -f rust/README.md ]; then - cp rust/README.md site_docs/rust/README.md -fi +cp bindings/rust/README.md site_docs/rust/README.md if [ -d blog ] && ls blog/*.md >/dev/null 2>&1; then cp blog/*.md site_docs/blog/ @@ -79,6 +80,12 @@ sedi 's|(cactus_index\.md)|(docs/cactus_index.md)|g' site_docs/index.md sedi 's|(finetuning\.md)|(docs/finetuning.md)|g' site_docs/index.md sedi 's|(compatibility\.md)|(docs/compatibility.md)|g' site_docs/index.md sedi 's|(/CONTRIBUTING\.md)|(CONTRIBUTING.md)|g' site_docs/index.md +sedi 's|(/bindings/swift/)|(swift/README.md)|g' site_docs/index.md +sedi 's|(/bindings/kotlin/)|(kotlin/README.md)|g' site_docs/index.md +sedi 's|(/bindings/python/)|(python/README.md)|g' site_docs/index.md +sedi 's|(/bindings/react-native/)|(react-native/README.md)|g' site_docs/index.md +sedi 's|(/bindings/flutter/)|(flutter/README.md)|g' site_docs/index.md +sedi 's|(/bindings/rust/)|(rust/README.md)|g' site_docs/index.md sedi 's|(/python/)|(python/README.md)|g' site_docs/index.md sedi 's|(/apple/)|(apple/README.md)|g' site_docs/index.md sedi 's|(/android/)|(android/README.md)|g' site_docs/index.md @@ -91,7 +98,7 @@ sedi 's|(/blog/lfm2\.5_350m\.md)|(blog/lfm2.5_350m.md)|g' site_docs/index.md sedi 's|(/blog/gemma4\.md)|(blog/gemma4.md)|g' site_docs/index.md sedi 's|(/blog/turboquant-h\.md)|(blog/turboquant-h.md)|g' site_docs/index.md sedi 's|(quickstart\.md)|(docs/quickstart.md)|g' site_docs/index.md -sedi 's|(choose-sdk\.md)|(docs/choose-sdk.md)|g' site_docs/index.md +sedi 's|(choose-bindings\.md)|(docs/choose-bindings.md)|g' site_docs/index.md for f in site_docs/docs/*.md; do sedi 's|(/docs/cactus_engine\.md)|(cactus_engine.md)|g' "$f" @@ -100,9 +107,15 @@ for f in site_docs/docs/*.md; do sedi 's|(/docs/finetuning\.md)|(finetuning.md)|g' "$f" sedi 's|(/docs/compatibility\.md)|(compatibility.md)|g' "$f" sedi 's|(/docs/quickstart\.md)|(quickstart.md)|g' "$f" - sedi 's|(/docs/choose-sdk\.md)|(choose-sdk.md)|g' "$f" + sedi 's|(/docs/choose-bindings\.md)|(choose-bindings.md)|g' "$f" sedi 's|(/docs/index\.md)|(../index.md)|g' "$f" sedi 's|(/CONTRIBUTING\.md)|(../CONTRIBUTING.md)|g' "$f" + sedi 's|(/bindings/swift/)|(../swift/README.md)|g' "$f" + sedi 's|(/bindings/kotlin/)|(../kotlin/README.md)|g' "$f" + sedi 's|(/bindings/python/)|(../python/README.md)|g' "$f" + sedi 's|(/bindings/react-native/)|(../react-native/README.md)|g' "$f" + sedi 's|(/bindings/flutter/)|(../flutter/README.md)|g' "$f" + sedi 's|(/bindings/rust/)|(../rust/README.md)|g' "$f" sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" @@ -110,14 +123,20 @@ for f in site_docs/docs/*.md; do sedi 's|(/rust/)|(../rust/README.md)|g' "$f" done -for f in site_docs/python/README.md site_docs/apple/README.md site_docs/android/README.md site_docs/flutter/README.md; do +for f in site_docs/python/README.md site_docs/apple/README.md site_docs/android/README.md site_docs/flutter/README.md site_docs/swift/README.md site_docs/kotlin/README.md; do sedi 's|(/docs/cactus_engine\.md)|(../docs/cactus_engine.md)|g' "$f" sedi 's|(/docs/cactus_graph\.md)|(../docs/cactus_graph.md)|g' "$f" sedi 's|(/docs/cactus_index\.md)|(../docs/cactus_index.md)|g' "$f" sedi 's|(/docs/finetuning\.md)|(../docs/finetuning.md)|g' "$f" sedi 's|(/docs/compatibility\.md)|(../docs/compatibility.md)|g' "$f" sedi 's|(/docs/quickstart\.md)|(../docs/quickstart.md)|g' "$f" - sedi 's|(/docs/choose-sdk\.md)|(../docs/choose-sdk.md)|g' "$f" + sedi 's|(/docs/choose-bindings\.md)|(../docs/choose-bindings.md)|g' "$f" + sedi 's|(/bindings/swift/)|(../swift/README.md)|g' "$f" + sedi 's|(/bindings/kotlin/)|(../kotlin/README.md)|g' "$f" + sedi 's|(/bindings/python/)|(../python/README.md)|g' "$f" + sedi 's|(/bindings/react-native/)|(../react-native/README.md)|g' "$f" + sedi 's|(/bindings/flutter/)|(../flutter/README.md)|g' "$f" + sedi 's|(/bindings/rust/)|(../rust/README.md)|g' "$f" sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" @@ -134,7 +153,13 @@ if [ -f site_docs/rust/README.md ]; then 's|(/docs/finetuning\.md)|(../docs/finetuning.md)|g' \ 's|(/docs/compatibility\.md)|(../docs/compatibility.md)|g' \ 's|(/docs/quickstart\.md)|(../docs/quickstart.md)|g' \ - 's|(/docs/choose-sdk\.md)|(../docs/choose-sdk.md)|g' \ + 's|(/docs/choose-bindings\.md)|(../docs/choose-bindings.md)|g' \ + 's|(/bindings/swift/)|(../swift/README.md)|g' \ + 's|(/bindings/kotlin/)|(../kotlin/README.md)|g' \ + 's|(/bindings/python/)|(../python/README.md)|g' \ + 's|(/bindings/react-native/)|(../react-native/README.md)|g' \ + 's|(/bindings/flutter/)|(../flutter/README.md)|g' \ + 's|(/bindings/rust/)|(../rust/README.md)|g' \ 's|(/python/)|(../python/README.md)|g' \ 's|(/apple/)|(../apple/README.md)|g' \ 's|(/android/)|(../android/README.md)|g' \ @@ -152,6 +177,12 @@ if ls site_docs/blog/*.md >/dev/null 2>&1; then sedi 's|(/docs/cactus_index\.md)|(../docs/cactus_index.md)|g' "$f" sedi 's|(/docs/finetuning\.md)|(../docs/finetuning.md)|g' "$f" sedi 's|(/docs/compatibility\.md)|(../docs/compatibility.md)|g' "$f" + sedi 's|(/bindings/swift/)|(../swift/README.md)|g' "$f" + sedi 's|(/bindings/kotlin/)|(../kotlin/README.md)|g' "$f" + sedi 's|(/bindings/python/)|(../python/README.md)|g' "$f" + sedi 's|(/bindings/react-native/)|(../react-native/README.md)|g' "$f" + sedi 's|(/bindings/flutter/)|(../flutter/README.md)|g' "$f" + sedi 's|(/bindings/rust/)|(../rust/README.md)|g' "$f" sedi 's|(/python/)|(../python/README.md)|g' "$f" sedi 's|(/apple/)|(../apple/README.md)|g' "$f" sedi 's|(/android/)|(../android/README.md)|g' "$f" @@ -184,6 +215,8 @@ fi for nav_path in \ "rust/README.md" \ + "swift/README.md" \ + "kotlin/README.md" \ "react-native/README.md" \ "blog/README.md" \ "blog/hybrid_transcription.md" \ diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index 636bb764b..3ff7868fe 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -1,14 +1,20 @@ +name: C++ CI + on: pull_request: paths: - cactus/** - - tests/** + - cactus-engine/** + - cactus-graph/** + - cactus-kernels/** - .github/workflows/cpp.yml push: branches: [main] paths: - cactus/** - - tests/** + - cactus-engine/** + - cactus-graph/** + - cactus-kernels/** - .github/workflows/cpp.yml workflow_dispatch: @@ -33,14 +39,14 @@ jobs: - uses: actions/cache@v4 with: path: ~/.ccache - key: ccache-${{ matrix.os }}-${{ github.ref_name }}-${{ hashFiles('cactus/CMakeLists.txt', 'tests/CMakeLists.txt', '.github/workflows/cpp.yml') }} + key: ccache-${{ matrix.os }}-${{ github.ref_name }}-${{ hashFiles('cactus/CMakeLists.txt', 'cactus-engine/tests/CMakeLists.txt', '.github/workflows/cpp.yml') }} restore-keys: | ccache-${{ matrix.os }}-${{ github.ref_name }}- ccache-${{ matrix.os }}- - run: ccache --zero-stats || true - run: cmake -G Ninja -S cactus -B cactus/build -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - run: cmake --build cactus/build --target cactus_ffi --parallel 4 - - run: cmake -G Ninja -S tests -B tests/build-ci -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache - - run: cmake --build tests/build-ci --target test_stt --parallel 4 - - run: ./tests/build-ci/test_stt + - run: cmake --build cactus/build --target cactus --parallel 4 + - run: cmake -G Ninja -S cactus-engine/tests -B cactus-engine/tests/build-ci -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER_LAUNCHER=ccache + - run: cmake --build cactus-engine/tests/build-ci --target test_stt --parallel 4 + - run: ./cactus-engine/tests/build-ci/test_stt - run: ccache --show-stats || true diff --git a/.github/workflows/dco.yml b/.github/workflows/dco.yml index 5a392485a..ed73abd3e 100644 --- a/.github/workflows/dco.yml +++ b/.github/workflows/dco.yml @@ -10,7 +10,7 @@ jobs: name: DCO Check steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 84ffd2a80..8150c1a4a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,8 +13,8 @@ on: - python/README.md - apple/README.md - android/README.md - - flutter/README.md - - rust/README.md + - bindings/flutter/README.md + - bindings/rust/README.md - README.md workflow_dispatch: diff --git a/.github/workflows/publish_to_hf.yml b/.github/workflows/publish_to_hf.yml deleted file mode 100644 index 2dad72ddc..000000000 --- a/.github/workflows/publish_to_hf.yml +++ /dev/null @@ -1,114 +0,0 @@ -name: Publish to Hugging Face - -on: - release: - types: [published] - workflow_dispatch: - inputs: - model: - description: Model to publish (leave empty to publish all) - required: false - type: string - version: - description: Version tag - required: true - type: string - org: - description: Organization - required: false - default: Cactus-Compute - type: string - bits: - description: CQ quantization bits (e.g. "4" or "3 4") - required: false - type: string - default: "4" - pipeline_tag: - description: Pipeline tag (e.g. text-generation) - required: false - type: string - tags: - description: Comma-separated tags (e.g. completion,tools) - required: false - type: string - description: - description: Model description - required: false - type: string - branch: - description: Branch override (uses version tag if empty) - required: false - type: string - -jobs: - publish: - runs-on: macos-latest - steps: - - name: Get version - id: version - run: | - if [ "${{ github.event_name }}" == "release" ]; then - echo "tag=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT - else - echo "tag=${{ github.event.inputs.version }}" >> $GITHUB_OUTPUT - fi - - - uses: actions/checkout@v4 - with: - ref: ${{ github.event.inputs.branch || steps.version.outputs.tag }} - - - uses: actions/setup-python@v5 - with: - python-version: '3.12' - - - name: Install dependencies - run: pip install -r python/requirements.txt - - - name: Publish models - run: | - cd python - ORG="${{ github.event.inputs.org || 'Cactus-Compute' }}" - VERSION="${{ steps.version.outputs.tag }}" - BITS="${{ github.event.inputs.bits || '4' }}" - - publish_model() { - local model=$1; shift - local bits=$1; shift - local pipeline_tag=$1; shift - local tags=$1; shift - local description="${1:-}" - local FLAGS=(--version "$VERSION" --org "$ORG" --model "$model" --bits $bits) - [ -n "$pipeline_tag" ] && FLAGS+=(--pipeline-tag "$pipeline_tag") - [ -n "$tags" ] && FLAGS+=(--tags "$tags") - python -m cactus.cli.publish_to_hf "${FLAGS[@]}" --description "$description" - } - - if [ -n "${{ github.event.inputs.model }}" ]; then - publish_model \ - "${{ github.event.inputs.model }}" \ - "$BITS" \ - "${{ github.event.inputs.pipeline_tag }}" \ - "${{ github.event.inputs.tags }}" \ - "${{ github.event.inputs.description }}" - else - FAILED_MODELS=() - while read -r config; do - model=$(echo "$config" | jq -r '.model') - bits=$(echo "$config" | jq -r 'if .bits | type == "array" then .bits | map(tostring) | join(" ") else (.bits // 4 | tostring) end') - pipeline_tag=$(echo "$config" | jq -r '.pipeline_tag // ""') - tags=$(echo "$config" | jq -r 'if .tags then .tags | join(",") else "" end') - description=$(echo "$config" | jq -r '.description // ""') - if ! publish_model "$model" "$bits" "$pipeline_tag" "$tags" "$description"; then - echo "::error::Failed to publish $model" - FAILED_MODELS+=("$model") - fi - done < <(jq -c '.[]' "$GITHUB_WORKSPACE/models.json") - - if [ ${#FAILED_MODELS[@]} -gt 0 ]; then - echo "::error::Failed models: ${FAILED_MODELS[*]}" - exit 1 - fi - fi - env: - HF_TOKEN: ${{ secrets.HF_TOKEN }} - HF_HUB_DISABLE_XET: "1" diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index bb39ea933..000000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,45 +0,0 @@ -on: - workflow_dispatch: - push: - branches: [main] - paths: [rust/**, cactus/**, .github/workflows/rust.yml] - pull_request: - paths: [rust/**, cactus/**, .github/workflows/rust.yml] - -permissions: - contents: read - -jobs: - test: - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - include: - - os: macos-15 - deps: brew install llvm - - os: ubuntu-24.04-arm - deps: sudo apt-get update && sudo apt-get install -y cmake build-essential libcurl4-openssl-dev libclang-dev - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: dtolnay/rust-toolchain@stable - - run: ${{ matrix.deps }} - - uses: Swatinem/rust-cache@v2 - with: - workspaces: rust - - run: pip3 install --break-system-packages huggingface-hub && pip3 install --break-system-packages -e python/ --no-deps - - uses: actions/cache@v4 - with: - path: weights/ - key: cactus-models-${{ matrix.os }}-v2 - - run: cactus download google/gemma-3-270m-it - - run: cactus download openai/whisper-small - - run: curl -fL -o "${{ runner.temp }}/jfk.wav" https://github.com/ggml-org/whisper.cpp/raw/refs/tags/v1.8.3/samples/jfk.wav - - run: cargo test --manifest-path rust/Cargo.toml --test llm -- --nocapture - env: - CACTUS_MODEL_PATH: ${{ github.workspace }}/weights/gemma-3-270m-it - - run: cargo test --manifest-path rust/Cargo.toml --test stt -- --nocapture - env: - CACTUS_STT_MODEL_PATH: ${{ github.workspace }}/weights/whisper-small - CACTUS_STT_AUDIO_PATH: ${{ runner.temp }}/jfk.wav diff --git a/.github/workflows/verify-pr.yml b/.github/workflows/verify-pr.yml index ef9420ab8..1f5e5e9fb 100644 --- a/.github/workflows/verify-pr.yml +++ b/.github/workflows/verify-pr.yml @@ -78,7 +78,7 @@ jobs: if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "python_changed=false" >> $GITHUB_OUTPUT else - CHANGED=$(gh pr diff ${{ github.event.pull_request.number }} --name-only | grep -cE '^python/(src/|requirements\.txt)' || true) + CHANGED=$(gh pr diff ${{ github.event.pull_request.number }} --name-only | grep -cE '^python/(cactus/|requirements\.txt|pyproject\.toml)' || true) echo "python_changed=$([[ $CHANGED -gt 0 ]] && echo true || echo false)" >> $GITHUB_OUTPUT fi @@ -132,7 +132,7 @@ jobs: - name: Build tests run: | - cd tests + cd cactus-engine/tests rm -rf build && mkdir -p build && cd build cmake .. -DCMAKE_RULE_MESSAGES=OFF -DCMAKE_VERBOSE_MAKEFILE=OFF make -j$(nproc) @@ -153,7 +153,7 @@ jobs: FAILED=0 for test in test_graph test_kernel test_kv_cache test_index test_model_loading; do echo "=== Running $test ===" - if ! ./tests/build/$test; then + if ! ./cactus-engine/tests/build/$test; then FAILED=$((FAILED + 1)) fi done @@ -215,11 +215,8 @@ jobs: CACTUS_NO_CLOUD_TELE: "1" run: | export ${{ matrix.env_var }}="${{ github.workspace }}/weights/${{ matrix.model_dir }}" - export CACTUS_TEST_ASSETS="${{ github.workspace }}/tests/assets" - export CACTUS_TEST_GOLDEN_FILE="${{ github.workspace }}/tests/golden/golden.json" + export CACTUS_TEST_ASSETS="${{ github.workspace }}/cactus-engine/tests/assets" + export CACTUS_TEST_GOLDEN_FILE="${{ github.workspace }}/cactus-engine/tests/golden/golden.json" export CACTUS_TEST_GOLDEN_FAMILY="${{ matrix.family }}" export CACTUS_TEST_GOLDEN_PRECISION="${{ matrix.precision }}" - if [ "${{ matrix.family }}" = "whisper" ] || [ "${{ matrix.family }}" = "moonshine" ]; then - export CACTUS_TEST_VAD_MODEL="${{ github.workspace }}/weights/silero-vad" - fi - ./tests/build/test_exhaustive + ./cactus-engine/tests/build/test_exhaustive diff --git a/README.md b/README.md index 4bdc6568c..9c25eaaf3 100644 --- a/README.md +++ b/README.md @@ -180,9 +180,6 @@ cactus build # default static lib | nvidia/parakeet-ctc-0.6b | 600M | 201.77 | 201.44 | 5214285 | yes | 0.0101 | 0.0930 | | nvidia/parakeet-tdt-0.6b-v3 | 600M | 718.91 | 718.82 | 3583333 | yes | 0.0359 | 0.0465 | | nvidia/parakeet-ctc-1.1b | 1.1B | 279.03 | 278.92 | 4562500 | yes | 0.0139 | 0.1628 | -| snakers4/silero-vad | - | - | - | - | - | - | - | -| pyannote/segmentation-3.0 | - | - | - | - | - | - | - | -| pyannote/wespeaker-voxceleb-resnet34-LM | - | - | - | - | - | - | - | ## Supported LLMs @@ -194,12 +191,10 @@ cactus build # default static lib | google/gemma-3-270m-it | completion | | google/functiongemma-270m-it | tools | | google/gemma-3-1b-it | completion, gated | -| google/gemma-4-E2B-it | completion, tools, embed, vision, speech| -| google/gemma-3n-E2B-it | completion, tools | -| google/gemma-4-E4B-it | completion, tools, embed, vision, speech| -| google/gemma-3n-E4B-it | completion, tools | | google/gemma-4-E2B-it | vision, audio, completion, tools, Apple NPU | | google/gemma-4-E4B-it | vision, audio, completion, tools, Apple NPU | +| google/gemma-3n-E2B-it | completion, tools | +| google/gemma-3n-E4B-it | completion, tools | | Qwen/Qwen3-0.6B | completion, tools, embed | | Qwen/Qwen3-Embedding-0.6B | embed | | Qwen/Qwen3.5-0.8B | vision, completion, tools, embed | @@ -255,9 +250,10 @@ cactus build # default static lib │ --status show key status │ │ --clear remove saved key │ │ │ -│ cactus run opens playground (auto downloads) │ -│ --token HF token (gated models) │ -│ --reconvert force reconversion from source │ +│ cactus run runs a transpiled Cactus bundle │ +│ --system system prompt │ +│ --prompt initial prompt │ +│ --thinking enable thinking/reasoning mode │ │ │ │ cactus transcribe [model] live mic transcription (parakeet-tdt-0.6b-v3) │ │ --file transcribe file instead of mic │ @@ -279,12 +275,12 @@ cactus build # default static lib │ --python shared lib for Python FFI │ │ │ │ cactus test run unit tests and benchmarks │ -│ --model default: LFM2-VL-450M │ -│ --transcribe_model default: moonshine-base │ -│ --benchmark use larger models │ +│ --model default: google/gemma-4-E2B-it │ │ --reconvert force reconversion from source │ -│ --no-rebuild skip building library │ -│ --llm / --stt / --performance run specific test suite │ +│ --llm / --vlm / --stt / --embed run specific test suite │ +│ --rag / --graph / --index run specific test suite │ +│ --kernel / --kv_cache run specific test suite │ +│ --performance run performance benchmarks │ │ --ios run on connected iPhone │ │ --android run on connected Android │ │ │ diff --git a/apple/BatteryTestApp/CactusBatteryTestApp.swift b/apple/BatteryTestApp/CactusBatteryTestApp.swift deleted file mode 100644 index d87271a9b..000000000 --- a/apple/BatteryTestApp/CactusBatteryTestApp.swift +++ /dev/null @@ -1,203 +0,0 @@ -import SwiftUI -import UIKit -import Combine -import Foundation - -@main -struct CactusBatteryTestApp: App { - init() { - UIDevice.current.isBatteryMonitoringEnabled = true - UIApplication.shared.isIdleTimerDisabled = true - } - var body: some Scene { - WindowGroup { ContentView() } - } -} - -enum Backend: String { case cpu = "CPU", gpu = "GPU", npu = "NPU", idle = "idle" } - -@MainActor -final class Runner: ObservableObject { - @Published var backend: Backend = .idle - @Published var iterations: Int = 0 - @Published var lastPrefillTps: Double = 0 - @Published var batteryLevel: Float = UIDevice.current.batteryLevel - @Published var batteryStateText: String = "" - @Published var elapsedSeconds: Int = 0 - @Published var status: String = "ready" - - private var task: Task? = nil - private var startBattery: Float = 0 - private var startTime: Date = .now - private var model: OpaquePointer? = nil - - private let modelPath: String = { - let fm = FileManager.default - if let resPath = Bundle.main.resourcePath { - let candidate = (resPath as NSString).appendingPathComponent("model") - if fm.fileExists(atPath: (candidate as NSString).appendingPathComponent("config.txt")) { - return candidate - } - } - if let p = Bundle.main.path(forResource: "model", ofType: nil) { return p } - if let p = ProcessInfo.processInfo.environment["CACTUS_MODEL"] { return p } - return (NSHomeDirectory() as NSString).appendingPathComponent("Documents/model") - }() - - private let prompt: String = """ - [ - {"role":"system","content":"You are a helpful assistant."}, - {"role":"user","content":"Summarize the following: The history of computing is a long and complex story. The earliest mechanical aids to calculation were the abacus and slide rule. In the 19th century, Charles Babbage designed the Analytical Engine, often considered the first general-purpose mechanical computer, though it was never fully built in his lifetime. Ada Lovelace wrote the first algorithm intended for processing on a machine, making her widely regarded as the first computer programmer. The 20th century brought rapid advances: vacuum tubes, transistors, integrated circuits, and microprocessors. Personal computers in the 1970s and 1980s, followed by networked computing and the internet in the 1990s, transformed society. Today computing is ubiquitous."} - ] - """ - - private let options: String = "{\"max_tokens\":1,\"telemetry_enabled\":false}" - - func loadModelIfNeeded() -> Bool { - if model != nil { return true } - status = "loading model from \(modelPath)..." - let m = modelPath.withCString { mp in - cactus_init(mp, nil, false) - } - guard let m = m else { - status = "failed to load model — set CACTUS_MODEL or include 'model' bundle dir" - return false - } - model = OpaquePointer(m) - status = "model loaded" - return true - } - - func start(_ b: Backend) { - if backend != .idle { stop(); return } - guard loadModelIfNeeded() else { return } - - switch b { - case .cpu: - cactus_npu_set_enabled(false) - cactus_mps_set_enabled(false) - case .gpu: - cactus_npu_set_enabled(false) - cactus_mps_set_enabled(true) - case .npu: - cactus_mps_set_enabled(false) - cactus_npu_set_enabled(true) - case .idle: return - } - - backend = b - iterations = 0 - lastPrefillTps = 0 - startBattery = UIDevice.current.batteryLevel - startTime = .now - status = "running \(b.rawValue)..." - - let modelPtr = self.model - let promptC = strdup(self.prompt)! - let optsC = strdup(self.options)! - - task = Task.detached(priority: .userInitiated) { - var buffer = [CChar](repeating: 0, count: 4096) - while !Task.isCancelled { - cactus_reset(modelPtr.map { UnsafeMutableRawPointer($0) }) - let rc = cactus_prefill(modelPtr.map { UnsafeMutableRawPointer($0) }, - promptC, - &buffer, - buffer.count, - optsC, - nil, nil, 0) - if rc <= 0 { - await MainActor.run { self.status = "prefill rc=\(rc) (stopping)" } - break - } - let resp = String(cString: buffer) - let tps = parsePrefillTps(resp) - await MainActor.run { - self.iterations += 1 - if tps > 0 { self.lastPrefillTps = tps } - self.batteryLevel = UIDevice.current.batteryLevel - self.elapsedSeconds = Int(Date().timeIntervalSince(self.startTime)) - self.batteryStateText = String(format: "%.1f%% drop over %ds", - (self.startBattery - self.batteryLevel) * 100, self.elapsedSeconds) - } - } - free(promptC) - free(optsC) - } - } - - func stop() { - task?.cancel() - task = nil - backend = .idle - status = "stopped" - } -} - -private func parsePrefillTps(_ s: String) -> Double { - guard let r = s.range(of: "\"prefill_tps\"") else { return 0 } - let after = s[r.upperBound...] - let digits = after.drop(while: { !($0.isNumber || $0 == "-") }) - let val = digits.prefix(while: { $0.isNumber || $0 == "." || $0 == "-" }) - return Double(val) ?? 0 -} - -struct ContentView: View { - @StateObject var runner = Runner() - var body: some View { - VStack(spacing: 18) { - Text("Cactus prefill battery test").font(.title2).bold().padding(.top, 28) - - VStack(alignment: .leading, spacing: 6) { - row("backend", runner.backend.rawValue) - row("iterations", "\(runner.iterations)") - row("prefill_tps", String(format: "%.1f", runner.lastPrefillTps)) - row("battery", String(format: "%.0f%%", runner.batteryLevel * 100)) - row("Δ", runner.batteryStateText) - row("elapsed", "\(runner.elapsedSeconds)s") - row("status", runner.status) - } - .font(.system(.body, design: .monospaced)) - .padding() - .background(Color(.secondarySystemBackground)) - .cornerRadius(10) - .padding(.horizontal) - - HStack(spacing: 12) { - Button { runner.start(.cpu) } label: { - Text(runner.backend == .cpu ? "STOP CPU" : "CPU") - .frame(maxWidth: .infinity).padding().bold() - } - .buttonStyle(.borderedProminent) - .tint(runner.backend == .cpu ? .red : .orange) - .disabled(runner.backend == .gpu || runner.backend == .npu) - - Button { runner.start(.gpu) } label: { - Text(runner.backend == .gpu ? "STOP GPU" : "GPU") - .frame(maxWidth: .infinity).padding().bold() - } - .buttonStyle(.borderedProminent) - .tint(runner.backend == .gpu ? .red : .blue) - .disabled(runner.backend == .cpu || runner.backend == .npu) - - Button { runner.start(.npu) } label: { - Text(runner.backend == .npu ? "STOP NPU" : "NPU") - .frame(maxWidth: .infinity).padding().bold() - } - .buttonStyle(.borderedProminent) - .tint(runner.backend == .npu ? .red : .green) - .disabled(runner.backend == .cpu || runner.backend == .gpu) - } - .padding(.horizontal) - - Spacer() - } - } - private func row(_ k: String, _ v: String) -> some View { - HStack { - Text(k).foregroundStyle(.secondary).frame(width: 100, alignment: .leading) - Text(v) - Spacer() - } - } -} diff --git a/apple/BatteryTestApp/CactusBridge.h b/apple/BatteryTestApp/CactusBridge.h deleted file mode 100644 index bfd3515b5..000000000 --- a/apple/BatteryTestApp/CactusBridge.h +++ /dev/null @@ -1,36 +0,0 @@ -#ifndef CACTUS_BRIDGE_H -#define CACTUS_BRIDGE_H - -#include -#include -#include - -#ifdef __cplusplus -extern "C" { -#endif - -typedef void* cactus_model_t; - -cactus_model_t cactus_init(const char* model_path, const char* corpus_dir, bool cache_index); -void cactus_destroy(cactus_model_t model); -void cactus_reset(cactus_model_t model); - -int cactus_prefill(cactus_model_t model, - const char* messages_json, - char* response_buffer, - size_t buffer_size, - const char* options_json, - const char* tools_json, - const uint8_t* pcm_buffer, - size_t pcm_buffer_size); - -void cactus_npu_set_enabled(bool enabled); -bool cactus_npu_enabled(void); -void cactus_mps_set_enabled(bool enabled); -bool cactus_mps_enabled(void); - -#ifdef __cplusplus -} -#endif - -#endif diff --git a/apple/BatteryTestApp/README.md b/apple/BatteryTestApp/README.md deleted file mode 100644 index 954b2e009..000000000 --- a/apple/BatteryTestApp/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Cactus Battery Test App - -Minimal SwiftUI app: two buttons (GPU / NPU). Tap one to run cactus prefill repeatedly on that backend until you tap again. Shows iteration count, last prefill_tps, battery level, and battery drop over elapsed time. - -## Files - -- `CactusBatteryTestApp.swift` — single-file SwiftUI app (App + ContentView + Runner) -- `CactusBridge.h` — Objective-C bridging header that exposes the cactus C API to Swift - -## Setup in Xcode (one-time) - -1. **Create a new Xcode project** - - File → New → Project → iOS → App - - Interface: SwiftUI, Language: Swift - - Name it whatever, e.g. `CactusBatteryTest` - - Save it inside this directory or wherever convenient - -2. **Drop in the source files** - - Delete the auto-generated `ContentView.swift` and `App.swift` - - Drag `CactusBatteryTestApp.swift` and `CactusBridge.h` into the project (check "Copy items if needed") - -3. **Set the bridging header** - - Project → Build Settings → "Objective-C Bridging Header" - - Set to `$(SRCROOT)/CactusBridge.h` (or whatever path) - -4. **Link `libcactus-device.a`** - - Build Phases → Link Binary With Libraries → + - - Add Other → Add Files → pick `apple/libcactus-device.a` from this repo - - Also add system frameworks: `Metal`, `MetalPerformanceShaders`, `Accelerate`, `CoreML`, `Foundation`, `Security`, `SystemConfiguration`, `CFNetwork` - - Add `libc++.tbd` for C++ stdlib - -5. **Provide a model** - The app looks for the model in this order: - - `Bundle.main.path(forResource: "model", ofType: nil)` — if you add a folder called `model` to the app bundle (drag the model directory into Xcode, choose "Create folder references" so it stays a folder), this is used - - `ProcessInfo.processInfo.environment["CACTUS_MODEL"]` — set in Xcode scheme → Run → Arguments → Environment Variables - - `~/Documents/model` on the device — if you push via Files app - - The model directory is the cactus-format weights folder (e.g. `weights/gemma-4-e2b-it/`). - -6. **Build target** - - Set scheme to your physical iPhone - - Build and run (⌘R) - -## How to run a battery test - -1. Charge phone to 100%, unplug -2. Disable background app refresh, set airplane mode, lock screen brightness at 50% -3. Open the app -4. Tap **GPU** — runs prefill on MPS continuously -5. Wait 15-30 minutes, note final battery level on the in-app display (`Δ` row) -6. Tap **STOP GPU** to stop -7. Reset (charge back, etc.) and repeat with **NPU** - -The on-screen `Δ` shows e.g. `2.3% drop over 945s` — that's your drain rate. - -## Notes - -- `cactus_prefill` is called with `max_tokens=1` and a fixed long prompt (~250 tokens). Each iteration is a full prefill of the prompt; decode is suppressed. -- The `Runner` calls `cactus_npu_set_enabled` / `cactus_mps_set_enabled` before each run to ensure the right backend handles the work. -- Both buttons are mutually exclusive — one disables while the other is running. -- For more accurate per-op energy (joules/token), use Xcode → Open Developer Tool → Instruments → Energy Log instead of (or alongside) the in-app battery readout. - -## Caveats - -- The v2 model loader currently throws "Scalar operations only support FP16 precision" during init for current Gemma 4 E2B weights. Until that's fixed in the v2 main branch, `cactus_init` will fail. The app will display "failed to load model" and you can't run the test. This is a v2 bug, not the GPU code; see project notes. -- `apple/libcactus-device.a` must be the MPS-enabled build. If unsure, run `nm libcactus-device.a | grep cactus_mps_set_enabled` — it should show a `T` symbol. diff --git a/benchmarks/python_baseline/results.json b/benchmarks/python_baseline/results.json deleted file mode 100644 index 8fa455076..000000000 --- a/benchmarks/python_baseline/results.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "captured_at": "2026-05-19T10:21:19", - "platform": "macOS-26.3-arm64-arm-64bit", - "python": "3.12.12", - "commit": "c8999fc40d709d451d1f71046e135e696ee57af1", - "note": "Pre-refactor Python orchestration microbench. Graph kernels are mocked: numbers reflect ONLY the per-call/per-iteration Python wrapper overhead that the C++ port replaces.", - "benches": [ - { - "name": "audio_preprocess.prepare_cactus_audio_features", - "description": "End-to-end audio feature wrapper (C++ frontend + Python reshape/padding)", - "ok": true, - "error": null, - "iterations": 10, - "total_ms": 626.2661, - "mean_ms": 62.6266, - "p50_ms": 62.7228, - "p95_ms": 62.2703, - "peak_rss_mb": 269.45, - "notes": { - "audio_file": "/Users/justinl/Desktop/GitRepos/cactus/cactus-engine/tests/assets/test.wav" - } - }, - { - "name": "multimodal_runtime._prepare_gemma4_native_image_tensors", - "description": "Pure-Python Gemma4 image preprocessing (PIL load+resize, NumPy patch extract)", - "ok": true, - "error": null, - "iterations": 10, - "total_ms": 83.9351, - "mean_ms": 8.3935, - "p50_ms": 8.157, - "p95_ms": 8.1007, - "peak_rss_mb": 304.84, - "notes": { - "image_file": "/Users/justinl/Desktop/GitRepos/cactus/cactus-engine/tests/assets/test_monkey.png" - } - }, - { - "name": "multimodal_runtime._load_image_inputs", - "description": "PIL Image.open + convert('RGB') + resize_static_image", - "ok": true, - "error": null, - "iterations": 10, - "total_ms": 13.5149, - "mean_ms": 1.3515, - "p50_ms": 1.3094, - "p95_ms": 1.2776, - "peak_rss_mb": 304.84, - "notes": { - "image_file": "/Users/justinl/Desktop/GitRepos/cactus/cactus-engine/tests/assets/test_monkey.png" - } - }, - { - "name": "tdt_runtime.inner_loop_step", - "description": "Per-step TDT Python wrapper overhead (argmax + state copy + emit logic), graph step mocked", - "ok": true, - "error": null, - "iterations": 200, - "total_ms": 7.9116, - "mean_ms": 0.0396, - "p50_ms": 0.0348, - "p95_ms": 0.0389, - "peak_rss_mb": 304.92, - "notes": { - "token_class_count": 1024, - "hidden_dim": 1024 - } - }, - { - "name": "component_bundle_runtime.cached_step_inner_loop", - "description": "Per-token cached-step decode Python overhead (argmax+stop+buffer fill+numpy copy), graph mocked", - "ok": true, - "error": null, - "iterations": 200, - "total_ms": 581.9351, - "mean_ms": 2.9097, - "p50_ms": 2.9505, - "p95_ms": 2.7146, - "peak_rss_mb": 304.92, - "notes": { - "vocab_size": 256128, - "tokens_per_iter": 32, - "hidden_dim": 2304, - "per_layer_dim": 32 - } - } - ] -} \ No newline at end of file diff --git a/benchmarks/python_baseline/run_baseline.py b/benchmarks/python_baseline/run_baseline.py deleted file mode 100644 index d1b080a60..000000000 --- a/benchmarks/python_baseline/run_baseline.py +++ /dev/null @@ -1,397 +0,0 @@ -"""Pre-refactor microbenchmark for the Python runtime orchestration code. - -The Python pipeline currently being ported to C++ is the *orchestration* -layer that wraps the C++ graph kernels — image preprocessing (PIL + -NumPy), the TDT per-step decode loop, and the cached-step causal-LM -decode loop. Graph execution itself is already C++, so this bench -isolates the Python overhead that the port will replace. - -We do not need real model weights for this: the C++ graph calls inside -each loop are mocked with a no-op or a tiny NumPy op that returns the -same shape the real graph would. That isolates Python wall time per -iteration, which is exactly what the port saves. - -Captures: total wall, per-iteration mean, p50/p95 (where iteration count -is meaningful), peak RSS. Output: ``results.json`` in this folder. -""" -from __future__ import annotations - -import argparse -import gc -import json -import os -import platform -import resource -import statistics -import sys -import time -import traceback -from dataclasses import dataclass, field -from pathlib import Path -from types import SimpleNamespace -from typing import Any, Callable - -REPO_ROOT = Path(__file__).resolve().parents[2] -ASSETS = REPO_ROOT / "cactus-engine" / "tests" / "assets" -DEFAULT_OUT = Path(__file__).resolve().parent / "results.json" - - -@dataclass -class BenchResult: - name: str - description: str - ok: bool - error: str | None - iterations: int - total_ms: float - mean_ms: float - p50_ms: float - p95_ms: float - peak_rss_mb: float - notes: dict[str, Any] = field(default_factory=dict) - - -def _peak_rss_mb() -> float: - usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss - if platform.system() == "Darwin": - return usage / (1024 * 1024) - return usage / 1024 - - -def _timed(fn: Callable[[], None], *, iterations: int) -> list[float]: - samples: list[float] = [] - for _ in range(iterations): - gc.collect() - start = time.perf_counter() - fn() - samples.append((time.perf_counter() - start) * 1000.0) - return samples - - -def _summarize(name: str, description: str, samples: list[float], notes: dict[str, Any]) -> BenchResult: - return BenchResult( - name=name, - description=description, - ok=True, - error=None, - iterations=len(samples), - total_ms=sum(samples), - mean_ms=statistics.fmean(samples), - p50_ms=statistics.median(samples), - p95_ms=samples[max(0, int(round(0.95 * (len(samples) - 1))))] if samples else 0.0, - peak_rss_mb=_peak_rss_mb(), - notes=notes, - ) - - -def _failed(name: str, description: str, exc: BaseException) -> BenchResult: - return BenchResult( - name=name, - description=description, - ok=False, - error=f"{type(exc).__name__}: {exc}\n{traceback.format_exc(limit=6)}", - iterations=0, - total_ms=0.0, - mean_ms=0.0, - p50_ms=0.0, - p95_ms=0.0, - peak_rss_mb=_peak_rss_mb(), - notes={}, - ) - - -def bench_audio_preprocess(iterations: int) -> BenchResult: - """Calls the public Cactus audio frontend on a real wav. Most of the - heavy lifting is already C++ via the bindings; this measures the - end-to-end wrapper cost (sample-rate detect, padding, reshape).""" - name = "audio_preprocess.prepare_cactus_audio_features" - desc = "End-to-end audio feature wrapper (C++ frontend + Python reshape/padding)" - try: - import torch - from cactus.transpile.audio_preprocess import prepare_cactus_audio_features - except Exception as exc: - return _failed(name, desc, exc) - - wav = ASSETS / "test.wav" - if not wav.exists(): - return _failed(name, desc, FileNotFoundError(str(wav))) - - def call() -> None: - prepare_cactus_audio_features( - str(wav), - model_type="parakeet_tdt", - expected_frames=None, - expected_mels=128, - torch_dtype=torch.float16, - layout="frames_mels", - ) - - try: - call() # warmup - samples = _timed(call, iterations=iterations) - except Exception as exc: - return _failed(name, desc, exc) - return _summarize(name, desc, samples, {"audio_file": str(wav)}) - - -def bench_image_preprocess_gemma4(iterations: int) -> BenchResult: - """Calls _prepare_gemma4_native_image_tensors with a synthetic processor. - All of this work is pure Python/NumPy/PIL today; it is exactly what - moves to C++ in the port.""" - name = "multimodal_runtime._prepare_gemma4_native_image_tensors" - desc = "Pure-Python Gemma4 image preprocessing (PIL load+resize, NumPy patch extract)" - try: - from cactus.transpile.multimodal_runtime import _prepare_gemma4_native_image_tensors - except Exception as exc: - return _failed(name, desc, exc) - - img = ASSETS / "test_monkey.png" - if not img.exists(): - return _failed(name, desc, FileNotFoundError(str(img))) - - processor = SimpleNamespace( - image_processor=SimpleNamespace( - patch_size=16, - pooling_kernel_size=3, - max_soft_tokens=280, - do_rescale=True, - do_normalize=True, - rescale_factor=1.0 / 255.0, - image_mean=[0.5, 0.5, 0.5], - image_std=[0.5, 0.5, 0.5], - ) - ) - image_files = (str(img),) - - def call() -> None: - out = _prepare_gemma4_native_image_tensors(processor, image_files) - assert out is not None - - try: - call() - samples = _timed(call, iterations=iterations) - except Exception as exc: - return _failed(name, desc, exc) - return _summarize(name, desc, samples, {"image_file": str(img)}) - - -def bench_image_load_basic(iterations: int) -> BenchResult: - name = "multimodal_runtime._load_image_inputs" - desc = "PIL Image.open + convert('RGB') + resize_static_image" - try: - from cactus.transpile.multimodal_runtime import _load_image_inputs - except Exception as exc: - return _failed(name, desc, exc) - img = ASSETS / "test_monkey.png" - if not img.exists(): - return _failed(name, desc, FileNotFoundError(str(img))) - - image_files = (str(img),) - - def call() -> None: - _load_image_inputs(image_files) - - try: - call() - samples = _timed(call, iterations=iterations) - except Exception as exc: - return _failed(name, desc, exc) - return _summarize(name, desc, samples, {"image_file": str(img)}) - - -def bench_tdt_inner_loop(iterations: int) -> BenchResult: - """Replicates the per-step Python overhead from tdt_runtime.py:151-196 - with a mock step() that returns precomputed logits. Each iteration - represents one inner-loop pass of the TDT decoder.""" - import numpy as np - - name = "tdt_runtime.inner_loop_step" - desc = "Per-step TDT Python wrapper overhead (argmax + state copy + emit logic), graph step mocked" - token_class_count = 1024 - num_durations = 5 - total_classes = token_class_count + num_durations - hidden_dim = 1024 - state_dim = 320 - - fake_logits = np.random.randn(1, total_classes).astype(np.float32) - fake_states = ( - np.zeros((1, 1, state_dim), dtype=np.float32), - np.zeros((1, 1, state_dim), dtype=np.float32), - ) - - def mock_step(frame, last_token, states): - return fake_logits, fake_states - - emitted: list[int] = [] - durations = list(range(num_durations)) - blank_id = token_class_count - 1 - states = fake_states - - def call() -> None: - last_token = 0 - symbols_added = 0 - frame = np.zeros((1, hidden_dim), dtype=np.float32) - local_states = states - while symbols_added < 1: - logits, next_states = mock_step(frame, last_token, local_states) - logits_array = np.asarray(logits, dtype=np.float32) - tc = total_classes - num_durations - token_scores = logits_array[:, :tc] - duration_scores = logits_array[:, tc:] - next_token = int(np.argmax(token_scores[0])) - duration_index = int(np.argmax(duration_scores[0])) - skip = int(durations[min(duration_index, len(durations) - 1)]) - if next_token != blank_id: - emitted.append(next_token) - last_token = next_token - local_states = tuple( - np.ascontiguousarray(np.asarray(state).copy()) for state in next_states - ) - symbols_added += 1 - if skip > 0: - break - - try: - call() - samples = _timed(call, iterations=iterations) - except Exception as exc: - return _failed(name, desc, exc) - return _summarize(name, desc, samples, {"token_class_count": token_class_count, "hidden_dim": hidden_dim}) - - -def bench_cached_step_inner_loop(iterations: int) -> BenchResult: - """Replicates the per-token Python overhead from the Gemma4 cached-step - decode loop in component_bundle_runtime.py:2207-2234. - - Mocks the underlying graph.execute() with a no-op; measures the - per-token Python wrapper cost (argmax, stop checks, buffer fill, - numpy conversions, encoder->decoder copy).""" - import numpy as np - - name = "component_bundle_runtime.cached_step_inner_loop" - desc = "Per-token cached-step decode Python overhead (argmax+stop+buffer fill+numpy copy), graph mocked" - vocab = 256128 - hidden = 2304 - per_layer = 32 - fake_logits = np.random.randn(1, 1, vocab).astype(np.float32) - fake_inputs_embeds = np.zeros((1, 1, hidden), dtype=np.float32) - fake_per_layer = np.zeros((1, 1, per_layer), dtype=np.float32) - fake_position = np.zeros((1, 1), dtype=np.int64) - - lm_encoder_input_ids = np.zeros((1, 1), dtype=np.int64) - lm_encoder_position = np.zeros((1, 1), dtype=np.int64) - decoder_inputs_embeds = np.zeros((1, 1, hidden), dtype=np.float32) - decoder_per_layer = np.zeros((1, 1, per_layer), dtype=np.float32) - decoder_position = np.zeros((1, 1), dtype=np.int64) - - stop_token_ids: set[int] = set() - encoded_stop_sequences: tuple = () - - def _trim_stop_suffix(_ids, _seqs): - return False - - def _run_step_token(token_id: int, position_id: int) -> np.ndarray: - lm_encoder_input_ids.fill(int(token_id)) - lm_encoder_position.fill(int(position_id)) - # graph.execute() — mocked as no-op - decoder_inputs_embeds[:] = fake_inputs_embeds - decoder_per_layer[:] = fake_per_layer - decoder_position[:] = fake_position - # decoder graph.execute() — mocked as no-op - return np.asarray(fake_logits) - - requested_tokens = 32 - - def call() -> None: - generated_ids: list[int] = [] - next_position_id = 0 - logits = _run_step_token(0, next_position_id) - for step_index in range(requested_tokens): - next_token_id = int(np.argmax(logits[0, -1])) - generated_ids.append(next_token_id) - if next_token_id in stop_token_ids: - break - if _trim_stop_suffix(generated_ids, encoded_stop_sequences): - break - if step_index + 1 >= requested_tokens: - break - logits = _run_step_token(next_token_id, next_position_id) - next_position_id += 1 - - try: - call() - samples = _timed(call, iterations=iterations) - except Exception as exc: - return _failed(name, desc, exc) - return _summarize( - name, - desc, - samples, - {"vocab_size": vocab, "tokens_per_iter": requested_tokens, "hidden_dim": hidden, "per_layer_dim": per_layer}, - ) - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--iterations-light", type=int, default=10) - parser.add_argument("--iterations-loop", type=int, default=200) - parser.add_argument("--out", type=Path, default=DEFAULT_OUT) - args = parser.parse_args() - - benches = [ - ("audio_preprocess", bench_audio_preprocess, args.iterations_light), - ("image_preprocess_gemma4", bench_image_preprocess_gemma4, args.iterations_light), - ("image_load_basic", bench_image_load_basic, args.iterations_light), - ("tdt_inner_loop", bench_tdt_inner_loop, args.iterations_loop), - ("cached_step_inner_loop", bench_cached_step_inner_loop, args.iterations_loop), - ] - - results: list[BenchResult] = [] - for label, fn, iters in benches: - print(f"[{label}] running ({iters} iterations)...", flush=True) - r = fn(iters) - status = "OK " if r.ok else "FAIL" - print( - f"[{label}] {status} mean={r.mean_ms:.4f}ms p50={r.p50_ms:.4f}ms " - f"p95={r.p95_ms:.4f}ms total={r.total_ms:.1f}ms peak_rss={r.peak_rss_mb:.1f}MB", - flush=True, - ) - if not r.ok: - print(r.error, file=sys.stderr) - results.append(r) - - payload = { - "captured_at": time.strftime("%Y-%m-%dT%H:%M:%S"), - "platform": platform.platform(), - "python": sys.version.split()[0], - "commit": os.popen(f"git -C {REPO_ROOT} rev-parse HEAD").read().strip(), - "note": ( - "Pre-refactor Python orchestration microbench. Graph kernels are " - "mocked: numbers reflect ONLY the per-call/per-iteration Python " - "wrapper overhead that the C++ port replaces." - ), - "benches": [ - { - "name": r.name, - "description": r.description, - "ok": r.ok, - "error": r.error, - "iterations": r.iterations, - "total_ms": round(r.total_ms, 4), - "mean_ms": round(r.mean_ms, 4), - "p50_ms": round(r.p50_ms, 4), - "p95_ms": round(r.p95_ms, 4), - "peak_rss_mb": round(r.peak_rss_mb, 2), - "notes": r.notes, - } - for r in results - ], - } - args.out.parent.mkdir(parents=True, exist_ok=True) - args.out.write_text(json.dumps(payload, indent=2)) - print(f"\nWrote {args.out}") - return 0 if all(r.ok for r in results) else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/benchmarks/python_vs_cpp/results.json b/benchmarks/python_vs_cpp/results.json deleted file mode 100644 index 599f3df56..000000000 --- a/benchmarks/python_vs_cpp/results.json +++ /dev/null @@ -1,89 +0,0 @@ -{ - "captured_at": "2026-05-20", - "branch": "justin/platform-apis", - "machine": "Apple Silicon (arm64 NEON)", - "comparison": { - "Parakeet TDT 0.6B (assets/test.wav, ~15s clip)": { - "python_v2": { - "total_ms_runs": [1470, 748, 711], - "total_ms_warm": 730, - "decoder_steps": 111, - "encoder_ms_warm": 600, - "response": "Hello, hello, hello. Just um quickly testing out creating a wave file through voice memos. Um the goal is to use this wave file to test out Whisper. Hopefully, this will transcribe properly. That's all I can hope for. Alright, here we go." - }, - "cpp_asr": { - "total_ms_runs": [1513, 679, 679], - "total_ms_warm": 679, - "decode_tokens": 88, - "response": "Hello, hello, hello. Just um quickly testing out creating a wave file through voice memos. Um the goal is to use this wave file to test out Whisper. Hopefully, this will transcribe properly. That's all I can hope for. Alright, here we go." - }, - "notes": "Response byte-identical. C++ warm total ~7% faster; both paths share the same C++ graph kernels — Python overhead is the per-step argmax/state-copy wrapper that the C++ TDT loop replaces. Python emits more decoder_steps because it counts skipped frames too." - }, - "LFM2-VL-450M text": { - "python_v2": { - "decode_tps": 149.6, - "ttft_ms": 320.6, - "total_ms": 467.7, - "decode_tokens": 23, - "response": "Hello! I'm doing well, thank you for asking. How about you? How are you doing today?" - }, - "cpp_chat": { - "decode_tps": 144.4, - "ttft_ms": 117, - "total_ms": 262, - "decode_tokens": 22, - "ram_mb": 420.6, - "response": "Hello! I'm doing well, thank you for asking. How about you? How are you doing today?" - }, - "notes": "Decode rates within noise. C++ TTFT is ~2.7x faster (no Python import overhead)." - }, - "LFM2-VL-450M vision (test_monkey.png)": { - "python_v2": { - "decode_tps": 147.7, - "ttft_ms": 1036.5, - "total_ms": 1111.0, - "decode_tokens": 12, - "response": "A close up of a brown face with yellow eyes." - }, - "cpp_chat": { - "decode_tps": 135.0, - "ttft_ms": 728, - "total_ms": 802, - "decode_tokens": 11, - "ram_mb": 2132.1, - "response": "A close up of a brown face with yellow eyes." - }, - "notes": "Response byte-identical. C++ TTFT ~30% faster. Decode rates within noise." - }, - "Gemma 4 E2B text": { - "python_v2": null, - "cpp_chat": { - "decode_tps": 29.7, - "ttft_ms": 1768, - "total_ms": 2777, - "decode_tokens": 31, - "ram_mb": 914.0, - "response": "I'm doing well, thank you for asking! I'm ready to assist you with whatever you need." - }, - "notes": "v2 Python cannot load this bundle (missing constant payload error). C++ only baseline." - }, - "Gemma 4 E2B vision (test_monkey.png)": { - "python_v2": null, - "cpp_chat": { - "decode_tps": 25.1, - "ttft_ms": 11133, - "total_ms": 12806, - "decode_tokens": 43, - "ram_mb": 9562.6, - "response": "The image is a close-up of the face of a marmoset, showing its nose and eyes. The marmoset has a prominent nose and appears to be smiling. The background is blurred green foliage." - }, - "notes": "Vision encoder over 2520 patches dominates TTFT (~11s). Decode rate stable at ~25 tok/s. Response identifies marmoset correctly." - } - }, - "untested": [ - "Streaming transcribe (cactus_stream_transcribe_*)", - "RAG query (cactus_rag_query) - needs corpus dir", - "VAD path (cactus_vad)", - "Image/audio embed (Cactus.image_embed / audio_embed) - LFM-VL embed FFI fails with EmbeddingFailed" - ] -} diff --git a/bindings/README.md b/bindings/README.md index c43d5a420..dd7d63599 100644 --- a/bindings/README.md +++ b/bindings/README.md @@ -1,9 +1,5 @@ # Bindings -Language-facing interop layers live here. - -These are bindings, not SDKs: - - `python/` uses `ctypes` - `swift/` uses a C module map - `kotlin/` uses JNI @@ -15,5 +11,3 @@ Platform-native build and packaging stay outside this folder: - [`android/`](/android/) builds Android native artifacts - [`apple/`](/apple/) builds Apple native artifacts - -Use `sdk` terminology only for packaged, consumer-ready layers with a stable public API. diff --git a/blog/README.md b/blog/README.md new file mode 100644 index 000000000..420c7d498 --- /dev/null +++ b/blog/README.md @@ -0,0 +1,10 @@ +# Cactus Blog + +| Post | Author | Description | +|------|--------|-------------| +| [TurboQuant-H](turboquant-h.md) | Karen Mosoyan & Henry Ndubuaku | Hadamard rotation for 2-bit per-layer embedding quantization: 4x PLI compression, 40% total model size reduction | +| [Gemma 4 on Cactus](gemma4.md) | Henry Ndubuaku | Native multimodal voice, vision, and audio on-device with hybrid cloud handoff | +| [Hybrid Transcription](hybrid_transcription.md) | Roman Shemet | Sub-150ms transcription with cloud-level accuracy using on-device/cloud hybrid inference | +| [On-Device Coding Agents](lfm2_24b_a2b.md) | Noah Cylich & Henry Ndubuaku | Running LFM2-24B MoE locally on Mac for coding use cases | +| [Ridiculously Fast Transcription](parakeet.md) | Satyajit Kumar & Henry Ndubuaku | 6M tok/sec decode speed with Parakeet CTC 1.1B | +| [LFM-2.5-350m on Cactus](lfm2.5_350m.md) | Henry Ndubuaku | 140 tok/sec single-core INT8 inference across seven devices, from Vision Pro to Raspberry Pi 5 | \ No newline at end of file diff --git a/blog/gemma4.md b/blog/gemma4.md new file mode 100644 index 000000000..c96f62dac --- /dev/null +++ b/blog/gemma4.md @@ -0,0 +1,141 @@ +--- +title: "Gemma 4 on Cactus: The first model you can talk to, show things, and trust to know when it needs help" +description: "Gemma 4 runs natively on your device with real-time voice, vision, and audio, and routes hard problems to the cloud when it should." +keywords: ["gemma4", "multimodal", "hybrid", "realtime inference", "on-device inference", "Google", "DeepMind"] +author: "Henry Ndubuaku & The Cactus Jacks" +date: 2026-04-01 +tags: ["gemma4", "cactus", "on-device", "hybrid inference", "multimodal", "voice AI"] +--- + +Gemma 4 is the first on-device model that genuinely works across text, vision, and audio in a single architecture, and does it well enough that you stop thinking about the model and start thinking about what to build with it. + +We shipped day-one support in Cactus. Here's what that means in practice and why we think this changes the trajectory of on-device AI. + +## What's actually new here + +Most "multimodal" models bolt on vision or audio as an afterthought. Gemma 4 is different. The vision encoder, audio encoder, and language model were trained together from the start. The audio conformer isn't a whisper-style bolt-on. It's a 300M-parameter encoder that feeds directly into the transformer's residual stream. Same for vision. The model doesn't "transcribe then think." It reasons over the raw modality. + +This matters because latency compounds. A pipeline that transcribes audio, then feeds text to an LLM, then generates a response has three serialized steps. Gemma 4 collapses that into one forward pass. On a modern ARM device, that means 0.3 seconds from the end of a 30-second voice clip to the first token of a response. Not 0.3 seconds for the transcription step. 0.3 seconds total. + +## Performance + +Cactus targets ARM across platforms: Apple Silicon Macs, iPhones, iPads, Vision Pro, and Android devices with ARM64 chipsets (Snapdragon, Dimensity, Tensor). On Apple hardware we additionally leverage the Neural Engine for vision and audio encoders, while on Android and Linux the runtime uses NEON, i8mm, and dot-product intrinsics. + +| Metric (M5 Mac/iPad/Vision Pro) | E2B | +|---|---| +| 4096-token prefill | 660 tok/s | +| 1024-token decode | 40 tok/s | +| 30s audio end-to-end | 0.3s | +| Image encode (ANE) | 0.7s | + +40 tok/s decode is faster than most people read. You're not waiting for the model. The model is waiting for you. + +## The benchmarks are hard to believe + +We normally don't lead with benchmarks because they rarely reflect real-world use. But these numbers demand attention. + +### LLM + +| Benchmark | E4B | E2B | Gemma 3 27B (no think) | +|---|---|---|---| +| MMLU Pro | 69.4% | 60.0% | 67.6% | +| AIME 2026 (no tools) | 42.5% | 37.5% | 20.8% | +| LiveCodeBench v6 | 52.0% | 44.0% | 29.1% | +| Codeforces ELO | 940 | 633 | 110 | +| GPQA Diamond | 58.6% | 43.4% | 42.4% | +| Tau2 (avg over 3) | 42.2% | 24.5% | 16.2% | +| BigBench Extra Hard | 33.1% | 21.9% | 19.3% | +| MMMLU | 76.6% | 67.4% | 70.7% | +| MRCR v2 8-needle 128k (avg) | 25.4% | 19.1% | 13.5% | + +E4B, a 4.5B-effective-parameter model running on your phone, outperforms Gemma 3 27B on nearly every benchmark. E2B, the smaller variant at 2.3B effective parameters, matches or beats it on half of them. The AIME and LiveCodeBench numbers are particularly striking: these are hard reasoning tasks where you'd expect scale to dominate. + +### Vision + +| Benchmark | E4B | E2B | Gemma 3 27B (no think) | +|---|---|---|---| +| MMMU Pro | 52.6% | 44.2% | 49.7% | +| OmniDocBench 1.5 (edit dist, lower=better) | 0.181 | 0.290 | 0.365 | +| MATH-Vision | 59.5% | 52.4% | 46.0% | +| MedXPertQA MM | 28.7% | 23.5% | - | + +E4B beats Gemma 3 27B on vision tasks across the board. Document understanding (OmniDocBench) is 2x better. This isn't a toy. It's genuinely useful for reading receipts, parsing handwritten notes, understanding diagrams. + +### Audio + +| Benchmark | E4B | E2B | +|---|---|---| +| CoVoST | 35.54 | 33.47 | +| FLEURS (lower=better) | 0.08 | 0.09 | + +## Model Architecture + +| Property | E2B | E4B | +|---|---|---| +| Total Parameters | 2.3B effective (5.1B w/ embeddings) | 4.5B effective (8B w/ embeddings) | +| Layers | 35 | 42 | +| Sliding Window | 512 tokens | 512 tokens | +| Context Length | 128K tokens | 128K tokens | +| Vocabulary Size | 262K | 262K | +| Supported Modalities | Text, Image, Audio | Text, Image, Audio | +| Vision Encoder | ~150M params | ~150M params | +| Audio Encoder | ~300M params | ~300M params | + +The architecture uses per-layer embeddings with AltUp, a technique that keeps most of the vocabulary knowledge in a shared embedding table while giving each layer a small specialized projection. This is how they fit 262K vocabulary tokens into a model that runs on a phone. The sliding window attention at 512 tokens with global attention every 5 layers gives you 128K context without the quadratic memory cost. + +## What this unlocks + +### Voice control that actually works + +We've all used voice assistants that feel like speaking into a form field. The voice gets transcribed, the text gets processed, and you get a response that could just as easily have been typed. Gemma 4 doesn't work that way. Because it reasons directly over the audio signal, it picks up on tone, hesitation, emphasis. It knows the difference between "delete that" spoken confidently and "delete that?" spoken as a question. + +On headsets running Cactus (Vision Pro, Quest, or any AR/VR platform with ARM compute), this means spatial voice control that responds to how you speak, not just what you say. Ask it to "move that over there" while looking at a 3D object and gesturing, and the model has the audio understanding to disambiguate "that" from context. The 0.3s latency means the interaction feels physical. You speak, things happen. + +### Voice agents that run locally + +If you're building a voice agent (a customer service bot, an in-car assistant, a medical triage system) you currently have two bad options. You can run everything in the cloud and deal with latency, cost, and privacy issues. Or you can cobble together a local pipeline of ASR + LLM + TTS and deal with error propagation and integration pain. + +Gemma 4 on Cactus gives you a third option. The model handles voice understanding natively. You pipe audio in, you get structured responses out. Tool calling works. The model can decide to call functions, hit APIs, or route to a cloud model when the task exceeds its capability. And because it runs locally, there's no per-request cost and no audio leaving the device. + +For healthcare, legal, and finance applications where audio data is sensitive, this isn't a nice-to-have. It's a compliance requirement. + +### Hybrid inference: knowing when to ask for help + +This is the part we're most excited about. Gemma 4 is good, but it's not omniscient. A 2B-parameter model isn't going to write a production database migration or debug a complex distributed system. What it can do is recognize when a task is beyond its capability and route it to a frontier cloud model. + +Cactus implements this as cloud handoff. The on-device model evaluates the complexity of the request, and if it determines it can't handle it confidently, it signals for handoff. The request goes to a cloud model (Claude, GPT-4, Gemini, your choice), the response comes back, and the user sees a seamless interaction. The on-device model handles the 80% of requests that are straightforward. The cloud handles the 20% that need heavy lifting. + +This means you can build apps that feel like they have frontier-model intelligence while keeping costs at a fraction of full cloud inference. And the user's casual conversations, voice notes, and image queries never leave their device. + +### Building on any platform + +Cactus runs Gemma 4 on macOS, iOS, Android, and Linux. + +## Try it + +```bash +brew install cactus-compute/cactus/cactus +cactus run google/gemma-4-E2B-it +``` + +The weights download automatically. INT4 quantized, optimized for ARM. Both E2B and E4B are available. + +## Build with it + +Cactus supports React Native, Flutter, Swift, Kotlin, Python, Rust, and C++. Pick the binding that fits your stack: + +| Platform | Binding | Install | +|---|---|---| +| React Native | [cactus-react-native](https://github.com/cactus-compute/cactus-react-native) | `npm install cactus-react-native` | +| Flutter | [cactus-flutter](https://cactus.dev/flutter/) | Dart FFI bindings | +| Swift (iOS/macOS) | [cactus-swift](https://cactus.dev/swift/) | XCFramework with NPU support | +| Kotlin (Android) | [cactus-kotlin](https://cactus.dev/kotlin/) | JNI + Kotlin Multiplatform | +| Python | [cactus-ai](https://cactus.dev/python/) | `pip install cactus-ai` | +| Rust | [cactus-sys](https://cactus.dev/rust/) | `cargo add cactus-sys` | +| C++ | [cactus.h](https://cactus.dev/docs/cactus_engine/) | Single header, link `libcactus.a` | + +Full quickstart with code examples for every binding: [cactus.dev/docs/quickstart](https://cactus.dev/docs/quickstart/) + +Pre-quantized weights for Gemma 4 and 30+ other models: [huggingface.co/cactus-compute](https://huggingface.co/cactus-compute) + +If you're building something with on-device multimodal AI (voice agents, VR interfaces, local-first apps) we want to hear about it. Open an issue on [GitHub](https://github.com/cactus-compute/cactus), or just start building. diff --git a/blog/hybrid_transcription.md b/blog/hybrid_transcription.md new file mode 100644 index 000000000..2d353dd68 --- /dev/null +++ b/blog/hybrid_transcription.md @@ -0,0 +1,81 @@ +--- +title: "Sub-150ms Transcription with Cloud-Level Accuracy: Why We Built a Hybrid Engine" +description: "How Cactus combines on-device and cloud inference for real-time speech transcription with sub-150ms latency and automatic cloud handoff for noisy audio." +keywords: ["hybrid AI", "speech transcription", "on-device inference", "cloud handoff", "Whisper", "voice interface", "edge AI"] +author: "Roman Shemet" +date: 2026-02-01 +tags: ["transcription", "hybrid AI", "edge AI", "whisper", "cloud handoff"] +--- + +# Sub-150ms transcription with cloud-level accuracy: Why we built a hybrid engine + +*By Roman Shemet* + +## The Voice Interface Dilemma +Voice is no longer a novelty; it is quickly becoming the default interface. From meeting transcriptions and dictation to voice notes and ambient computing, users expect to speak to their devices. + +But building a seamless voice experience is hard. If you've ever built a voice product, you are intimately familiar with the iron triangle of AI: Cheap, Accurate, Fast. + +Historically, you could only pick two: +- **Fast and Accurate?** It won't be cheap. You'll need heavy cloud compute. +- **Fast and Cheap?** It won't be accurate. You're running tiny, compromised models. +- **Cheap and Accurate?** It won't be fast. You're waiting in API queues. + +The logic is simple. High-accuracy transcription requires large models. Large models require massive cloud resources. This introduces network latency and high infrastructure costs. + +Or, you can run small models on-device. The latency is practically zero, and compute is free. But smaller models struggle with noisy environments, accents, and complex vocabulary. + +You either pay the "cloud tax" in latency and cost, or you pay the "edge tax" in accuracy. + +## Enter Hybrid AI: The Best of Both Worlds +We didn't want to choose, so we spent the last few months building a third option at Cactus. We call it Hybrid AI. + +> What if we had small, fast on-device models that were self-aware enough to **know** when they are struggling? + +Instead of routing 100% of user audio to an expensive cloud API, the Cactus engine processes speech locally by default. Our on-device inference provides real-time transcription with sub-150ms latency. + +However, when the engine detects messy audio (think background noise, static, or multiple speakers), it automatically hands that specific segment off to a larger model in the cloud to clean it up. + +It's like having a brilliant intern who processes 80% of the work instantly, but knows exactly when to call the senior engineer to review the tricky edge cases. + +## See it in Action +Here is a look at the hybrid engine running locally, seamlessly managing the local-to-cloud transcription flow: + +[![Realtime Hybrid Transcription With Cactus](https://img.youtube.com/vi/0SldBm8rGkE/maxresdefault.jpg)](https://youtu.be/0SldBm8rGkE?si=TLLYnDuGEG8Rqexw) + +As a Mac user, you can pull down the CLI and test the latency and accuracy locally right now: + +```bash +brew install cactus-compute/cactus/cactus +cactus transcribe +``` + +## Built for the Edge, from the Ground Up +Achieving this required relying on our industry-leading on-device engine. We built Cactus from the ground up exactly for this. + +Because the engine sits directly on the metal, it is optimized for edge constraints. It targets ARM-based CPUs and NPUs natively, resulting in super low RAM usage (e.g. Moonshine transcription for example, runs with sub-10MB RAM utilization). This is a critical factor for mobile and wearable devices where memory is strictly rationed by the OS. + +By keeping the footprint minimal, the engine operates silently in the background without draining battery or causing OS-level memory warnings, making it viable for continuous, ambient listening. + +## A Single API for Every Platform +While the underlying engine is low-level C++, we know that product teams need to move fast in their native environments. We designed the integration to be a drop-in experience regardless of your stack. + +The interface remains the same whether you are a low-level C++ engineer, or a mobile developer. You can integrate the Cactus Hybrid AI engine directly into your stack: +- **Mobile:** React Native, Flutter, Kotlin, Swift +- **Systems:** C++, Rust +- **Scripting:** Python + +At the top level, the implementation is beautifully simple: initialize the model, pass the audio stream, and receive real-time text. + +A few levels deeper, developers have granular control. You can configure the engine's behavior to define a specific Word Error Rate (WER) threshold, target a strict cloud-handoff ratio, or establish hard cost limits per user. + +Check out the open-source engine and documentation on GitHub. + +## See Also + +- [Cactus Engine API Reference](/docs/cactus_engine.md) — Full transcription API docs (`cactus_transcribe`, `cactus_stream_transcribe_*`) +- [Python Binding](/python/) — Python bindings with streaming transcription support +- [Swift Binding](/bindings/swift/) — Swift API with streaming transcription +- [Kotlin Binding](/bindings/kotlin/) — Kotlin API with streaming transcription support +- [Flutter Binding](/flutter/) — Flutter bindings with streaming transcription +- [LFM2 24B Review](/blog/lfm2_24b_a2b.md) — Running large MoE models locally with Cactus \ No newline at end of file diff --git a/blog/lfm2.5_350m.md b/blog/lfm2.5_350m.md new file mode 100644 index 000000000..75ca391c7 --- /dev/null +++ b/blog/lfm2.5_350m.md @@ -0,0 +1,204 @@ +--- +title: "LFM-2.5-350m on Cactus: 140 tok/sec, Single Core, 355 MB" +description: "Benchmarking Liquid's LFM-2.5-350m across seven devices with Cactus. INT8 quantization, single-core CPU decode, zero-copy loading, and why this configuration makes on-device inference practical." +keywords: ["LFM-2.5", "on-device inference", "INT8 quantization", "ARM CPU", "mobile AI", "edge inference", "Cactus"] +author: "Henry Ndubuaku" +date: 2026-03-31 +tags: ["LFM", "INT8", "edge AI", "mobile inference", "ARM", "benchmarks"] +--- + +# LFM-2.5-350m on Cactus: 140 tok/sec, Single Core, 355 MB + +*By Henry Ndubuaku* + +We ran Liquid's LFM-2.5-350m through the Cactus inference engine across seven devices, from a Vision Pro to a Raspberry Pi 5. +The configuration: 1024-token prefill, 100-token decode, INT8 quantization, CPU-only, single-core decode. +The model file is 355 MB on Cactus Compressed format. + +## Device Benchmarks + +### Apple +| Device | Prefill (tok/s) | Decode (tok/s) | RAM | +|---|---|---|---| +| Vision Pro | 2,067 | 140 | 85 MB | +| iPhone 17 Pro | 797 | 140 | 75 MB | +| iPhone 13 Mini | 496 | 88 | 56 MB | + +### Android +| Device | Prefill (tok/s) | Decode (tok/s) | RAM | +|---|---|---|---| +| Galaxy S25 Ultra | 660 | 100 | 322 MB | +| Google Pixel 6a | 208 | 42 | 328 MB | +| Galaxy A56 | 110 | 24 | 330 MB | + +### Other +| Device | Prefill (tok/s) | Decode (tok/s) | RAM | +|---|---|---|---| +| Raspberry Pi 5 | 200 | 30 | 300 MB | + +Two things stand out in this data. First, the decode speeds: 140 tok/sec on a single core is fast, +most people read at roughly 4-5 words per second, so even the Raspberry Pi's 30 tok/sec is well ahead of human consumption. +Second, the RAM gap between Apple and Android: 56-85 MB versus 300-330 MB for the same 355 MB model. +That gap is not a bug. It's a direct consequence of how the two platforms handle memory-mapped files, which we'll get into in the Cactus format section below. + +## Why CPU, Not GPU or NPU + +It is tempting to reach for the GPU. On paper, mobile GPUs have 10-50x the FLOPS of a single CPU core. +But inference on a 350M-parameter model is not a FLOPS problem, it's a memory bandwidth problem. +During decode, you generate one token at a time, which means you read the entire weight matrix for each token but only perform a single matrix-vector multiply. +The arithmetic intensity (FLOPS per byte loaded) is extremely low. Without batched decode, GPU compute units spend most of their time stalled on memory fetches. +GPU excels when arithmetic intensity is high, large batch sizes, long matrix-matrix multiplies. +For single-token decode, the GPU's compute units sit mostly idle while waiting for memory. + +There's also the practical problem: mobile GPU inference requires Metal on iOS and Vulkan or OpenCL on Android. +That's two separate shader codepaths to maintain, with different memory models, different synchronization primitives, and different performance characteristics per vendor. +And GPU inference holds a persistent GPU context, which competes with the UI rendering pipeline and drains battery even when the inference workload is light. +GPUs are also energy-inefficient. On Macs and PCs this barely matters since they're plugged into a power source most of the time, and GPU inference works well there. But mobile devices run on battery, and every watt spent on GPU inference cuts directly into the user's runtime. The GPU path that works fine on a MacBook becomes a battery killer on a phone. + +NPU is a more interesting target. Apple's Neural Engine and Qualcomm's Hexagon DSP both offer high-throughput inference at good power efficiency. +The problem is access. Apple's Neural Engine requires CoreML model conversion, which imposes its own quantization and graph constraints. +You don't control the execution schedule, the memory layout, or the precision semantics. +The quantisation techniques available through CoreML severely lag the state-of-the-art methods used by GGML and Cactus. +Qualcomm's QNN SDK has similar restrictions (limited quantisation scheme support) and not all Android devices ship Qualcomm chips. +Both lock you into a vendor-specific format that may not support your model architecture at all, and neither works on budget devices that lack the NPU entirely. + +CPU is the universal target. Every ARM device, flagship, budget, wearable, Raspberry Pi, and most recent ones support DOTPROD or I8MM extensions. +A single well-optimized CPU codepath works everywhere, the main challenge is energy-inefficiency compared to NPU. + +ARM is also closing this gap. The progression from NEON to DOTPROD to I8MM to SME2 reflects a deliberate push to bring matrix-processing efficiency onto the CPU itself. Each generation narrows the energy and throughput gap with dedicated NPUs. SME2, available on Apple's M4 and upcoming ARMv9.2 cores, adds streaming matrix operations that approach NPU-class efficiency for INT8 workloads while remaining fully programmable. Cactus already compiles SME2 codepaths where available. The bet is that general-purpose CPU silicon will continue absorbing the capabilities that once justified a separate accelerator. + +## Why Single Core for Decode + +Decode is memory-bandwidth-bound. For a 350M INT8 model, each decode step reads roughly 355 MB of weights to produce a single token. +On an iPhone 17 Pro with ~60 GB/s memory bandwidth, the theoretical maximum decode speed is around 169 tok/sec, +our measured 140 tok/sec is 83% of that ceiling, the haircut comes from our choice to stream weights from storage for memory efficiency, reducing OS chances of throttling background inference. +Adding more cores doesn't give you more memory bandwidth; it just adds synchronization overhead. + +Prefill is different. With 1024 tokens in the input, each weight load amortizes across 1024 multiply-accumulate operations. +The arithmetic intensity is 1024x higher than decode, making prefill genuinely compute-bound. +That's why we use multi-threaded prefill, the Vision Pro's 2,067 tok/sec prefill speed reflects all available cores working in parallel. + +But there's a deeper reason single-core decode matters for mobile: background execution. +On both iOS and Android, the OS aggressively throttles multi-core workloads that run in the background. +iOS will suspend or terminate background tasks that consume excessive CPU across multiple cores. +Android's thermal management will cap frequency on the performance cores. +We found that single-core decode workload, by contrast, looks like a lightweight background task to the OS scheduler. +It doesn't trigger thermal throttling, doesn't compete with foreground apps, and doesn't get killed. +This is the difference between inference as a demo and inference as a product feature, the model needs to run while the user is doing something else. + +## Why INT8 + +At 350M parameters, quantization precision matters more than at larger scales. +Quantization works by approximating continuous weight values with discrete integers, and the approximation error is relative to the dynamic range of each weight group. +Smaller models have fewer parameters absorbing this error, so each parameter carries more of the model's capacity. +INT4 quantization at 350M parameters produces measurable accuracy degradation, our internal testing shows 2-4 point drops on MMLU compared to INT8, which is near-lossless at this scale. + +Cactus uses grouped affine quantization: weights are divided into groups of 32 elements, and each group gets its own scale factor. +For each group, the scale is computed as `max(|w|) / 127`, and each weight is rounded to the nearest integer in [-128, 127]. +During inference, the INT8 values are multiplied by their group's scale to reconstruct the approximate FP32 value. +Per-group scaling is critical, a single outlier weight in a per-tensor scheme would compress the effective precision of every other weight in that tensor. + +The performance benefit is concrete. On ARM CPUs with I8MM support (ARMv8.2-a and later), the `SMMLA` instruction computes a 2x8 by 8x2 INT8 matrix multiply in a single instruction, accumulating into INT32. +This is a native hardware operation, not a software emulation of low-precision arithmetic. +On devices without I8MM (like the Pixel 6a's Tensor G1), Cactus falls back to DOTPROD instructions, which compute 4-element INT8 dot products. +Both paths are substantially faster than FP16 arithmetic because they process more elements per instruction and use less memory bandwidth per parameter. + +## The Cactus Format and Zero-Copy Loading + +The RAM numbers in our benchmarks tell a story about memory architecture. +The LFM-2.5-350m model file is 355 MB. On Apple devices, runtime RAM usage is 56-85 MB. On Android, it's 300-330 MB. +The model is the same. The quantization is the same. The difference is in how the operating system handles memory-mapped files. + +The Cactus format (identified by the magic number `0x54434143`, "CACT") is a binary container designed around `mmap()`. +Rather than loading the entire model into a heap allocation, Cactus maps the file directly into virtual address space with `mmap(PROT_READ, MAP_SHARED)`. +The weights stay on disk. When the inference engine accesses a weight tensor, the OS kernel page-faults it into physical memory on demand. +When memory pressure rises, the OS can evict those pages, they're backed by the file, so no writeback is needed. + +On Apple Silicon, this works exceptionally well. +The unified memory architecture means there's no distinction between CPU memory and file cache, +a memory-mapped page that's been paged in *is* the physical memory the CPU reads from. +The OS can manage a working set that's a fraction of the total model size because not all layers are active simultaneously during a single decode step. + +On Android, the behavior varies by kernel version and vendor. +Many Android devices aggressively copy memory-mapped pages into anonymous memory due to SELinux policies and vendor-specific memory management. +The result is that `mmap()` often degrades to something closer to `malloc() + read()`, explaining the 300+ MB footprint. + +The format itself is designed for SIMD-friendly access. +Weight data is aligned to 32-byte boundaries (matching ARM NEON register width), +and an optional block-interleaved layout groups 4 columns of weights together so that a single vector load fetches data for 4 output neurons simultaneously. +This layout eliminates the need for runtime transposition and ensures every memory access is a sequential, aligned read, optimal for both cache prefetching and NEON/I8MM instruction operands. + +## Model Quality + +A fast small model is only useful if it's accurate enough. Here's how the LFM-2.5 family compares to similarly-sized open models: + +| Benchmark | LFM2-350M | LFM2-700M | LFM2-1.2B | Qwen3-0.6B | Qwen3-1.7B | Llama-3.2-1B | gemma-3-1b-it | +|---|---|---|---|---|---|---|---| +| MMLU | 43.43 | 49.90 | **55.23** | 44.93 | **59.11** | 46.60 | 40.08 | +| GPQA | **27.46** | **28.48** | **31.47** | 22.14 | 27.72 | 28.84 | 21.07 | +| IFEval | **65.12** | **72.23** | **74.89** | 64.24 | 73.98 | 52.39 | 62.90 | +| IFBench | 16.41 | **20.56** | **20.70** | 19.75 | **21.27** | 16.86 | 17.72 | +| GSM8K | 30.10 | 46.40 | 58.30 | 36.47 | 51.40 | 35.71 | **59.59** | +| MGSM | 29.52 | 45.36 | 55.04 | 41.28 | **66.56** | 29.12 | 43.60 | +| MMMLU | **37.99** | **43.28** | **46.73** | 30.84 | 46.51 | 38.15 | 34.43 | + +The numbers worth examining closely: + +**IFEval (instruction following):** LFM2-350M scores 65.12, versus 52.39 for Llama-3.2-1B (a model 3x its size) and 64.24 for Qwen3-0.6B (nearly 2x its size). For on-device applications, +where the model is typically following structured instructions from an app rather than engaging in open-ended conversation, +instruction following is arguably the most important benchmark. The LFM2-1.2B's 74.89 is within a point of Qwen3-1.7B's 73.98, at 70% of the parameters. + +**GPQA (graduate-level reasoning):** LFM2-350M at 27.46 beats both Qwen3-0.6B (22.14) and gemma-3-1b-it (21.07). +This is notable because GPQA tests reasoning depth, not just pattern matching, it suggests the LFM architecture is efficient at encoding reasoning capability per parameter. + +**MMMLU (multilingual knowledge):** LFM2-350M scores 37.99 versus Qwen3-0.6B's 30.84. +A 7-point lead over a model nearly twice the size suggests strong multilingual data efficiency. The full LFM2-1.2B hits 46.73, matching Qwen3-1.7B's 46.51. + +**Where LFM2-350M trails:** GSM8K (30.10) and MGSM (29.52) math benchmarks. gemma-3-1b-it leads GSM8K at 59.59, and Qwen3-1.7B leads MGSM at 66.56. +Math is where the parameter count ceiling hits hardest, since multi-step arithmetic requires retaining intermediate state that larger models can distribute across more capacity. +For pure math tasks at this scale, a larger model is the right answer. + +## What This Adds Up To + +The LFM-2.5-350m at INT8 on Cactus is a 355 MB model that decodes at 88 tok/sec on a three-year-old iPhone 13 Mini using 56 MB of RAM on a single CPU core. +It follows instructions better than models 3x its size. It runs as a background process without triggering OS throttling. +These are not theoretical numbers on a development board with a fan. They're measured on production consumer devices, under the constraints that real mobile apps face; +memory pressure from other apps, thermal limits, background execution policies, 4G download speeds for model delivery. +The question for on-device inference has always been whether the quality-speed-size tradeoff lands in a useful region. +At 350M parameters, LFM-2.5 on Cactus suggests it does: accurate enough for structured on-device tasks, fast enough to be invisible to the user, and small enough to ship inside an app. + +## Where is Cactus headed? + +On-device inference on a Mac or PC is a solved problem. You have gigabytes of RAM, a fast SSD, a multi-core CPU with wide SIMD, and a stable OS that doesn't kill your process for using too much battery. +Any reasonably written inference engine will run well on a MacBook and GPUs are a great idea, given these devices are plugged in anyway. + +Mobile devices, wearables, and custom hardware are a different story entirely. The ecosystem is incredibly fragmented. +You're dealing with hundreds of ARM chip variants across Apple, Qualcomm, MediaTek, Samsung, and Broadcom, each with different ISA extensions, different memory bandwidth, different thermal envelopes. +Android alone ships on devices ranging from 3 GB of RAM to 16 GB, with kernel versions spanning half a decade. +Wearables have even tighter constraints. Custom embedded hardware has its own. +A model that runs fine on a Galaxy S25 Ultra may crash on a Galaxy A56, not because the code is wrong, but because the memory management behavior differs at the kernel level. + +This fragmentation is what makes on-device AI difficult in production. It's a research and engineering problem that compounds at scale: every new device, every OS update, +every vendor-specific memory policy is another edge case that can break inference for some slice of your user base. +For enterprises shipping AI features to millions of devices, this is a hard pill. +The choice often comes down to small models that fit everywhere but are too weak to be useful, or capable models with large files that fail on half the device population. + +Cactus exists because we took on this specific slice of the problem; I spent the last few years building foundation models and inference for tiny devices. +The team's approach is systematic: profile every major ARM variant, write dedicated kernel paths for each ISA extension tier (NEON, DOTPROD, I8MM, SME2), +design the model format around the worst-case memory behavior (Android's mmap limitations), and test on real production devices, not emulators. +This it not-trivial; battling with chip manufactures with different brand visions, model developers chasing benchmarks over useability and more. + +The result is that businesses shipping with Cactus don't need to worry about inference failing silently on a budget Samsung, or a model that's accurate on benchmarks but too large for half their install base. +The kernel, the format, and the quantization strategy are all designed so that the same model file works correctly across the full device spectrum, with predictable RAM, predictable speed, and predictable quality. +On-device market is quite noisy, with hobbyists driving decisions, but production use is largely unsolved, Cactus is a bit unconventional but rightfully so. + +LFM-2.5-350m is a step in the right direction, hence this dedicated post from me personally. +Not just that LFM-2.5-350m is fast on one device, but on Cactus, it brings frontier intelligence to very different devices, +from a Vision Pro to a Raspberry Pi 5, with no special casing and no device-specific builds. + +## See Also + +- [Cactus Engine API Reference](/docs/cactus_engine.md) - Full C API docs for completion, tool calling, and cloud handoff +- [LFM2-24B-A2B](/blog/lfm2_24b_a2b.md) - Running LFM2 24B MoE locally on Mac for coding use cases +- [Hybrid Transcription](/blog/hybrid_transcription.md) - On-device/cloud hybrid speech transcription with Cactus +- [Ridiculously Fast Transcription](/blog/parakeet.md) - 6M tok/sec decode speed with Parakeet CTC 1.1B \ No newline at end of file diff --git a/blog/lfm2_24b_a2b.md b/blog/lfm2_24b_a2b.md new file mode 100644 index 000000000..2fde03658 --- /dev/null +++ b/blog/lfm2_24b_a2b.md @@ -0,0 +1,220 @@ +--- +title: "The Sweet Spot for Mac Code Use: Reviewing LFM2 24B MoE A2B with Cactus" +description: "Review of LiquidAI's LFM2-24B-A2B mixture-of-experts model running locally on Mac with Cactus. Architecture breakdown, benchmarks, and coding agent use cases." +keywords: ["LFM2", "mixture of experts", "MoE", "on-device coding", "Mac inference", "LiquidAI", "Apple Silicon"] +author: "Noah Cylich and Henry Ndubuaku" +date: 2026-02-15 +tags: ["LFM2", "MoE", "coding agents", "Apple Silicon", "Python", "function calling"] +--- + +# The Sweet Spot for Mac Code Use: Reviewing LFM2 24B MoE A2B with Cactus + +*By Noah Cylich and Henry Ndubuaku* + +[![Video Title](https://img.youtube.com/vi/-duR4gh_O10/maxresdefault.jpg)](https://youtu.be/-duR4gh_O10) + +LFM2-24B-A2B is a really great next step to see over the LFM2-8B-A1B model. The model features 24B total parameters, but only activates a sparse subset of 2B during inference. This allows it to be competitive in inference speed to 2B dense models, while delivering far greater performance. + +> "LFM2-24B-A2B excels at coding, keen to see on-device coding agents built with these." +> — Henry Ndubuaku, Cactus Co-founder & CTO + +## Architecture Breakdown + +Going into more depth about the model, there's really a lot to appreciate with all the architectural work LFM has accomplished, here's the breakdown: + +1. **GQA**: Grouped-query attention is the industry standard choice for efficient LLMs, their choice of a group size of 4 means that the KV cache is 4x smaller than standard, baseline attention. +2. **Gated Convolution**: This is the signature design choice of the Liquid series of models and efficiently adds parameters and expressiveness without much compute cost. +3. **Efficient Vocab**: The small vocab size of 65k is actually a strength for these models, as the final matmul vocab projection is the slowest static part of every model and is extremely parameter efficient. Gemma3 270m for instance dedicated 170m params just to its vocab projection since it has a vocab of 250k tokens. +4. **MoE**: Mixture of experts is the most important choice for this model that really separates it from Liquid's prior work, it scales up parameters without sacrificing speed. + +Together with Cactus, these choices enable lightning fast inference at low energy. Ultimately, despite being 24B params, only 200mb of running memory is used, while generating 25 TPS with our m4 pro chips with 48gb of ram. + + +## Model Architecture Diagram + +``` + ┌───────────────┐ + │ Linear │ + │Tied w/ Embed. │ + └───────┬───────┘ + │ + ┌─────┴─────┐ ┌───────────────────────┐ + │ Norm │ │ Gated Short │ + └─────┬─────┘ │ Convolution Block │ + │ │ │ + ┌──────────────────┐ ┌─────────⊕─────────────┐ │ ↑ │ + │ SwiGLU Expert │ │ │ │ │ ┌───┴───┐ │ + │ │ │ ┌───────┴───────────┐ │ │ │Linear │ │ + │ ↑ │ │ │ MoE Block │ │ │ └───┬───┘ │ + │ ┌───┴───┐ │ │ │ ↑ │ │ │ │ │ + │ │Linear │ │ │ │ ⊕ │ │ │ ┌──────►⊗ │ + │ └───┬───┘ │ │ │ ↗ ↗ ↖ ↖ │ │ │ │ ↑ │ + │ │ │ │ │ ⊗ ⊗ ⊗ ⊗ │ │ │ ┌───┴────┐ │ │ + │ ⊗ ◄───┐ │◄------------┤ │ ╎ ╎ ╎ ╎ │ │ ┌------►│ │ Conv1D │ │ │ + │ ↑ │ │ │ │ ↑ ↑ ↑ ↑ │ │ ╎ │ └───┬────┘ │ │ + │ │ ┌──┴──┐│ │ │ E1...E4...E9...E64│ │ ╎ │ │ │ │ + │ │ │SiLU ││ │ │ ↑ ↑ ↑ ↑ │ │ ╎ │ ⊗ ◄───┐ │ │ + │ │ └──┬──┘│ │ ├─┴────┴─────┴────┴─┤ │ ╎ │ ↑ │ │ │ + │ ┌───┴───┐ │ │ │ │ ┌───────────┐ │ │ ╎ │ B X C │ + │ │Linear ├─┘ │ │ │ │ Router │ │ │ ╎ │ ↑ ↑ ↑ │ + │ └───┬───┘ │ │ │ └─────┬─────┘ │ │ ╎ │ ┌──┴─────┴─┴──────┐ │ + │ ↑ │ │ │ │ │ │ ╎ │ │ Linear │ │ + └────────│─────────┘ │ └─────────┴─────────┘ │ ╎ │ └────────┬────────┘ │ + │ │ │ ╎ │ ↑ | + └───────────│───────────┘ × Num of Layers ╎ └───────────│───────────┘ + │ ╎ + ┌─────┴─────┐ ╎ + │ Norm │ ╎ ┌───────────────────────┐ + └─────┬─────┘ ╎ │ GQA Block │ + │ ╎ │ │ + ┌─────────⊕─────────┐ ╎ │ ↑ │ + │ │ │ ╎ │ ┌───┴───┐ │ + │ ┌───────┴───────┐ │ │ │ │Linear │ │ + │ │Sequence Block │ │ │ │ └───┬───┘ │ + │ └───────┬───────┘ ├-------------------┤ OR │ │ │ + │ │ │ │ │ ┌─────────┴─────────┐ │ + └─────────│─────────┘ │ │ │ Grouped Query │ │ + │ └------►│ │ Attention │ │ + ┌─────┴─────┐ │ └─┬───────┬───────┬─┘ │ + │ Norm │ │ Q K V │ + └─────┬─────┘ │ ↑ ↑ ↑ │ + │ │ ┌─┴──┐ ┌─┴──┐ │ │ + ┌─────┴─────┐ │ │Norm│ │Norm│ │ │ + │ Embedding │ │ └─┬──┘ └─┬──┘ │ │ + └─────┬─────┘ │ ↑ ↑ │ │ + │ │ ┌─┴───────┴───────┴─┐ │ + Input │ │ Linear │ │ + │ └─────────┬─────────┘ │ + │ ↑ │ + └───────────│───────────┘ +``` + +## Getting Started with LFM2-24B on Cactus + +Ready to run LFM2-24B locally on your Mac? Here's how to get up and running. + +### Prerequisites + +- macOS with Apple Silicon and 16GB+ RAM (M1 or later recommended; M4 Pro with 48GB RAM for best results) +- Python 3.10+ +- CMake (`brew install cmake`) +- Git + +### 1. Clone and Build + +```bash +git clone https://github.com/cactus-compute/cactus.git +cd cactus + +# Build the Cactus engine (shared library for Python FFI) +cactus build --python +``` + +### 2. Download the Model + +Cactus handles downloading and converting HuggingFace models to its optimized binary format with INT4/INT8 quantization, all in one command: + +```bash +cactus download LiquidAI/LFM2-24B-A2B +``` + +### 3. Chat Interactively + +The fastest way to start chatting with the model: + +```bash +cactus run LiquidAI/LFM2-24B-A2B +``` + +This builds, downloads (if needed), and launches an interactive chat session. + +### 4. Use the Python API + +For building your own applications and agents, use the Python FFI bindings directly: + +```python +import json +from cactus import cactus_init, cactus_complete, cactus_reset, cactus_destroy + +# Load the model +model = cactus_init("weights/lfm2-24b-a2b", None, False) + +# Simple chat completion +messages = json.dumps([{"role": "user", "content": "Write a Python function to sort a list"}]) +response = json.loads(cactus_complete(model, messages, None, None, None)) + +print(response["response"]) # Generated text +print(f"{response['decode_tps']:.1f} tokens/sec") + +# Streaming with a callback +def on_token(token, token_id): + print(token, end="", flush=True) + +cactus_complete(model, messages, None, None, on_token) + +# Clean up +cactus_reset(model) +cactus_destroy(model) +``` + +### 5. Function Calling for Agents + +Cactus supports tool use out of the box, a key building block for on-device coding agents: + +```python +tools = [ + { + "type": "function", + "function": { + "name": "run_code", + "description": "Execute Python code and return the output", + "parameters": { + "type": "object", + "properties": { + "code": {"type": "string", "description": "Python code to execute"} + }, + "required": ["code"] + } + } + } +] + +messages = json.dumps([{"role": "user", "content": "Calculate the factorial of 10"}]) +response = json.loads(cactus_complete(model, messages, None, json.dumps(tools), None)) + +if response["function_calls"]: + print(response["function_calls"]) # Model's tool invocation +``` + +### Cloud Handoff + +Cactus measures model confidence during generation. When the model isn't confident enough for a query, the response signals `cloud_handoff: true`, letting your agent route complex requests to a cloud API while keeping simple ones fast and local: + +```python +options = json.dumps({"confidence_threshold": 0.7}) +response = json.loads(cactus_complete(model, messages, options, None, None)) + +if response["cloud_handoff"]: + # Route to cloud API for this query + pass +else: + print(response["response"]) +``` + +This hybrid local-cloud pattern is what makes on-device coding agents practical: fast local inference for the majority of tasks, with automatic escalation when needed. + +## Conclusion + +LFM2-24B-A2B represents a compelling sweet spot for on-device coding. The MoE architecture activates just 2B of its 24B parameters per token, delivering quality that punches well above its compute class while keeping inference fast and memory-lean at ~200MB of running RAM. Paired with Cactus's INT4 quantization, SIMD-optimized kernels, and built-in function calling, this is a model you can actually build local coding agents on top of today. + +The pieces are coming together: models that are smart enough to be useful, efficient enough to run on a laptop, and runtimes that make it all accessible through a few lines of Python. Whether you're building a code assistant that works offline, a privacy-first dev tool, or just experimenting with what's possible without a cloud API, LFM2-24B with Cactus is a great place to start. + +Give it a try, build something, and let us know what you think. + +## See Also + +- [Cactus Engine API Reference](/docs/cactus_engine.md) — Full C API docs for completion, tool calling, and cloud handoff +- [Python Binding](/python/) — Python bindings used in the examples above +- [Fine-tuning Guide](/docs/finetuning.md) — Deploy your own LoRA fine-tunes to mobile +- [Hybrid Transcription](/blog/hybrid_transcription.md) — On-device/cloud hybrid speech transcription with Cactus +- [Runtime Compatibility](/docs/compatibility.md) — Weight versioning across Cactus releases \ No newline at end of file diff --git a/blog/parakeet.md b/blog/parakeet.md new file mode 100644 index 000000000..dd62694bf --- /dev/null +++ b/blog/parakeet.md @@ -0,0 +1,395 @@ +--- +title: "Ridiculously Fast On-Device Transcription: Reviewing Parakeet CTC 1.1B with Cactus" +description: "Review of NVIDIA's Parakeet-CTC-1.1B model running locally on Mac with Cactus. Architecture breakdown, benchmarks, and transcription use cases." +keywords: ["Parakeet CTC 1.1B", "FastConformer", "speech-to-text", "on-device transcription", "Apple Silicon"] +author: "Satyajit Kumar and Henry Ndubuaku" +date: 2026-02-26 +tags: ["Parakeet", "ASR", "speech-to-text", "Apple Silicon", "Transcription"] +--- + +# Ridiculously Fast On-Device Transcription: Reviewing Parakeet CTC 1.1B with Cactus + +*By Satyajit Kumar and Henry Ndubuaku* + +[![Video Title](https://img.youtube.com/vi/x0t83tDr5S0/maxresdefault.jpg)](https://youtu.be/x0t83tDr5S0) + +Parakeet CTC 1.1B is NVIDIA’s non-autoregressive English speech-to-text model built on FastConformer. At only **1.1 billion parameters**, it is small enough to run entirely on-device while still delivering state-of-the-art transcription quality. It uses Limited Context Attention in the encoder and a lightweight CTC projection head instead of an autoregressive decoder, which makes the decoding stage extremely efficient. Using Cactus we achieve up to **6 million tokens/second** decode speed with **sub-200 ms end-to-end latency** on Apple Silicon, fast enough for real-time, always-on transcription without a cloud round-trip. + +## Architecture Details + +Parakeet CTC 1.1B is built on NVIDIA's FastConformer encoder and optimized for non-autoregressive ASR. At a high level: + +1. **Audio front-end (mel + subsampling):** Input audio is converted to log-mel features, then an 8x depthwise-separable convolutional subsampler reduces sequence length before the encoder stack. +2. **FastConformer encoder blocks:** The encoder combines Conformer layers with **Limited Context Attention (LCA)** for local efficiency and periodic **Global Tokens (GT)** so long-range context is still preserved. +3. **CTC projection head:** Instead of an autoregressive decoder, Parakeet projects encoder states directly to token logits and uses **CTC** decoding (blank/repeat collapse), making inference highly parallel and low latency. + +This architecture is why Parakeet works well for both real-time and batch transcription: most compute is in the encoder pass, and decoding stays lightweight. + +## Model Architecture Diagram + +```text + ┌───────────────────────┐ + │ CTC Collapse │ + │ remove blanks / merge │ + │ repeated labels │ + └───────────┬───────────┘ + ▲ + ┌───────────┴───────────┐ + │ CTC Projection Head │ + │ Conv1D / Linear → V │ + └───────────┬───────────┘ + ▲ + ┌───────────┴───────────┐ + │ Norm │ + └───────────┬───────────┘ + ▲ + ┌────────────────────────⊕───────────────────────┐ + │ │ │ + │ FastConformer Encoder Stack │ + │ × Num Layers │ + │ │ + │ ┌────────────────────────────────────────┐ │ + │ │ FastConformer Block │ │ + │ │ │ │ + │ │ ┌──────────────┐ │ │ + │ │ │ FFN │ │ │ + │ │ │ Linear │ │ │ + │ │ │ SwiGLU/Act │ │ │ + │ │ │ Linear │ │ │ + │ │ └──────┬───────┘ │ │ + │ │ │ │ │ + │ │ ⊕ │ │ + │ │ │ │ │ + │ │ ┌──────┴───────┐ │ │ + │ │ │ Conv Module │ │ │ + │ │ │ Pointwise │ │ │ + │ │ │ Depthwise │ │ │ + │ │ │ Pointwise │ │ │ + │ │ └──────┬───────┘ │ │ + │ │ │ │ │ + │ │ ⊕ │ │ + │ │ │ │ │ + │ │ ┌───────────────┴──────────────┐ │ │ + │ │ │ Limited Context Attention │ │ │ + │ │ │ local / sliding window │ │ │ + │ │ │ │ │ │ + │ │ │ Q K V │ │ │ + │ │ │ ↑ ↑ ↑ │ │ │ + │ │ │ ┌────┴────────┴────────┴───┐ │ │ │ + │ │ │ │ Linear │ │ │ │ + │ │ │ └─────────────┬────────────┘ │ │ │ + │ │ └───────────────┼──────────────┘ │ │ + │ │ │ │ │ + │ │ ⊕ │ │ + │ │ │ │ │ + │ │ ┌───────────────┴──────────────┐ │ │ + │ │ │ FFN │ │ │ + │ │ │ Linear → Act → Linear │ │ │ + │ │ └──────────────────────────────┘ │ │ + │ │ │ │ + │ └────────────────────────────────────────┘ │ + └────────────────────────┬───────────────────────┘ + ▲ + ┌───────────┴───────────┐ + │ Conv Subsampling / │ + │ Sequence Reduction │ + │ (time downsample) │ + └───────────┬───────────┘ + ▲ + ┌───────────┴───────────┐ + │ Mel-Spectrogram / │ + │ Acoustic Features │ + └───────────┬───────────┘ + ▲ + ┌───────────┴───────────┐ + │ 16 kHz Audio │ + │ Waveform In │ + └───────────────────────┘ +``` + +## Getting Started with Parakeet-CTC-1.1B on Cactus + +### Quick Start (Homebrew) + +The fastest way to try Parakeet: two commands, sub-200 ms latency: + +```bash +brew install cactus-compute/cactus/cactus +cactus transcribe nvidia/parakeet-ctc-1.1b +``` + +That's it. Cactus downloads the 1.1B model, quantizes it, and starts a live transcription session from your microphone. To transcribe a file instead: + +```bash +cactus transcribe nvidia/parakeet-ctc-1.1b --file /path/to/your/file.wav +``` + +### Building from Source + +If you need the Python, Rust, or C libraries for integration, build from source: + +#### Prerequisites + +- macOS with Apple Silicon and 16GB+ RAM (M1 or later recommended) +- Python 3.10+ +- CMake (`brew install cmake`) +- Git + +#### Clone and Build + +```bash +git clone https://github.com/cactus-compute/cactus.git +cd cactus + +# Build the Cactus engine (shared library for Python FFI) +cactus build --python +``` + +#### Download the Model + +Cactus handles downloading and converting HuggingFace models to its optimized binary format with INT4/INT8 quantization, all in one command: + +```bash +cactus download nvidia/parakeet-ctc-1.1b +``` + +### 4. Use the [Python Binding](/python/) + +For integrating Parakeet into your own applications, use the Python FFI bindings directly: + +```python +import json +from cactus import cactus_init, cactus_transcribe, cactus_destroy + +model = cactus_init("weights/parakeet-ctc-1.1b", None, False) + +result = json.loads( + cactus_transcribe(model, "/path/to/audio.wav", None, None, None, None) +) + +if not result["success"]: + raise RuntimeError(result["error"]) + +print("\n\nFinal transcript:") +print(result["response"]) +print(f"Decode speed: {result['decode_tps']:.1f} tokens/sec") + +cactus_destroy(model) +``` + +Cactus also supports streaming for constant transcription. Below is a code snippet that uses the computer's mic to transcribe audio on-device. Note that sample rate is 16000 hz for Parakeet, and that each block is one second long to allow the model to have enough context to accurately transcribe the audio stream. + +Minimal streaming example using your computer mic (`Ctrl+C` to stop): + +```python +import json +import queue + +import sounddevice as sd +from cactus import ( + cactus_init, + cactus_stream_transcribe_start, + cactus_stream_transcribe_process, + cactus_stream_transcribe_stop, + cactus_destroy, +) + +model = cactus_init("weights/parakeet-ctc-1.1b", None, False) +stream = cactus_stream_transcribe_start( + model, + json.dumps({"min_chunk_size": 16000, "language": "en"}), +) + +audio_q = queue.Queue() + +def on_audio(indata, _frames, _time_info, status): + audio_q.put(bytes(indata)) + +print("Listening... press Ctrl+C to stop.") +with sd.RawInputStream( + samplerate=16000, + blocksize=16000, # 1 second callbacks + channels=1, + dtype="int16", + callback=on_audio, +): + try: + while True: + pcm_chunk = audio_q.get() + out = json.loads(cactus_stream_transcribe_process(stream, pcm_chunk)) + if out.get("confirmed"): + print(out["confirmed"], end="\n", flush=True) + if out.get("pending"): + print(out["pending"], end="\r", flush=True) + except KeyboardInterrupt: + pass + +final = json.loads(cactus_stream_transcribe_stop(stream)) +print(final["confirmed"], end="\n", flush=True) + +cactus_destroy(model) +``` + +### 5. Use the [C API](/docs/cactus_engine.md) + +The C API is the base layer all other bindings build on. Link against `libcactus` and include the FFI header: + +```c +#include "cactus_engine.h" +#include +#include + +int main() { + cactus_model_t model = cactus_init("weights/parakeet-ctc-1.1b", NULL, false); + + char response[16384]; + int rc = cactus_transcribe( + model, "audio.wav", NULL, + response, sizeof(response), + NULL, NULL, NULL, NULL, 0 + ); + + if (rc >= 0) printf("Transcript: %s\n", response); + + cactus_destroy(model); + return 0; +} +``` + +Streaming works the same way — start a stream, feed PCM chunks, then stop: + +```c +cactus_stream_transcribe_t stream = cactus_stream_transcribe_start( + model, + "{\"min_chunk_size\": 16000, \"language\": \"en\"}" +); + +char buf[8192]; +// In your audio callback, feed 16-bit 16kHz mono PCM: +cactus_stream_transcribe_process(stream, pcm_data, pcm_size, buf, sizeof(buf)); +printf("Partial: %s\n", buf); + +cactus_stream_transcribe_stop(stream, buf, sizeof(buf)); +printf("Final: %s\n", buf); +``` + +### 6. Use the [Rust Binding](/rust/) + +Add `cactus-sys` to your `Cargo.toml` and call the FFI bindings directly: + +```rust +use std::ffi::CString; +use std::os::raw::c_char; +use std::ptr; + +fn main() { + let model_path = CString::new("weights/parakeet-ctc-1.1b").unwrap(); + let audio_path = CString::new("audio.wav").unwrap(); + + let model = unsafe { + cactus_sys::cactus_init(model_path.as_ptr(), ptr::null(), false) + }; + + let mut buf = vec![0u8; 16384]; + let rc = unsafe { + cactus_sys::cactus_transcribe( + model, + audio_path.as_ptr(), + ptr::null(), + buf.as_mut_ptr() as *mut c_char, + buf.len(), + ptr::null(), None, ptr::null_mut(), + ptr::null(), 0, + ) + }; + + if rc >= 0 { + let response = unsafe { std::ffi::CStr::from_ptr(buf.as_ptr() as *const c_char).to_string_lossy() }; + println!("Transcript: {}", response); + } + + unsafe { cactus_sys::cactus_destroy(model) }; +} +``` + +### 7. Use the [Swift Binding](/bindings/swift/) + +The Swift binding exposes top-level functions that map directly to the C FFI: + +```swift +import Foundation + +let model = try cactusInit("weights/parakeet-ctc-1.1b", nil, false) + +// File-based transcription +let resultJson = try cactusTranscribe(model, "/path/to/audio.wav", nil, nil, nil, nil) +print(resultJson) + +// Streaming transcription +let stream = try cactusStreamTranscribeStart(model, "{\"min_chunk_size\": 16000}") +// Feed PCM chunks from your audio source (16-bit, 16kHz, mono) +let partial = try cactusStreamTranscribeProcess(stream, audioChunk) +print("Partial: \(partial)") + +let finalResult = try cactusStreamTranscribeStop(stream) +print("Final: \(finalResult)") + +cactusDestroy(model) +``` + +### 8. Use the [Kotlin Binding](/bindings/kotlin/) + +The Kotlin binding exposes top-level functions that map directly to the C FFI: + +```kotlin +import com.cactus.* + +val model = cactusInit("weights/parakeet-ctc-1.1b", null, false) + +// File-based transcription +val resultJson = cactusTranscribe(model, "/path/to/audio.wav", null, null, null, null) +println(resultJson) + +// Streaming transcription +val stream = cactusStreamTranscribeStart(model, "{\"min_chunk_size\": 16000}") +// Feed PCM chunks from your audio source (16-bit, 16kHz, mono) +val partial = cactusStreamTranscribeProcess(stream, audioChunk) +println("Partial: $partial") + +val finalResult = cactusStreamTranscribeStop(stream) +println("Final: $finalResult") + +cactusDestroy(model) +``` + +### 9. Use the [Flutter Binding](/flutter/) + +The Flutter binding brings Cactus transcription to iOS, macOS, and Android: + +```dart +import 'cactus.dart'; + +final model = cactusInit('weights/parakeet-ctc-1.1b', null, false); + +// File-based transcription +final resultJson = cactusTranscribe(model, '/path/to/audio.wav', null, null, null, null); +print(resultJson); + +// Streaming transcription +final stream = cactusStreamTranscribeStart(model, '{"min_chunk_size": 16000}'); +final partial = cactusStreamTranscribeProcess(stream, audioChunk); +print('Partial: $partial'); + +final finalResult = cactusStreamTranscribeStop(stream); +print('Final: $finalResult'); + +cactusDestroy(model); +``` + +## See Also + +- [Cactus Engine API Reference](/docs/cactus_engine.md) — Full C API docs for completion, tool calling, and cloud handoff +- [Python Binding](/python/) — Python bindings used in the examples above +- [Hybrid Transcription](/blog/hybrid_transcription.md) — On-device/cloud hybrid speech transcription with Cactus +- [LFM2-24B-A2B](/blog/lfm2_24b_a2b.md) - Reviewing LFM2 24B MoE A2B with Cactus +- [Runtime Compatibility](/docs/compatibility.md) — Weight versioning across Cactus releases diff --git a/blog/turboquant-h.md b/blog/turboquant-h.md new file mode 100644 index 000000000..fccb48c89 --- /dev/null +++ b/blog/turboquant-h.md @@ -0,0 +1,296 @@ +--- +title: "TurboQuant-H: Hadamard Rotation for 2-Bit Embedding Quantization" +description: "TurboQuant compresses KV cache to 1-3 bits. We introduce TurboQuant-H, a simplified offline variant using Hadamard rotation and per-group Lloyd-Max codebooks, applied to per-layer embedding tables where it matters most: models where embeddings are 60%+ of total weight storage." +keywords: ["TurboQuant-H", "TurboQuant", "vector quantization", "embeddings", "2-bit", "on-device AI", "Gemma 4", "Gemma 3n", "per-layer embeddings", "AltUp", "Hadamard", "memory reduction"] +author: "Karen Mosoyan & Henry Ndubuaku" +date: 2026-04-21 +tags: ["quantization", "embeddings", "TurboQuant-H", "Gemma", "on-device", "memory optimization"] +--- + +# TurboQuant-H: Hadamard Rotation for 2-Bit Embedding Quantization + +*By Karen Mosoyan & Henry Ndubuaku* + +*(karen@cactuscompute.com, henry@cactuscompute.com)* + +## Abstract + +TurboQuant (Zandieh et al., ICLR 2026) compresses KV cache vectors to 1-3 bits via random orthogonal rotation, optimal scalar quantization, and QJL bias correction. We introduce **TurboQuant-H**, a simplified offline variant that replaces random rotation with Hadamard rotation, uses per-group Lloyd-Max codebooks, and drops the QJL correction stage. We apply TurboQuant-H to per-layer input (PLI) embedding tables in Gemma 4 E2B, where embeddings constitute 60.6% of total model weight. On Gemma 4 E2B, TurboQuant-H compresses PLI weights from 2,496 MB to 624 MB (4x) at 2.125 effective bits per dimension, reducing total LLM storage by 40% (4,790 MB → 2,918 MB) with a perplexity increase of 0.06 (1.85 → 1.91) and no measured speed regression. + +## 1. Introduction + +TurboQuant (ICLR 2026) compresses KV cache vectors to 1-3 bits with near-zero quality loss. The technique is elegant: rotate vectors with a random orthogonal matrix, exploit the resulting Beta distribution to apply optimal scalar quantizers per coordinate, then correct inner product bias with a 1-bit QJL residual. The paper demonstrates quality neutrality at 3.5 bits and marginal degradation at 2.5 bits on Llama-3.1-8B and Ministral-7B. + +But TurboQuant was designed for KV cache, vectors generated at runtime during inference. There's a catch: mobile devices and wearables need small models, which we found to significantly degrade when KV cache goes below INT4. We in fact keep KV cache at INT8 on Cactus to ensure correctness. This makes applying TurboQuant to Cactus KV workloads tricky. + +However, with the emergence of per-layer embedding architectures (each layer has its own embedding lookup), these embeddings dominate the parameter count of models like the Gemma E-series. For instance, Gemma E2B has 2.3B effective parameters but 5.1B total, because the per-layer embeddings alone account for the difference. That bloats memory and storage footprint by more than 2x. There is a need to re-visit embedding quantisation. + +## 2. Background: Per-Layer Embeddings Dominate Model Storage + +Most quantization research focuses on linear layer weights and activations. Embeddings are treated as untouchable lookup tables, typically kept at FP16 or at best INT8 while everything else goes to INT4. This made sense when embeddings were a small fraction of total parameters. That assumption broke with per-layer embedding architectures. + +Gemma 4 E2B uses AltUp, a technique where each of the 35 transformer layers gets its own embedding projection from the 262K-token vocabulary. Instead of one shared embedding table, you have a shared table plus a per-layer table. The numbers on Cactus's current INT4 weights: + +| Component | Size | % of Model | +|---|---|---| +| `token_embeddings` (shared) | 408 MB | 8.7% | +| `embed_tokens_per_layer` (35 layers) | 2,496 MB | 52.1% | +| **Total embedding storage** | **2,904 MB** | **60.6%** | +| All other weights (attention, FFN, norms, encoders) | 1,886 MB | 39.4% | +| **Total model** | **4,790 MB** | 100% | + +The per-layer embedding table is 2.5 GB. More than half the model. This is not unique to Gemma 4. The AltUp design pattern, where per-layer vocabulary projections replace a single shared embedding, is becoming standard for models that need large vocabularies (262K tokens for multilingual coverage) without proportionally large hidden dimensions. Gemma 3n uses the same architecture. Any model that follows this pattern will be embedding-dominated. + +## 3. TurboQuant-H + +TurboQuant-H shares the core insight from TurboQuant; rotation concentrates coordinates into a well-behaved distribution, enabling aggressive scalar quantization, but simplifies the pipeline for offline weight quantization. + +### 3.1 Comparison with TurboQuant + +| | TurboQuant (Zandieh et al.) | TurboQuant-H (this work) | +|---|---|---| +| **Target** | KV cache (runtime activations) | Embedding weight tables (offline) | +| **Rotation** | Random orthogonal matrix via QR of Gaussian, $O(d^2)$ | Normalized Hadamard matrix, $O(N \log N)$, symmetric = self-inverse | +| **Quantizer** | Per-coordinate scalar quantizer (precomputed for Beta distribution) | Per-position Lloyd-Max codebook (trained on actual weight distribution) | +| **Codebook** | Implicit (quantization levels derived from Beta CDF) | Explicit FP16 centroids per position group (0.125 bits overhead at group-128) | +| **Bias correction** | Two-stage: MSE quantizer at $b-1$ bits + 1-bit QJL residual | Single-stage: no QJL correction | +| **When it runs** | Every forward pass during inference | Once during weight conversion | +| **Bit width** | 2.5-bit and 3.5-bit | 2-bit (+0.125 codebook overhead ≈ 2.125 effective) | + +### 3.2 Formal Description + +Let $\mathbf{E} \in \mathbb{R}^{V \times D}$ be the PLI embedding matrix with vocabulary size $V = 262{,}144$ and embedding dimension $D = 8{,}190$. We partition each row into $P = \lceil D/G \rceil = 64$ positional groups of $G = 128$ contiguous elements. The $p$-th positional group of vocabulary row $v$ is denoted $\mathbf{x}_{v,p} \in \mathbb{R}^G$. + +**Quantization** (offline, during weight conversion): + +**Step 1: Hadamard rotation.** For each row $v$ and position $p$, rotate: + +$$\hat{\mathbf{x}}_{v,p} = \bar{\mathbf{H}}_G \cdot \mathbf{x}_{v,p}$$ + +where $\bar{\mathbf{H}}_G = \frac{1}{\sqrt{G}} \mathbf{H}_G$ is the $G \times G$ normalized Hadamard matrix, satisfying $\bar{\mathbf{H}}_G^T \bar{\mathbf{H}}_G = \mathbf{I}$ and $\bar{\mathbf{H}}_G = \bar{\mathbf{H}}_G^T$. The $\frac{1}{\sqrt{G}}$ normalization ensures the transform is its own inverse. + +**Step 2: Codebook training.** For each positional group $p \in \{1, \ldots, P\}$, collect the rotated values from all $V$ vocabulary rows and train a Lloyd-Max codebook $\mathcal{C}_p = \{c_1, c_2, c_3, c_4\}$ (at $b = 2$ bits, 4 centroids) by minimizing: + +$$\mathcal{C}_p^* = \arg\min_{\mathcal{C}} \sum_{v=1}^{V} \sum_{i=1}^{G} \min_{c \in \mathcal{C}} \left( \hat{x}_{v,p,i} - c \right)^2$$ + +This trains each codebook on $V \times G = 262{,}144 \times 128 \approx 33.6\text{M}$ data points, giving the Lloyd-Max algorithm sufficient statistics for accurate centroid placement. Each positional group gets its own codebook because the weight distributions vary across positions in the embedding dimension. We tried a single joint codebook shared across all positions; per-position was consistently better. + +The codebook values are stored at FP16 (16 bits per centroid), contributing: + +$$\frac{2^b \cdot 16}{G} = \frac{4 \cdot 16}{128} = 0.125 \text{ bits/element overhead}$$ + +**Step 3: Quantize by proximity.** Each rotated element maps to its nearest centroid: + +$$q_{v,p,i} = \arg\min_{j \in \{1,\ldots,2^b\}} \left| \hat{x}_{v,p,i} - c_j \right|$$ + +Store the 2-bit indices $q_{v,p,i}$ and the $P = 64$ FP16 codebooks $\{\mathcal{C}_1, \ldots, \mathcal{C}_P\}$. + +**Dequantization** (at inference): + +$$\tilde{\mathbf{x}}_{v,p} = \bar{\mathbf{H}}_G \cdot \text{scatter}(\mathcal{C}_p, \mathbf{q}_{v,p})$$ + +where $\text{scatter}(\mathcal{C}_p, \mathbf{q}_{v,p})_i = c_{q_{v,p,i}}$ maps indices back to centroids. Since $\bar{\mathbf{H}}_G$ is symmetric and orthogonal, the inverse rotation is the same forward transform. No transpose is needed. + +**Effective bit rate:** + +$$b_{\text{eff}} = b + \frac{2^b \cdot 16}{G} = 2 + 0.125 = 2.125 \text{ bits/element}$$ + +### 3.3 The Quantization Pipeline + +``` +QUANTIZATION (offline, during cactus convert) +============================================== + +PLI Matrix E (262K x 8190) + | + v ++-----------------------+ +| Partition into | Each row -> 64 positional groups +| groups of G=128 | of 128 contiguous elements ++-----------+-----------+ + | + v ++-----------------------+ +| Hadamard rotation | x_hat = (1/sqrt(G)) * H_128 * x +| per group | O(G log G) butterfly ++-----------+-----------+ + | + v ++-----------------------+ +| Lloyd-Max codebook | Train 4 centroids (2-bit) per position +| per position | across all 262K vocab rows +| | C_p = {c1, c2, c3, c4} in FP16 ++-----------+-----------+ + | + v ++-----------------------+ +| Quantize by | q = argmin_j |x_hat_i - c_j| +| proximity | Store 2-bit indices per element ++-----------+-----------+ + | + v +Output: 2-bit index tensor + 64 FP16 codebooks +Effective: 2.125 bits/element + + +DEQUANTIZATION (at inference, per token) +========================================= + +Token IDs + | + v ++-----------------------+ +| Gather 2-bit indices | Look up row from compressed table +| + codebook per pos. | ~3.8x less bandwidth than INT8 ++-----------+-----------+ + | + v ++-----------------------+ +| Scatter codebook | Replace 2-bit indices with FP16 +| values | centroid values from C_p ++-----------+-----------+ + | + v ++-----------------------+ +| Hadamard rotation | x_tilde = (1/sqrt(G)) * H_128 * scatter(...) +| (same as forward, | H_bar is symmetric: H_bar = H_bar^T = H_bar^-1 +| no transpose needed) | O(G log G) butterfly per group ++-----------+-----------+ + | + v +FP16 embedding -> feed to transformer layer +``` + +### 3.4 Design Decisions + +**Why Hadamard instead of random orthogonal?** The normalized Hadamard matrix $\bar{\mathbf{H}}_G = \frac{1}{\sqrt{G}} \mathbf{H}_G$ is deterministic, $O(N \log N)$ to apply via the butterfly factorization (same structure as the FFT), and its own inverse ($\bar{\mathbf{H}} = \bar{\mathbf{H}}^T = \bar{\mathbf{H}}^{-1}$). For offline weight quantization we don't need the data-oblivious guarantees of a random rotation, we have full access to the weight data at conversion time. The Hadamard rotation still concentrates coordinates, which is all we need to make low-bit scalar quantization work. Note: the unnormalized Hadamard satisfies $\mathbf{H}\mathbf{H}^T = G \cdot \mathbf{I}$, so the $\frac{1}{\sqrt{G}}$ factor is essential for the self-inverse property. + +**Why no QJL correction?** TurboQuant's second stage exists because MSE-optimal quantizers introduce multiplicative bias in inner product estimation. At 1-bit, $\mathbb{E}[\langle \mathbf{y}, Q(\mathbf{x}) \rangle] = \frac{2}{\pi} \langle \mathbf{y}, \mathbf{x} \rangle$, a 36% shrinkage. The QJL residual corrects this at the cost of 1 additional bit per dimension. But we're quantizing at 2 bits with a trained codebook, not a precomputed scalar quantizer. The per-position Lloyd-Max codebook already minimizes distortion over the actual weight distribution, and the Hadamard rotation ensures the codebook sees well-spread inputs. At 2 bits with group-128, the inner product bias is small enough that the downstream perplexity impact is negligible (PPL 1.91 vs 1.85). Adding QJL would cost an extra bit per dimension for a correction that isn't needed at this operating point. + +**Why per-position codebooks instead of per-coordinate?** TurboQuant can use a single precomputed quantizer because random rotation makes all coordinates identically distributed (each coordinate of a uniform unit-sphere vector follows a known distribution that converges to $\mathcal{N}(0, 1/d)$ in high dimensions). Hadamard rotation concentrates coordinates but doesn't make them identically distributed — there are structured patterns from the butterfly network. Per-position codebooks (one codebook per group of 128 dimensions, trained across all 262K vocabulary rows at that position) adapt to these patterns. We tried a single joint codebook shared across all positions; per-position was consistently better. + +**Why group size 128?** We swept group sizes from 32 to 512. The qualitative tradeoffs: + +| Group size | Codebook overhead (bits/elem) | Hadamard cost | Quality | +|---|---|---|---| +| 32 | 0.500 | fastest | degraded (high overhead eats bit budget) | +| 64 | 0.250 | fast | good | +| **128** | **0.125** | **fast** | **best (sweet spot)** | +| 256 | 0.063 | moderate | slightly worse (less uniform within group) | +| 512 | 0.031 | slow | worse (distribution spreads too thin) | + +Group-128 gives 0.125 bits overhead, a fast butterfly, and the best quality. Detailed per-group-size PPL numbers are a target for future work. + +## 4. Results + +### 4.1 Perplexity + +Evaluated on Gemma 4 E2B. Evaluation set: 128 self-generated WildChat-1M completions from our `trajectories.jsonl` calibration set, completion-only NLL, 24,438 scored tokens. + +| Variant | Avg bits | PPL | +|---|---|---| +| HuggingFace BF16 | 16 | 1.2892 | +| Cactus default (INT4 linears + INT8 PLI + INT8 token-emb) | ~6.3 | 1.8547 | +| **Cactus + TurboQuant-H PLI** | **~3.8** | **1.9111** | + +Perplexity moves from 1.85 to 1.91. A delta of 0.06 PPL on a 24K-token eval set, within noise for practical use. No measured speed regression. + +### 4.2 Disk Footprint + +| Variant | Size (MB) | Factor | +|---|---|---| +| HuggingFace FP16 snapshot | ~10,240 | 1.00× | +| Cactus default (INT4 linears + INT8 PLI + INT8 emb) | ~4,790 | 0.47× | +| **Cactus + TurboQuant-H PLI** | **2,918** | **0.29×** | + +The PLI table specifically: **2,496 MB → 624 MB, a 4× reduction.** + +Total LLM weight reduction: **40%** from the Cactus baseline. Including the vision and audio encoders (untouched by this change), the overall model reduction is **30%**. + +For Gemma 4 E2B, that's the difference between a 4.8 GB model and a 2.9 GB model. On a 4 GB RAM Android device, that's the difference between fitting and not fitting. + +For Gemma-270m, where the embedding table (295M params) is larger than all other weights combined, we expect the same technique to cut total model size roughly in half. Validation on Gemma-270m is planned future work. + +## 5. Inference Path + +### 5.1 Before (Cactus default) + +``` +token_ids → gather from INT8 table → dequantize to FP16 → feed to transformer +``` + +### 5.2 After (Cactus TurboQuant-H) + +``` +token_ids → gather 2-bit indices → scatter codebook → Hadamard rotation → FP16 → transformer +``` + +### 5.3 Overhead Analysis + +The Hadamard butterfly on a group of $G = 128$ elements has $\log_2(G) = 7$ stages. Each stage performs $G/2 = 64$ add/subtract pairs (the butterfly operations). Over 7 stages that is $7 \times 64 = 448$ add/sub operations per group. On ARM NEON (128-bit SIMD, 8 FP16 lanes), each stage processes the group in $G / (2 \times 8) = 8$ vector instructions, giving $7 \times 8 = 56$ vector operations per group. + +For a single PLI embedding row of $D = 8{,}190$ elements: + +$$\text{Groups per row} = \lceil 8190 / 128 \rceil = 64$$ + +$$\text{Total vector ops} = 64 \times 56 = 3{,}584$$ + +At one vector add/sub per cycle on a 2 GHz A15 NEON unit, this completes in under 2 microseconds per embedding lookup. The gather from a table that is $8 / 2.125 \approx 3.8\times$ smaller in memory more than compensates for the rotation cost in bandwidth savings. + +## 6. Related Work + +**TurboQuant** (Zandieh et al., ICLR 2026) introduced data-oblivious vector quantization with random rotation and QJL correction for KV cache compression, achieving near-optimal distortion rates within a constant factor of $\approx 2.7$ from information-theoretic lower bounds. + +**QuIP#** (Tseng et al., 2024) uses the randomized Hadamard transform for weight quantization of linear layers, but does not address embedding tables or per-layer embeddings. + +**GPTQ, AWQ** focus on linear layer weight quantization with calibration data. These methods do not handle embedding tables, which are pure lookup operations with no gradient flow during inference. + +To our knowledge, TurboQuant-H is the first application of rotation-based vector quantization specifically to per-layer embedding tables, which is where the technique yields the largest storage benefit due to the embedding-dominated weight distribution in AltUp architectures. + +## 7. Next Steps + +This is a research preview of ongoing work at Cactus. Here's what's coming: + +1. **Downstream task validation.** The 0.06 PPL increase is promising, but we're running full evals (MMLU, IFEval, GPQA) across Gemma-270m, Gemma 3n, and Gemma 4 E4B to confirm TurboQuant-H holds up on real tasks, not just perplexity. +2. **Extend to shared `token_embeddings`.** TurboQuant-H currently targets PLI tables only. Applying it to the shared embedding table (408 MB) is straightforward and would push total compression further. +3. **Quantitative group size sweep.** Section 3.4 reports qualitative tradeoffs. We're collecting per-group-size PPL numbers to give a rigorous recommendation. +4. **1-bit with QJL correction.** At 1-bit, the theoretical compression reaches 8x on PLI tables. We're evaluating whether reintroducing the QJL residual stage at this extreme bit width recovers enough quality to be practical. +5. **Ship TurboQuant-H weights.** Integrate the quantization path into `cactus convert` and publish pre-quantized weights on HuggingFace for all supported Gemma E-series models. + +## Try It + +Run Gemma 4 today on Cactus: + +```bash +brew install cactus-compute/cactus/cactus +cactus run google/gemma-4-E2B-it +``` + +TurboQuant-H PLI weights will ship in an upcoming release. If you're working on embedding quantization or have thoughts on extending this to the shared token embedding table, open an issue on [GitHub](https://github.com/cactus-compute/cactus). + +## Citation + +If you use TurboQuant-H in your research, please cite: + +```bibtex +@article{turboquant-h, + title = {TurboQuant-H: Hadamard Rotation for 2-Bit Embedding + Quantization in Embedding-Dominated Models}, + author = {Mosoyan, Karen and Ndubuaku, Henry}, + year = {2026}, + url = {https://docs.cactuscompute.com/latest/blog/turboquant-h/}, + note = {Cactus Compute Research Preview} +} +``` + +## References + +- Zandieh, Daliri, Hadian, Mirrokni. [TurboQuant: Online Vector Quantization with Near-optimal Distortion Rate](https://arxiv.org/abs/2504.19874). ICLR 2026. +- Tseng et al. QuIP#: Even Better LLM Quantization with Hadamard Incoherence and Lattice Codebooks. 2024. +- Google Gemma Team. Gemma 3n / Gemma 4 Technical Reports. 2026. + +## See Also + +- [Gemma 4 on Cactus](/blog/gemma4.md) — Day-one multimodal support with vision, audio, and hybrid inference +- [LFM-2.5-350m on Cactus](/blog/lfm2.5_350m.md) — INT8 quantization deep dive and zero-copy loading +- [Cactus Engine API](/docs/cactus_engine.md) — Full C API reference diff --git a/cactus-kernels/src/lstm.cpp b/cactus-kernels/src/lstm.cpp index 53bb69e6f..327f02012 100644 --- a/cactus-kernels/src/lstm.cpp +++ b/cactus-kernels/src/lstm.cpp @@ -2,6 +2,7 @@ #include "threading.h" #include #include +#include #include void cactus_lstm_cell_f16( diff --git a/cactus-kernels/src/matmul.cpp b/cactus-kernels/src/matmul.cpp index 347a19b9f..b244388b3 100644 --- a/cactus-kernels/src/matmul.cpp +++ b/cactus-kernels/src/matmul.cpp @@ -1396,7 +1396,7 @@ static void tq_preexpand_weights_interleaved( uint8x8_t hi = vshr_n_u8(bytes_r, 4); uint8x16_t combined = vcombine_u8(lo, hi); uint8x16_t reordered = vqtbl1q_u8(combined, reorder4); - exp4[r][v] = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(reordered)); + exp4[r][v] = vqtbl1q_s8(cb_lut, reordered); } } } else if (bits == 2) { @@ -1408,7 +1408,7 @@ static void tq_preexpand_weights_interleaved( uint8x16_t row_bytes = vqtbl1q_u8(bytes, pick_r); uint8x16_t shifted = vshlq_u8(row_bytes, shifts2); uint8x16_t idx = vandq_u8(shifted, mask2); - exp4[r][c] = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(idx)); + exp4[r][c] = vqtbl1q_s8(cb_lut, idx); } } } else if (bits == 1) { @@ -1433,7 +1433,7 @@ static void tq_preexpand_weights_interleaved( } for (size_t r = 0; r < 4; ++r) for (uint32_t v = 0; v < n_vecs; ++v) - exp4[r][v] = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vld1q_u8(row_idx[r] + v * 16))); + exp4[r][v] = vqtbl1q_s8(cb_lut, vld1q_u8(row_idx[r] + v * 16)); } else if (bits == 3) { const uint32_t chunks = gs / 4; alignas(16) uint8_t row_idx[4][256]; @@ -1451,7 +1451,7 @@ static void tq_preexpand_weights_interleaved( } for (size_t r = 0; r < 4; ++r) for (uint32_t v = 0; v < n_vecs; ++v) - exp4[r][v] = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vld1q_u8(row_idx[r] + v * 16))); + exp4[r][v] = vqtbl1q_s8(cb_lut, vld1q_u8(row_idx[r] + v * 16)); } for (size_t ni = valid_n; ni < 4; ++ni) @@ -2133,13 +2133,13 @@ void cactus_quant_4bit_gemv_interleaved( const uint8_t* c0 = (P) + (kb / 8 + 0) * 16; \ const uint8_t* c1 = (P) + (kb / 8 + 1) * 16; \ uint8x16_t b0 = vld1q_u8(c0); \ - int8x16_t wl0 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vandq_u8(b0, lo_mask))); \ - int8x16_t wh0 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vshrq_n_u8(b0, 4))); \ + int8x16_t wl0 = vqtbl1q_s8(cb_lut, vandq_u8(b0, lo_mask)); \ + int8x16_t wh0 = vqtbl1q_s8(cb_lut, vshrq_n_u8(b0, 4)); \ DA = vdotq_laneq_s32(DA, wl0, a_v, 0); \ DB = vdotq_laneq_s32(DB, wh0, a_v, 1); \ uint8x16_t b1 = vld1q_u8(c1); \ - int8x16_t wl1 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vandq_u8(b1, lo_mask))); \ - int8x16_t wh1 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vshrq_n_u8(b1, 4))); \ + int8x16_t wl1 = vqtbl1q_s8(cb_lut, vandq_u8(b1, lo_mask)); \ + int8x16_t wh1 = vqtbl1q_s8(cb_lut, vshrq_n_u8(b1, 4)); \ DA = vdotq_laneq_s32(DA, wl1, a_v, 2); \ DB = vdotq_laneq_s32(DB, wh1, a_v, 3); \ } while (0) @@ -2176,13 +2176,13 @@ void cactus_quant_4bit_gemv_interleaved( for (uint32_t kb = 0; kb < gs; kb += 16) { int8x16_t a_v = vld1q_s8(a_grp + kb); uint8x16_t b0 = vld1q_u8(p_base + (kb / 8 + 0) * 16); - int8x16_t wl0 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vandq_u8(b0, lo_mask))); - int8x16_t wh0 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vshrq_n_u8(b0, 4))); + int8x16_t wl0 = vqtbl1q_s8(cb_lut, vandq_u8(b0, lo_mask)); + int8x16_t wh0 = vqtbl1q_s8(cb_lut, vshrq_n_u8(b0, 4)); dot_a = vdotq_laneq_s32(dot_a, wl0, a_v, 0); dot_b = vdotq_laneq_s32(dot_b, wh0, a_v, 1); uint8x16_t b1 = vld1q_u8(p_base + (kb / 8 + 1) * 16); - int8x16_t wl1 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vandq_u8(b1, lo_mask))); - int8x16_t wh1 = vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(vshrq_n_u8(b1, 4))); + int8x16_t wl1 = vqtbl1q_s8(cb_lut, vandq_u8(b1, lo_mask)); + int8x16_t wh1 = vqtbl1q_s8(cb_lut, vshrq_n_u8(b1, 4)); dot_a = vdotq_laneq_s32(dot_a, wl1, a_v, 2); dot_b = vdotq_laneq_s32(dot_b, wh1, a_v, 3); } @@ -2333,7 +2333,7 @@ void cactus_quant_2bit_gemv_interleaved( uint8x16_t spread = vqtbl1q_u8(bytes, lookup); uint8x16_t shifted = vshlq_u8(spread, shifts); uint8x16_t idx = vandq_u8(shifted, idx_mask); - return vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(idx)); + return vqtbl1q_s8(cb_lut, idx); }; int8x16_t w0 = unpack_set(lookup_s0); @@ -2425,7 +2425,7 @@ void cactus_quant_1bit_gemv_interleaved( uint8x16_t spread = vqtbl1q_u8(bytes, lookup); uint8x16_t shifted = vshlq_u8(spread, sh); uint8x16_t idx = vandq_u8(shifted, idx_mask); - return vqtbl1q_s8(cb_lut, vreinterpretq_s8_u8(idx)); + return vqtbl1q_s8(cb_lut, idx); }; int8x16_t w0_lo = unpack_kq(lookup_s0, shifts_lo); diff --git a/docs/cactus_engine.md b/docs/cactus_engine.md new file mode 100644 index 000000000..706767ef8 --- /dev/null +++ b/docs/cactus_engine.md @@ -0,0 +1,1691 @@ +--- +title: "Cactus Engine FFI API Reference" +description: "C API documentation for Cactus on-device AI inference engine. Supports text completion, vision, transcription, embeddings, RAG, tool calling, and cloud handoff." +keywords: ["on-device AI", "mobile inference", "LLM API", "C FFI", "edge AI", "transcription", "embeddings", "RAG", "tool calling"] +--- + +# Cactus Engine FFI Documentation + +The Cactus Engine provides a clean C FFI (Foreign Function Interface) for integrating the LLM inference engine into various applications. This documentation covers all available functions, their parameters, and usage examples. + +## Getting Started + +Before using the Cactus Engine, you need to download model weights: + +```bash +./setup +cactus download LiquidAI/LFM2-1.2B +cactus download LiquidAI/LFM2-VL-450M +cactus download openai/whisper-small +cactus download UsefulSensors/moonshine-base --precision FP16 + +# Optional: set your Cactus Cloud API key for automatic cloud fallback +cactus auth +``` + +Weights are saved to the `weights/` directory and can be loaded using `cactus_init()`. +Moonshine requires FP16 precision when downloading and running. + +## Types + +### `cactus_model_t` +An opaque pointer type representing a loaded model instance. This handle is used throughout the API to reference a specific model. + +```c +typedef void* cactus_model_t; +``` + +### `cactus_index_t` +An opaque pointer type representing a vector index instance. + +```c +typedef void* cactus_index_t; +``` + +### `cactus_stream_transcribe_t` +An opaque pointer type representing a streaming transcription session. + +```c +typedef void* cactus_stream_transcribe_t; +``` + +### `cactus_token_callback` +Callback function type for streaming token generation. Called for each generated token during completion. + +```c +typedef void (*cactus_token_callback)( + const char* token, // The generated token text + uint32_t token_id, // The token's ID in the vocabulary + void* user_data // User-provided context data +); +``` + +### `cactus_log_callback_t` +Callback function type for log messages. Installed via `cactus_log_set_callback`. + +```c +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); +``` + +## Core Functions + +### `cactus_init` +Initializes a model from disk and prepares it for inference. + +```c +cactus_model_t cactus_init( + const char* model_path, // Path to the model directory + const char* corpus_dir, // Optional path to corpus directory for RAG (can be NULL) + bool cache_index // false = always rebuild index, true = load cached if available +); +``` + +**Returns:** Model handle on success, NULL on failure + +**Example:** +```c +cactus_model_t model = cactus_init("../../weights/qwen3-600m", NULL, false); +if (!model) { + fprintf(stderr, "Failed to initialize model\n"); + return -1; +} + +// with RAG corpus +cactus_model_t rag_model = cactus_init("../../weights/lfm2-rag", "./documents", true); +``` + +### `cactus_complete` +Performs text completion with optional streaming and tool support. + +```c +int cactus_complete( + cactus_model_t model, // Model handle + const char* messages_json, // JSON array of messages + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional generation options (can be NULL) + const char* tools_json, // Optional tools definition (can be NULL) + cactus_token_callback callback, // Optional streaming callback (can be NULL) + void* user_data, // User data for callback (can be NULL) + const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL) + size_t pcm_buffer_size // Size of PCM buffer in bytes (0 when not used) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Message Format:** +```json +[ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "What is your name?"} +] +``` + +**Messages with Images (for VLM models):** +```json +[ + {"role": "user", "content": "Describe this image", "images": ["/path/to/image.jpg"]} +] +``` + +**Messages with Audio (for multimodal models like Gemma4):** +```json +[ + {"role": "user", "content": "Transcribe the audio.", "audio": ["/path/to/audio.wav"]} +] +``` + +**Messages with Images and Audio:** +```json +[ + {"role": "user", "content": "Describe the image and transcribe the audio.", "images": ["/path/to/image.jpg"], "audio": ["/path/to/audio.wav"]} +] +``` + +**Options Format:** +```json +{ + "max_tokens": 256, + "temperature": 0.7, + "top_p": 0.95, + "min_p": 0.15, + "repetition_penalty": 1.1, + "top_k": 40, + "stop_sequences": ["<|im_end|>", ""], + "include_stop_sequences": false, + "force_tools": false, + "tool_rag_top_k": 2, + "confidence_threshold": 0.7, + "auto_handoff": true, + "cloud_timeout_ms": 15000, + "handoff_with_images": true, + "enable_thinking_if_supported": false +} +``` + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `max_tokens` | int | 100 | Maximum tokens to generate | +| `temperature` | float | 0.0 | Sampling temperature | +| `top_p` | float | 0.0 | Top-p (nucleus) sampling | +| `top_k` | int | 0 | Top-k sampling | +| `min_p` | float | 0.15 | Minimum probability threshold relative to max probability | +| `repetition_penalty` | float | 1.1 | Penalize previously generated tokens (1.0 disables) | +| `stop_sequences` | array | [] | Stop generation on these strings | +| `include_stop_sequences` | bool | false | Include stop sequence tokens in the response | +| `force_tools` | bool | false | Constrain output to tool call format | +| `tool_rag_top_k` | int | 2 | Select top-k relevant tools via Tool RAG (0 = disabled, use all tools) | +| `confidence_threshold` | float | 0.7 | Minimum confidence for local generation; triggers cloud_handoff when below | +| `auto_handoff` | bool | true | Automatically attempt cloud handoff when confidence is low | +| `cloud_timeout_ms` | int | 15000 | Timeout in milliseconds for cloud handoff requests | +| `handoff_with_images` | bool | true | Allow cloud handoff for requests that include images | +| `enable_thinking_if_supported` | bool | false | Enable chain-of-thought thinking blocks for models that support it | + +**Response Format:** +```json +{ + "success": true, + "error": null, + "cloud_handoff": false, + "response": "I am an AI assistant.", + "function_calls": [], + "segments": [], + "confidence": 0.85, + "time_to_first_token_ms": 150.5, + "total_time_ms": 1250.3, + "prefill_tps": 166.1, + "decode_tps": 45.2, + "ram_usage_mb": 245.67, + "prefill_tokens": 25, + "decode_tokens": 8, + "total_tokens": 33 +} +``` + +The `thinking` field is only present in the JSON when the model produced a chain-of-thought block: +```json +{ + "success": true, + "error": null, + "cloud_handoff": false, + "response": "The answer is 4.", + "thinking": "Let me consider this... 2+2 equals 4.", + "function_calls": [], + "segments": [], + "confidence": 0.91, + "time_to_first_token_ms": 150.5, + "total_time_ms": 1250.3, + "prefill_tps": 166.1, + "decode_tps": 45.2, + "ram_usage_mb": 245.67, + "prefill_tokens": 25, + "decode_tokens": 8, + "total_tokens": 33 +} +``` + +**Cloud Handoff Response** (when model detects low confidence and cloud handoff succeeds): +```json +{ + "success": true, + "error": null, + "cloud_handoff": true, + "response": "Cloud-provided answer.", + "function_calls": [], + "segments": [], + "confidence": 0.18, + "time_to_first_token_ms": 45.2, + "total_time_ms": 45.2, + "prefill_tps": 619.5, + "decode_tps": 0.0, + "ram_usage_mb": 245.67, + "prefill_tokens": 28, + "decode_tokens": 0, + "total_tokens": 28 +} +``` + +When `cloud_handoff` is true, the model's confidence dropped below `confidence_threshold` (default: 0.7) and the response was fulfilled by a cloud-based model. The `response` field contains the cloud-provided answer. + +**Error Response:** +```json +{ + "success": false, + "error": "Error message here", + "cloud_handoff": false, + "response": null, + "function_calls": [], + "confidence": 0.0, + "time_to_first_token_ms": 0.0, + "total_time_ms": 0.0, + "prefill_tps": 0.0, + "decode_tps": 0.0, + "ram_usage_mb": 245.67, + "prefill_tokens": 0, + "decode_tokens": 0, + "total_tokens": 0 +} +``` + +Note: `ram_usage_mb` reflects actual current RAM usage even in error responses. + +**Response with Function Call:** +```json +{ + "success": true, + "error": null, + "cloud_handoff": false, + "response": "", + "function_calls": [ + { + "name": "get_weather", + "arguments": {"location": "San Francisco, CA, USA"} + } + ], + "segments": [], + "confidence": 0.92, + "time_to_first_token_ms": 120.0, + "total_time_ms": 450.5, + "prefill_tps": 375.0, + "decode_tps": 38.5, + "ram_usage_mb": 245.67, + "prefill_tokens": 45, + "decode_tokens": 15, + "total_tokens": 60 +} +``` + +**Example with Streaming:** +```c +void streaming_callback(const char* token, uint32_t token_id, void* user_data) { + printf("%s", token); + fflush(stdout); +} + +const char* messages = "[{\"role\": \"user\", \"content\": \"Tell me a story\"}]"; + +char response[8192]; +int result = cactus_complete(model, messages, response, sizeof(response), + NULL, NULL, streaming_callback, NULL, NULL, 0); +``` + +### `cactus_prefill` +Pre-processes input text and populates the KV cache without generating output tokens. This reduces latency for future calls to `cactus_complete`. + +```c +int cactus_prefill( + cactus_model_t model, // Model handle + const char* messages_json, // JSON array of messages + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional generation options (can be NULL) + const char* tools_json, // Optional tools definition (can be NULL) + const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL) + size_t pcm_buffer_size // Size of PCM buffer in bytes (0 when not used) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error. + +**Message Format:** Same as `cactus_complete` (see above) + +**Options Format:** Same as `cactus_complete` (see above) + +**Response Format:** +```json +{ + "success": true, + "error": null, + "prefill_tokens": 25, + "prefill_tps": 166.1, + "total_time_ms": 150.5, + "ram_usage_mb": 245.67 +} +``` + +**Error Response:** +```json +{ + "success": false, + "error": "Error message here", + "prefill_tokens": 0, + "prefill_tps": 0.0, + "total_time_ms": 0.0, + "ram_usage_mb": 245.67 +} +``` + +**Example:** +```c +const char* tools = R"([{ + "type": "function", + "function": { + "name": "get_weather", + "description": "Get weather for a location", + "parameters": { + "type": "object", + "properties": { + "location": {"type": "string", "description": "City, State, Country"} + }, + "required": ["location"] + } + } +}])"; + +const char* base_messages = R"([ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "What is the weather in Paris?" }, + { "role": "assistant", "content": "<|tool_call_start|>get_weather(location=\"Paris\")<|tool_call_end|>" }, + { "role": "tool", "content": "{\"name\": \"get_weather\", \"content\": \"Sunny, 72°F\"}" }, + { "role": "assistant", "content": "It's sunny and 72°F in Paris!" } +])"; + +char prefill_response[1024]; +cactus_prefill(model, base_messages, prefill_response, sizeof(prefill_response), NULL, tools, NULL, 0); + +const char* completion_messages = R"([ + { "role": "system", "content": "You are a helpful assistant." }, + { "role": "user", "content": "What is the weather in Paris?" }, + { "role": "assistant", "content": "<|tool_call_start|>get_weather(location=\"Paris\")<|tool_call_end|>" }, + { "role": "tool", "content": "{\"name\": \"get_weather\", \"content\": \"Sunny, 72°F\"}" }, + { "role": "assistant", "content": "It's sunny and 72°F in Paris!" }, + { "role": "user", "content": "What about SF?" } +])"; +char response[4096]; +cactus_complete(model, completion_messages, response, sizeof(response), NULL, tools, NULL, NULL, NULL, 0); +``` + +### `cactus_tokenize` +Tokenizes text into token IDs using the model's tokenizer. + +```c +int cactus_tokenize( + cactus_model_t model, // Model handle + const char* text, // Text to tokenize + uint32_t* token_buffer, // Buffer for token IDs + size_t token_buffer_len, // Maximum number of tokens buffer can hold + size_t* out_token_len // Output: actual number of tokens +); +``` + +**Returns:** 0 on success; -1 on invalid parameters or tokenization error; -2 if `token_buffer_len` is smaller than the number of tokens produced (but `*out_token_len` is still set to the required count). Pass `NULL` for `token_buffer` and `0` for `token_buffer_len` to query the token count without copying. + +**Example:** +```c +const char* text = "Hello, world!"; +uint32_t tokens[256]; +size_t num_tokens = 0; + +int result = cactus_tokenize(model, text, tokens, 256, &num_tokens); +if (result == 0) { + printf("Tokenized into %zu tokens: ", num_tokens); + for (size_t i = 0; i < num_tokens; i++) { + printf("%u ", tokens[i]); + } + printf("\n"); +} +``` + +### `cactus_score_window` +Scores a window of tokens for perplexity calculation or token probability analysis. + +```c +int cactus_score_window( + cactus_model_t model, // Model handle + const uint32_t* tokens, // Array of token IDs + size_t token_len, // Total number of tokens + size_t start, // Start index of window to score + size_t end, // End index of window to score + size_t context, // Context window size + char* response_buffer, // Buffer for response JSON + size_t buffer_size // Size of response buffer +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Response Format:** +```json +{ + "success": true, + "logprob": -12.3456789012, + "tokens": 4 +} +``` + +- `logprob`: Total log-probability of the scored token window +- `tokens`: Number of tokens scored in the window + +**Example:** +```c +uint32_t tokens[256]; +size_t num_tokens; +cactus_tokenize(model, "The quick brown fox", tokens, 256, &num_tokens); + +char response[4096]; +int result = cactus_score_window(model, tokens, num_tokens, 0, num_tokens, 512, + response, sizeof(response)); +if (result >= 0) { + printf("Scores: %s\n", response); +} +``` + +### `cactus_transcribe` +Transcribes audio to text. Supports Whisper, Moonshine, and Parakeet models. Supports both file-based and buffer-based audio input. + +```c +int cactus_transcribe( + cactus_model_t model, // Model handle (Whisper, Moonshine, or Parakeet model) + const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer + const char* prompt, // Optional prompt to guide transcription (can be NULL) + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional transcription options (can be NULL) + cactus_token_callback callback, // Optional streaming callback (can be NULL) + void* user_data, // User data for callback (can be NULL) + const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL if using file) + size_t pcm_buffer_size // Size of PCM buffer in bytes (must be even and >= 2) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. + +**Options Format:** +```json +{ + "max_tokens": 448, + "temperature": 0.0, + "top_p": 0.0, + "top_k": 0, + "use_vad": true, + "cloud_handoff_threshold": 0.0, + "custom_vocabulary": ["word1", "word2"], + "vocabulary_boost": 5.0 +} +``` + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `max_tokens` | int | auto | Maximum tokens to generate; defaults to an estimate based on audio length | +| `temperature` | float | 0.0 | Sampling temperature | +| `top_p` | float | 0.0 | Top-p (nucleus) sampling | +| `top_k` | int | 0 | Top-k sampling | +| `use_vad` | bool | true | Split audio using voice activity detection before transcribing | +| `cloud_handoff_threshold` | float | model default | Maximum token entropy norm above which cloud handoff is flagged | +| `custom_vocabulary` | array | [] | Words or phrases to boost recognition probability | +| `vocabulary_boost` | float | 5.0 | Log-probability bias for custom_vocabulary tokens (0.0–20.0) | + +**Response Format:** +```json +{ + "success": true, + "error": null, + "cloud_handoff": false, + "response": "Transcribed text here.", + "function_calls": [], + "segments": [ + {"start": 0.0, "end": 2.5, "text": "Transcribed text here."} + ], + "confidence": 0.92, + "time_to_first_token_ms": 120.0, + "total_time_ms": 450.0, + "prefill_tps": 50.0, + "decode_tps": 30.0, + "ram_usage_mb": 512.34, + "prefill_tokens": 10, + "decode_tokens": 15, + "total_tokens": 25 +} +``` + +- `response`: Full transcription text +- `segments`: Array of `{"start": float, "end": float, "text": string}` objects with timestamps (seconds). Whisper produces phrase-level segments from timestamp tokens; Parakeet TDT produces word-level segments from native TDT frame timing; Parakeet CTC and Moonshine produce one segment per transcription window (consecutive VAD speech regions grouped up to 30 seconds), with `start`/`end` reflecting the window's boundaries in the source audio. +- `cloud_handoff`: true when `cloud_handoff_threshold > 0`, the transcribed text is non-empty and longer than 5 characters, and the peak token entropy norm exceeded `cloud_handoff_threshold` + +**Example (file-based):** +```c +cactus_model_t whisper = cactus_init("../../weights/whisper-small", NULL, false); + +char response[16384]; +int result = cactus_transcribe(whisper, "audio.wav", NULL, + response, sizeof(response), NULL, NULL, NULL, + NULL, 0); +if (result >= 0) { + printf("Transcription: %s\n", response); +} +``` + +**Example (buffer-based):** +```c +uint8_t* pcm_data = load_audio_buffer("audio.wav", &pcm_size); // 16kHz, mono, 16-bit + +char response[16384]; +int result = cactus_transcribe(whisper, NULL, NULL, + response, sizeof(response), NULL, NULL, NULL, + pcm_data, pcm_size); +``` + +**Transcription Options Format:** +```json +{ + "max_tokens": 100, + "custom_vocabulary": ["Omeprazole", "HIPAA", "Cactus"], + "vocabulary_boost": 3.0 +} +``` + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `max_tokens` | int | 448 | Maximum tokens to generate | +| `custom_vocabulary` | array | [] | List of words or phrases to bias the decoder toward. Useful for proper nouns, acronyms, medical terms, and domain-specific jargon. | +| `vocabulary_boost` | float | 5.0 | Logit bias strength applied to tokens from `custom_vocabulary`. Clamped to 0.0–20.0. Higher values make the listed words more likely to appear. | + +**Note:** Custom vocabulary biasing is supported for Whisper and Moonshine models. Each vocabulary entry is tokenized into sub-tokens and the boost is applied per-token at each decoder step. + +**Example (with custom vocabulary):** +```c +cactus_model_t whisper = cactus_init("../../weights/whisper-small", NULL, false); + +const char* options = "{\"custom_vocabulary\": [\"Omeprazole\", \"HIPAA\", \"Cactus\"], \"vocabulary_boost\": 3.0}"; + +char response[16384]; +int result = cactus_transcribe(whisper, "medical_notes.wav", NULL, + response, sizeof(response), options, NULL, NULL, + NULL, 0); +if (result > 0) { + printf("Transcription: %s\n", response); +} +``` + +### `cactus_stream_transcribe_t` +An opaque pointer type representing a streaming transcription session. Used for real-time audio transcription with incremental confirmation. + +```c +typedef void* cactus_stream_transcribe_t; +``` + +### `cactus_stream_transcribe_start` +Initializes a new streaming transcription session with optional configuration. + +```c +cactus_stream_transcribe_t cactus_stream_transcribe_start( + cactus_model_t model, // Model handle + const char* options_json // Optional configuration (can be NULL) +); +``` + +**Returns:** Stream handle on success, NULL on failure + +**Options Format:** +```json +{ + "min_chunk_size": 32000, + "language": "en", + "custom_vocabulary": ["Omeprazole", "HIPAA", "Cactus"], + "vocabulary_boost": 3.0 +} +``` + +- `min_chunk_size`: Minimum number of audio samples (as int16 samples) required before a transcription processing step is triggered. Default: 32000 +- `language`: ISO 639-1 language code (e.g., "en", "es", "fr", "de"). Default: "en". Ignored for non-Whisper models. +- `custom_vocabulary`: List of words or phrases to bias the decoder toward. Useful for proper nouns, acronyms, and domain-specific terms. The bias is applied for the lifetime of the stream session. +- `vocabulary_boost`: Logit bias strength for `custom_vocabulary` tokens. Default: 5.0. Clamped to 0.0–20.0. + +**Example:** +```c +cactus_model_t whisper = cactus_init("../../weights/whisper-small", NULL, false); + +cactus_stream_transcribe_t stream = cactus_stream_transcribe_start(whisper, "{\"min_chunk_size\": 32000, \"language\": \"en\"}"); +if (!stream) { + fprintf(stderr, "Failed to start stream: %s\n", cactus_get_last_error()); + return -1; +} +``` + +**Example (with custom vocabulary):** +```c +const char* options = "{\"confirmation_threshold\": 0.99, \"custom_vocabulary\": [\"Omeprazole\", \"HIPAA\"], \"vocabulary_boost\": 5.0}"; +cactus_stream_transcribe_t stream = cactus_stream_transcribe_start(whisper, options); +``` + +### `cactus_stream_transcribe_process` +Processes audio chunk and returns confirmed and pending transcription results. + +```c +int cactus_stream_transcribe_process( + cactus_stream_transcribe_t stream, // Stream handle + const uint8_t* pcm_buffer, // Raw PCM audio (16-bit, 16kHz, mono) + size_t pcm_buffer_size, // Size of PCM buffer in bytes + char* response_buffer, // Buffer for response JSON + size_t buffer_size // Size of response buffer +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +When the accumulated audio has not yet reached `min_chunk_size`, a minimal buffering response is returned immediately (no inference is run): +```json +{"success": true, "confirmed": "", "pending": ""} +``` + +When a transcription step is triggered, the full response is returned: + +**Response Format:** +```json +{ + "success": true, + "buffer_duration_ms": 1000.0, + "error": null, + "cloud_handoff": false, + "cloud_job_id": 0, + "cloud_result_job_id": 0, + "cloud_result": "", + "cloud_result_used_cloud": false, + "cloud_result_error": null, + "cloud_result_source": "fallback", + "confirmed_local": "text confirmed from local model", + "confirmed": "text confirmed (may be from cloud if cloud was used)", + "pending": "current transcription result", + "segments": [], + "function_calls": [], + "confidence": 0.95, + "time_to_first_token_ms": 150.5, + "total_time_ms": 450.2, + "prefill_tps": 100.0, + "decode_tps": 50.0, + "ram_usage_mb": 512.5, + "prefill_tokens": 100, + "decode_tokens": 50, + "total_tokens": 150 +} +``` + +- `buffer_duration_ms`: Duration of the confirmed audio that has been consumed from the buffer (milliseconds) +- `confirmed`: Confirmed transcription text from this chunk; if a cloud job returned a result it may reflect the cloud transcript +- `confirmed_local`: The confirmed text as produced by the local model (before any cloud override) +- `pending`: Current (not yet confirmed) transcription result from the latest inference pass +- `segments`: Array of `{"start": float, "end": float, "text": string}` objects representing transcription segments with timestamps (in seconds, relative to the start of the stream). Whisper produces phrase-level segments; Parakeet TDT produces word-level segments; Parakeet CTC and Moonshine produce one segment per transcription window (consecutive VAD speech regions grouped up to 30 seconds). +- `cloud_handoff`: Whether a cloud transcription job was queued for the confirmed audio +- `cloud_job_id`: ID of the cloud job queued in this call (0 if none) +- `cloud_result_job_id`: ID of the cloud job whose result is returned in this response (0 if none ready) +- `cloud_result`: Transcript returned by the completed cloud job (empty if no result ready) +- `cloud_result_used_cloud`: Whether the completed cloud job actually reached a cloud API +- `cloud_result_error`: Error message from the cloud job, null if none +- `cloud_result_source`: `"cloud"` or `"fallback"` for the completed cloud job +- `error`: Error message if any, null otherwise +- `function_calls`: Array of function calls if any +- `confidence`, timing, and token metrics: Model performance statistics from the underlying transcription call + +**Example:** +```c +uint8_t audio_chunk[32000]; // 1 second at 16kHz, 16-bit +char response[32768]; + +int result = cactus_stream_transcribe_process(stream, audio_chunk, sizeof(audio_chunk), response, sizeof(response)); +if (result >= 0) { + printf("Response: %s\n", response); +} +``` + +### `cactus_stream_transcribe_stop` +Stops the streaming session and returns any remaining confirmed transcription. Releases all resources. + +```c +int cactus_stream_transcribe_stop( + cactus_stream_transcribe_t stream, // Stream handle + char* response_buffer, // Buffer for response JSON (can be NULL) + size_t buffer_size // Size of response buffer (can be 0) +); +``` + +**Returns:** Number of bytes written on success, 0 if no response buffer provided, negative value on error + +**Response Format:** +```json +{ + "success": true, + "confirmed": "Final confirmed transcription chunk" +} +``` + +**Example:** +```c +char final_response[32768]; +int result = cactus_stream_transcribe_stop(stream, final_response, sizeof(final_response)); +if (result >= 0) { + printf("Final: %s\n", final_response); +} + +// Or simply cleanup resources without response +cactus_stream_transcribe_stop(stream, NULL, 0); +``` + +### `cactus_diarize` +Runs speaker diarization on audio using the pyannote/segmentation-3.0 model. Supports both file-based and buffer-based audio input. + +```c +int cactus_diarize( + cactus_model_t model, // Model handle (must be PyAnnote model) + const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional JSON options (can be NULL) + const uint8_t* pcm_buffer, // Optional raw int16 PCM buffer (can be NULL if using file) + size_t pcm_buffer_size // Size of PCM buffer in bytes (must be even and >= 2) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Options (`options_json`):** +| Field | Type | Default | Description | +|---|---|---|---| +| `step_ms` | int | 1000 | Sliding window stride in milliseconds. Smaller = more overlap and smoother output, larger = faster. | +| `threshold` | float | none | If set, zeroes out per-speaker scores below this value. Equivalent to `segmentation.threshold` in the Python pipeline. | +| `num_speakers` | int | none | Keep only the N most active speakers (by total activity), zeroing out the rest. | +| `min_speakers` | int | none | Lower bound on the number of active speakers to retain. | +| `max_speakers` | int | none | Upper bound on the number of active speakers to retain. | +| `raw_powerset` | bool | false | Return raw 7-class powerset scores instead of 3-speaker probabilities. When true, speaker filtering and thresholding are skipped. | + +**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. + +The model processes 10-second windows (160,000 samples at 16 kHz) with configurable step. Shorter input is zero-padded. Output scores are a flat array of T × num_speakers float32 values in row-major order (index `f*num_speakers+s`), where T is the total number of output frames. When `raw_powerset` is false (default), num_speakers is 3 and each value is the Hamming-weighted mean of per-speaker probabilities in [0, 1]. When `raw_powerset` is true, num_speakers is 7 and values are raw powerset class scores. + +**Response Format:** +```json +{ + "success": true, + "error": null, + "num_speakers": 3, + "scores": [0.0, 0.1, ...], + "total_time_ms": 12.34, + "ram_usage_mb": 256.0 +} +``` + +**Example:** +```c +cactus_model_t pyannote = cactus_init("../../weights/segmentation-3.0", NULL, false); + +char response[1 << 20]; +int result = cactus_diarize(pyannote, "audio.wav", response, sizeof(response), "{\"step_ms\":500}", NULL, 0); + +if (result >= 0) { + printf("Response: %s\n", response); +} +``` + +### `cactus_embed_speaker` +Extracts a speaker embedding vector from audio using the WeSpeaker ResNet34-LM model. Supports both file-based and buffer-based audio input. Filter bank features are computed internally from raw audio. + +```c +int cactus_embed_speaker( + cactus_model_t model, // Model handle (must be WeSpeaker model) + const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional JSON options (can be NULL, reserved for future use) + const uint8_t* pcm_buffer, // Optional raw int16 PCM buffer (can be NULL if using file) + size_t pcm_buffer_size, // Size of PCM buffer in bytes (must be even and >= 2) + const float* mask_weights, // Optional per-frame mask weights for weighted embedding (can be NULL) + size_t mask_num_frames // Number of mask weight frames (0 if mask_weights is NULL) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. Output is a 256-dimensional speaker embedding. When `mask_weights` is provided, weighted stats pooling is used to extract a speaker-specific embedding — the mask weights are resampled to match the model's internal temporal resolution. + +**Response Format:** +```json +{ + "success": true, + "error": null, + "embedding": [0.123, -0.456, ...], + "total_time_ms": 8.12, + "ram_usage_mb": 128.0 +} +``` + +**Example:** +```c +cactus_model_t wespeaker = cactus_init("../../weights/wespeaker-voxceleb-resnet34-lm", NULL, false); + +char response[1 << 16]; +int result = cactus_embed_speaker(wespeaker, "audio.wav", response, sizeof(response), NULL, NULL, 0, NULL, 0); + +if (result >= 0) { + printf("Response: %s\n", response); +} +``` + +### `cactus_detect_language` +Detects the spoken language in an audio file or PCM buffer. + +```c +int cactus_detect_language( + cactus_model_t model, // Model handle (must be Whisper model) + const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional options (can be NULL) + const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL if using file) + size_t pcm_buffer_size // Size of PCM buffer in bytes (must be even and >= 2) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. Only Whisper models are supported; passing any other model type returns -1. + +**Options Format:** +```json +{ + "use_vad": true +} +``` + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `use_vad` | bool | true | Filter audio through VAD before language detection (requires model initialized with a VAD component) | + +**Response Format:** +```json +{ + "success": true, + "error": null, + "language": "en", + "language_token": "<|en|>", + "token_id": 50259, + "confidence": 0.9812, + "entropy": 0.0188, + "total_time_ms": 234.56, + "ram_usage_mb": 512.34 +} +``` + +- `language`: ISO 639-1 language code, or `"unknown"` if detection failed +- `language_token`: Raw token text emitted by the model for the language (e.g. `"<|en|>"`) +- `token_id`: Vocabulary token ID of the language token +- `confidence`: Detection confidence (0.0–1.0), derived as `1.0 - entropy` +- `entropy`: Normalized entropy of the sampled token +- `total_time_ms`: Total detection time in milliseconds +- `ram_usage_mb`: Current process RAM usage + +**Example:** +```c +cactus_model_t whisper = cactus_init("../../weights/whisper-small", NULL, false); + +char response[1024]; +int result = cactus_detect_language(whisper, "audio.wav", response, sizeof(response), NULL, NULL, 0); +if (result >= 0) { + printf("Detected language: %s\n", response); +} +``` + +### `cactus_vad` +Detects speech segments in audio using Voice Activity Detection. Supports both file-based and buffer-based audio input. + +```c +int cactus_vad( + cactus_model_t model, // Model handle (must be VAD model) + const char* audio_file_path, // Path to WAV file (16-bit PCM) - can be NULL if using pcm_buffer + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + const char* options_json, // Optional VAD options (can be NULL) + const uint8_t* pcm_buffer, // Optional raw PCM audio buffer (can be NULL if using file) + size_t pcm_buffer_size // Size of PCM buffer in bytes (must be even and >= 2) +); +``` + +**Returns:** Number of bytes written to response_buffer on success, negative value on error + +**Note:** Exactly one of `audio_file_path` or `pcm_buffer` must be provided; passing both or neither returns -1. The file path must point to a 16-bit PCM WAV file. The `pcm_buffer` must contain 16-bit signed PCM samples at 16 kHz and `pcm_buffer_size` must be even and at least 2. + +**Options Format:** +```json +{ + "threshold": 0.5, + "neg_threshold": 0.0, + "min_speech_duration_ms": 250, + "max_speech_duration_s": 30.0, + "min_silence_duration_ms": 100, + "speech_pad_ms": 30, + "window_size_samples": 512, + "min_silence_at_max_speech": 98, + "use_max_poss_sil_at_max_speech": true, + "sampling_rate": 16000 +} +``` + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `threshold` | float | 0.5 | Speech probability threshold (0.0–1.0) | +| `neg_threshold` | float | 0.0 | Threshold below which a frame is considered non-speech; 0.0 means auto-compute as `max(threshold - 0.15, 0.01)` | +| `min_speech_duration_ms` | int | 250 | Minimum speech segment duration in milliseconds | +| `max_speech_duration_s` | float | infinity | Maximum speech segment duration in seconds | +| `min_silence_duration_ms` | int | 100 | Minimum silence duration to split segments | +| `speech_pad_ms` | int | 30 | Padding added to each end of a speech segment in milliseconds | +| `window_size_samples` | int | 512 | Window size for VAD processing | +| `min_silence_at_max_speech` | int | 98 | Minimum silence duration in milliseconds at which a segment may be split when max_speech_duration_s is reached | +| `use_max_poss_sil_at_max_speech` | bool | true | Use maximum possible silence at max speech duration | +| `sampling_rate` | int | 16000 | Audio sampling rate in Hz | + +**Response Format:** +```json +{ + "success": true, + "error": null, + "segments": [ + {"start": 0, "end": 16000}, + {"start": 32000, "end": 48000} + ], + "total_time_ms": 12.34, + "ram_usage_mb": 45.67 +} +``` + +- `segments`: Array of `{"start": int, "end": int}` objects, where values are sample indices (not seconds) + +**Example:** +```c +cactus_model_t vad = cactus_init("../../weights/silero-vad", NULL, false); + +char response[4096]; +int result = cactus_vad(vad, "audio.wav", response, sizeof(response), NULL, NULL, 0); + +if (result >= 0) { + printf("Response: %s\n", response); +} +``` + +### `cactus_embed` +Generates text embeddings for semantic search, similarity, and RAG applications. + +```c +int cactus_embed( + cactus_model_t model, // Model handle + const char* text, // Text to embed + float* embeddings_buffer, // Buffer for embedding vector + size_t buffer_size, // Size of embeddings_buffer in bytes + size_t* embedding_dim, // Output: actual embedding dimensions + bool normalize // Whether to L2-normalize the output vector +); +``` + +**Returns:** Number of float elements written to embeddings_buffer on success; -1 on invalid parameters, tokenization error, or other failure; -2 if `buffer_size` (in bytes) is smaller than `embedding_dim * sizeof(float)` + +**Example:** +```c +const char* text = "The quick brown fox jumps over the lazy dog"; +float embeddings[2048]; +size_t actual_dim = 0; + +int result = cactus_embed(model, text, embeddings, sizeof(embeddings), &actual_dim, true); +if (result >= 0) { + printf("Generated %zu-dimensional embedding\n", actual_dim); +} +``` + +**Note:** Set `normalize` to `true` for cosine similarity comparisons (recommended for most use cases). + +### `cactus_image_embed` +Generates embeddings for images, useful for multimodal retrieval tasks. + +```c +int cactus_image_embed( + cactus_model_t model, // Model handle (must support vision) + const char* image_path, // Path to image file + float* embeddings_buffer, // Buffer for embedding vector + size_t buffer_size, // Size of embeddings_buffer in bytes + size_t* embedding_dim // Output: actual embedding dimensions +); +``` + +**Returns:** Number of float elements written to embeddings_buffer on success; -1 on invalid parameters or embedding failure; -2 if `buffer_size` (in bytes) is smaller than `embedding_dim * sizeof(float)` + +**Example:** +```c +float image_embeddings[1024]; +size_t dim = 0; + +int result = cactus_image_embed(model, "photo.jpg", image_embeddings, sizeof(image_embeddings), &dim); +if (result >= 0) { + printf("Image embedding dimension: %zu\n", dim); +} +``` + +### `cactus_audio_embed` +Generates embeddings for audio files, useful for audio retrieval and classification. + +```c +int cactus_audio_embed( + cactus_model_t model, // Model handle (must support audio) + const char* audio_path, // Path to audio file + float* embeddings_buffer, // Buffer for embedding vector + size_t buffer_size, // Size of embeddings_buffer in bytes + size_t* embedding_dim // Output: actual embedding dimensions +); +``` + +**Returns:** Number of float elements written to embeddings_buffer on success; -1 on invalid parameters or embedding failure; -2 if `buffer_size` (in bytes) is smaller than `embedding_dim * sizeof(float)` + +**Example:** +```c +float audio_embeddings[768]; +size_t dim = 0; + +int result = cactus_audio_embed(model, "speech.wav", audio_embeddings, sizeof(audio_embeddings), &dim); +``` + +### `cactus_stop` +Stops ongoing generation. Useful for implementing early stopping based on custom logic. + +```c +void cactus_stop(cactus_model_t model); +``` + +**Example with Controlled Generation:** +```c +struct ControlData { + cactus_model_t model; + int token_count; + int max_tokens; +}; + +void control_callback(const char* token, uint32_t token_id, void* user_data) { + struct ControlData* data = (struct ControlData*)user_data; + printf("%s", token); + data->token_count++; + + // Stop after reaching limit + if (data->token_count >= data->max_tokens) { + cactus_stop(data->model); + } +} + +struct ControlData control = {model, 0, 50}; +cactus_complete(model, messages, response, sizeof(response), + NULL, NULL, control_callback, &control, NULL, 0); +``` + +### `cactus_reset` +Resets the model's internal state, clearing KV cache and any cached context. + +```c +void cactus_reset(cactus_model_t model); +``` + +**Use Cases:** +- Starting a new conversation +- Clearing context between unrelated requests +- Recovering from errors +- Freeing memory after long conversations + +### `cactus_rag_query` +Queries the RAG corpus and returns relevant text chunks. Requires model to be initialized with a corpus directory. + +```c +int cactus_rag_query( + cactus_model_t model, // Model handle (must have corpus_dir set) + const char* query, // Query text + char* response_buffer, // Buffer for response JSON + size_t buffer_size, // Size of response buffer + size_t top_k // Number of chunks to retrieve +); +``` + +**Returns:** Number of bytes written to response_buffer on success; 0 when the query cannot be executed (no corpus index, no tokenizer, empty query, or dimension mismatch) — response_buffer contains `{"chunks":[],"error":"..."}` in those cases; also 0 when the query executes but returns no results — response_buffer contains `{"chunks":[]}` with no `error` field; -1 on error (invalid params, buffer too small, or exception) + +**Response Format:** +```json +{ + "chunks": [ + {"score": 0.85, "source": "document.txt", "content": "Relevant chunk 1..."}, + {"score": 0.72, "source": "document.txt", "content": "Relevant chunk 2..."} + ] +} +``` + +When the query cannot be executed (no corpus index, no tokenizer, empty query, or dimension mismatch), `chunks` is empty and an `error` field is present: +```json +{ + "chunks": [], + "error": "No corpus index loaded" +} +``` + +**Example:** +```c +// Initialize model with corpus +cactus_model_t model = cactus_init("path/to/model", "./documents", true); + +// Query for relevant chunks +char response[65536]; +int result = cactus_rag_query(model, "What is machine learning?", + response, sizeof(response), 5); +if (result >= 0) { + printf("Retrieved chunks: %s\n", response); +} +``` + +### `cactus_destroy` +Releases all resources associated with the model. + +```c +void cactus_destroy(cactus_model_t model); +``` + +**Important:** Always call this when done with a model to prevent memory leaks. + +## Utility Functions + +### `cactus_get_last_error` +Returns the last error message from the Cactus engine. + +```c +const char* cactus_get_last_error(void); +``` + +**Returns:** Error message string (never NULL; empty string if no error) + +**Example:** +```c +cactus_model_t model = cactus_init("invalid/path", NULL, false); +if (!model) { + const char* error = cactus_get_last_error(); + fprintf(stderr, "Error: %s\n", error); +} +``` + +## Vector Index APIs + +The vector index APIs provide persistent storage and retrieval of embeddings for RAG (Retrieval-Augmented Generation) applications. + +### `cactus_index_init` +Initializes or opens a vector index from disk. + +```c +cactus_index_t cactus_index_init( + const char* index_dir, // Path to index directory + size_t embedding_dim // Dimension of embeddings to store +); +``` + +**Returns:** Index handle on success, NULL on failure + +**Example:** +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) { + fprintf(stderr, "Failed to initialize index\n"); + return -1; +} +``` + +### `cactus_index_add` +Adds documents with their embeddings to the index. + +```c +int cactus_index_add( + cactus_index_t index, // Index handle + const int* ids, // Array of document IDs + const char** documents, // Array of document texts + const char** metadatas, // Array of metadata JSON strings (can be NULL) + const float** embeddings, // Array of embedding vectors + size_t count, // Number of documents to add + size_t embedding_dim // Dimension of each embedding +); +``` + +**Returns:** 0 on success, negative value on error + +**Example:** +```c +int ids[] = {1, 2, 3}; +const char* docs[] = {"Hello world", "Foo bar", "Test document"}; +const char* metas[] = {"{\"source\":\"a\"}", "{\"source\":\"b\"}", NULL}; + +float emb1[768], emb2[768], emb3[768]; +const float* embeddings[] = {emb1, emb2, emb3}; + +int result = cactus_index_add(index, ids, docs, metas, embeddings, 3, 768); +``` + +### `cactus_index_delete` +Deletes documents from the index by ID. + +```c +int cactus_index_delete( + cactus_index_t index, // Index handle + const int* ids, // Array of document IDs to delete + size_t ids_count // Number of IDs +); +``` + +**Returns:** 0 on success, negative value on error + +**Example:** +```c +int ids_to_delete[] = {1, 3}; +cactus_index_delete(index, ids_to_delete, 2); +``` + +### `cactus_index_get` +Retrieves documents by their IDs. + +```c +int cactus_index_get( + cactus_index_t index, // Index handle + const int* ids, // Array of document IDs to retrieve + size_t ids_count, // Number of IDs + char** document_buffers, // Output: document text buffers + size_t* document_buffer_sizes, // Sizes of document buffers (in bytes) + char** metadata_buffers, // Output: metadata JSON buffers + size_t* metadata_buffer_sizes, // Sizes of metadata buffers (in bytes) + float** embedding_buffers, // Output: embedding buffers + size_t* embedding_buffer_sizes // Sizes of embedding buffers (in float elements, not bytes) +); +``` + +**Returns:** 0 on success, negative value on error + +### `cactus_index_query` +Queries the index for similar documents using embedding vectors. + +```c +int cactus_index_query( + cactus_index_t index, // Index handle + const float** embeddings, // Array of query embeddings + size_t embeddings_count, // Number of query embeddings + size_t embedding_dim, // Dimension of each embedding + const char* options_json, // Query options (e.g., {"top_k": 10, "score_threshold": 0.5}) + int** id_buffers, // Output: arrays of result IDs + size_t* id_buffer_sizes, // In: capacity of each id_buffer; Out: actual result count written + float** score_buffers, // Output: arrays of similarity scores + size_t* score_buffer_sizes // In: capacity of each score_buffer; Out: actual result count written +); +``` + +**Returns:** 0 on success, negative value on error + +**Options JSON fields:** + +| Option | Type | Default | Description | +|--------|------|---------|-------------| +| `top_k` | int | 10 | Maximum number of results to return per query | +| `score_threshold` | float | -1.0 | Minimum similarity score threshold; results below this are excluded (-1.0 disables filtering) | + +**Example:** +```c +float query_emb[768]; +size_t dim; +cactus_embed(model, "search query", query_emb, sizeof(query_emb), &dim, true); + +const float* queries[] = {query_emb}; +int result_ids[10]; +float result_scores[10]; +int* id_bufs[] = {result_ids}; +float* score_bufs[] = {result_scores}; +size_t id_sizes[] = {10}; +size_t score_sizes[] = {10}; + +cactus_index_query(index, queries, 1, 768, "{\"top_k\": 10}", + id_bufs, id_sizes, score_bufs, score_sizes); + +// id_sizes[0] is updated to the actual number of results returned +for (size_t i = 0; i < id_sizes[0]; i++) { + printf("ID: %d, Score: %.4f\n", result_ids[i], result_scores[i]); +} +``` + +### `cactus_index_compact` +Compacts the index to optimize storage and query performance. + +```c +int cactus_index_compact(cactus_index_t index); +``` + +**Returns:** 0 on success, negative value on error + +**Example:** +```c +cactus_index_compact(index); +``` + +### `cactus_index_destroy` +Releases all resources associated with the index. + +```c +void cactus_index_destroy(cactus_index_t index); +``` + +**Important:** Always call this when done with an index to ensure data is persisted. + +### Complete RAG Example + +```c +#include "cactus_engine.h" + +int main() { + cactus_model_t embed_model = cactus_init("path/to/embed-model", NULL, false); + cactus_index_t index = cactus_index_init("./rag_index", 768); + + const char* docs[] = { + "The capital of France is Paris.", + "Python is a programming language.", + "The Earth orbits the Sun." + }; + int ids[] = {1, 2, 3}; + float emb1[768], emb2[768], emb3[768]; + size_t dim; + + cactus_embed(embed_model, docs[0], emb1, sizeof(emb1), &dim, true); + cactus_embed(embed_model, docs[1], emb2, sizeof(emb2), &dim, true); + cactus_embed(embed_model, docs[2], emb3, sizeof(emb3), &dim, true); + + const float* embeddings[] = {emb1, emb2, emb3}; + cactus_index_add(index, ids, docs, NULL, embeddings, 3, 768); + + float query_emb[768]; + cactus_embed(embed_model, "What is the capital of France?", query_emb, sizeof(query_emb), &dim, true); + + const float* queries[] = {query_emb}; + int result_ids[3]; + float result_scores[3]; + int* id_bufs[] = {result_ids}; + float* score_bufs[] = {result_scores}; + size_t id_sizes[] = {3}; + size_t score_sizes[] = {3}; + + cactus_index_query(index, queries, 1, 768, "{\"top_k\": 3}", + id_bufs, id_sizes, score_bufs, score_sizes); + + printf("Top result ID: %d (score: %.4f)\n", result_ids[0], result_scores[0]); + + cactus_index_destroy(index); + cactus_destroy(embed_model); + return 0; +} +``` + +## Complete Examples + +### Basic Conversation +```c +#include "cactus_engine.h" +#include + +int main() { + cactus_model_t model = cactus_init("path/to/model", NULL, false); + if (!model) return -1; + + const char* messages = + "[{\"role\": \"system\", \"content\": \"You are a helpful assistant.\"}," + " {\"role\": \"user\", \"content\": \"Hello!\"}," + " {\"role\": \"assistant\", \"content\": \"Hello! How can I help you today?\"}," + " {\"role\": \"user\", \"content\": \"What's 2+2?\"}]"; + + char response[4096]; + int result = cactus_complete(model, messages, response, + sizeof(response), NULL, NULL, NULL, NULL, NULL, 0); + if (result >= 0) { + printf("Response: %s\n", response); + } + + cactus_destroy(model); + return 0; +} +``` + +### Vision-Language Model (VLM) +```c +#include "cactus_engine.h" + +int main() { + cactus_model_t vlm = cactus_init("path/to/lfm2-vlm", NULL, false); + if (!vlm) return -1; + + const char* messages = + "[{\"role\": \"user\"," + " \"content\": \"What do you see in this image?\"," + " \"images\": [\"/path/to/photo.jpg\"]}]"; + + char response[8192]; + int result = cactus_complete(vlm, messages, response, sizeof(response), + NULL, NULL, NULL, NULL, NULL, 0); + if (result >= 0) { + printf("%s\n", response); + } + + cactus_destroy(vlm); + return 0; +} +``` + +### Tool Calling +```c +const char* tools = + "[{\"function\": {" + " \"name\": \"get_weather\"," + " \"description\": \"Get weather for a location\"," + " \"parameters\": {" + " \"type\": \"object\"," + " \"properties\": {" + " \"location\": {\"type\": \"string\", \"description\": \"City, State, Country\"}" + " }," + " \"required\": [\"location\"]" + " }" + "}}]"; + +const char* messages = "[{\"role\": \"user\", \"content\": \"What's the weather in Paris?\"}]"; + +char response[4096]; +int result = cactus_complete(model, messages, response, sizeof(response), + NULL, tools, NULL, NULL, NULL, 0); +printf("Response: %s\n", response); +``` + +### Computing Similarity with Embeddings +```c +float compute_cosine_similarity(cactus_model_t model, const char* text1, const char* text2) { + float embeddings1[2048], embeddings2[2048]; + size_t dim1, dim2; + + cactus_embed(model, text1, embeddings1, sizeof(embeddings1), &dim1, true); + cactus_embed(model, text2, embeddings2, sizeof(embeddings2), &dim2, true); + + // with normalized embeddings, cosine similarity = dot product + float dot_product = 0.0f; + for (size_t i = 0; i < dim1; i++) { + dot_product += embeddings1[i] * embeddings2[i]; + } + return dot_product; +} + +float similarity = compute_cosine_similarity(embed_model, + "The cat sat on the mat", "A feline rested on the rug"); +printf("Similarity: %.4f\n", similarity); +``` + +### Audio Transcription with Whisper +```c +#include "cactus_engine.h" +#include + +void transcription_callback(const char* token, uint32_t token_id, void* user_data) { + printf("%s", token); + fflush(stdout); +} + +int main() { + cactus_model_t whisper = cactus_init("path/to/whisper-small", NULL, false); + if (!whisper) return -1; + + char response[32768]; + int result = cactus_transcribe(whisper, "meeting.wav", NULL, + response, sizeof(response), NULL, + transcription_callback, NULL, NULL, 0); + printf("\n\nFull response: %s\n", response); + + cactus_destroy(whisper); + return 0; +} +``` + +### Multimodal Retrieval +```c +#include "cactus_engine.h" +#include + +int find_similar_image(cactus_model_t model, const char* query, + const char** image_paths, int num_images) { + float query_embed[1024]; + size_t query_dim; + cactus_embed(model, query, query_embed, sizeof(query_embed), &query_dim, true); + + float best_score = -1.0f; + int best_idx = -1; + + for (int i = 0; i < num_images; i++) { + float img_embed[1024]; + size_t img_dim; + cactus_image_embed(model, image_paths[i], img_embed, sizeof(img_embed), &img_dim); + + float dot = 0, norm_q = 0, norm_i = 0; + for (size_t j = 0; j < query_dim; j++) { + dot += query_embed[j] * img_embed[j]; + norm_q += query_embed[j] * query_embed[j]; + norm_i += img_embed[j] * img_embed[j]; + } + float score = dot / (sqrtf(norm_q) * sqrtf(norm_i)); + + if (score > best_score) { + best_score = score; + best_idx = i; + } + } + return best_idx; +} +``` + +## Environment Variables + +| Variable | Default | Description | +|----------|---------|-------------| +| `CACTUS_KV_WINDOW_SIZE` | 512 | Sliding window size for KV cache | +| `CACTUS_KV_SINK_SIZE` | 4 | Number of attention sink tokens to preserve | + +**Example:** +```bash +export CACTUS_KV_WINDOW_SIZE=1024 +export CACTUS_KV_SINK_SIZE=8 +./my_app +``` + +## Best Practices + +1. **Always Check Return Values**: Functions return negative values on error +2. **Buffer Sizes**: Use large response buffers (8192+ bytes recommended) +3. **Memory Management**: Always call `cactus_destroy()` when done +4. **Thread Safety**: Each model instance should be used from a single thread +5. **Context Management**: Use `cactus_reset()` between unrelated conversations +6. **Streaming**: Implement callbacks for better user experience with long generations +7. **Reuse Models**: Initialize once, use multiple times for efficiency + +## Error Handling + +Most functions return: +- Positive values or 0 on success +- Negative values on error + +Common error scenarios: +- Invalid model path +- Insufficient buffer size +- Malformed JSON input +- Unsupported operation for model type +- Out of memory + +## Performance Tips + +1. **Reuse Model Instances**: Initialize once, use multiple times +2. **Streaming for UX**: Use callbacks for responsive user interfaces +3. **Early Stopping**: Use `cactus_stop()` to avoid unnecessary generation +4. **Batch Embeddings**: When possible, process multiple texts in sequence without resetting +5. **KV Cache Tuning**: Adjust `CACTUS_KV_WINDOW_SIZE` based on your context needs + +## Logging + +### `cactus_log_set_level` +Sets the minimum log level. Messages below this level are suppressed. + +```c +void cactus_log_set_level(int level); +// level: 0=DEBUG, 1=INFO, 2=WARN (default), 3=ERROR, 4=NONE +``` + +### `cactus_log_set_callback` +Installs a callback to receive log messages. Pass NULL to remove the callback. + +```c +typedef void (*cactus_log_callback_t)(int level, const char* component, const char* message, void* user_data); + +void cactus_log_set_callback(cactus_log_callback_t callback, void* user_data); +``` + +**Example:** +```c +void my_log(int level, const char* component, const char* message, void* user_data) { + printf("[%d] %s: %s\n", level, component, message); +} + +cactus_log_set_level(1); // INFO and above +cactus_log_set_callback(my_log, NULL); +``` + +## Telemetry + +These functions configure anonymous usage telemetry sent to Cactus Compute. Telemetry is opt-out and contains no user data. + +### `cactus_set_telemetry_environment` +Identifies the calling framework and cache directory. + +```c +void cactus_set_telemetry_environment(const char* framework, const char* cache_location, const char* version); +``` + +### `cactus_set_app_id` +Associates telemetry events with an application identifier. + +```c +void cactus_set_app_id(const char* app_id); +``` + +### `cactus_telemetry_flush` +Flushes pending telemetry events. + +```c +void cactus_telemetry_flush(void); +``` + +### `cactus_telemetry_shutdown` +Flushes and shuts down the telemetry subsystem. Call before process exit. + +```c +void cactus_telemetry_shutdown(void); +``` + +## See Also + +- [Cactus Graph API](/docs/cactus_graph.md) — Low-level computational graph for custom tensor operations +- [Cactus Index API](/docs/cactus_index.md) — On-device vector database for RAG applications +- [Fine-tuning Guide](/docs/finetuning.md) — Deploy Unsloth LoRA fine-tunes to mobile +- [Runtime Compatibility](/docs/compatibility.md) — Weight versioning across releases +- [Python Binding](/python/) — Python bindings for the Engine API +- [Swift Binding](/bindings/swift/) — Swift bindings for iOS and macOS +- [Kotlin Binding](/bindings/kotlin/) — Kotlin Multiplatform bindings +- [Flutter Binding](/flutter/) — Dart FFI bindings for mobile apps +- [Rust Binding](/rust/) — Rust FFI bindings via bindgen diff --git a/docs/cactus_graph.md b/docs/cactus_graph.md new file mode 100644 index 000000000..a3c6fc50e --- /dev/null +++ b/docs/cactus_graph.md @@ -0,0 +1,599 @@ +--- +title: "Cactus Graph API Documentation" +description: "Computational graph framework for building and executing tensor operations on mobile devices. Supports matrix multiplication, attention, normalization, and INT8/FP16/CQ1-CQ4 precision." +keywords: ["computation graph", "tensor operations", "mobile AI", "matrix multiplication", "attention mechanism", "INT8", "FP16", "CQ1", "CQ4"] +--- + +# Cactus Graph API Documentation + +The Cactus Graph API provides a computational graph framework for building and executing tensor operations. It supports multiple precision types, broadcasting, and optimized execution for neural network inference. + +## Table of Contents +- [Setup](#setup) +- [Core Concepts](#core-concepts) +- [Getting Started](#getting-started) +- [Tensor Operations](#tensor-operations) +- [Advanced Features](#advanced-features) +- [Complete Examples](#complete-examples) +- [Best Practices](#best-practices) + +## Setup + +Before using the Cactus Graph API, set up your development environment: + +```bash +# Setup the environment and install dependencies +source ./setup + +# Build the Cactus library +cactus build + +# Run tests to verify everything works +cactus test +``` + +## Core Concepts + +### Precision Types +The framework supports the following precision types for tensors: + +```cpp +enum class Precision { + INT8, + FP16, + FP32, + CQ1, + CQ2, + CQ3, + CQ4 +}; +``` + +**Note:** CQ1–CQ4 are Cactus custom quantization formats. + +### Graph Construction +The `CactusGraph` class manages the computational graph: + +```cpp +CactusGraph graph; +size_t input = graph.input({2, 3}, Precision::INT8); +size_t result = graph.add(input, another_input); +graph.execute(); +void* output = graph.get_output(result); +``` + +### Test Fixtures +For testing, use the provided fixtures that handle memory management: + +```cpp +TestUtils::Int8TestFixture fixture("My Test"); +TestUtils::FP16TestFixture fixture("Float Test"); +``` + +## Getting Started + +### Basic Example +```cpp +#include "cactus_graph.h" + +CactusGraph graph; +size_t a = graph.input({4}, Precision::INT8); +size_t b = graph.input({4}, Precision::INT8); +size_t sum = graph.add(a, b); + +std::vector data_a = {1, 2, 3, 4}; +std::vector data_b = {5, 6, 7, 8}; +graph.set_input(a, data_a.data(), Precision::INT8); +graph.set_input(b, data_b.data(), Precision::INT8); + +graph.execute(); +int8_t* result = static_cast(graph.get_output(sum)); // [6, 8, 10, 12] +``` + +## Tensor Operations + +### Basic Arithmetic + +#### Element-wise Operations +```cpp +size_t add_result = graph.add(a, b); // a + b +size_t sub_result = graph.subtract(a, b); // a - b +size_t mul_result = graph.multiply(a, b); // a * b +size_t div_result = graph.divide(a, b); // a / b +``` + +#### Scalar Operations +```cpp +size_t scalar_add = graph.scalar_add(input, 5.0f); // input + 5 +size_t scalar_sub = graph.scalar_subtract(input, 2.0f); // input - 2 +size_t scalar_mul = graph.scalar_multiply(input, 3.0f); // input * 3 +size_t scalar_div = graph.scalar_divide(input, 2.0f); // input / 2 +``` + +#### Mathematical Functions +```cpp +size_t exp_result = graph.scalar_exp(input); // e^input +size_t sqrt_result = graph.scalar_sqrt(input); // √input +size_t cos_result = graph.scalar_cos(input); // cos(input) +size_t sin_result = graph.scalar_sin(input); // sin(input) +size_t log_result = graph.scalar_log(input); // ln(input) +``` + +### Matrix Operations + +#### Matrix Multiplication +```cpp +// Standard matmul: (2,3) x (3,4) = (2,4) +size_t a = graph.input({2, 3}, Precision::FP16); +size_t b = graph.input({3, 4}, Precision::FP16); +size_t result = graph.matmul(a, b); + +// With pre-transposed right-hand side +size_t result = graph.matmul(a, b, true); +``` + +#### Transpose +```cpp +size_t transposed = graph.transpose(input); // (2,3) -> (3,2) +``` + +#### Reshape +```cpp +size_t reshaped = graph.reshape(input, {6, 1}); // (2,3) -> (6,1) +``` + +### Reduction Operations + +```cpp +size_t sum_all = graph.sum(input, -1); // -1 for all elements +size_t sum_axis0 = graph.sum(input, 0); +size_t mean_all = graph.mean(input, -1); +size_t var = graph.variance(input, axis); +size_t min_val = graph.min(input, axis); +size_t max_val = graph.max(input, axis); +``` + +### Neural Network Operations + +#### Layer Normalization +```cpp +size_t weight = graph.input({hidden_size}, Precision::FP16); +size_t bias = graph.input({hidden_size}, Precision::FP16); +size_t normalized = graph.layernorm(input, weight, bias, 1e-5f); +``` + +#### RMS Normalization +```cpp +size_t weight = graph.input({hidden_size}, Precision::FP16); +size_t normalized = graph.rms_norm(input, weight, 1e-5f); +``` + +#### Softmax +```cpp +size_t softmax_result = graph.softmax(input, -1); +``` + +#### Attention Mechanism +```cpp +size_t attention_out = graph.attention(query, key, value, scale); +size_t attention_out = graph.attention(query, key, value, scale, position_offset); +size_t attention_out = graph.attention(query, key, value, scale, position_offset, window_size); +``` + +#### Rotary Position Embedding (RoPE) +```cpp +size_t rope_output = graph.rope(input, theta, position_offset); +``` + +#### Activation Functions +```cpp +size_t silu_out = graph.silu(input); +size_t gelu_out = graph.gelu(input); +size_t gelu_erf_out = graph.gelu_erf(input); // GeLU with erf approximation +size_t sigmoid_out = graph.sigmoid(input); +size_t tanh_out = graph.tanh(input); +size_t relu_out = graph.relu(input); +size_t glu_out = graph.glu(input, axis); // Gated Linear Unit +``` + +#### Convolution Operations +```cpp +// 1D convolutions +size_t conv1d_out = graph.conv1d(input, weight, has_bias, bias, stride); +size_t conv1d_k3_out = graph.conv1d_k3(input, weight, stride); +size_t conv1d_causal = graph.conv1d_causal(input, weight, kernel_size, dilation); +size_t conv1d_pointwise = graph.conv1d_pointwise(input, weight, has_bias, bias); +size_t conv1d_depthwise = graph.conv1d_same_depthwise_k9(input, weight, has_bias, bias); + +// 2D convolutions +size_t conv2d_out = graph.conv2d_k3s2p1(input, weight, has_bias, bias); +size_t conv2d_dw = graph.conv2d_depthwise_k3s2p1(input, weight, has_bias, bias); +size_t conv2d_pw = graph.conv2d_pointwise_1x1(input, weight, has_bias, bias); +``` + +#### Normalization +```cpp +size_t groupnorm_out = graph.groupnorm(input, weight, bias, num_groups, epsilon); +size_t batchnorm_out = graph.batchnorm(input, weight, bias, running_mean, running_var, axis, epsilon); +``` + +#### Recurrent & Sequence Operations +```cpp +size_t lstm_out = graph.lstm_cell(input, h_prev, c_prev, weight_ih, weight_hh, bias_ih, bias_hh); +size_t deltanet_out = graph.gated_deltanet_decode(query, key, value, gate_log, beta, initial_state, scale); +size_t deltanet_prefill = graph.gated_deltanet_prefill(query, key, value, gate_log, beta, initial_state, chunk_size, scale); +``` + +#### Mixture of Experts (MoE) +```cpp +size_t moe_gated = graph.moe_layer_gated(hidden, routing_probs, topk_indices, + w1_weights, w3_weights, w2_weights, + num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor); + +size_t moe_ungated = graph.moe_layer_ungated(hidden, routing_probs, topk_indices, + w1_weights, w2_weights, + num_experts, num_experts_per_tok, normalize_routing, epsilon, routed_scaling_factor, activation); +``` + +#### Signal Processing +```cpp +size_t stft_out = graph.stft(input, weight, stride, num_fft_bins); +``` + +### Indexing and Gathering + +#### Gather Operation +```cpp +size_t embeddings = graph.input({vocab_size, embed_dim}, Precision::FP16); +size_t indices = graph.input({batch_size, seq_len}, Precision::INT8); +size_t gathered = graph.gather(embeddings, indices); +``` + +#### Embedding Lookup +```cpp +size_t embedded = graph.embedding(embedding_tensor, indices); +size_t embedded = graph.embedding("embeddings.bin", indices); // memory-mapped +``` + +#### Memory-Mapped Weights +```cpp +size_t mmap_embed = graph.mmap_embeddings("embeddings.bin"); +size_t weights = graph.mmap_weights("model_weights.bin"); +``` + +### Advanced Operations + +#### Concatenation +```cpp +size_t concatenated = graph.concat(tensor1, tensor2, axis); +size_t multi_cat = graph.cat({tensor1, tensor2, tensor3}, axis); // cat multiple tensors +``` + +#### Slicing +```cpp +size_t sliced = graph.slice(input, axis, start, length); +``` + +#### Indexing +```cpp +size_t indexed = graph.index(input, index_value, dimension); +``` + +#### Top-K Selection +```cpp +size_t topk_values = graph.topk(input, k); +``` + +#### Persistent Nodes +```cpp +size_t persistent = graph.persistent(source_node); // cache result across executions +``` + +#### AltUp Operations +```cpp +size_t prediction = graph.altup_predict(coefs, streams, num_streams); +size_t correction = graph.altup_correct(coefs, innovation, predictions, num_predictions); +``` + +#### Bilinear Interpolation +```cpp +size_t interpolated = graph.bilinear_interpolation(pos_embeds, dst_height, dst_width); +``` + +#### Sampling +```cpp +size_t sampled = graph.sample(logits, temperature, top_p, top_k); +``` + +## Advanced Features + +### Broadcasting +The framework automatically handles broadcasting for compatible shapes: + +```cpp +size_t tensor = graph.input({2, 3}, Precision::INT8); +size_t scalar = graph.input({1}, Precision::INT8); +size_t result = graph.add(tensor, scalar); // {1} -> {2,3} + +size_t a = graph.input({2, 3}, Precision::INT8); +size_t b = graph.input({2, 1}, Precision::INT8); +size_t result = graph.add(a, b); // {2,1} -> {2,3} + +size_t a = graph.input({2, 2, 3}, Precision::INT8); +size_t b = graph.input({2, 3}, Precision::INT8); +size_t result = graph.add(a, b); // {2,3} -> {2,2,3} +``` + +### Precision Conversion +```cpp +size_t int8_tensor = graph.input({4}, Precision::INT8); +size_t fp16_tensor = graph.precision_cast(int8_tensor, Precision::FP16); +graph.set_quantization_scale(node_id, scale); +``` + +### Graph Persistence + +#### Saving graphs +```cpp +const std::string filename = "test_graph_save_load.cg"; + +CactusGraph graph; +size_t input_a = graph.input({2, 3}, Precision::FP16); +size_t input_b = graph.input({2, 3}, Precision::FP16); +size_t sum_id = graph.add(input_a, input_b); +graph.save(filename); + +``` +#### Loading Graphs +```cpp +CactusGraph loaded = CactusGraph::load(filename); +std::vector<__fp16> data_a = {1, 2, 3, 4, 5, 6}; +std::vector<__fp16> data_b = {10, 20, 30, 40, 50, 60}; +loaded.set_input(0, data_a.data(), Precision::FP16); +loaded.set_input(1, data_b.data(), Precision::FP16); +loaded.execute(); +``` + +#### Saving Nodes +```cpp +GraphFile::save_node(graph, node_id, "output.bin"); +``` + +#### Loading Nodes +```cpp +CactusGraph new_graph; +auto loaded = GraphFile::load_into_graph(new_graph, "output.bin"); +size_t node_id = loaded.node_id; +std::vector shape = loaded.shape; +Precision precision = loaded.precision; +``` + +### Graph Management + +#### Execution +```cpp +graph.execute(); +graph.execute("profile_output.json"); // with profiling +``` + +#### Reset Operations +```cpp +graph.hard_reset(); // clear all nodes and buffers +graph.soft_reset(); // clear only buffers, keep graph structure +``` + +## Complete Examples + +### Building a Simple Neural Network Layer +```cpp +CactusGraph graph; + +size_t input = graph.input({2, 4}, Precision::FP16); +size_t weight = graph.input({4, 8}, Precision::FP16); +size_t bias = graph.input({8}, Precision::FP16); + +size_t linear = graph.matmul(input, weight); +size_t with_bias = graph.add(linear, bias); +size_t activated = graph.gelu(with_bias); + +size_t ln_weight = graph.input({8}, Precision::FP16); +size_t ln_bias = graph.input({8}, Precision::FP16); +size_t output = graph.layernorm(activated, ln_weight, ln_bias); +``` + +### Implementing Multi-Head Attention +```cpp +CactusGraph graph; + +size_t hidden_dim = 512; +size_t num_heads = 8; +size_t head_dim = hidden_dim / num_heads; +size_t seq_len = 32; + +size_t input = graph.input({1, seq_len, hidden_dim}, Precision::FP16); +size_t q_weight = graph.input({hidden_dim, hidden_dim}, Precision::FP16); +size_t k_weight = graph.input({hidden_dim, hidden_dim}, Precision::FP16); +size_t v_weight = graph.input({hidden_dim, hidden_dim}, Precision::FP16); + +size_t query = graph.matmul(input, q_weight); +size_t key = graph.matmul(input, k_weight); +size_t value = graph.matmul(input, v_weight); + +query = graph.reshape(query, {1, seq_len, num_heads, head_dim}); +key = graph.reshape(key, {1, seq_len, num_heads, head_dim}); +value = graph.reshape(value, {1, seq_len, num_heads, head_dim}); + +float scale = 1.0f / sqrt(head_dim); +size_t attention_out = graph.attention(query, key, value, scale); +``` + +### Working with Embeddings +```cpp +CactusGraph graph; + +size_t vocab_size = 50000; +size_t embed_dim = 768; +size_t tokens = graph.input({2, 10}, Precision::INT8); + +size_t embed_table = graph.input({vocab_size, embed_dim}, Precision::FP16); +size_t embeddings = graph.gather(embed_table, tokens); + +// or memory-mapped for large models +size_t mmap_table = graph.mmap_embeddings("vocab_embeddings.bin"); +size_t embeddings = graph.gather(mmap_table, tokens); + +size_t pos_embed = graph.input({1, 10, embed_dim}, Precision::FP16); +size_t final_embed = graph.add(embeddings, pos_embed); +``` + +### Similarity Computation +```cpp +TestUtils::FP16TestFixture fixture("Similarity"); + +size_t text1 = fixture.create_input({1, 768}, Precision::FP16); +size_t text2 = fixture.create_input({1, 768}, Precision::FP16); + +// L2 norms +size_t norm1 = fixture.graph().scalar_sqrt( + fixture.graph().sum(fixture.graph().multiply(text1, text1), -1)); +size_t norm2 = fixture.graph().scalar_sqrt( + fixture.graph().sum(fixture.graph().multiply(text2, text2), -1)); + +// cosine similarity = dot(a,b) / (norm(a) * norm(b)) +size_t dot_product = fixture.graph().sum(fixture.graph().multiply(text1, text2), -1); +size_t similarity = fixture.graph().divide(dot_product, fixture.graph().multiply(norm1, norm2)); +``` + +## Best Practices + +### Memory Management +1. **Use appropriate precision**: CQ1-CQ4/INT8 for memory efficiency, FP16 for accuracy +2. **Memory-map large tensors**: Use `mmap_embeddings()` for vocabulary tables +3. **Reset graphs**: Call `hard_reset()` when switching between different models +4. **External buffers**: Use `set_external_input()` to avoid copying large inputs + +### Performance Optimization +1. **Batch operations**: Process multiple samples together +2. **Pre-transpose weights**: Use `pretransposed_rhs=true` for matmul when possible +3. **Fused operations**: The framework automatically fuses compatible operations +4. **Backend selection**: Use NPU backend for supported operations: + ```cpp + size_t result = graph.matmul(a, b, false, ComputeBackend::NPU); + ``` + +### Graph Construction +1. **Build once, execute many**: Construct the graph once, run with different inputs +2. **Validate shapes**: Ensure tensor shapes are compatible before operations +3. **Handle broadcasts**: Be aware of automatic broadcasting rules +4. **Profile execution**: Use `execute("profile.json")` to identify bottlenecks + +### Testing +1. **Use test fixtures**: Leverage provided fixtures for automatic cleanup +2. **Verify outputs**: Use `verify_output()` methods for tolerance-based comparison +3. **Test edge cases**: Include tests for broadcasting, empty tensors, and large inputs +4. **Check precision**: Test operations with different precision types + +### Contributing Graph Operations +1. ** Define the op in core graph types ** +Add the new OpType in `cactus-graph/cactus_graph.h` +If the op needs additional parameters, add the fields to OpParams in the same file + +2. ** Add a graph builder API ** +Add a builder method in `cactus-graph/src/builder.cpp` and its declaration in +`cactus-graph/cactus_graph.h` +Follow the pattern of existing builder methods, e.g. for a new "relu" op: +```cpp +size_t CactusGraph::relu(size_t input) { + OpParams params; + return add_node(OpType::RELU, {input}, params); +} +``` +3. ** Implement the op in the execution engine ** +Implement the kernel or graph op code in the relevant file, usually in `cactus-kernels/src/` +Register the new op in the dispatch table in `cactus-graph/src/execute.cpp` for the supported backends (CPU, NPU) + +4. ** Export op in FFI bindings ** +- header: `cactus-engine/cactus_engine.h` +- implementation: `cactus-engine/src/graph_ffi.cpp` + +5. ** Add python ctypes declaration ** +Add `_lib.cactus_graph_my_new_op.argtypes/restype` in `python/cactus/bindings/cactus.py` + +6. ** Add python graph wrapper ** +Add `Graph.my_new_op(...)` in `python/cactus/bindings/graph.py`, and optionally a Tensor + convenience method. + +7. ** Add serialization schema entry if needed ** +If your op has extra parameters that need to be saved/loaded that are not in the +default node, add new ParamField enum values. + +If the op has any graph-persistent params: + +- add any new ParamField enum values in cactus-graph/src/param_io.cpp +- add read/write logic for those fields +- add the op’s schema entry in `op_schema(...)` + +If the op has no params, you may not need to touch schema beyond maybe adding an +empty schema entry. + +The syntax pattern there is: +``` +{OpType::MY_NEW_OP, { + {ParamField::Alpha, FieldPersistence::Persistent}, + {ParamField::Mode, FieldPersistence::Persistent}, +}}, +``` +8. ** Add test coverage ** +Add unit tests to `tests/test_graph.cpp` covering the native graph function, and +add python tests in `python/tests/test_graph.py` covering the Python API and end-to-end execution. + +and then support those fields in `write_field(...)` / `read_field(...)`. + +If a field is runtime-only, mark it RuntimeOnly instead of Persistent. + + + +### Error Handling +```cpp +try { + CactusGraph graph; + // ... build and execute graph +} catch (const std::exception& e) { + std::cerr << "Graph error: " << e.what() << std::endl; +} +``` + +## Common Patterns + +### Sequential Processing +```cpp +CactusGraph graph; +size_t x = graph.input({batch, dim}, Precision::FP16); +x = graph.linear(x, weight1, bias1); +x = graph.gelu(x); +x = graph.layernorm(x, ln_weight1, ln_bias1); +x = graph.linear(x, weight2, bias2); +``` + +### Residual Connections +```cpp +size_t input = graph.input({batch, dim}, Precision::FP16); +size_t processed = graph.matmul(input, weight); +processed = graph.gelu(processed); +size_t output = graph.add(input, processed); +``` + +### Multi-Path Processing +```cpp +size_t input = graph.input({batch, dim}, Precision::FP16); +size_t path1 = graph.matmul(input, weight1); +path1 = graph.silu(path1); +size_t path2 = graph.matmul(input, weight2); +size_t output = graph.multiply(path1, path2); +``` + +## See Also + +- [Cactus Engine API](/docs/cactus_engine.md) — High-level inference API built on top of Cactus Graph +- [Cactus Index API](/docs/cactus_index.md) — On-device vector database for embedding storage and search +- [Runtime Compatibility](/docs/compatibility.md) — Weight versioning across releases diff --git a/docs/cactus_index.md b/docs/cactus_index.md new file mode 100644 index 000000000..455de9c7d --- /dev/null +++ b/docs/cactus_index.md @@ -0,0 +1,573 @@ +--- +title: "Cactus Index FFI API Reference" +description: "On-device vector database C API for storing and querying embeddings. Supports cosine similarity search, memory-mapped files, and RAG applications." +keywords: ["vector database", "embeddings", "cosine similarity", "RAG", "on-device search", "C FFI"] +--- + +# Cactus Index FFI Documentation + +The Cactus Index provides a clean C FFI (Foreign Function Interface) for integrating a vector database into various applications. This documentation covers all available functions, their parameters, and usage examples. + +## Getting Started + +The index uses memory-mapped files: +- `index.bin`: Embeddings (FP16) and metadata pointers +- `data.bin`: Document content and metadata (UTF-8) + +All embeddings are automatically normalized to unit length + +## Types + +### `cactus_index_t` +An opaque pointer type representing an index instance. This handle is used throughout the API to reference a specific index. + +```c +typedef void* cactus_index_t; +``` + +## Core Functions + +### `cactus_index_init` +Initializes or opens an index with specified embedding dimension. + +```c +cactus_index_t cactus_index_init( + const char* index_dir, + size_t embedding_dim +); +``` + +**Parameters:** +- `index_dir`: Directory path where index files will be stored +- `embedding_dim`: Dimension of embeddings (must match for existing index) + +**Returns:** Index handle on success, NULL on failure + +**Example:** +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) { + fprintf(stderr, "Failed to initialize index\n"); + return -1; +} +``` + +### `cactus_index_add` +Adds documents to the index. + +```c +int cactus_index_add( + cactus_index_t index, + const int* ids, + const char** documents, + const char** metadatas, + const float** embeddings, + size_t count, + size_t embedding_dim +); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) +- `ids`: Array of unique document IDs (required) +- `documents`: Array of document content strings (required, UTF-8) +- `metadatas`: Array of metadata strings (optional, can be NULL; UTF-8) +- `embeddings`: Array of pointers to embedding vectors. (required, none can be NULL) +- `count`: Number of documents to add (must be > 0) +- `embedding_dim`: Dimension of embeddings (must match index, must be > 0) + +**Returns:** 0 on success, -1 on error + +**Constraints:** +- Document IDs must be unique integers +- Content max 65535 bytes per document +- Metadata max 65535 bytes per document +- Embedding dimension must match index dimension from init +- `ids`, `documents`, and `embeddings` arrays must be non-null +- Each individual `embeddings[i]` pointer must be non-null +- Individual `documents[i]` can be NULL (stored as empty string) +- `metadatas` array can be NULL, or individual `metadatas[i]` can be NULL + +**Example:** +```c +int ids[] = {1, 2}; +const char* docs[] = {"AI is transforming technology", "Machine learning enables predictions"}; +const char* metas[] = {"{\"source\":\"wiki\"}", "{\"source\":\"blog\"}"}; + +float emb1[768] = {0.1, 0.2, 0.3, /* ... */}; +float emb2[768] = {0.4, 0.5, 0.6, /* ... */}; +const float* embeddings[] = {emb1, emb2}; + +int result = cactus_index_add(index, ids, docs, metas, embeddings, 2, 768); +if (result != 0) { + fprintf(stderr, "Failed to add documents: %s\n", cactus_get_last_error()); +} +``` + +**Example without metadata:** +```c +int ids[] = {1, 2}; +const char* docs[] = {"Document one", "Document two"}; + +float emb1[768] = {0.1, 0.2, 0.3, /* ... */}; +float emb2[768] = {0.4, 0.5, 0.6, /* ... */}; +const float* embeddings[] = {emb1, emb2}; + +// Pass NULL for metadatas +int result = cactus_index_add(index, ids, docs, NULL, embeddings, 2, 768); +``` + +### `cactus_index_delete` +Soft deletes documents. Space reclaimed via `cactus_index_compact`. + +```c +int cactus_index_delete( + cactus_index_t index, + const int* ids, + size_t ids_count +); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) +- `ids`: Array of document IDs to delete (required) +- `ids_count`: Number of document IDs (must be > 0) + +**Returns:** 0 on success, -1 on error + +**Example:** +```c +int ids[] = {1, 2, 3}; +int result = cactus_index_delete(index, ids, 3); +if (result != 0) { + fprintf(stderr, "Failed to delete documents: %s\n", cactus_get_last_error()); +} +``` + +### `cactus_index_get` +Retrieves documents by IDs. Fetch only the fields you need by passing NULL for unused buffers. + +```c +int cactus_index_get( + cactus_index_t index, + const int* ids, + size_t ids_count, + char** document_buffers, + size_t* document_buffer_sizes, + char** metadata_buffers, + size_t* metadata_buffer_sizes, + float** embedding_buffers, + size_t* embedding_buffer_sizes +); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) +- `ids`: Array of document IDs to retrieve (required) +- `ids_count`: Number of document IDs (must be > 0) +- `document_buffers`: Array of pre-allocated buffers for document content (optional, can be NULL) +- `document_buffer_sizes`: Array serving dual purpose (required if `document_buffers` is not NULL): + - **Input**: Initial capacity of each buffer in bytes + - **Output**: Actual size of data written to each buffer (including null terminator) +- `metadata_buffers`: Array of pre-allocated buffers for metadata (optional, can be NULL) +- `metadata_buffer_sizes`: Array serving dual purpose (required if `metadata_buffers` is not NULL): + - **Input**: Initial capacity of each buffer in bytes + - **Output**: Actual size of data written to each buffer (including null terminator) +- `embedding_buffers`: Array of pre-allocated buffers for embeddings (optional, can be NULL) +- `embedding_buffer_sizes`: Array for embedding buffer sizes (required if `embedding_buffers` is not NULL) + - **Input**: Capacity in number of floats + - **Output**: Actual number of floats written + +**Returns:** 0 on success, -1 on error (e.g., buffer too small, no data copied) + +**Example - Retrieve all fields:** +```c +int ids[] = {1, 2}; +char* docs[2]; +char* metas[2]; +float* embs[2]; + +// Allocate buffers +for (int i = 0; i < 2; i++) { + docs[i] = malloc(65536); + metas[i] = malloc(65536); + embs[i] = malloc(768 * sizeof(float)); +} + +size_t doc_sizes[2] = {65536, 65536}; +size_t meta_sizes[2] = {65536, 65536}; +size_t emb_sizes[2] = {768, 768}; // Number of floats + +int result = cactus_index_get(index, ids, 2, + docs, doc_sizes, + metas, meta_sizes, + embs, emb_sizes); + +if (result == 0) { + printf("Retrieved documents successfully\n"); + for (int i = 0; i < 2; i++) { + printf("Doc %d (%zu bytes): %s\n", ids[i], doc_sizes[i], docs[i]); + printf("Embedding dim: %zu floats\n", emb_sizes[i]); + } +} +``` + +**Example - Retrieve only documents (no metadata or embeddings):** +```c +int ids[] = {1, 2, 3}; +char* docs[3]; + +for (int i = 0; i < 3; i++) { + docs[i] = malloc(65536); +} + +size_t doc_sizes[3] = {65536, 65536, 65536}; + +// Pass NULL for metadata and embeddings +int result = cactus_index_get(index, ids, 3, + docs, doc_sizes, + NULL, NULL, // No metadata + NULL, NULL); // No embeddings + +if (result == 0) { + for (int i = 0; i < 3; i++) { + printf("%s\n", docs[i]); + } +} +``` + +### `cactus_index_query` +Similarity search using cosine similarity. Results sorted by score (highest first). + +```c +int cactus_index_query( + cactus_index_t index, + const float** embeddings, + size_t embeddings_count, + size_t embedding_dim, + const char* options_json, + int** id_buffers, + size_t* id_buffer_sizes, + float** score_buffers, + size_t* score_buffer_sizes +); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) +- `embeddings`: Array of query embedding pointers (required, none can be NULL) +- `embeddings_count`: Number of query embeddings (must be > 0) +- `embedding_dim`: Dimension of embeddings (must match index, must be > 0) +- `options_json`: JSON string with query options (optional, can be NULL for defaults) +- `id_buffers`: Array of pre-allocated buffers for result document IDs (required, one buffer per query) +- `id_buffer_sizes`: Array serving dual purpose (required): + - **Input**: Maximum capacity of each buffer (number of results) + - **Output**: Actual number of results per query +- `score_buffers`: Array of pre-allocated buffers for result scores (required, one buffer per query) +- `score_buffer_sizes`: Array serving dual purpose (required): + - **Input**: Maximum capacity of each buffer (number of results) + - **Output**: Actual number of results per query + +**Options JSON Format:** +```json +{ + "top_k": 10, + "score_threshold": 0.7 +} +``` + +**Defaults:** `top_k`: 10, `score_threshold`: -1.0 (no filtering) + +**Returns:** 0 on success, -1 on error (if buffers too small, no data copied) + +**Example:** +```c +float query1[768] = {/* ... */}; +float query2[768] = {/* ... */}; +const float* queries[] = {query1, query2}; + +// Allocate result buffers (2 queries, max 5 results each) +int* ids[2]; +float* scores[2]; +for (int i = 0; i < 2; i++) { + ids[i] = (int*)malloc(5 * sizeof(int)); + scores[i] = (float*)malloc(5 * sizeof(float)); +} + +size_t id_sizes[2] = {5, 5}; +size_t score_sizes[2] = {5, 5}; + +int result = cactus_index_query(index, queries, 2, 768, "{\"top_k\":5}", + ids, id_sizes, + scores, score_sizes); + +if (result == 0) { + for (int q = 0; q < 2; q++) { + printf("Query %d: %zu results\n", q, id_sizes[q]); + for (size_t i = 0; i < id_sizes[q]; i++) { + printf(" ID: %d, Score: %.3f\n", ids[q][i], scores[q][i]); + } + } +} +``` + +### `cactus_index_compact` +Removes deleted documents and reclaims disk space. + +```c +int cactus_index_compact(cactus_index_t index); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) + +**Returns:** 0 on success, -1 on error + +**Example:** +```c +int result = cactus_index_compact(index); +if (result != 0) { + fprintf(stderr, "Compaction failed: %s\n", cactus_get_last_error()); +} +``` + +### `cactus_index_destroy` +Releases all resources associated with the index. + +```c +void cactus_index_destroy(cactus_index_t index); +``` + +**Parameters:** +- `index`: Index handle from `cactus_index_init` (required) + +**Example:** +```c +cactus_index_destroy(index); +index = NULL; +``` + +## Examples + +### Create and Populate +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) { + fprintf(stderr, "Failed to initialize index\n"); + return -1; +} + +int ids[] = {1, 2}; +const char* docs[] = { + "AI is transforming technology", + "Machine learning enables predictions" +}; +const char* metas[] = {"{\"source\":\"wiki\"}", "{\"source\":\"blog\"}"}; + +float emb1[768] = {0.1, 0.2, 0.3, /* ... */}; +float emb2[768] = {0.4, 0.5, 0.6, /* ... */}; +const float* embeddings[] = {emb1, emb2}; + +int result = cactus_index_add(index, ids, docs, metas, embeddings, 2, 768); +if (result != 0) { + fprintf(stderr, "Failed to add documents: %s\n", cactus_get_last_error()); +} + +cactus_index_destroy(index); +``` + +### Similarity Search +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) { + fprintf(stderr, "Failed to open index: %s\n", cactus_get_last_error()); + return -1; +} + +float query_embedding[768] = {0.1, 0.2, 0.3, /* ... */}; +const float* queries[] = {query_embedding}; + +// Allocate buffers for 1 query with max 10 results +int* ids[1]; +float* scores[1]; +ids[0] = (int*)malloc(10 * sizeof(int)); +scores[0] = (float*)malloc(10 * sizeof(float)); + +size_t id_sizes[1] = {10}; +size_t score_sizes[1] = {10}; + +int result = cactus_index_query(index, queries, 1, 768, + "{\"top_k\":10,\"score_threshold\":0.7}", + ids, id_sizes, + scores, score_sizes); + +if (result == 0) { + printf("Found %zu results:\n", id_sizes[0]); + for (size_t i = 0; i < id_sizes[0]; i++) { + printf(" ID: %d, Score: %.3f\n", ids[0][i], scores[0][i]); + } +} + +cactus_index_destroy(index); +``` + +### RAG (Retrieval-Augmented Generation) +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) return -1; + +// Search for relevant documents +float query_embedding[768] = {0.1, 0.2, 0.3, /* ... */}; +const float* queries[] = {query_embedding}; + +// Allocate query result buffers +int* result_ids[1]; +float* result_scores[1]; +result_ids[0] = (int*)malloc(10 * sizeof(int)); +result_scores[0] = (float*)malloc(10 * sizeof(float)); + +size_t id_sizes[1] = {10}; +size_t score_sizes[1] = {10}; + +int query_result = cactus_index_query(index, queries, 1, 768, + "{\"top_k\":3,\"score_threshold\":0.5}", + result_ids, id_sizes, + result_scores, score_sizes); + +if (query_result == 0) { + // Allocate buffers for retrieving documents + size_t num_results = id_sizes[0]; + char* docs[10]; + + for (size_t i = 0; i < num_results; i++) { + docs[i] = (char*)malloc(65536); + } + + size_t doc_sizes[10]; + for (size_t i = 0; i < num_results; i++) { + doc_sizes[i] = 65536; + } + + // Retrieve only documents (no metadata or embeddings needed for RAG) + int get_result = cactus_index_get(index, result_ids[0], num_results, + docs, doc_sizes, + NULL, NULL, // No metadata + NULL, NULL); // No embeddings + + if (get_result == 0) { + // Build context from retrieved documents + char context[32768] = ""; + for (size_t i = 0; i < num_results; i++) { + strcat(context, docs[i]); + strcat(context, "\n\n"); + } + + printf("Context: %s\n", context); + } +} + +cactus_index_destroy(index); +``` + +### Delete and Compact +```c +cactus_index_t index = cactus_index_init("./my_index", 768); +if (!index) return -1; + +int ids[] = {1, 3, 5}; +int deleted = cactus_index_delete(index, ids, 3); +if (deleted == 0) { + printf("Documents deleted successfully\n"); +} + +// Compact to reclaim space +int compact_result = cactus_index_compact(index); +if (compact_result == 0) { + printf("Index compacted successfully\n"); +} + +cactus_index_destroy(index); +``` + +### Migrate Embedding Model +```c +// Open old index and create new index with different dimensions +cactus_index_t old_index = cactus_index_init("./old_index", 768); +cactus_index_t new_index = cactus_index_init("./new_index", 1536); + +if (!old_index || !new_index) { + fprintf(stderr, "Failed to open indexes\n"); + return -1; +} + +// Get all documents from old index +int all_doc_ids[] = {1, 2, 3, 4, 5}; +int num_docs = 5; + +// Allocate buffers for old documents +char* old_docs[5]; +char* old_metas[5]; + +for (int i = 0; i < num_docs; i++) { + old_docs[i] = malloc(65536); + old_metas[i] = malloc(65536); +} + +size_t doc_sizes[5], meta_sizes[5]; +for (int i = 0; i < num_docs; i++) { + doc_sizes[i] = 65536; + meta_sizes[i] = 65536; +} + +int get_result = cactus_index_get(old_index, all_doc_ids, num_docs, + old_docs, doc_sizes, + old_metas, meta_sizes, + NULL, NULL); // Don't need old embeddings + +if (get_result == 0) { + // Regenerate embeddings with new model (1536 dimensions) + float new_embs[5][1536]; + const float* new_emb_ptrs[5]; + + for (int i = 0; i < num_docs; i++) { + // Generate new embedding for old_docs[i] + // ... embedding generation code ... + new_emb_ptrs[i] = new_embs[i]; + } + + // Add to new index + const char* doc_ptrs[5]; + const char* meta_ptrs[5]; + for (int i = 0; i < num_docs; i++) { + doc_ptrs[i] = old_docs[i]; + meta_ptrs[i] = old_metas[i]; + } + + cactus_index_add(new_index, all_doc_ids, doc_ptrs, meta_ptrs, + new_emb_ptrs, num_docs, 1536); +} + +cactus_index_destroy(old_index); +cactus_index_destroy(new_index); +``` + +## Best Practices + +1. **Return Values**: Check all returns (0 = success, -1 = error) +2. **Buffers**: + - `document_buffer_sizes` and `metadata_buffer_sizes`: bytes + - `embedding_buffer_sizes`: number of floats (not bytes) + - Pass NULL for unused buffers in `cactus_index_get` +3. **Memory**: Always call `cactus_index_destroy()` when done +4. **Thread Safety**: One index instance per thread +5. **Batching**: Add 100-1000 documents per call for best performance +6. **Errors**: Use `cactus_get_last_error()` for error details + +## See Also + +- [Cactus Engine API](/docs/cactus_engine.md) — LLM inference, embeddings (`cactus_embed`), and RAG query APIs +- [Cactus Graph API](/docs/cactus_graph.md) — Low-level computational graph for custom tensor operations +- [Python Binding](/python/) — Python bindings with vector index support +- [Swift Binding](/bindings/swift/) — Swift vector index functions +- [Kotlin Binding](/bindings/kotlin/) — Kotlin vector index functions +- [Flutter Binding](/flutter/) — Dart vector index functions diff --git a/docs/choose-bindings.md b/docs/choose-bindings.md new file mode 100644 index 000000000..78789f27c --- /dev/null +++ b/docs/choose-bindings.md @@ -0,0 +1,49 @@ +# Choose Your Binding + +Not sure which binding to use? + +Pick the right one for your platform and use case: + +| | React Native | Flutter | Kotlin | Swift | Python | Rust | CLI | C++ | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| **Platforms** | iOS, Android | iOS, Android, macOS | Android, iOS (KMP) | iOS, macOS | Arm Linux, macOS | All | macOS, Linux | All | +| **Install** | npm | build | build | build | build | cargo | brew / source | header | +| **LLM** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Streaming** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Vision** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Audio** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Transcription** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Diarization** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Function Calling** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **RAG / Embeddings** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | +| **Cloud Fallback** | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | :white_check_mark: | + +## Quick Recommendations + +!!! tip "Building a mobile app?" + - **React Native** -- already have a React Native project? Just `npm install` and go + - **Flutter** -- cross-platform mobile + Mac with full native bindings + - **Kotlin** -- native Android apps or Kotlin Multiplatform + - **Swift** -- native iOS/macOS apps with Apple NPU acceleration + +!!! tip "Server-side or scripting?" + - **Python** -- server-side inference, batch processing, or rapid prototyping + - **CLI** -- quick model testing and interactive sessions without writing code + +!!! tip "Embedding in a native app?" + - **C++** -- game engines, native desktop apps, or any C/C++ project + - **Rust** -- systems-level integration with safe FFI bindings + +## Binding Documentation + +- **[React Native](https://github.com/cactus-compute/cactus-react-native)** -- npm package with React hooks (`useCactusLM`) +- **[Python](/python/)** -- Module-level FFI bindings, mirrors the C API +- **[Swift](/bindings/swift/)** -- XCFramework for iOS/macOS with NPU support +- **[Kotlin](/bindings/kotlin/)** -- JNI bindings + Kotlin Multiplatform support +- **[Flutter](/flutter/)** -- Dart FFI bindings for Android, iOS, and macOS +- **[Rust](/rust/)** -- Auto-generated FFI bindings via bindgen +- **[C++ / Engine API](/docs/cactus_engine.md)** -- Direct C FFI for maximum control + +## Getting Started + +Once you've picked your binding, head to the **[Quickstart](quickstart.md)** to install and run your first completion. diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 000000000..9c800202a --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,38 @@ +--- +title: "Runtime & Weights Compatibility" +description: "How Cactus runtime versions map to model weight versions on HuggingFace. Explains versioning, compatibility checks, and when to re-download weights." +keywords: ["versioning", "compatibility", "model weights", "HuggingFace", "Cactus runtime"] +--- + +# Runtime & Weights Compatibility + +Some Cactus releases change the internal weight format. When this happens, cached weights from an older version will not load with a newer runtime and must be re-downloaded. + +Breaking weight changes are called out in the [release notes](https://github.com/cactus-compute/cactus/releases). + +## How Versioning Works + +Weights are published to [Hugging Face](https://huggingface.co/Cactus-Compute) and **only re-tagged when they actually change**. If a release does not affect the weight format, the previous tag remains — no new upload. + +``` +Runtime v1.7 -> weights tagged v1.7 on HF +Runtime v1.8 -> no new tag (unchanged) - still use v1.7 +... +Runtime v1.14 -> no new tag - still use v1.7 +Runtime v1.15 -> new tag v1.15 (changed!) - must update +``` + +**The rule:** use the latest HF weight tag that is ≤ your runtime version. + +## Checking Compatibility + +1. Open your model on [huggingface.co/Cactus-Compute](https://huggingface.co/Cactus-Compute) +2. Click **Files and versions → open branch dropdown from Main** +3. Find the latest tag that is ≤ your runtime version +4. If your local weights use an older tag, re-download them + +## See Also + +- [Cactus Engine API](/docs/cactus_engine.md) — Full inference API reference +- [Fine-tuning Guide](/docs/finetuning.md) — Convert and deploy custom fine-tunes +- [HuggingFace Weights](https://huggingface.co/Cactus-Compute) — Official Cactus model weights diff --git a/docs/finetuning.md b/docs/finetuning.md new file mode 100644 index 000000000..c15db43e5 --- /dev/null +++ b/docs/finetuning.md @@ -0,0 +1,190 @@ +--- +title: "Deploying Unsloth Fine-Tunes to Cactus for Phones" +description: "Step-by-step guide to training LoRA fine-tunes with Unsloth and deploying them to iOS and Android devices using Cactus inference engine." +keywords: ["fine-tuning", "LoRA", "Unsloth", "mobile deployment", "on-device AI", "iOS", "Android", "Qwen", "Gemma"] +--- + +# Deploying Unsloth Fine-Tunes to Cactus for Phones + +- Cactus is an inference engine for mobile devices, macs and ARM chips like Raspberry Pi. +- At INT8, Cactus runs `Qwen3-0.6B` and `LFM2-1.2B` at `60-70 toks/sec` on iPhone 17 Pro, `13-18 toks/sec` on budget Pixel 6a. +- INT4 quantization provides ~50% memory reduction with minimal quality loss. +- Task-Specific INT8 tunes of `Gemma3-270m` hit `150 toks/sec` on iPhone 17 Pro and `23 toks/sec` on Raspberry Pi. + +## Quick Start + +### 1. Train (Google Colab / GPU) + +Use the provided notebook or your own Unsloth training script: + +```python +from unsloth import FastLanguageModel + +model, tokenizer = FastLanguageModel.from_pretrained( + model_name="unsloth/gemma-3-4b-it", + max_seq_length=2048, + load_in_4bit=True, +) + +model = FastLanguageModel.get_peft_model( + model, + r=16, + target_modules=["q_proj", "k_proj", "v_proj", "o_proj", + "gate_proj", "up_proj", "down_proj"], + lora_alpha=16, + lora_dropout=0, + use_gradient_checkpointing="unsloth", +) + +# ... train with SFTTrainer ... + +# Save adapter +model.save_pretrained("my-lora-adapter") +tokenizer.save_pretrained("my-lora-adapter") + +# Push to Hub (optional) +model.push_to_hub("username/my-lora-adapter") +``` + +### 2. Setup Cactus + +```bash +git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup +``` +Logo + +### 3. Convert for Cactus + +```bash +# From local adapter: Use the correct base model! +cactus convert Qwen/Qwen3-0.6B ./my-qwen3-0.6b --lora ./my-lora-adapter + +# From HuggingFace Hub: Use the correct base model! +cactus convert Qwen/Qwen3-0.6B ./my-qwen3-0.6b --lora username/my-lora-adapter + +``` +Logo + +### 4. Run + +Build the native library, then test your model on Mac: + +```bash +cactus build +cactus run ./my-qwen3-0.6b +``` +Logo + +### 5. Use in iOS/macOS App + +Build the native library: + +```bash +cactus build --apple +``` +```bash +Build complete! +Total time: 58 seconds +Static libraries: + Device: /Users/henry/Desktop/cactus/apple/libcactus-device.a + Simulator: /Users/henry/Desktop/cactus/apple/libcactus-simulator.a +XCFrameworks: + iOS: /Users/henry/Desktop/cactus/apple/cactus-ios.xcframework + macOS: /Users/henry/Desktop/cactus/apple/cactus-macos.xcframework +Apple build complete! +(venv) henry@Henrys-MacBook-Air cactus % +``` + +Link `cactus-ios.xcframework` to your Xcode project, then: + +```swift +import Foundation +import cactus + +// Load model from app bundle +let modelPath = Bundle.main.path(forResource: "my-model", ofType: nil)! +let model = try cactusInit(modelPath, nil, false) + +// Run completion +let messages = "[{\"role\":\"user\",\"content\":\"Hello!\"}]" +let resultJson = try cactusComplete(model, messages, nil, nil, nil) +if let data = resultJson.data(using: .utf8), + let obj = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let response = obj["response"] as? String { + print(response) +} + +// Cleanup +cactusDestroy(model) +``` +Logo + +You can now build iOS apps using the following code, +but to see performance on any device while testing, +run cactus tests by plugging any iphone to your Mac then running: + +```bash +cactus test --model --ios +``` + +Cactus demo apps will eventually expand to using your custom fine-tunes. +Also, `cactus run` will allow plugging in a phone, +such that the interactive session use the phone chips, +this way you can test before fully building out your apps. + +### 6. Use in Android App + +Build the native library: + +```bash +cactus build --android +``` +```bash +Build complete! +Shared library location: /Users/henry/Desktop/cactus/android/libcactus.so +Static library location: /Users/henry/Desktop/cactus/android/libcactus.a +Android build complete! +(venv) henry@Henrys-MacBook-Air cactus % +``` + +Copy `libcactus.so` to `app/src/main/jniLibs/arm64-v8a/`, then: + +```kotlin +import com.cactus.* +import org.json.JSONObject + +val model = cactusInit("/data/local/tmp/my-model", null, false) + +// Run completion +val resultJson = cactusComplete(model, """[{"role":"user","content":"Hello!"}]""", null, null, null) +val response = JSONObject(resultJson).getString("response") +println(response) +cactusDestroy(model) +``` + +You can now build Android apps using the following code, +but to see performance on any device while testing, +run cactus tests by plugging any android phone to your Mac then running: + +```bash +cactus test --model --android +``` + +Cactus demo apps will eventually expand to using your custom fine-tunes. +Also, `cactus run` will allow plugging in a phone, +such that the interactive session use the phone chips, +this way you can test before fully building out your apps. + +## Resources + +- Supported Base Models: `Qwen3, Qwen3.5, Gemma3, LFM2, LFM2.5` +- Full API reference: [Cactus Engine](https://github.com/cactus-compute/cactus/blob/main/docs/cactus_engine.md) +- Learn more and report bugs: [Cactus](https://github.com/cactus-compute/cactus/tree/main) + +## See Also + +- [Cactus Engine API](/docs/cactus_engine.md) — Full C API reference for inference, streaming, and tool calling +- [Runtime Compatibility](/docs/compatibility.md) — Ensure your weights match your Cactus runtime version +- [Python Binding](/python/) — Use fine-tuned models from Python +- [Swift Binding](/bindings/swift/) — Deploy fine-tuned models in iOS/macOS apps +- [Kotlin Binding](/bindings/kotlin/) — Deploy fine-tuned models in Android apps diff --git a/docs/quickstart.md b/docs/quickstart.md new file mode 100644 index 000000000..6c7a0f43f --- /dev/null +++ b/docs/quickstart.md @@ -0,0 +1,193 @@ +# Quickstart + +Install Cactus and run your first on-device AI completion. + +## Installation + +=== "React Native" + + ```bash + npm install cactus-react-native react-native-nitro-modules + ``` + +=== "Flutter" + + --8<-- "flutter/README.md:install" + + ### Platform Integration + + --8<-- "flutter/README.md:integration" + +=== "Kotlin" + + --8<-- "android/README.md:install" + + ### Platform Integration + + --8<-- "android/README.md:integration" + +=== "Swift" + + --8<-- "apple/README.md:install" + + ### Platform Integration + + --8<-- "apple/README.md:integration" + +=== "Python" + + --8<-- "python/README.md:install" + +=== "Rust" + + --8<-- "rust/README.md:install" + +=== "CLI" + + **Homebrew (macOS):** + + ```bash + brew install cactus-compute/cactus/cactus + ``` + + **From Source (macOS):** + + ```bash + git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup + ``` + + **From Source (Linux):** + + ```bash + sudo apt-get install python3 python3-venv python3-pip cmake build-essential libcurl4-openssl-dev + git clone https://github.com/cactus-compute/cactus && cd cactus && source ./setup + ``` + +=== "C++" + + Include the Cactus header in your project: + + ```cpp + #include + ``` + + See the [Cactus repository](https://github.com/cactus-compute/cactus) for CMake build instructions. + +--- + +## Your First Completion + +=== "React Native" + + ```tsx + import { useCactusLM } from 'cactus-react-native'; + + const App = () => { + const cactusLM = useCactusLM(); + + useEffect(() => { + if (!cactusLM.isDownloaded) { + cactusLM.download(); + } + }, []); + + const handleGenerate = () => { + cactusLM.complete({ + messages: [{ role: 'user', content: 'What is the capital of France?' }], + }); + }; + + if (cactusLM.isDownloading) { + return Downloading: {Math.round(cactusLM.downloadProgress * 100)}%; + } + + return ( + <> +