diff --git a/.bazelignore b/.bazelignore new file mode 100644 index 0000000..9b8aeea --- /dev/null +++ b/.bazelignore @@ -0,0 +1,12 @@ +# Local experiments and unused vendor trees; excluded from //... discovery. +share/google_benchmark +share/benchmarks +third_party/cblas +third_party/csparse +third_party/hdf5 +third_party/jburkardt +third_party/matmul +third_party/mesch12b +third_party/mm_offloading +third_party/spmatrix +third_party/spblas diff --git a/.bazelrc b/.bazelrc index ee67f6b..a8ad33b 100644 --- a/.bazelrc +++ b/.bazelrc @@ -135,10 +135,7 @@ build:openmp_only --define=MATRIX_BACKEND=openmp # macOS defaults: OpenMP + Accelerate. MPS stays available as an optional # runtime backend via sm_set_backend(SM_BACKEND_MPS). build:macos --config=accelerate -build:macos --repo_env=SUITESPARSE_ROOT=/opt/homebrew/opt/suite-sparse # Linux defaults: OpenMP + OpenBLAS. build:linux --config=openblas -build:linux --repo_env=OPENBLAS_ROOT=/home/linuxbrew/.linuxbrew/opt/openblas -build:linux --repo_env=SUITESPARSE_ROOT=/home/linuxbrew/.linuxbrew/opt/suite-sparse diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..b13d146 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +9.3.0 diff --git a/.clangd b/.clangd index 1cd346f..f699c51 100644 --- a/.clangd +++ b/.clangd @@ -1,23 +1,5 @@ CompileFlags: Add: - # matio Include paths - - "-Ithird_party/matio" - - "-Ibazel-out/darwin_arm64-fastbuild/bin/external/+http_archive+matio" - - "-Ibazel-out/darwin_arm64-fastbuild/bin/external/+http_archive+matio/src" - - "-Iexternal/+http_archive+matio" - - "-Iexternal/+http_archive+matio/src" - # matrix public headers - "-Iapp/matrix/include" - # third-party benchmark dependencies used by share/google_benchmark - - "-Ithird_party/matmul/include" - - "-Ithird_party/mm_offloading" - # bazel external repo symlink for google benchmark headers - - "-Iexternal/google_benchmark+/include" - ---- -If: - PathMatch: ^share/google_benchmark/benchmarks/legacy/.* -Diagnostics: - Suppress: - - pp_file_not_found - - unused-includes + - "-Iapp/vector/include" + - "-Iapp/tensor/include" diff --git a/.envrc b/.envrc deleted file mode 100644 index bcdb166..0000000 --- a/.envrc +++ /dev/null @@ -1 +0,0 @@ -conda activate py-phys diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 14716a7..051d35a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,23 +2,50 @@ name: BUILD on: push: - branches: - - master + branches: [master] pull_request: - branches: - - master + branches: [master] + workflow_dispatch: -jobs: - test: - # virtual environments: https://github.com/actions/virtual-environments - runs-on: macos-latest - - steps: - - name: Checkout - uses: actions/checkout@v3 - - - name: Test the code (unit tests) - run: bazel test //tests/... +permissions: + contents: read - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v3 +jobs: + test: + name: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [macos-latest, ubuntu-24.04] + runs-on: ${{ matrix.os }} + timeout-minutes: 45 + env: + OPENBLAS_NUM_THREADS: "1" + OMP_NUM_THREADS: "2" + steps: + - uses: actions/checkout@v7 + - uses: bazel-contrib/setup-bazel@0.19.1 + with: + bazelisk-cache: true + disk-cache: true + repository-cache: true + - name: Prepare system OpenBLAS and LAPACKE + if: runner.os == 'Linux' + shell: bash + run: | + sudo apt-get update + sudo apt-get install -y libopenblas-pthread-dev liblapacke-dev gfortran ruby + prefix="$RUNNER_TEMP/mmatrix-openblas" + multiarch="$(gcc -print-multiarch)" + mkdir -p "$prefix/include" "$prefix/lib" + for header in /usr/include/"$multiarch"/openblas-pthread/*.h /usr/include/lapack*.h; do + ln -s "$header" "$prefix/include/$(basename "$header")" + done + ln -s /usr/lib/"$multiarch"/openblas-pthread/libopenblas.a "$prefix/lib/libopenblas.a" + ln -s /usr/lib/"$multiarch"/liblapacke.a "$prefix/lib/liblapacke.a" + echo "OPENBLAS_ROOT=$prefix" >> "$GITHUB_ENV" + - name: Build and test all published targets + run: bazel test //... --test_output=errors --color=no --curses=no + - name: Verify installed libraries + shell: bash + run: bash tools/ci/test_install.sh diff --git a/.gitignore b/.gitignore index ff2cba8..e2ffc4b 100644 --- a/.gitignore +++ b/.gitignore @@ -79,10 +79,7 @@ Temporary Items /bazel-* /bazel.iml /output/ -.bazelversion -.bazelignore user.bazelrc -MODULE.bazel.lock #------------------------------------------------------------------------------- # Temporary files @@ -108,6 +105,7 @@ MODULE.bazel.lock /share/simple_benchmark/* !/share/simple_benchmark/BUILD !/share/simple_benchmark/bench_sm_mps.c +!/share/simple_benchmark/simple_bench.h /lib/* /_archive/* /archive/* @@ -129,15 +127,12 @@ MODULE.bazel.lock /third_party/cblas #------------------------------------------------------------------------------- -*.in .clang-tidy -.clangd .envrc compile_commands.json *.png StyleGuide.md -MODULE.bazel.lock profile.gz /.venv-bench/ @@ -145,3 +140,8 @@ profile.gz TENSOR_ARCHITECTURE_ANALYSIS.md TENSOR_PATTERNS_QUICK_REF.md app/tensor/docs/mps-performance.md + +# Generated benchmark and documentation output +*.profraw +*.profdata +/site/ diff --git a/BUILD b/BUILD index 3373d58..e9c2bf6 100644 --- a/BUILD +++ b/BUILD @@ -4,24 +4,22 @@ load("@mkdocs//:defs.bzl", "mkdocs_build", "mkdocs_config", "mkdocs_serve") load("@rules_docs//docgen:defs.bzl", "docs") load("//tools/install:def.bzl", "installer") -exports_files(["compile_commands.json"]) - installer( name = "matrix_installer", data = [ - "//:LICENSE.txt", # if available - "//app/matrix", # the target to be installed - "//app/matrix:matrix_header", # must be collected in a filegroup + "//:LICENSE.txt", + "//app/matrix", + "//app/matrix:matrix_header", "//app/vector", "//app/vector:vector_header", - "@log", - "@matio", - "@openblas//:install_files", - "@openmp//:libomp", - "@pcg", - "@suitesparse//:install_files", - "@zlib", - ], + ] + select({ + "//app/matrix:macos": [ + "//app/matrix:sm_mps", + "//app/tensor:st", + "//app/tensor:tensor_header", + ], + "//conditions:default": [], + }), system_integration = False, # set "True" symlink to /usr/local/lib and /usr/local/include ) @@ -43,6 +41,7 @@ alias( docs( name = "docs_src", srcs = glob(["doc/**/*.md"]) + [ + "CHANGELOG.md", "//doc/examples:tensor_basic.md", ], readme_content = "MMatrix Dokumentation.", diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..1590fd6 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,36 @@ +# Changelog + +## 0.4.0 (unreleased) + +Changes since `v0.3.1`. + +### Added + +- Vector modules for float/double, strided views, sparse vectors, geometry, and vector/tensor bridges. +- Flat-buffer SGD, momentum, RMSProp, Adam, and gradient norm clipping kernels. +- Shared NN activation/backward kernels, fused GEMM + bias + activation, and row/loss kernels. +- Tensor storage abstraction, Metal buffer support, and f32/bf16 conversion utilities. +- Interpreter kernels: broadcast binary arithmetic, LayerNorm, ReduceMax, batched matmul, gather, and materializing permute. +- IEEE binary16 scalar/bulk conversion helpers. +- GPU-resident matrix GEMM streams/plans and reproducible benchmark sources. + +### Changed / migration + +- SuiteSparse is no longer a library or build dependency. Sparse matrices continue to use COO with a lazy CSR row cache. +- Removed `dms_to_cs`, `dms_from_cs`, and `cs_to_dms`, and the CSparse cache fields in `DoubleSparseMatrix`. Code accessing those APIs/fields must migrate; recompile consumers. +- Matrix/tensor conversions now live in `st_convert.h` and `//app/tensor:st`, rather than `m_convert.h`. `st_from_sm` creates a rank-2 `[rows, cols]` tensor; `sm_from_st` requires a rank-2 f32 tensor. +- Tensor implementation files are grouped into core, backend, runtime, and operation directories. Depend on public Bazel targets rather than source paths. +- Bazel version and module lockfile are included for reproducible dependency resolution. Linux OpenBLAS remains a system installation. + +### Fixed during release preparation + +- Produce ordinary Linux object archives rather than forcing LLVM LTO; check system-linker consumption and choose the platform archive bundler. +- Link separately packaged LAPACKE on Linux and preserve NN-kernel NaN/thread semantics under optimized builds. + +- Exercise actual OpenMP branches in interpreter thread-comparison tests and remove an unmatched floating-point pragma. +- Reject size overflows before gather/permute byte copies; validate permutation indices before shape access; preserve arbitrary unaligned byte payloads. +- Remove stale SuiteSparse and local HDF5 wiring, obsolete vendor build wrappers, and benchmark targets whose sources were not published. +- Include the MPS benchmark helper header in the source release and exclude local experiments from Bazel discovery. +- Install transitive library archives and required external headers; include tensor artifacts on macOS; use portable macOS/Linux install flags. +- Add macOS/Ubuntu CI coverage with standard/bundle installation consumer checks. +- Repair CI test discovery and document the platform boundaries and new API entrypoints. diff --git a/MODULE.bazel b/MODULE.bazel index a757d98..a23578c 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -13,7 +13,7 @@ For more details, please check https://github.com/bazelbuild/bazel/issues/18958 module( name = "mmatrix", - version = "0.1.0", + version = "0.4.0", ) ###################################################################### @@ -22,10 +22,9 @@ module( bazel_dep(name = "platforms", version = "1.0.0") bazel_dep(name = "zlib", version = "1.3.1.bcr.5") -bazel_dep(name = "apple_support", version = "1.24.2") +bazel_dep(name = "apple_support", version = "2.5.2") bazel_dep(name = "bazel_skylib", version = "1.8.2") bazel_dep(name = "rules_shell", version = "0.6.1") -bazel_dep(name = "google_benchmark", version = "1.9.2") bazel_dep(name = "rules_cc", version = "0.2.17") bazel_dep(name = "rules_python", version = "1.7.0") bazel_dep(name = "rules_docs", version = "0.3.0") @@ -44,7 +43,7 @@ pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( hub_name = "pypi", python_version = "3.11", - requirements_lock = "//doc:requirements.txt", + requirements_lock = "//:doc/requirements.txt", ) use_repo(pip, "pypi") @@ -91,25 +90,7 @@ bazel_dep(name = "wolfd_bazel_compile_commands", version = "0.5.2", dev_dependen host_local_library = use_repo_rule("//tools:host_local_library.bzl", "host_local_library") -############################# -# Hombrew Local Libraries (MacOS) -############################# - -# define new_local_repository rule for local installed non-Bazel libraries: -new_local_repository = use_repo_rule("@bazel_tools//tools/build_defs/repo:local.bzl", "new_local_repository") - -# due to license issues in private lib folder. -new_local_repository( - name = "hdf5", - path = "third_party/hdf5", -) - -new_local_repository( - name = "log", - build_file = "@//:third_party/log.c/log.c.BUILD", - path = "third_party/log.c", -) - +# System OpenBLAS (only fetched when the OpenBLAS backend is selected). # openblas is an optimized BLAS library based on GotoBLAS2 1.13 BSD version. host_local_library( name = "openblas", @@ -122,16 +103,3 @@ host_local_library( "lib", ], ) - -# Optional Homebrew SuiteSparse wrapper for the GraphBLAS benchmark only. -host_local_library( - name = "suitesparse_graphblas_host", - build_file = "@//:third_party/suitesparse.BUILD", - env_var = "SUITESPARSE_ROOT", - linux_path = "/home/linuxbrew/.linuxbrew/opt/suite-sparse", - macos_path = "/opt/homebrew/opt/suite-sparse", - required_subdirs = [ - "include", - "lib", - ], -) diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock new file mode 100644 index 0000000..6f52b00 --- /dev/null +++ b/MODULE.bazel.lock @@ -0,0 +1,689 @@ +{ + "lockFileVersion": 28, + "registryFileHashes": { + "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", + "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", + "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", + "https://bcr.bazel.build/modules/abseil-cpp/20230125.1/MODULE.bazel": "89047429cb0207707b2dface14ba7f8df85273d484c2572755be4bab7ce9c3a0", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "1c8cec495288dccd14fdae6e3f95f772c1c91857047a098fad772034264cc8cb", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.0/MODULE.bazel": "d253ae36a8bd9ee3c5955384096ccb6baf16a1b1e93e858370da0a3b94f77c16", + "https://bcr.bazel.build/modules/abseil-cpp/20230802.1/MODULE.bazel": "fa92e2eb41a04df73cdabeec37107316f7e5272650f81d6cc096418fe647b915", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.1/MODULE.bazel": "37bcdb4440fbb61df6a1c296ae01b327f19e9bb521f9b8e26ec854b6f97309ed", + "https://bcr.bazel.build/modules/abseil-cpp/20240116.2/MODULE.bazel": "73939767a4686cd9a520d16af5ab440071ed75cec1a876bf2fcfaf1f71987a16", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.0/MODULE.bazel": "d1086e248cda6576862b4b3fe9ad76a214e08c189af5b42557a6e1888812c5d5", + "https://bcr.bazel.build/modules/abseil-cpp/20250127.1/MODULE.bazel": "c4a89e7ceb9bf1e25cf84a9f830ff6b817b72874088bf5141b314726e46a57c1", + "https://bcr.bazel.build/modules/abseil-cpp/20250512.1/MODULE.bazel": "d209fdb6f36ffaf61c509fcc81b19e81b411a999a934a032e10cd009a0226215", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/MODULE.bazel": "51f2312901470cdab0dbdf3b88c40cd21c62a7ed58a3de45b365ddc5b11bcab2", + "https://bcr.bazel.build/modules/abseil-cpp/20250814.1/source.json": "cea3901d7e299da7320700abbaafe57a65d039f10d0d7ea601c4a66938ea4b0c", + "https://bcr.bazel.build/modules/apple_support/1.11.1/MODULE.bazel": "1843d7cd8a58369a444fc6000e7304425fba600ff641592161d9f15b179fb896", + "https://bcr.bazel.build/modules/apple_support/1.15.1/MODULE.bazel": "a0556fefca0b1bb2de8567b8827518f94db6a6e7e7d632b4c48dc5f865bc7c85", + "https://bcr.bazel.build/modules/apple_support/1.21.0/MODULE.bazel": "ac1824ed5edf17dee2fdd4927ada30c9f8c3b520be1b5fd02a5da15bc10bff3e", + "https://bcr.bazel.build/modules/apple_support/1.21.1/MODULE.bazel": "5809fa3efab15d1f3c3c635af6974044bac8a4919c62238cce06acee8a8c11f1", + "https://bcr.bazel.build/modules/apple_support/2.5.2/MODULE.bazel": "bb1b7081e390a8d3885f642cb0073dfcf8e7a36525e079fa8f5835af7079c0fe", + "https://bcr.bazel.build/modules/apple_support/2.5.2/source.json": "dd51ee925a550e92b55eb41c4c809a967da311a2d2520241252a238a48b39c4c", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "2b31ffcc9bdc8295b2167e07a757dbbc9ac8906e7028e5170a3708cecaac119f", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "253d739ba126f62a5767d832765b12b59e9f8d2bc88cc1572f4a73e46eb298ca", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.19.3/source.json": "ffab9254c65ba945f8369297ad97ca0dec213d3adc6e07877e23a48624a8b456", + "https://bcr.bazel.build/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "812d2dd42f65dca362152101fbec418029cc8fd34cbad1a2fde905383d705838", + "https://bcr.bazel.build/modules/bazel_features/1.0.0/MODULE.bazel": "d7f022dc887efb96e1ee51cec7b2e48d41e36ff59a6e4f216c40e4029e1585bf", + "https://bcr.bazel.build/modules/bazel_features/1.1.1/MODULE.bazel": "27b8c79ef57efe08efccbd9dd6ef70d61b4798320b8d3c134fd571f78963dbcd", + "https://bcr.bazel.build/modules/bazel_features/1.10.0/MODULE.bazel": "f75e8807570484a99be90abcd52b5e1f390362c258bcb73106f4544957a48101", + "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", + "https://bcr.bazel.build/modules/bazel_features/1.15.0/MODULE.bazel": "d38ff6e517149dc509406aca0db3ad1efdd890a85e049585b7234d04238e2a4d", + "https://bcr.bazel.build/modules/bazel_features/1.17.0/MODULE.bazel": "039de32d21b816b47bd42c778e0454217e9c9caac4a3cf8e15c7231ee3ddee4d", + "https://bcr.bazel.build/modules/bazel_features/1.18.0/MODULE.bazel": "1be0ae2557ab3a72a57aeb31b29be347bcdc5d2b1eb1e70f39e3851a7e97041a", + "https://bcr.bazel.build/modules/bazel_features/1.19.0/MODULE.bazel": "59adcdf28230d220f0067b1f435b8537dd033bfff8db21335ef9217919c7fb58", + "https://bcr.bazel.build/modules/bazel_features/1.21.0/MODULE.bazel": "675642261665d8eea09989aa3b8afb5c37627f1be178382c320d1b46afba5e3b", + "https://bcr.bazel.build/modules/bazel_features/1.23.0/MODULE.bazel": "fd1ac84bc4e97a5a0816b7fd7d4d4f6d837b0047cf4cbd81652d616af3a6591a", + "https://bcr.bazel.build/modules/bazel_features/1.27.0/MODULE.bazel": "621eeee06c4458a9121d1f104efb80f39d34deff4984e778359c60eaf1a8cb65", + "https://bcr.bazel.build/modules/bazel_features/1.28.0/MODULE.bazel": "4b4200e6cbf8fa335b2c3f43e1d6ef3e240319c33d43d60cc0fbd4b87ece299d", + "https://bcr.bazel.build/modules/bazel_features/1.3.0/MODULE.bazel": "cdcafe83ec318cda34e02948e81d790aab8df7a929cec6f6969f13a489ccecd9", + "https://bcr.bazel.build/modules/bazel_features/1.30.0/MODULE.bazel": "a14b62d05969a293b80257e72e597c2da7f717e1e69fa8b339703ed6731bec87", + "https://bcr.bazel.build/modules/bazel_features/1.33.0/MODULE.bazel": "8b8dc9d2a4c88609409c3191165bccec0e4cb044cd7a72ccbe826583303459f6", + "https://bcr.bazel.build/modules/bazel_features/1.34.0/MODULE.bazel": "e8475ad7c8965542e0c7aac8af68eb48c4af904be3d614b6aa6274c092c2ea1e", + "https://bcr.bazel.build/modules/bazel_features/1.38.0/MODULE.bazel": "f9b8a9c890ebd216b4049fd12a31d3c2602e3403c7af636b04fbbd7453edc9c9", + "https://bcr.bazel.build/modules/bazel_features/1.4.1/MODULE.bazel": "e45b6bb2350aff3e442ae1111c555e27eac1d915e77775f6fdc4b351b758b5d7", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/MODULE.bazel": "275a59b5406ff18c01739860aa70ad7ccb3cfb474579411decca11c93b951080", + "https://bcr.bazel.build/modules/bazel_features/1.42.1/source.json": "fcd4396b2df85f64f2b3bb436ad870793ecf39180f1d796f913cc9276d355309", + "https://bcr.bazel.build/modules/bazel_features/1.9.0/MODULE.bazel": "885151d58d90d8d9c811eb75e3288c11f850e1d6b481a8c9f766adee4712358b", + "https://bcr.bazel.build/modules/bazel_features/1.9.1/MODULE.bazel": "8f679097876a9b609ad1f60249c49d68bfab783dd9be012faf9d82547b14815a", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "d6e00979a98ac14ada5e31c8794708b41434d461e7e7ca39b59b765e6d233b18", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/MODULE.bazel": "22b70b80ac89ad3f3772526cd9feee2fa412c2b01933fea7ed13238a448d370d", + "https://bcr.bazel.build/modules/bazel_lib/3.0.0/source.json": "895f21909c6fba01d7c17914bb6c8e135982275a1b18cdaa4e62272217ef1751", + "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", + "https://bcr.bazel.build/modules/bazel_skylib/1.1.1/MODULE.bazel": "1add3e7d93ff2e6998f9e118022c84d163917d912f5afafb3058e3d2f1545b5e", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.0/MODULE.bazel": "44fe84260e454ed94ad326352a698422dbe372b21a1ac9f3eab76eb531223686", + "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", + "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.1/MODULE.bazel": "a0dcb779424be33100dcae821e9e27e4f2901d9dfd5333efe5ac6a8d7ab75e1d", + "https://bcr.bazel.build/modules/bazel_skylib/1.4.2/MODULE.bazel": "3bd40978e7a1fac911d5989e6b09d8f64921865a45822d8b09e815eaa726a651", + "https://bcr.bazel.build/modules/bazel_skylib/1.5.0/MODULE.bazel": "32880f5e2945ce6a03d1fbd588e9198c0a959bb42297b2cfaf1685b7bc32e138", + "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.0/MODULE.bazel": "0db596f4563de7938de764cc8deeabec291f55e8ec15299718b93c4423e9796d", + "https://bcr.bazel.build/modules/bazel_skylib/1.7.1/MODULE.bazel": "3120d80c5861aa616222ec015332e5f8d3171e062e3e804a2a0253e1be26e59b", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.1/MODULE.bazel": "88ade7293becda963e0e3ea33e7d54d3425127e0a326e0d17da085a5f1f03ff6", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/MODULE.bazel": "69ad6927098316848b34a9142bcc975e018ba27f08c4ff403f50c1b6e646ca67", + "https://bcr.bazel.build/modules/bazel_skylib/1.8.2/source.json": "34a3c8bcf233b835eb74be9d628899bb32999d3e0eadef1947a0a562a2b16ffb", + "https://bcr.bazel.build/modules/bazelrc-preset.bzl/1.6.0/MODULE.bazel": "11bc946a2e1a9c1bea066e44a704091b482e6049164b75b2381e8419bf055cf1", + "https://bcr.bazel.build/modules/bazelrc-preset.bzl/1.6.0/source.json": "ad6cee6b42b5def78f6b3082c0bd7041a330003cbace330aaf1d3f94a122c48b", + "https://bcr.bazel.build/modules/buildozer/8.5.1/MODULE.bazel": "a35d9561b3fc5b18797c330793e99e3b834a473d5fbd3d7d7634aafc9bdb6f8f", + "https://bcr.bazel.build/modules/buildozer/8.5.1/source.json": "e3386e6ff4529f2442800dee47ad28d3e6487f36a1f75ae39ae56c70f0cd2fbd", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "cdf8cbe5ee750db04b78878c9633cc76e80dcf4416cbe982ac3a9222f80713c8", + "https://bcr.bazel.build/modules/gawk/5.3.2.bcr.1/source.json": "fa7b512dfcb5eafd90ce3959cf42a2a6fe96144ebbb4b3b3928054895f2afac2", + "https://bcr.bazel.build/modules/google_benchmark/1.8.2/MODULE.bazel": "a70cf1bba851000ba93b58ae2f6d76490a9feb74192e57ab8e8ff13c34ec50cb", + "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", + "https://bcr.bazel.build/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "22c31a561553727960057361aa33bf20fb2e98584bc4fec007906e27053f80c6", + "https://bcr.bazel.build/modules/googletest/1.14.0/MODULE.bazel": "cfbcbf3e6eac06ef9d85900f64424708cc08687d1b527f0ef65aa7517af8118f", + "https://bcr.bazel.build/modules/googletest/1.15.2/MODULE.bazel": "6de1edc1d26cafb0ea1a6ab3f4d4192d91a312fd2d360b63adaa213cd00b2108", + "https://bcr.bazel.build/modules/googletest/1.17.0/MODULE.bazel": "dbec758171594a705933a29fcf69293d2468c49ec1f2ebca65c36f504d72df46", + "https://bcr.bazel.build/modules/googletest/1.17.0/source.json": "38e4454b25fc30f15439c0378e57909ab1fd0a443158aa35aec685da727cd713", + "https://bcr.bazel.build/modules/helly25_bzl/0.3.1/MODULE.bazel": "3a4be20f6fc13be32ad44643b8252ef5af09eee936f1d943cd4fd7867fa92826", + "https://bcr.bazel.build/modules/helly25_bzl/0.3.1/source.json": "b129ab1828492de2c163785bbeb4065c166de52d932524b4317beb5b7f917994", + "https://bcr.bazel.build/modules/jq.bzl/0.1.0/MODULE.bazel": "2ce69b1af49952cd4121a9c3055faa679e748ce774c7f1fda9657f936cae902f", + "https://bcr.bazel.build/modules/jq.bzl/0.4.0/MODULE.bazel": "a7b39b37589f2b0dad53fd6c1ccaabbdb290330caa920d7ef3e6aad068cd4ab2", + "https://bcr.bazel.build/modules/jq.bzl/0.4.0/source.json": "52ec7530c4618e03f634b30ff719814a68d7d39c235938b7aa2abbfe1eb1c52c", + "https://bcr.bazel.build/modules/jsoncpp/1.9.5/MODULE.bazel": "31271aedc59e815656f5736f282bb7509a97c7ecb43e927ac1a37966e0578075", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/MODULE.bazel": "2f8d20d3b7d54143213c4dfc3d98225c42de7d666011528dc8fe91591e2e17b0", + "https://bcr.bazel.build/modules/jsoncpp/1.9.6/source.json": "a04756d367a2126c3541682864ecec52f92cdee80a35735a3cb249ce015ca000", + "https://bcr.bazel.build/modules/libpfm/4.11.0/MODULE.bazel": "45061ff025b301940f1e30d2c16bea596c25b176c8b6b3087e92615adbd52902", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/MODULE.bazel": "6f7b417dcc794d9add9e556673ad25cb3ba835224290f4f848f8e2db1e1fca74", + "https://bcr.bazel.build/modules/nlohmann_json/3.6.1/source.json": "f448c6e8963fdfa7eb831457df83ad63d3d6355018f6574fb017e8169deb43a9", + "https://bcr.bazel.build/modules/package_metadata/0.0.2/MODULE.bazel": "fb8d25550742674d63d7b250063d4580ca530499f045d70748b1b142081ebb92", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/MODULE.bazel": "341dab6f417197494517d54c8e557c0baee1de7aec83543a4fbefe57900acb7e", + "https://bcr.bazel.build/modules/package_metadata/0.0.6/source.json": "9581d8b22db43550ac75ecc314ee4fa0a33400bfdc77d1317d8af6b18dca7756", + "https://bcr.bazel.build/modules/platforms/0.0.10/MODULE.bazel": "8cb8efaf200bdeb2150d93e162c40f388529a25852b332cec879373771e48ed5", + "https://bcr.bazel.build/modules/platforms/0.0.11/MODULE.bazel": "0daefc49732e227caa8bfa834d65dc52e8cc18a2faf80df25e8caea151a9413f", + "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", + "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", + "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", + "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", + "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", + "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", + "https://bcr.bazel.build/modules/platforms/1.0.0/MODULE.bazel": "f05feb42b48f1b3c225e4ccf351f367be0371411a803198ec34a389fb22aa580", + "https://bcr.bazel.build/modules/platforms/1.0.0/source.json": "f4ff1fd412e0246fd38c82328eb209130ead81d62dcd5a9e40910f867f733d96", + "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", + "https://bcr.bazel.build/modules/protobuf/27.0/MODULE.bazel": "7873b60be88844a0a1d8f80b9d5d20cfbd8495a689b8763e76c6372998d3f64c", + "https://bcr.bazel.build/modules/protobuf/27.1/MODULE.bazel": "703a7b614728bb06647f965264967a8ef1c39e09e8f167b3ca0bb1fd80449c0d", + "https://bcr.bazel.build/modules/protobuf/29.0-rc2/MODULE.bazel": "6241d35983510143049943fc0d57937937122baf1b287862f9dc8590fc4c37df", + "https://bcr.bazel.build/modules/protobuf/29.0-rc3/MODULE.bazel": "33c2dfa286578573afc55a7acaea3cada4122b9631007c594bf0729f41c8de92", + "https://bcr.bazel.build/modules/protobuf/29.1/MODULE.bazel": "557c3457560ff49e122ed76c0bc3397a64af9574691cb8201b4e46d4ab2ecb95", + "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", + "https://bcr.bazel.build/modules/protobuf/31.0/MODULE.bazel": "cce0f305064713b382211c4c5af1d4fc1466e1fbc9059aa0941801970f6add5c", + "https://bcr.bazel.build/modules/protobuf/32.1/MODULE.bazel": "89cd2866a9cb07fee9ff74c41ceace11554f32e0d849de4e23ac55515cfada4d", + "https://bcr.bazel.build/modules/protobuf/33.4/MODULE.bazel": "114775b816b38b6d0ca620450d6b02550c60ceedfdc8d9a229833b34a223dc42", + "https://bcr.bazel.build/modules/protobuf/33.4/source.json": "555f8686b4c7d6b5ba731fbea13bf656b4bfd9a7ff629c1d9d3f6e1d6155de79", + "https://bcr.bazel.build/modules/pybind11_bazel/2.11.1/MODULE.bazel": "88af1c246226d87e65be78ed49ecd1e6f5e98648558c14ce99176da041dc378e", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/MODULE.bazel": "e6f4c20442eaa7c90d7190d8dc539d0ab422f95c65a57cc59562170c58ae3d34", + "https://bcr.bazel.build/modules/pybind11_bazel/2.12.0/source.json": "6900fdc8a9e95866b8c0d4ad4aba4d4236317b5c1cd04c502df3f0d33afed680", + "https://bcr.bazel.build/modules/re2/2023-09-01/MODULE.bazel": "cb3d511531b16cfc78a225a9e2136007a48cf8a677e4264baeab57fe78a80206", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "b4963dda9b31080be1905ef085ecd7dd6cd47c05c79b9cdf83ade83ab2ab271a", + "https://bcr.bazel.build/modules/re2/2024-07-02.bcr.1/source.json": "2ff292be6ef3340325ce8a045ecc326e92cbfab47c7cbab4bd85d28971b97ac4", + "https://bcr.bazel.build/modules/re2/2024-07-02/MODULE.bazel": "0eadc4395959969297cbcf31a249ff457f2f1d456228c67719480205aa306daa", + "https://bcr.bazel.build/modules/rules_android/0.1.1/MODULE.bazel": "48809ab0091b07ad0182defb787c4c5328bd3a278938415c00a7b69b50c4d3a8", + "https://bcr.bazel.build/modules/rules_android/0.1.1/source.json": "e6986b41626ee10bdc864937ffb6d6bf275bb5b9c65120e6137d56e6331f089e", + "https://bcr.bazel.build/modules/rules_apple/3.16.0/MODULE.bazel": "0d1caf0b8375942ce98ea944be754a18874041e4e0459401d925577624d3a54a", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/MODULE.bazel": "76e10fd4a48038d3fc7c5dc6e63b7063bbf5304a2e3bd42edda6ec660eebea68", + "https://bcr.bazel.build/modules/rules_apple/4.1.0/source.json": "8ee81e1708756f81b343a5eb2b2f0b953f1d25c4ab3d4a68dc02754872e80715", + "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", + "https://bcr.bazel.build/modules/rules_cc/0.0.10/MODULE.bazel": "ec1705118f7eaedd6e118508d3d26deba2a4e76476ada7e0e3965211be012002", + "https://bcr.bazel.build/modules/rules_cc/0.0.13/MODULE.bazel": "0e8529ed7b323dad0775ff924d2ae5af7640b23553dfcd4d34344c7e7a867191", + "https://bcr.bazel.build/modules/rules_cc/0.0.15/MODULE.bazel": "6704c35f7b4a72502ee81f61bf88706b54f06b3cbe5558ac17e2e14666cd5dcc", + "https://bcr.bazel.build/modules/rules_cc/0.0.16/MODULE.bazel": "7661303b8fc1b4d7f532e54e9d6565771fea666fbdf839e0a86affcd02defe87", + "https://bcr.bazel.build/modules/rules_cc/0.0.17/MODULE.bazel": "2ae1d8f4238ec67d7185d8861cb0a2cdf4bc608697c331b95bf990e69b62e64a", + "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", + "https://bcr.bazel.build/modules/rules_cc/0.0.6/MODULE.bazel": "abf360251023dfe3efcef65ab9d56beefa8394d4176dd29529750e1c57eaa33f", + "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", + "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", + "https://bcr.bazel.build/modules/rules_cc/0.1.1/MODULE.bazel": "2f0222a6f229f0bf44cd711dc13c858dad98c62d52bd51d8fc3a764a83125513", + "https://bcr.bazel.build/modules/rules_cc/0.1.2/MODULE.bazel": "557ddc3a96858ec0d465a87c0a931054d7dcfd6583af2c7ed3baf494407fd8d0", + "https://bcr.bazel.build/modules/rules_cc/0.1.5/MODULE.bazel": "88dfc9361e8b5ae1008ac38f7cdfd45ad738e4fa676a3ad67d19204f045a1fd8", + "https://bcr.bazel.build/modules/rules_cc/0.2.0/MODULE.bazel": "b5c17f90458caae90d2ccd114c81970062946f49f355610ed89bebf954f5783c", + "https://bcr.bazel.build/modules/rules_cc/0.2.11/MODULE.bazel": "e94f24f065bf2191dba2dace951814378b66a94bb3bcc48077492fe0508059b5", + "https://bcr.bazel.build/modules/rules_cc/0.2.13/MODULE.bazel": "eecdd666eda6be16a8d9dc15e44b5c75133405e820f620a234acc4b1fdc5aa37", + "https://bcr.bazel.build/modules/rules_cc/0.2.14/MODULE.bazel": "353c99ed148887ee89c54a17d4100ae7e7e436593d104b668476019023b58df8", + "https://bcr.bazel.build/modules/rules_cc/0.2.15/MODULE.bazel": "6a0a4a75a57aa6dc888300d848053a58c6b12a29f89d4304e1c41448514ec6e8", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/MODULE.bazel": "1849602c86cb60da8613d2de887f9566a6d354a6df6d7009f9d04a14402f9a84", + "https://bcr.bazel.build/modules/rules_cc/0.2.17/source.json": "3832f45d145354049137c0090df04629d9c2b5493dc5c2bf46f1834040133a07", + "https://bcr.bazel.build/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "c9e8c682bf75b0e7c704166d79b599f93b72cfca5ad7477df596947891feeef6", + "https://bcr.bazel.build/modules/rules_fuzzing/0.5.2/MODULE.bazel": "40c97d1144356f52905566c55811f13b299453a14ac7769dfba2ac38192337a8", + "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", + "https://bcr.bazel.build/modules/rules_java/5.3.5/MODULE.bazel": "a4ec4f2db570171e3e5eb753276ee4b389bae16b96207e9d3230895c99644b86", + "https://bcr.bazel.build/modules/rules_java/6.3.0/MODULE.bazel": "a97c7678c19f236a956ad260d59c86e10a463badb7eb2eda787490f4c969b963", + "https://bcr.bazel.build/modules/rules_java/6.5.2/MODULE.bazel": "1d440d262d0e08453fa0c4d8f699ba81609ed0e9a9a0f02cd10b3e7942e61e31", + "https://bcr.bazel.build/modules/rules_java/7.10.0/MODULE.bazel": "530c3beb3067e870561739f1144329a21c851ff771cd752a49e06e3dc9c2e71a", + "https://bcr.bazel.build/modules/rules_java/7.12.2/MODULE.bazel": "579c505165ee757a4280ef83cda0150eea193eed3bef50b1004ba88b99da6de6", + "https://bcr.bazel.build/modules/rules_java/7.2.0/MODULE.bazel": "06c0334c9be61e6cef2c8c84a7800cef502063269a5af25ceb100b192453d4ab", + "https://bcr.bazel.build/modules/rules_java/7.6.1/MODULE.bazel": "2f14b7e8a1aa2f67ae92bc69d1ec0fa8d9f827c4e17ff5e5f02e91caa3b2d0fe", + "https://bcr.bazel.build/modules/rules_java/8.3.2/MODULE.bazel": "7336d5511ad5af0b8615fdc7477535a2e4e723a357b6713af439fe8cf0195017", + "https://bcr.bazel.build/modules/rules_java/8.5.1/MODULE.bazel": "d8a9e38cc5228881f7055a6079f6f7821a073df3744d441978e7a43e20226939", + "https://bcr.bazel.build/modules/rules_java/8.6.1/MODULE.bazel": "f4808e2ab5b0197f094cabce9f4b006a27766beb6a9975931da07099560ca9c2", + "https://bcr.bazel.build/modules/rules_java/9.1.0/MODULE.bazel": "ee63f27e36a3fada80342869361182f120a9819c74320e8e65b1e04ba0cd7a9d", + "https://bcr.bazel.build/modules/rules_java/9.1.0/source.json": "da589573c1dee2c9ac4a568b301269a2e8191110ff0345c1a959fa7ea6c4dfd6", + "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", + "https://bcr.bazel.build/modules/rules_jvm_external/5.1/MODULE.bazel": "33f6f999e03183f7d088c9be518a63467dfd0be94a11d0055fe2d210f89aa909", + "https://bcr.bazel.build/modules/rules_jvm_external/5.2/MODULE.bazel": "d9351ba35217ad0de03816ef3ed63f89d411349353077348a45348b096615036", + "https://bcr.bazel.build/modules/rules_jvm_external/6.3/MODULE.bazel": "c998e060b85f71e00de5ec552019347c8bca255062c990ac02d051bb80a38df0", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/MODULE.bazel": "e717beabc4d091ecb2c803c2d341b88590e9116b8bf7947915eeb33aab4f96dd", + "https://bcr.bazel.build/modules/rules_jvm_external/6.7/source.json": "5426f412d0a7fc6b611643376c7e4a82dec991491b9ce5cb1cfdd25fe2e92be4", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/MODULE.bazel": "d269a01a18ee74d0335450b10f62c9ed81f2321d7958a2934e44272fe82dcef3", + "https://bcr.bazel.build/modules/rules_kotlin/1.9.6/source.json": "2faa4794364282db7c06600b7e5e34867a564ae91bda7cae7c29c64e9466b7d5", + "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", + "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", + "https://bcr.bazel.build/modules/rules_license/1.0.0/MODULE.bazel": "a7fda60eefdf3d8c827262ba499957e4df06f659330bbe6cdbdb975b768bb65c", + "https://bcr.bazel.build/modules/rules_license/1.0.0/source.json": "a52c89e54cc311196e478f8382df91c15f7a2bfdf4c6cd0e2675cc2ff0b56efb", + "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", + "https://bcr.bazel.build/modules/rules_pkg/1.0.1/MODULE.bazel": "5b1df97dbc29623bccdf2b0dcd0f5cb08e2f2c9050aab1092fd39a41e82686ff", + "https://bcr.bazel.build/modules/rules_pkg/1.0.1/source.json": "bd82e5d7b9ce2d31e380dd9f50c111d678c3bdaca190cb76b0e1c71b05e1ba8a", + "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", + "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", + "https://bcr.bazel.build/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "1e5b502e2e1a9e825eef74476a5a1ee524a92297085015a052510b09a1a09483", + "https://bcr.bazel.build/modules/rules_proto/6.0.2/MODULE.bazel": "ce916b775a62b90b61888052a416ccdda405212b6aaeb39522f7dc53431a5e73", + "https://bcr.bazel.build/modules/rules_proto/7.0.2/MODULE.bazel": "bf81793bd6d2ad89a37a40693e56c61b0ee30f7a7fdbaf3eabbf5f39de47dea2", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/MODULE.bazel": "002d62d9108f75bb807cd56245d45648f38275cb3a99dcd45dfb864c5d74cb96", + "https://bcr.bazel.build/modules/rules_proto/7.1.0/source.json": "39f89066c12c24097854e8f57ab8558929f9c8d474d34b2c00ac04630ad8940e", + "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", + "https://bcr.bazel.build/modules/rules_python/0.23.1/MODULE.bazel": "49ffccf0511cb8414de28321f5fcf2a31312b47c40cc21577144b7447f2bf300", + "https://bcr.bazel.build/modules/rules_python/0.25.0/MODULE.bazel": "72f1506841c920a1afec76975b35312410eea3aa7b63267436bfb1dd91d2d382", + "https://bcr.bazel.build/modules/rules_python/0.28.0/MODULE.bazel": "cba2573d870babc976664a912539b320cbaa7114cd3e8f053c720171cde331ed", + "https://bcr.bazel.build/modules/rules_python/0.31.0/MODULE.bazel": "93a43dc47ee570e6ec9f5779b2e64c1476a6ce921c48cc9a1678a91dd5f8fd58", + "https://bcr.bazel.build/modules/rules_python/0.33.2/MODULE.bazel": "3e036c4ad8d804a4dad897d333d8dce200d943df4827cb849840055be8d2e937", + "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", + "https://bcr.bazel.build/modules/rules_python/1.0.0/MODULE.bazel": "898a3d999c22caa585eb062b600f88654bf92efb204fa346fb55f6f8edffca43", + "https://bcr.bazel.build/modules/rules_python/1.3.0/MODULE.bazel": "8361d57eafb67c09b75bf4bbe6be360e1b8f4f18118ab48037f2bd50aa2ccb13", + "https://bcr.bazel.build/modules/rules_python/1.4.1/MODULE.bazel": "8991ad45bdc25018301d6b7e1d3626afc3c8af8aaf4bc04f23d0b99c938b73a6", + "https://bcr.bazel.build/modules/rules_python/1.6.0/MODULE.bazel": "7e04ad8f8d5bea40451cf80b1bd8262552aa73f841415d20db96b7241bd027d8", + "https://bcr.bazel.build/modules/rules_python/1.6.3/MODULE.bazel": "a7b80c42cb3de5ee2a5fa1abc119684593704fcd2fec83165ebe615dec76574f", + "https://bcr.bazel.build/modules/rules_python/1.7.0/MODULE.bazel": "d01f995ecd137abf30238ad9ce97f8fc3ac57289c8b24bd0bf53324d937a14f8", + "https://bcr.bazel.build/modules/rules_python/1.7.0/source.json": "028a084b65dcf8f4dc4f82f8778dbe65df133f234b316828a82e060d81bdce32", + "https://bcr.bazel.build/modules/rules_shell/0.2.0/MODULE.bazel": "fda8a652ab3c7d8fee214de05e7a9916d8b28082234e8d2c0094505c5268ed3c", + "https://bcr.bazel.build/modules/rules_shell/0.3.0/MODULE.bazel": "de4402cd12f4cc8fda2354fce179fdb068c0b9ca1ec2d2b17b3e21b24c1a937b", + "https://bcr.bazel.build/modules/rules_shell/0.4.1/MODULE.bazel": "00e501db01bbf4e3e1dd1595959092c2fadf2087b2852d3f553b5370f5633592", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/MODULE.bazel": "72e76b0eea4e81611ef5452aa82b3da34caca0c8b7b5c0c9584338aa93bae26b", + "https://bcr.bazel.build/modules/rules_shell/0.6.1/source.json": "20ec05cd5e592055e214b2da8ccb283c7f2a421ea0dc2acbf1aa792e11c03d0c", + "https://bcr.bazel.build/modules/rules_swift/1.16.0/MODULE.bazel": "4a09f199545a60d09895e8281362b1ff3bb08bbde69c6fc87aff5b92fcc916ca", + "https://bcr.bazel.build/modules/rules_swift/2.1.1/MODULE.bazel": "494900a80f944fc7aa61500c2073d9729dff0b764f0e89b824eb746959bc1046", + "https://bcr.bazel.build/modules/rules_swift/2.4.0/MODULE.bazel": "1639617eb1ede28d774d967a738b4a68b0accb40650beadb57c21846beab5efd", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/MODULE.bazel": "72c8f5cf9d26427cee6c76c8e3853eb46ce6b0412a081b2b6db6e8ad56267400", + "https://bcr.bazel.build/modules/rules_swift/3.1.2/source.json": "e85761f3098a6faf40b8187695e3de6d97944e98abd0d8ce579cb2daf6319a66", + "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", + "https://bcr.bazel.build/modules/stardoc/0.5.3/MODULE.bazel": "c7f6948dae6999bf0db32c1858ae345f112cacf98f174c7a8bb707e41b974f1c", + "https://bcr.bazel.build/modules/stardoc/0.6.2/MODULE.bazel": "7060193196395f5dd668eda046ccbeacebfd98efc77fed418dbe2b82ffaa39fd", + "https://bcr.bazel.build/modules/stardoc/0.7.0/MODULE.bazel": "05e3d6d30c099b6770e97da986c53bd31844d7f13d41412480ea265ac9e8079c", + "https://bcr.bazel.build/modules/stardoc/0.7.2/MODULE.bazel": "fc152419aa2ea0f51c29583fab1e8c99ddefd5b3778421845606ee628629e0e5", + "https://bcr.bazel.build/modules/stardoc/0.7.2/source.json": "58b029e5e901d6802967754adf0a9056747e8176f017cfe3607c0851f4d42216", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "5e463fbfba7b1701d957555ed45097d7f984211330106ccd1352c6e0af0dcf91", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "75aab2373a4bbe2a1260b9bf2a1ebbdbf872d3bd36f80bff058dccd82e89422f", + "https://bcr.bazel.build/modules/swift_argument_parser/1.3.1.2/source.json": "5fba48bbe0ba48761f9e9f75f92876cafb5d07c0ce059cc7a8027416de94a05b", + "https://bcr.bazel.build/modules/tar.bzl/0.2.1/MODULE.bazel": "52d1c00a80a8cc67acbd01649e83d8dd6a9dc426a6c0b754a04fe8c219c76468", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/MODULE.bazel": "a3584b4edcfafcabd9b0ef9819808f05b372957bbdff41601429d5fd0aac2e7c", + "https://bcr.bazel.build/modules/tar.bzl/0.6.0/source.json": "4a620381df075a16cb3a7ed57bd1d05f7480222394c64a20fa51bdb636fda658", + "https://bcr.bazel.build/modules/toolchains_llvm/1.6.0/MODULE.bazel": "39603859cafb1c6830160fcd6370552e836790e6abb2bfb8d13bff53c0c10a64", + "https://bcr.bazel.build/modules/toolchains_llvm/1.6.0/source.json": "6bd3ef95a288dd2bb1582eca332af850c9a5428a23bb92cb1c57c2dfe6cb7369", + "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", + "https://bcr.bazel.build/modules/wolfd_bazel_compile_commands/0.5.2/MODULE.bazel": "74527ae5c60c4974e5b6c9fc2f39b78af47b44559b1913324c8185ff4f2f3900", + "https://bcr.bazel.build/modules/wolfd_bazel_compile_commands/0.5.2/source.json": "bbf0b6d7d2a2e58d238ef580fbf3072018e1c9447c9e2295b6a2b32d87a74338", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/MODULE.bazel": "9039681f9bcb8958ee2c87ffc74bdafba9f4369096a2b5634b88abc0eaefa072", + "https://bcr.bazel.build/modules/yq.bzl/0.1.1/source.json": "2d2bad780a9f2b9195a4a370314d2c17ae95eaa745cefc2e12fbc49759b15aa3", + "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "eec517b5bbe5492629466e11dae908d043364302283de25581e3eb944326c4ca", + "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.5/source.json": "22bc55c47af97246cfc093d0acf683a7869377de362b5d1c552c2c2e16b7a806", + "https://bcr.bazel.build/modules/zlib/1.3.1/MODULE.bazel": "751c9940dcfe869f5f7274e1295422a34623555916eb98c174c1e945594bf198", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/bazel_registry.json": "722f186218945df57898ef6431105b0100eaeb456422246bd0c8cad3b9314ce7", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20210324.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20211102.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20230125.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20230802.0.bcr.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20230802.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20230802.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20240116.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20240116.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20250127.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20250127.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20250512.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/abseil-cpp/20250814.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/apple_support/1.11.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/apple_support/1.15.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/apple_support/1.21.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/apple_support/1.21.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/apple_support/2.5.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/aspect_bazel_lib/2.14.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/aspect_bazel_lib/2.19.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/aspect_bazel_lib/2.8.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.10.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.11.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.15.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.17.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.18.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.19.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.21.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.23.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.27.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.28.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.3.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.30.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.33.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.34.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.38.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.4.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.42.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.9.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_features/1.9.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_lib/3.0.0-rc.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_lib/3.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.0.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.2.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.2.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.3.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.4.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.4.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.5.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.6.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.7.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.7.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.8.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazel_skylib/1.8.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/bazelrc-preset.bzl/1.6.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/buildozer/8.5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/gawk/5.3.2.bcr.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/google_benchmark/1.8.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/googletest/1.11.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/googletest/1.14.0.bcr.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/googletest/1.14.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/googletest/1.15.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/googletest/1.17.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/helly25_bzl/0.3.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/jq.bzl/0.1.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/jq.bzl/0.4.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/jsoncpp/1.9.5/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/jsoncpp/1.9.6/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/libpfm/4.11.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/matio/1.5.27/MODULE.bazel": "4a7068393ef968578decd4d97238acf75bdabad88ac3d012855a38177b6dfbdd", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/matio/1.5.27/source.json": "57d9441ca006ed0ba70cb30abfbfb696f7534513adfff32e7d6f00ea39c41007", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/nlohmann_json/3.6.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/openmp/22.1.1/MODULE.bazel": "b83e508bd67c447eaf904856678709e0aec0e84dbc30166c431438f5841b1431", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/openmp/22.1.1/source.json": "8e3a846ba821cb88f42f52f1d30c5c0709a89d7adf9e0e806fdff2b899b13e10", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/package_metadata/0.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/package_metadata/0.0.6/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/pcg/83252d9c23df9c82ecb42210afed61a7b42402d7/MODULE.bazel": "6b7ec4dc813580a4fdb08c4286ac18307ca5dad6be5bfc207343d23864bd95ce", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/pcg/83252d9c23df9c82ecb42210afed61a7b42402d7/source.json": "28d0abf435a8b5f7c6b75bde959008b7101f538237f5a5a2f965fba44fd7f27e", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.10/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.11/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.4/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.5/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.6/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.7/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.8/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/0.0.9/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/platforms/1.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/21.7/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/27.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/27.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/29.0-rc2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/29.0-rc3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/29.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/3.19.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/31.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/32.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/protobuf/33.4/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/pybind11_bazel/2.11.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/pybind11_bazel/2.12.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/re2/2023-09-01/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/re2/2024-07-02.bcr.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/re2/2024-07-02/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_android/0.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_apple/3.16.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_apple/4.1.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.10/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.13/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.15/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.16/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.17/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.6/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.8/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.0.9/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.1.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.1.5/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.11/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.13/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.14/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.15/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_cc/0.2.17/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_foreign_cc/0.9.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_fuzzing/0.5.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/4.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/5.3.5/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/6.3.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/6.5.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/7.10.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/7.12.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/7.2.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/7.6.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/8.3.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/8.5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/8.6.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_java/9.1.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_jvm_external/4.4.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_jvm_external/5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_jvm_external/5.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_jvm_external/6.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_jvm_external/6.7/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_kotlin/1.9.6/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_license/0.0.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_license/0.0.7/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_license/1.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_pkg/0.7.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_pkg/1.0.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/4.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/6.0.0-rc1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/6.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/7.0.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_proto/7.1.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.10.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.23.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.25.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.28.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.31.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.33.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/0.4.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.0.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.3.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.4.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.6.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.6.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_python/1.7.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_shell/0.2.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_shell/0.3.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_shell/0.4.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_shell/0.6.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_swift/1.16.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_swift/2.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_swift/2.4.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/rules_swift/3.1.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/stardoc/0.5.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/stardoc/0.5.3/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/stardoc/0.6.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/stardoc/0.7.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/stardoc/0.7.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/swift_argument_parser/1.3.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/swift_argument_parser/1.3.1.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/tar.bzl/0.2.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/tar.bzl/0.6.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/throw_the_switch/2.6.1-bazel.1/MODULE.bazel": "e641f4d6271f21d464018aef3f9e3820f5956d0ea2e7e11c3517f90aa1ac8d03", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/throw_the_switch/2.6.1-bazel.1/source.json": "19293b7c2b489c9745c9f9fbd7504580375df0004a23ef93d0c6ee6b91d0293a", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/toolchains_llvm/1.6.0/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/wolfd_bazel_compile_commands/0.5.2/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/yq.bzl/0.1.1/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/zlib/1.2.11/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/zlib/1.3.1.bcr.5/MODULE.bazel": "not found", + "https://raw.githubusercontent.com/tragisch/bazel-registry/main/modules/zlib/1.3.1/MODULE.bazel": "not found" + }, + "selectedYankedVersions": {}, + "moduleExtensions": { + "@@rules_kotlin+//src/main/starlark/core/repositories:bzlmod_setup.bzl%rules_kotlin_extensions": { + "general": { + "bzlTransitiveDigest": "WDuVCwqV73DvcHB5Of+02lhUZgPxr9MqtpvMRyUi0eQ=", + "usagesDigest": "QI2z8ZUR+mqtbwsf2fLqYdJAkPOHdOV+tF2yVAUgRzw=", + "recordedInputs": [ + "REPO_MAPPING:rules_kotlin+,bazel_tools bazel_tools" + ], + "generatedRepoSpecs": { + "com_github_jetbrains_kotlin_git": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_compiler_git_repository", + "attributes": { + "urls": [ + "https://github.com/JetBrains/kotlin/releases/download/v1.9.23/kotlin-compiler-1.9.23.zip" + ], + "sha256": "93137d3aab9afa9b27cb06a824c2324195c6b6f6179d8a8653f440f5bd58be88" + } + }, + "com_github_jetbrains_kotlin": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:compiler.bzl%kotlin_capabilities_repository", + "attributes": { + "git_repository_name": "com_github_jetbrains_kotlin_git", + "compiler_version": "1.9.23" + } + }, + "com_github_google_ksp": { + "repoRuleId": "@@rules_kotlin+//src/main/starlark/core/repositories:ksp.bzl%ksp_compiler_plugin_repository", + "attributes": { + "urls": [ + "https://github.com/google/ksp/releases/download/1.9.23-1.0.20/artifacts.zip" + ], + "sha256": "ee0618755913ef7fd6511288a232e8fad24838b9af6ea73972a76e81053c8c2d", + "strip_version": "1.9.23-1.0.20" + } + }, + "com_github_pinterest_ktlint": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_file", + "attributes": { + "sha256": "01b2e0ef893383a50dbeb13970fe7fa3be36ca3e83259e01649945b09d736985", + "urls": [ + "https://github.com/pinterest/ktlint/releases/download/1.3.0/ktlint" + ], + "executable": true + } + }, + "rules_android": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "sha256": "cd06d15dd8bb59926e4d65f9003bfc20f9da4b2519985c27e190cddc8b7a7806", + "strip_prefix": "rules_android-0.1.1", + "urls": [ + "https://github.com/bazelbuild/rules_android/archive/v0.1.1.zip" + ] + } + } + } + } + }, + "@@rules_python+//python/extensions:config.bzl%config": { + "general": { + "bzlTransitiveDigest": "L+2ZCpb7GIJqvWaXpzkfk/80d916sIFVmSPzrSm6of8=", + "usagesDigest": "ZVSXMAGpD+xzVNPuvF1IoLBkty7TROO0+akMapt1pAg=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,pypi__build rules_python++config+pypi__build", + "REPO_MAPPING:rules_python+,pypi__click rules_python++config+pypi__click", + "REPO_MAPPING:rules_python+,pypi__colorama rules_python++config+pypi__colorama", + "REPO_MAPPING:rules_python+,pypi__importlib_metadata rules_python++config+pypi__importlib_metadata", + "REPO_MAPPING:rules_python+,pypi__installer rules_python++config+pypi__installer", + "REPO_MAPPING:rules_python+,pypi__more_itertools rules_python++config+pypi__more_itertools", + "REPO_MAPPING:rules_python+,pypi__packaging rules_python++config+pypi__packaging", + "REPO_MAPPING:rules_python+,pypi__pep517 rules_python++config+pypi__pep517", + "REPO_MAPPING:rules_python+,pypi__pip rules_python++config+pypi__pip", + "REPO_MAPPING:rules_python+,pypi__pip_tools rules_python++config+pypi__pip_tools", + "REPO_MAPPING:rules_python+,pypi__pyproject_hooks rules_python++config+pypi__pyproject_hooks", + "REPO_MAPPING:rules_python+,pypi__setuptools rules_python++config+pypi__setuptools", + "REPO_MAPPING:rules_python+,pypi__tomli rules_python++config+pypi__tomli", + "REPO_MAPPING:rules_python+,pypi__wheel rules_python++config+pypi__wheel", + "REPO_MAPPING:rules_python+,pypi__zipp rules_python++config+pypi__zipp" + ], + "generatedRepoSpecs": { + "rules_python_internal": { + "repoRuleId": "@@rules_python+//python/private:internal_config_repo.bzl%internal_config_repo", + "attributes": { + "transition_setting_generators": {}, + "transition_settings": [] + } + }, + "pypi__build": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e2/03/f3c8ba0a6b6e30d7d18c40faab90807c9bb5e9a1e3b2fe2008af624a9c97/build-1.2.1-py3-none-any.whl", + "sha256": "75e10f767a433d9a86e50d83f418e83efc18ede923ee5ff7df93b6cb0306c5d4", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__click": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/00/2e/d53fa4befbf2cfa713304affc7ca780ce4fc1fd8710527771b58311a3229/click-8.1.7-py3-none-any.whl", + "sha256": "ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__colorama": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", + "sha256": "4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__importlib_metadata": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/2d/0a/679461c511447ffaf176567d5c496d1de27cbe34a87df6677d7171b2fbd4/importlib_metadata-7.1.0-py3-none-any.whl", + "sha256": "30962b96c0c223483ed6cc7280e7f0199feb01a0e40cfae4d4450fc6fab1f570", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__installer": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl", + "sha256": "05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__more_itertools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/50/e2/8e10e465ee3987bb7c9ab69efb91d867d93959095f4807db102d07995d94/more_itertools-10.2.0-py3-none-any.whl", + "sha256": "686b06abe565edfab151cb8fd385a05651e1fdf8f0a14191e4439283421f8684", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__packaging": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/49/df/1fceb2f8900f8639e278b056416d49134fb8d84c5942ffaa01ad34782422/packaging-24.0-py3-none-any.whl", + "sha256": "2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pep517": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/25/6e/ca4a5434eb0e502210f591b97537d322546e4833dcb4d470a48c375c5540/pep517-0.13.1-py3-none-any.whl", + "sha256": "31b206f67165b3536dd577c5c3f1518e8fbaf38cbc57efff8369a392feff1721", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/8a/6a/19e9fe04fca059ccf770861c7d5721ab4c2aebc539889e97c7977528a53b/pip-24.0-py3-none-any.whl", + "sha256": "ba0d021a166865d2265246961bec0152ff124de910c5cc39f1156ce3fa7c69dc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pip_tools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/0d/dc/38f4ce065e92c66f058ea7a368a9c5de4e702272b479c0992059f7693941/pip_tools-7.4.1-py3-none-any.whl", + "sha256": "4c690e5fbae2f21e87843e89c26191f0d9454f362d8acdbd695716493ec8b3a9", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__pyproject_hooks": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/ae/f3/431b9d5fe7d14af7a32340792ef43b8a714e7726f1d7b69cc4e8e7a3f1d7/pyproject_hooks-1.1.0-py3-none-any.whl", + "sha256": "7ceeefe9aec63a1064c18d939bdc3adf2d8aa1988a510afec15151578b232aa2", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__setuptools": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/90/99/158ad0609729111163fc1f674a5a42f2605371a4cf036d0441070e2f7455/setuptools-78.1.1-py3-none-any.whl", + "sha256": "c3a9c4211ff4c309edb8b8c4f1cbfa7ae324c4ba9f91ff254e3d305b9fd54561", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__tomli": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/97/75/10a9ebee3fd790d20926a90a2547f0bf78f371b2f13aa822c759680ca7b9/tomli-2.0.1-py3-none-any.whl", + "sha256": "939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__wheel": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/7d/cd/d7460c9a869b16c3dd4e1e403cce337df165368c71d6af229a74699622ce/wheel-0.43.0-py3-none-any.whl", + "sha256": "55c570405f142630c6b9f72fe09d9b67cf1477fcf543ae5b8dcb1f5b7377da81", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + }, + "pypi__zipp": { + "repoRuleId": "@@bazel_tools//tools/build_defs/repo:http.bzl%http_archive", + "attributes": { + "url": "https://files.pythonhosted.org/packages/da/55/a03fd7240714916507e1fcf7ae355bd9d9ed2e6db492595f1a67f61681be/zipp-3.18.2-py3-none-any.whl", + "sha256": "dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e", + "type": "zip", + "build_file_content": "package(default_visibility = [\"//visibility:public\"])\n\nload(\"@rules_python//python:py_library.bzl\", \"py_library\")\n\npy_library(\n name = \"lib\",\n srcs = glob([\"**/*.py\"]),\n data = glob([\"**/*\"], exclude=[\n # These entries include those put into user-installed dependencies by\n # data_exclude to avoid non-determinism.\n \"**/*.py\",\n \"**/*.pyc\",\n \"**/*.pyc.*\", # During pyc creation, temp files named *.pyc.NNN are created\n \"**/*.dist-info/RECORD\",\n \"BUILD\",\n \"WORKSPACE\",\n ]),\n # This makes this directory a top-level in the python import\n # search path for anything that depends on this.\n imports = [\".\"],\n)\n" + } + } + } + } + }, + "@@rules_python+//python/uv:uv.bzl%uv": { + "general": { + "bzlTransitiveDigest": "IgkgxNrtlAec7OnaFrWoCC5NQhHcY/aIz4jsqcNMm8Y=", + "usagesDigest": "H8dQoNZcoqP+Mu0tHZTi4KHATzvNkM5ePuEqoQdklIU=", + "recordedInputs": [ + "REPO_MAPPING:rules_python+,bazel_tools bazel_tools", + "REPO_MAPPING:rules_python+,platforms platforms" + ], + "generatedRepoSpecs": { + "uv": { + "repoRuleId": "@@rules_python+//python/uv/private:uv_toolchains_repo.bzl%uv_toolchains_repo", + "attributes": { + "toolchain_type": "'@@rules_python+//python/uv:uv_toolchain_type'", + "toolchain_names": [ + "none" + ], + "toolchain_implementations": { + "none": "'@@rules_python+//python:none'" + }, + "toolchain_compatible_with": { + "none": [ + "@platforms//:incompatible" + ] + }, + "toolchain_target_settings": {} + } + } + } + } + } + }, + "facts": {}, + "factsVersions": {} +} diff --git a/README.md b/README.md index 3b02fef..4321002 100644 --- a/README.md +++ b/README.md @@ -1,94 +1,97 @@ # MMatrix -MMatrix is a C project for matrix and tensor operations, built with Bazel. -It targets macOS and Linux, with CPU acceleration (BLAS/OpenMP) and optional MPS/Metal support on Apple Silicon. +MMatrix is a C library for dense and sparse matrix, vector, and tensor operations, built with Bazel. +The next release candidate is **0.4.0**; see [CHANGELOG.md](CHANGELOG.md) for changes and migration notes. ## What is included -- `app/matrix`: dense (`sm`, `dm`) and sparse (`dms`) matrix modules -- `app/vector`: dense (`sv`, `dv`), strided view (`vv`), sparse (`dvs`) and fixed-size geometry (`vec3`) vector modules -- `app/matrix:sm_mps`: explicit Apple Silicon MPS path for GPU-resident large - dense GEMMs; check around `1024^3`, recommended for `2048^3`-class matrices. -- `app/tensor`: N-dimensional tensor module (`st`) with CPU and optional MPS backend (only on Apple Silicon). +- `app/matrix`: dense float (`sm`), dense double (`dm`), sparse COO (`dms`), matrix conversions, and MAT/MatrixMarket I/O. +- `app/vector`: dense (`sv`, `dv`), strided (`vv`), sparse (`dvs`), fixed-size geometry (`vec3`), and flat-buffer optimizer kernels (`sv_optim`). +- `app/tensor`: N-dimensional f32/bf16 tensors, shape/view operations, convolution, pooling, BatchNorm, and graph-interpreter kernels (broadcast arithmetic, LayerNorm, ReduceMax, batched matmul, gather, and materializing permute). Binary16 conversion is a separate raw-buffer API, not a tensor storage dtype. +- `app/matrix:sm_mps`: explicit Apple Silicon MPS path for GPU-resident dense GEMMs, with streams and reusable plans. -## Build +## Platforms and dependencies -Install system libraries on macOS (Homebrew): +| Platform | Modules | Default CPU backend | +| --- | --- | --- | +| macOS ARM64 (Apple Silicon) | Matrix, vector, tensor; optional MPS/Metal execution | Accelerate + OpenMP | +| Linux x86-64 | Matrix and vector | OpenBLAS + OpenMP | + +The tensor target currently requires macOS frameworks, including for CPU execution. +Linux support does not include the tensor module. Release validation status is recorded in [doc/release-0.4.0.md](doc/release-0.4.0.md). + +Install Bazelisk and the platform's C/C++ development tools. The repository pins Bazel in `.bazelversion`. +On macOS, install Xcode command line tools (`xcode-select --install`). No Homebrew SuiteSparse installation is required. + +Bzlmod resolves LLVM, OpenMP, matio, zlib, and the test/documentation tooling through the configured registries. +`log.c` is included under `third_party/log.c`. OpenBLAS is a **system dependency** when selecting the OpenBLAS backend: ```bash -brew install suite-sparse +# Homebrew, including Linuxbrew: +brew install openblas +# For another installation prefix, pass it explicitly: +bazel build //app/matrix --config=openblas --repo_env=OPENBLAS_ROOT=/path/to/openblas ``` -All other third-party dependencies such as `openmp`, `matio`, `openblas` [on Linux, Apple Silicon uses Accelerate] and `llvm` compiler are resolved via Bzlmod from the configured Bazel registries. +The default OpenBLAS prefixes are `/opt/homebrew/opt/openblas` on macOS and `/home/linuxbrew/.linuxbrew/opt/openblas` on Linux. +The OpenBLAS prefix must expose headers under `include/` and `libopenblas.a` under `lib/`. +LAPACKE must be included in OpenBLAS or supplied as `lib/liblapacke.a`; the Ubuntu CI job stages the distribution packages into this layout. +SuiteSparse and the former CSparse conversion API have been removed. -Build everything: +## Build and test ```bash bazel build //... +bazel test //... ``` -Build specific modules: +Build individual modules: ```bash -bazel build //app/matrix -bazel build //app/vector +bazel build //app/matrix //app/vector +# macOS only: bazel build //app/tensor ``` -Optional backend configs (matrix): +Matrix backend configs: ```bash -bazel build //app/matrix --config=accelerate -bazel build //app/matrix --config=openblas +bazel build //app/matrix --config=accelerate # macOS +bazel build //app/matrix --config=openblas # Linux bazel build //app/matrix --config=openmp_only ``` -## Test - -```bash -bazel test //... -``` - -Unit tests are in `app/matrix/tests/` and `app/tensor/tests/`. - +Unit tests live alongside each module in `app/matrix/tests/`, `app/vector/tests/`, and `app/tensor/tests/`. +Benchmarks under `app/*/tests/` and the published MPS benchmark under `share/simple_benchmark/` are buildable targets. +Local vendor experiments and legacy benchmarks are excluded from recursive discovery with `.bazelignore`. ## Documentation -Build docs: - -`bazel build //:docs` - -Serve docs locally: - -`bazel run //:docs_serve` - -Regenerate Tensor + Matrix API pages from public header docs (Doxygen XML -> Markdown): - -`bazel run //:docs_refresh_api` - -The short aliases `//:docs` and `//:docs_serve` are the recommended user entrypoints. - -## Install artifacts +```bash +bazel build //:docs +bazel run //:docs_serve +``` -Install headers and libraries to a custom directory: +Regenerate matrix and tensor API pages from public header comments (requires Doxygen and Python 3 on PATH): ```bash -bazel run //:matrix_installer -- $(PWD)/lib/matrix +bazel run //:docs_refresh_api ``` -Modes: +## Install artifacts + +Install public headers and libraries, including their link dependencies, to a custom directory: ```bash -bazel run //:matrix_installer -- --mode=standard $(PWD)/lib/matrix -bazel run //:matrix_installer -- --mode=bundle $(PWD)/lib/matrix_bundle +bazel run //:matrix_installer -- "$PWD/lib/matrix" +bazel run //:matrix_installer -- --mode=bundle "$PWD/lib/matrix_bundle" ``` -## Platforms - -- macOS ARM64 (Apple Silicon) -- Linux x86-64 +Standard mode keeps separate libraries; bundle mode combines static archives into `libmatrix_full.a`. +On macOS the installation includes tensor and MPS libraries; Linux installs matrix and vector libraries. +System frameworks still need to be linked by consumers. On Linux, link separate static archives as a dependency group (`-Wl,--start-group ... -Wl,--end-group`) together with the system C++, Fortran, math, threading, and dynamic-loader runtimes; `tools/ci/test_install.sh` contains a complete example. Installation does not create a portable binary SDK: +compiler/runtime compatibility and third-party notices must be checked before redistributing binary bundles. ## License -MIT (see `LICENSE.txt`). -Third-party licenses are provided by the respective upstream projects. +MIT (see `LICENSE.txt`). Third-party dependencies retain their upstream licenses. diff --git a/app/matrix/BUILD b/app/matrix/BUILD index 3e595fc..afd9285 100644 --- a/app/matrix/BUILD +++ b/app/matrix/BUILD @@ -1,3 +1,4 @@ +load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") load("@rules_cc//cc:objc_library.bzl", "objc_library") load("//tools:unity_test.bzl", "unity_test") @@ -24,6 +25,7 @@ config_setting( constraint_values = [ "@platforms//os:macos", ], + visibility = ["//visibility:public"], ) config_setting( @@ -118,7 +120,6 @@ perf_copts = select({ "-fomit-frame-pointer", "-march=native", "-mtune=native", - "-flto", "-fno-math-errno", "-fno-trapping-math", "-falign-functions=32", @@ -185,7 +186,7 @@ default_deps = [ "@matio", "@openmp//:libomp", "@pcg", - "@log", + "//third_party/log.c:log", ] ############################# @@ -266,10 +267,18 @@ cc_library( cc_library( name = "sm", - srcs = ["src/sm.c"], - hdrs = ["include/sm.h"], + srcs = [ + "src/sm.c", + ], + hdrs = [ + "include/sm.h", + "src/sm_kernels.h", + ], copts = default_copts + perf_copts + package_copts + openmp_copts, - includes = ["include"], + includes = [ + "include", + "src", + ], linkopts = openmp_linkopts + accelerate_linkopts + openblas_linkopts, target_compatible_with = select({ ":backend_accelerate": ["@platforms//os:macos"], @@ -279,12 +288,34 @@ cc_library( visibility = ["//visibility:public"], deps = default_deps + [ ":m_rng", + ":sm_kernels", ] + select({ ":backend_openblas_linux": ["@openblas"], "//conditions:default": [], }), ) +cc_library( + name = "sm_kernels", + srcs = ["src/sm_kernels.c"], + hdrs = [ + "include/sm.h", + "src/sm_kernels.h", + ], + # NaN propagation and thread-independent results are part of this API. + copts = default_copts + perf_copts + package_copts + openmp_copts + [ + "-fno-fast-math", + "-ffp-contract=off", + ], + includes = [ + "include", + "src", + ], + linkopts = openmp_linkopts + accelerate_linkopts, + visibility = ["//visibility:private"], + deps = ["@openmp//:libomp"], +) + objc_library( name = "sm_mps", srcs = ["src/sm_mps.m"], @@ -345,6 +376,26 @@ unity_test( deps = ["@//app/matrix:sm"], ) +unity_test( + name = "test_sm_nn", + srcs = ["tests/test_sm_nn.c"], + my_config = ["tests/my_config.yml"], + tags = ["smoke"], + deps = [":sm"], +) + +cc_binary( + name = "bench_sm_gemm_bias_act", + srcs = ["tests/bench_sm_gemm_bias_act.c"], + deps = [":sm"], +) + +cc_binary( + name = "bench_sm_rowwise", + srcs = ["tests/bench_sm_rowwise.c"], + deps = [":sm"], +) + unity_test( name = "test_dms", srcs = ["tests/test_dms.c"], diff --git a/app/matrix/include/sm.h b/app/matrix/include/sm.h index 05b542e..fb75e33 100644 --- a/app/matrix/include/sm.h +++ b/app/matrix/include/sm.h @@ -351,6 +351,39 @@ typedef enum SmTranspose { SM_TRANSPOSE = 1, } SmTranspose; +/** @brief Activation applied by fused matrix kernels. */ +typedef enum SmActivation { + SM_ACT_NONE = 0, + SM_ACT_RELU = 1, + SM_ACT_SIGMOID = 2, + SM_ACT_TANH = 3, +} SmActivation; + +/** @brief Apply ReLU in-place to every matrix element. */ +bool sm_apply_relu(FloatMatrix *mat); +/** @brief Apply sigmoid in-place to every matrix element. */ +bool sm_apply_sigmoid(FloatMatrix *mat); +/** @brief Apply hyperbolic tangent in-place to every matrix element. */ +bool sm_apply_tanh(FloatMatrix *mat); +/** @brief Apply tanh-approximation GELU in-place. */ +bool sm_apply_gelu_tanh(FloatMatrix *mat); +/** @brief Apply erf-form GELU in-place. */ +bool sm_apply_gelu_erf(FloatMatrix *mat); +/** @brief Apply numerically stable softmax independently to each row. */ +bool sm_apply_softmax(FloatMatrix *mat); + +/** @brief Multiply @p grad in-place by the ReLU mask at @p activation. */ +bool sm_apply_relu_backward(const FloatMatrix *activation, FloatMatrix *grad); +/** @brief Multiply @p grad in-place by the sigmoid derivative at @p activation. */ +bool sm_apply_sigmoid_backward(const FloatMatrix *activation, + FloatMatrix *grad); +/** @brief Multiply @p grad in-place by the tanh derivative at @p activation. */ +bool sm_apply_tanh_backward(const FloatMatrix *activation, FloatMatrix *grad); +/** @brief Multiply @p grad in-place by the tanh-GELU derivative at @p input. */ +bool sm_apply_gelu_tanh_backward(const FloatMatrix *input, FloatMatrix *grad); +/** @brief Multiply @p grad in-place by the erf-GELU derivative at @p input. */ +bool sm_apply_gelu_erf_backward(const FloatMatrix *input, FloatMatrix *grad); + /** * @brief BLAS-style GEMM kernel. * @param C Output matrix (shape must be compatible with A and B). @@ -367,6 +400,22 @@ bool sm_gemm(FloatMatrix *C, float alpha, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, float beta); +/** + * @brief Fused GEMM + optional row bias + activation. + * @param C Output matrix, shape `M x N`. + * @param A Left input matrix. + * @param trans_a Transpose mode for A. + * @param B Right input matrix. + * @param trans_b Transpose mode for B. + * @param bias Bias row (`1 x N`), per-element bias (`M x N`), or NULL. + * @param activation Activation to apply after bias. + * @return true on success, false on invalid input or incompatible shapes. + */ +bool sm_gemm_bias_act(FloatMatrix *C, const FloatMatrix *A, + SmTranspose trans_a, const FloatMatrix *B, + SmTranspose trans_b, const FloatMatrix *bias, + SmActivation activation); + /** * @brief Fused kernel: GEMM + optional bias + ReLU activation. * @param C Output matrix to accumulate result into. @@ -382,6 +431,39 @@ bool sm_gemm_bias_relu(FloatMatrix *C, const FloatMatrix *A, SmTranspose trans_a const FloatMatrix *B, SmTranspose trans_b, const FloatMatrix *bias); +/** + * @brief Add a `1 x cols` row to every row of a matrix in place. + * @return true on success, false on null input or shape mismatch. + */ +bool sm_add_rowwise(FloatMatrix *mat, const FloatMatrix *row); +/** + * @brief Sum matrix rows in fixed row order. + * @return New `1 x cols` matrix, or NULL on invalid input/allocation failure. + */ +FloatMatrix *sm_sum_rows(const FloatMatrix *mat); +/** + * @brief Return the first maximal column index for every row as float values. + * @return New `rows x 1` matrix, or NULL on invalid input/allocation failure. + */ +FloatMatrix *sm_argmax_rows(const FloatMatrix *mat); +/** + * @brief In-place softmax-cross-entropy gradient `predicted - target`. + * @return true on success, false on null input or shape mismatch. + */ +bool sm_softmax_ce_grad(const FloatMatrix *predicted, + const FloatMatrix *target, FloatMatrix *grad); +/** + * @brief Mean squared error, matching nm_lib reduction semantics. + * @return Loss value, or -1.0f for invalid/empty inputs. + */ +float sm_loss_mse(const FloatMatrix *predicted, const FloatMatrix *target); +/** + * @brief Cross-entropy loss, averaged over rows and clamped at 1e-7. + * @return Loss value, or -1.0f for invalid inputs. + */ +float sm_loss_cross_entropy(const FloatMatrix *predicted, + const FloatMatrix *target); + /**************************************/ /* Matrix In-Place Ops */ /**************************************/ diff --git a/app/matrix/src/sm.c b/app/matrix/src/sm.c index a23ca62..21fb385 100644 --- a/app/matrix/src/sm.c +++ b/app/matrix/src/sm.c @@ -8,6 +8,7 @@ #include "sm.h" #include "m_rng.h" +#include "sm_kernels.h" #include #include @@ -572,6 +573,102 @@ FloatMatrix *sm_get_last_col(const FloatMatrix *mat) { return sm_get_col(mat, mat->cols - 1); } +static bool sm_kernel_matrix_valid(const FloatMatrix *mat) { + return mat != NULL && mat->values != NULL && mat->rows > 0 && mat->cols > 0 && + mat->rows <= SIZE_MAX / mat->cols && + mat->capacity >= mat->rows * mat->cols; +} + +static bool sm_kernel_matrices_compatible(const FloatMatrix *a, + const FloatMatrix *b) { + return sm_kernel_matrix_valid(a) && sm_kernel_matrix_valid(b) && + a->rows == b->rows && a->cols == b->cols; +} + +bool sm_apply_relu(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_activation(mat->values, mat->rows * mat->cols, + SM_ACT_RELU); +} + +bool sm_apply_sigmoid(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_sigmoid(mat->values, mat->rows * mat->cols); +} + +bool sm_apply_tanh(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_tanh(mat->values, mat->rows * mat->cols); +} + +bool sm_apply_gelu_tanh(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_gelu_tanh(mat->values, mat->rows * mat->cols); +} + +bool sm_apply_gelu_erf(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_gelu_erf(mat->values, mat->rows * mat->cols); +} + +bool sm_apply_softmax(FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return false; + } + return sm_kernel_apply_softmax(mat->values, mat->rows, mat->cols); +} + +bool sm_apply_relu_backward(const FloatMatrix *activation, FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(activation, grad)) { + return false; + } + return sm_kernel_relu_backward(activation->values, grad->values, + activation->rows * activation->cols); +} + +bool sm_apply_sigmoid_backward(const FloatMatrix *activation, + FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(activation, grad)) { + return false; + } + return sm_kernel_sigmoid_backward(activation->values, grad->values, + activation->rows * activation->cols); +} + +bool sm_apply_tanh_backward(const FloatMatrix *activation, FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(activation, grad)) { + return false; + } + return sm_kernel_tanh_backward(activation->values, grad->values, + activation->rows * activation->cols); +} + +bool sm_apply_gelu_tanh_backward(const FloatMatrix *input, FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(input, grad)) { + return false; + } + return sm_kernel_gelu_tanh_backward(input->values, grad->values, + input->rows * input->cols); +} + +bool sm_apply_gelu_erf_backward(const FloatMatrix *input, FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(input, grad)) { + return false; + } + return sm_kernel_gelu_erf_backward(input->values, grad->values, + input->rows * input->cols); +} + // Performance & usage guidance: see .github/skills/sm-gemm-optimization/SKILL.md bool sm_gemm(FloatMatrix *C, float alpha, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, @@ -666,57 +763,182 @@ bool sm_gemm(FloatMatrix *C, float alpha, const FloatMatrix *A, return true; } +bool sm_gemm_bias_act(FloatMatrix *C, const FloatMatrix *A, + SmTranspose trans_a, const FloatMatrix *B, + SmTranspose trans_b, const FloatMatrix *bias, + SmActivation activation) { + if (!sm_kernel_matrix_valid(C) || !sm_kernel_matrix_valid(A) || + !sm_kernel_matrix_valid(B)) { + return false; + } + if (activation < SM_ACT_NONE || activation > SM_ACT_TANH) { + return false; + } + + const size_t rows = trans_a == SM_TRANSPOSE ? A->cols : A->rows; + const size_t inner_a = trans_a == SM_TRANSPOSE ? A->rows : A->cols; + const size_t inner_b = trans_b == SM_TRANSPOSE ? B->cols : B->rows; + const size_t cols = trans_b == SM_TRANSPOSE ? B->rows : B->cols; + if (inner_a != inner_b || C->rows != rows || C->cols != cols) { + return false; + } + if (bias && (!sm_kernel_matrix_valid(bias) || bias->cols != cols || + (bias->rows != 1 && bias->rows != rows))) { + return false; + } + + if (!sm_gemm(C, 1.0f, A, trans_a, B, trans_b, 0.0f)) { + return false; + } + if (bias) { + return sm_kernel_add_bias_activation(C->values, bias->values, rows, cols, + bias->rows, activation); + } + return sm_kernel_apply_activation(C->values, rows * cols, activation); +} + bool sm_gemm_bias_relu(FloatMatrix *C, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, const FloatMatrix *bias) { - if (!sm_gemm(C, 1.0f, A, trans_a, B, trans_b, 0.0f)) { + return sm_gemm_bias_act(C, A, trans_a, B, trans_b, bias, SM_ACT_RELU); +} + +bool sm_add_rowwise(FloatMatrix *mat, const FloatMatrix *row) { + if (!sm_kernel_matrix_valid(mat) || !sm_kernel_matrix_valid(row) || + row->rows != 1 || row->cols != mat->cols) { return false; } - if (!C || !C->values) { - log_error("Error: sm_gemm_bias_relu received NULL output."); + const size_t count = mat->rows * mat->cols; + if (count > SIZE_MAX / sizeof(float)) { return false; } - const size_t rows = C->rows; - const size_t cols = C->cols; - float *c = C->values; - - if (bias) { - if (!bias->values || bias->cols != cols || - !(bias->rows == 1 || bias->rows == rows)) { - log_error("Error: sm_gemm_bias_relu bias shape mismatch."); + const float *bias = row->values; + float *bias_copy = NULL; + const uintptr_t mat_begin = (uintptr_t)mat->values; + const uintptr_t row_begin = (uintptr_t)row->values; + const size_t mat_bytes = count * sizeof(float); + const size_t row_bytes = row->cols * sizeof(float); + if (mat_begin <= UINTPTR_MAX - mat_bytes && + row_begin <= UINTPTR_MAX - row_bytes && + row_begin < mat_begin + mat_bytes && mat_begin < row_begin + row_bytes) { + bias_copy = (float *)malloc(row_bytes); + if (!bias_copy) { return false; } + memcpy(bias_copy, row->values, row_bytes); + bias = bias_copy; + } - const float *b = bias->values; - if (bias->rows == 1) { - for (size_t i = 0; i < rows; ++i) { - size_t row_offset = i * cols; - for (size_t j = 0; j < cols; ++j) { - float v = c[row_offset + j] + b[j]; - c[row_offset + j] = (v > 0.0f) ? v : 0.0f; - } - } - } else { - for (size_t i = 0; i < rows; ++i) { - size_t row_offset = i * cols; - size_t bias_offset = i * cols; - for (size_t j = 0; j < cols; ++j) { - float v = c[row_offset + j] + b[bias_offset + j]; - c[row_offset + j] = (v > 0.0f) ? v : 0.0f; - } - } +#if defined(USE_ACCELERATE) + for (size_t r = 0; r < mat->rows; ++r) { + vDSP_vadd(mat->values + r * mat->cols, 1, bias, 1, + mat->values + r * mat->cols, 1, (vDSP_Length)mat->cols); + } +#else +#pragma omp parallel for simd schedule(static) if (count > 10000) + for (size_t i = 0; i < count; ++i) { + mat->values[i] += bias[i % mat->cols]; + } +#endif + + free(bias_copy); + return true; +} + +FloatMatrix *sm_sum_rows(const FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return NULL; + } + + FloatMatrix *sum = sm_create(1, mat->cols); + if (!sum) { + return NULL; + } + +#pragma omp parallel for schedule(static) if (mat->rows * mat->cols > 10000) + for (size_t col = 0; col < mat->cols; ++col) { + float value = 0.0f; + for (size_t row = 0; row < mat->rows; ++row) { + value += mat->values[row * mat->cols + col]; } - } else { - for (size_t i = 0; i < rows * cols; ++i) { - if (c[i] < 0.0f) c[i] = 0.0f; + sum->values[col] = value; + } + return sum; +} + +FloatMatrix *sm_argmax_rows(const FloatMatrix *mat) { + if (!sm_kernel_matrix_valid(mat)) { + return NULL; + } + + FloatMatrix *argmax = sm_create(mat->rows, 1); + if (!argmax) { + return NULL; + } + +#pragma omp parallel for schedule(static) if (mat->rows * mat->cols > 10000) + for (size_t row = 0; row < mat->rows; ++row) { + size_t best = 0; + float best_value = mat->values[row * mat->cols]; + for (size_t col = 1; col < mat->cols; ++col) { + const float value = mat->values[row * mat->cols + col]; + if (value > best_value) { + best = col; + best_value = value; + } } + argmax->values[row] = (float)best; } + return argmax; +} +bool sm_softmax_ce_grad(const FloatMatrix *predicted, + const FloatMatrix *target, FloatMatrix *grad) { + if (!sm_kernel_matrices_compatible(predicted, target) || + !sm_kernel_matrices_compatible(predicted, grad)) { + return false; + } + + const size_t count = predicted->rows * predicted->cols; +#pragma omp parallel for simd schedule(static) if (count > 10000) + for (size_t i = 0; i < count; ++i) { + grad->values[i] = predicted->values[i] - target->values[i]; + } return true; } +float sm_loss_mse(const FloatMatrix *predicted, const FloatMatrix *target) { + if (!sm_kernel_matrices_compatible(predicted, target)) { + return -1.0f; + } + + const size_t count = predicted->rows * predicted->cols; + float loss = 0.0f; + for (size_t i = 0; i < count; ++i) { + const float difference = predicted->values[i] - target->values[i]; + loss += difference * difference; + } + return loss / (float)count; +} + +float sm_loss_cross_entropy(const FloatMatrix *predicted, + const FloatMatrix *target) { + if (!sm_kernel_matrices_compatible(predicted, target) || + predicted->rows == 0) { + return -1.0f; + } + + const size_t count = predicted->rows * predicted->cols; + float loss = 0.0f; + for (size_t i = 0; i < count; ++i) { + const float probability = fmaxf(predicted->values[i], 1.0e-7f); + loss += -target->values[i] * logf(probability); + } + return loss / (float)predicted->rows; +} + FloatMatrix *sm_multiply(const FloatMatrix *mat1, const FloatMatrix *mat2) { if (!mat1 || !mat2 || mat1->cols != mat2->rows) { log_error("Error: invalid matrix dimensions.\n"); diff --git a/app/matrix/src/sm_kernels.c b/app/matrix/src/sm_kernels.c new file mode 100644 index 0000000..cddb37c --- /dev/null +++ b/app/matrix/src/sm_kernels.c @@ -0,0 +1,387 @@ +#include "sm_kernels.h" + +#include +#include +#include +#include + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpedantic" +#endif +#include +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#if defined(USE_ACCELERATE) +#include + +static void sm_kernel_vvexpf(float *values, size_t length) { + for (size_t offset = 0; offset < length;) { + const size_t remaining = length - offset; + const int count = (int)(remaining > (size_t)INT32_MAX + ? (size_t)INT32_MAX + : remaining); + vvexpf(values + offset, values + offset, &count); + offset += (size_t)count; + } +} + +static void sm_kernel_vvtanhf(float *values, size_t length) { + for (size_t offset = 0; offset < length;) { + const size_t remaining = length - offset; + const int count = (int)(remaining > (size_t)INT32_MAX + ? (size_t)INT32_MAX + : remaining); + vvtanhf(values + offset, values + offset, &count); + offset += (size_t)count; + } +} +#endif + +static bool sm_kernel_valid_buffer(const float *values, size_t length) { + return values != NULL && length > 0; +} + +#if defined(__clang__) +__attribute__((noinline, optnone)) +#elif defined(__GNUC__) +__attribute__((noinline, optimize("no-fast-math"))) +#endif +static bool sm_kernel_is_nan_bits(const float *value) { + uint32_t bits = 0; + memcpy(&bits, value, sizeof(bits)); + return (bits & 0x7f800000u) == 0x7f800000u && + (bits & 0x007fffffu) != 0; +} + +bool sm_kernel_apply_sigmoid(float *values, size_t length) { + if (!sm_kernel_valid_buffer(values, length)) { + return false; + } + +#if defined(USE_ACCELERATE) + vDSP_vneg(values, 1, values, 1, (vDSP_Length)length); + sm_kernel_vvexpf(values, length); + const float one = 1.0f; + vDSP_vsadd(values, 1, &one, values, 1, (vDSP_Length)length); + for (size_t i = 0; i < length; ++i) { + values[i] = 1.0f / values[i]; + } +#else +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + values[i] = 1.0f / (1.0f + expf(-values[i])); + } +#endif + return true; +} + +bool sm_kernel_apply_tanh(float *values, size_t length) { + if (!sm_kernel_valid_buffer(values, length)) { + return false; + } + +#if defined(USE_ACCELERATE) + sm_kernel_vvtanhf(values, length); +#else +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + values[i] = tanhf(values[i]); + } +#endif + return true; +} + +bool sm_kernel_apply_gelu_tanh(float *values, size_t length) { + if (!sm_kernel_valid_buffer(values, length)) { + return false; + } + + const float scale = 0.7978845608f; + const float cubic = 0.044715f; +#if defined(USE_ACCELERATE) + enum { BLOCK = 256 }; + const size_t blocks = (length + BLOCK - 1) / BLOCK; +#pragma omp parallel for schedule(static) if (length > 10000) + for (size_t block = 0; block < blocks; ++block) { + float original[BLOCK]; + float transformed[BLOCK]; + const size_t offset = block * BLOCK; + const size_t count = length - offset < BLOCK ? length - offset : BLOCK; + for (size_t j = 0; j < count; ++j) { + const float x = values[offset + j]; + original[j] = x; + transformed[j] = scale * (x + cubic * x * x * x); + } + sm_kernel_vvtanhf(transformed, count); + for (size_t j = 0; j < count; ++j) { + values[offset + j] = + 0.5f * original[j] * (1.0f + transformed[j]); + } + } +#else +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float x = values[i]; + values[i] = 0.5f * x * (1.0f + tanhf(scale * (x + cubic * x * x * x))); + } +#endif + return true; +} + +bool sm_kernel_apply_gelu_erf(float *values, size_t length) { + if (!sm_kernel_valid_buffer(values, length)) { + return false; + } + +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float x = values[i]; + values[i] = 0.5f * x * (1.0f + erff(x * 0.7071067812f)); + } + return true; +} + +bool sm_kernel_apply_activation(float *values, size_t length, + SmActivation activation) { + if (activation == SM_ACT_NONE) { + return values != NULL && length > 0; + } + if (!sm_kernel_valid_buffer(values, length)) { + return false; + } + + switch (activation) { + case SM_ACT_RELU: +#if defined(USE_ACCELERATE) + { + const float zero = 0.0f; + vDSP_vthr(values, 1, &zero, values, 1, (vDSP_Length)length); + } +#else +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + if (values[i] < 0.0f) { + values[i] = 0.0f; + } + } +#endif + return true; + case SM_ACT_SIGMOID: + return sm_kernel_apply_sigmoid(values, length); + case SM_ACT_TANH: + return sm_kernel_apply_tanh(values, length); + case SM_ACT_NONE: + return true; + } + return false; +} + +bool sm_kernel_apply_softmax(float *values, size_t rows, size_t cols) { + if (values == NULL || rows == 0 || cols == 0 || + rows > SIZE_MAX / cols) { + return false; + } + +#if defined(USE_ACCELERATE) + for (size_t row = 0; row < rows; ++row) { + float *current = values + row * cols; + float maximum = 0.0f; + vDSP_maxv(current, 1, &maximum, (vDSP_Length)cols); + const float negative_maximum = -maximum; + vDSP_vsadd(current, 1, &negative_maximum, current, 1, (vDSP_Length)cols); + sm_kernel_vvexpf(current, cols); + float sum = 0.0f; + vDSP_sve(current, 1, &sum, (vDSP_Length)cols); + if (sum == 0.0f) { + continue; + } + const float inverse = 1.0f / sum; + vDSP_vsmul(current, 1, &inverse, current, 1, (vDSP_Length)cols); + } +#else +#pragma omp parallel for schedule(static) if (rows > 1 && rows * cols > 10000) + for (size_t row = 0; row < rows; ++row) { + float *current = values + row * cols; + float maximum = current[0]; + for (size_t col = 1; col < cols; ++col) { + if (current[col] > maximum) { + maximum = current[col]; + } + } + + float sum = 0.0f; + for (size_t col = 0; col < cols; ++col) { + current[col] = expf(current[col] - maximum); + sum += current[col]; + } + if (sum == 0.0f) { + continue; + } + for (size_t col = 0; col < cols; ++col) { + current[col] /= sum; + } + } +#endif + return true; +} + +bool sm_kernel_add_bias_activation(float *values, const float *bias, + size_t rows, size_t cols, size_t bias_rows, + SmActivation activation) { + if (values == NULL || bias == NULL || rows == 0 || cols == 0 || + rows > SIZE_MAX / cols || (bias_rows != 1 && bias_rows != rows) || + (activation != SM_ACT_NONE && activation != SM_ACT_RELU && + activation != SM_ACT_SIGMOID && activation != SM_ACT_TANH)) { + return false; + } + + const size_t count = rows * cols; +#if defined(USE_ACCELERATE) + if (activation == SM_ACT_NONE) { + for (size_t row = 0; row < rows; ++row) { + const float *bias_row = bias + (bias_rows == 1 ? 0 : row * cols); + vDSP_vadd(values + row * cols, 1, bias_row, 1, values + row * cols, 1, + (vDSP_Length)cols); + } + return true; + } + if (activation == SM_ACT_RELU) { + for (size_t row = 0; row < rows; ++row) { + const float *bias_row = bias + (bias_rows == 1 ? 0 : row * cols); + vDSP_vadd(values + row * cols, 1, bias_row, 1, values + row * cols, 1, + (vDSP_Length)cols); + } + const float zero = 0.0f; + vDSP_vthr(values, 1, &zero, values, 1, (vDSP_Length)count); + return true; + } + + enum { BLOCK = 256 }; + const size_t blocks_per_row = (cols + BLOCK - 1) / BLOCK; + const size_t blocks = rows * blocks_per_row; +#pragma omp parallel for schedule(static) if (count > 65536) + for (size_t block = 0; block < blocks; ++block) { + float work[BLOCK]; + const size_t row = block / blocks_per_row; + const size_t col_offset = (block % blocks_per_row) * BLOCK; + const size_t length = cols - col_offset < BLOCK ? cols - col_offset : BLOCK; + const float *bias_row = bias + (bias_rows == 1 ? 0 : row * cols); + float *output = values + row * cols + col_offset; + for (size_t j = 0; j < length; ++j) { + work[j] = output[j] + bias_row[col_offset + j]; + } + (void)sm_kernel_apply_activation(work, length, activation); + memcpy(output, work, length * sizeof(float)); + } +#else +#pragma omp parallel for schedule(static) if (count > 65536) + for (size_t i = 0; i < count; ++i) { + const float bias_value = bias_rows == 1 ? bias[i % cols] : bias[i]; + if (activation == SM_ACT_RELU && sm_kernel_is_nan_bits(values + i)) { + continue; + } + if (activation == SM_ACT_RELU && sm_kernel_is_nan_bits(&bias_value)) { + values[i] = bias_value; + continue; + } + float value = values[i] + bias_value; + switch (activation) { + case SM_ACT_NONE: + break; + case SM_ACT_RELU: + if (value < 0.0f) { + value = 0.0f; + } + break; + case SM_ACT_SIGMOID: + value = 1.0f / (1.0f + expf(-value)); + break; + case SM_ACT_TANH: + value = tanhf(value); + break; + } + values[i] = value; + } +#endif + return true; +} + +bool sm_kernel_relu_backward(const float *activation, float *grad, + size_t length) { + if (activation == NULL || grad == NULL || length == 0) { + return false; + } +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + if (activation[i] <= 0.0f) { + grad[i] = 0.0f; + } + } + return true; +} + +bool sm_kernel_sigmoid_backward(const float *activation, float *grad, + size_t length) { + if (activation == NULL || grad == NULL || length == 0) { + return false; + } +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float sigmoid = activation[i]; + grad[i] *= sigmoid * (1.0f - sigmoid); + } + return true; +} + +bool sm_kernel_tanh_backward(const float *activation, float *grad, + size_t length) { + if (activation == NULL || grad == NULL || length == 0) { + return false; + } +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float tanh_value = activation[i]; + grad[i] *= 1.0f - tanh_value * tanh_value; + } + return true; +} + +bool sm_kernel_gelu_tanh_backward(const float *input, float *grad, + size_t length) { + if (input == NULL || grad == NULL || length == 0) { + return false; + } + const float scale = 0.7978845608f; + const float cubic = 0.044715f; +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float x = input[i]; + const float u = scale * (x + cubic * x * x * x); + const float t = tanhf(u); + const float derivative = + 0.5f * (1.0f + t) + + 0.5f * x * (1.0f - t * t) * scale * (1.0f + 3.0f * cubic * x * x); + grad[i] *= derivative; + } + return true; +} + +bool sm_kernel_gelu_erf_backward(const float *input, float *grad, + size_t length) { + if (input == NULL || grad == NULL || length == 0) { + return false; + } + const float inv_sqrt_two = 0.7071067812f; + const float inv_sqrt_two_pi = 0.3989422804f; +#pragma omp parallel for simd schedule(static) if (length > 10000) + for (size_t i = 0; i < length; ++i) { + const float x = input[i]; + const float cdf = 0.5f * (1.0f + erff(x * inv_sqrt_two)); + const float pdf = inv_sqrt_two_pi * expf(-0.5f * x * x); + grad[i] *= cdf + x * pdf; + } + return true; +} \ No newline at end of file diff --git a/app/matrix/src/sm_kernels.h b/app/matrix/src/sm_kernels.h new file mode 100644 index 0000000..1c59ef8 --- /dev/null +++ b/app/matrix/src/sm_kernels.h @@ -0,0 +1,28 @@ +#ifndef SM_KERNELS_H +#define SM_KERNELS_H + +#include "sm.h" + +bool sm_kernel_apply_activation(float *values, size_t length, + SmActivation activation); +bool sm_kernel_apply_sigmoid(float *values, size_t length); +bool sm_kernel_apply_tanh(float *values, size_t length); +bool sm_kernel_apply_gelu_tanh(float *values, size_t length); +bool sm_kernel_apply_gelu_erf(float *values, size_t length); +bool sm_kernel_apply_softmax(float *values, size_t rows, size_t cols); +bool sm_kernel_add_bias_activation(float *values, const float *bias, + size_t rows, size_t cols, size_t bias_rows, + SmActivation activation); + +bool sm_kernel_relu_backward(const float *activation, float *grad, + size_t length); +bool sm_kernel_sigmoid_backward(const float *activation, float *grad, + size_t length); +bool sm_kernel_tanh_backward(const float *activation, float *grad, + size_t length); +bool sm_kernel_gelu_tanh_backward(const float *input, float *grad, + size_t length); +bool sm_kernel_gelu_erf_backward(const float *input, float *grad, + size_t length); + +#endif // SM_KERNELS_H \ No newline at end of file diff --git a/app/matrix/tests/bench_sm_gemm_bias_act.c b/app/matrix/tests/bench_sm_gemm_bias_act.c new file mode 100644 index 0000000..723cf2a --- /dev/null +++ b/app/matrix/tests/bench_sm_gemm_bias_act.c @@ -0,0 +1,147 @@ +/* + * Small dense NN benchmark: GEMM + row bias + activation versus fused API. + * Run: bazel run //app/matrix:bench_sm_gemm_bias_act + */ + +#include "sm.h" + +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions(size_t n) { + if (n <= 32) return 100; + if (n <= 64) return 24; + if (n <= 128) return 6; + return 2; +} + +static bool apply_activation(FloatMatrix *out, SmActivation activation) { + switch (activation) { + case SM_ACT_NONE: return true; + case SM_ACT_RELU: return sm_apply_relu(out); + case SM_ACT_SIGMOID: return sm_apply_sigmoid(out); + case SM_ACT_TANH: return sm_apply_tanh(out); + } + return false; +} + +static bool split_path(FloatMatrix *out, const FloatMatrix *a, + const FloatMatrix *b, const FloatMatrix *bias, + SmActivation activation) { + if (!sm_gemm(out, 1.0f, a, SM_NO_TRANSPOSE, b, SM_NO_TRANSPOSE, 0.0f)) { + return false; + } + for (size_t row = 0; row < out->rows; ++row) { + for (size_t col = 0; col < out->cols; ++col) { + out->values[row * out->cols + col] += bias->values[col]; + } + } + return apply_activation(out, activation); +} + +static const char *activation_name(SmActivation activation) { + switch (activation) { + case SM_ACT_NONE: return "none"; + case SM_ACT_RELU: return "relu"; + case SM_ACT_SIGMOID: return "sigmoid"; + case SM_ACT_TANH: return "tanh"; + } + return "unknown"; +} + +static double measure(bool fused, FloatMatrix *out, const FloatMatrix *a, + const FloatMatrix *b, const FloatMatrix *bias, + SmActivation activation, size_t reps) { + double samples[9]; + for (size_t sample = 0; sample < 9; ++sample) { + const uint64_t start = now_ns(); + for (size_t i = 0; i < reps; ++i) { + const bool ok = fused + ? sm_gemm_bias_act(out, a, SM_NO_TRANSPOSE, b, SM_NO_TRANSPOSE, + bias, activation) + : split_path(out, a, b, bias, activation); + if (!ok) return -1.0; + } + samples[sample] = + (double)(now_ns() - start) / 1000.0 / (double)reps; + } + return median(samples, 9); +} + +int main(void) { + const size_t sizes[] = {32, 64, 128, 256}; + const SmActivation activations[] = { + SM_ACT_RELU, SM_ACT_SIGMOID, SM_ACT_TANH}; + const int max_threads = omp_get_max_threads(); + const int thread_counts[] = {1, max_threads}; + omp_set_dynamic(0); + + printf("activation,n,threads,split_us,fused_us,speedup\n"); + for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); ++si) { + const size_t n = sizes[si]; + FloatMatrix *a = sm_create(n, n); + FloatMatrix *b = sm_create(n, n); + FloatMatrix *bias = sm_create(1, n); + FloatMatrix *out = sm_create(n, n); + if (!a || !b || !bias || !out) { + fprintf(stderr, "allocation failed for n=%zu\n", n); + sm_destroy(a); + sm_destroy(b); + sm_destroy(bias); + sm_destroy(out); + return 1; + } + for (size_t i = 0; i < n * n; ++i) { + a->values[i] = (float)((i * 17u) % 101u) * 0.01f - 0.5f; + b->values[i] = (float)((i * 29u) % 97u) * 0.01f - 0.48f; + } + for (size_t i = 0; i < n; ++i) { + bias->values[i] = (float)i * 0.001f - 0.1f; + } + + for (size_t ti = 0; ti < 2; ++ti) { + const int threads = thread_counts[ti]; + if (ti == 1 && threads == thread_counts[0]) continue; + omp_set_num_threads(threads); + for (size_t ai = 0; ai < sizeof(activations) / sizeof(activations[0]); + ++ai) { + const size_t reps = repetitions(n); + const double split = measure(false, out, a, b, bias, activations[ai], + reps); + const double fused = measure(true, out, a, b, bias, activations[ai], + reps); + printf("%s,%zu,%d,%.3f,%.3f,%.2fx\n", + activation_name(activations[ai]), n, threads, split, fused, + fused > 0.0 ? split / fused : 0.0); + } + } + sm_destroy(out); + sm_destroy(bias); + sm_destroy(b); + sm_destroy(a); + } + omp_set_num_threads(max_threads); + return 0; +} \ No newline at end of file diff --git a/app/matrix/tests/bench_sm_rowwise.c b/app/matrix/tests/bench_sm_rowwise.c new file mode 100644 index 0000000..269163b --- /dev/null +++ b/app/matrix/tests/bench_sm_rowwise.c @@ -0,0 +1,161 @@ +/* + * Row-helper benchmarks against scalar nm_lib-style loops. + * Run: bazel run //app/matrix:bench_sm_rowwise + */ + +#include "sm.h" + +#include +#include +#include +#include +#include + +typedef enum RowOp { + ROW_ADD, + ROW_SUM, + ROW_ARGMAX, +} RowOp; + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions(size_t count) { + if (count <= 8192) return 128; + if (count <= 65536) return 32; + return 4; +} + +static FloatMatrix *scalar_sum_rows(const FloatMatrix *mat) { + FloatMatrix *out = sm_create(1, mat->cols); + if (!out) return NULL; + for (size_t col = 0; col < mat->cols; ++col) { + float sum = 0.0f; + for (size_t row = 0; row < mat->rows; ++row) { + sum += mat->values[row * mat->cols + col]; + } + out->values[col] = sum; + } + return out; +} + +static FloatMatrix *scalar_argmax_rows(const FloatMatrix *mat) { + FloatMatrix *out = sm_create(mat->rows, 1); + if (!out) return NULL; + for (size_t row = 0; row < mat->rows; ++row) { + size_t best = 0; + for (size_t col = 1; col < mat->cols; ++col) { + if (mat->values[row * mat->cols + col] > + mat->values[row * mat->cols + best]) { + best = col; + } + } + out->values[row] = (float)best; + } + return out; +} + +static const char *operation_name(RowOp op) { + switch (op) { + case ROW_ADD: return "add_rowwise"; + case ROW_SUM: return "sum_rows"; + case ROW_ARGMAX: return "argmax_rows"; + } + return "unknown"; +} + +static double measure(bool optimized, RowOp op, FloatMatrix *mat, + const FloatMatrix *row, size_t reps) { + double samples[7]; + for (size_t sample = 0; sample < 7; ++sample) { + const uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + if (op == ROW_ADD) { + if (optimized) { + if (!sm_add_rowwise(mat, row)) return -1.0; + } else { + for (size_t i = 0; i < mat->rows; ++i) { + for (size_t j = 0; j < mat->cols; ++j) { + mat->values[i * mat->cols + j] += row->values[j]; + } + } + } + } else { + FloatMatrix *out = NULL; + if (optimized) { + out = op == ROW_SUM ? sm_sum_rows(mat) : sm_argmax_rows(mat); + } else { + out = op == ROW_SUM ? scalar_sum_rows(mat) : scalar_argmax_rows(mat); + } + if (!out) return -1.0; + sm_destroy(out); + } + } + samples[sample] = (double)(now_ns() - start) / 1000.0 / (double)reps; + } + return median(samples, 7); +} + +int main(void) { + const size_t row_counts[] = {32, 256, 1024, 4096}; + const size_t cols = 256; + const int max_threads = omp_get_max_threads(); + const int thread_counts[] = {1, max_threads}; + omp_set_dynamic(0); + + printf("operation,rows,cols,threads,scalar_us,sm_us,speedup\n"); + for (size_t si = 0; si < sizeof(row_counts) / sizeof(row_counts[0]); ++si) { + const size_t rows = row_counts[si]; + const size_t count = rows * cols; + FloatMatrix *mat = sm_create(rows, cols); + FloatMatrix *bias = sm_create(1, cols); + if (!mat || !bias) { + fprintf(stderr, "allocation failed for %zux%zu\n", rows, cols); + sm_destroy(mat); + sm_destroy(bias); + return 1; + } + for (size_t i = 0; i < count; ++i) { + mat->values[i] = (float)((i * 31u) % 997u) * 0.001f; + } + for (size_t i = 0; i < cols; ++i) { + bias->values[i] = (float)i * 0.0001f; + } + + for (size_t ti = 0; ti < 2; ++ti) { + const int threads = thread_counts[ti]; + if (ti == 1 && threads == thread_counts[0]) continue; + omp_set_num_threads(threads); + for (int op = ROW_ADD; op <= ROW_ARGMAX; ++op) { + const size_t reps = repetitions(count); + const double before = + measure(false, (RowOp)op, mat, bias, reps); + const double after = + measure(true, (RowOp)op, mat, bias, reps); + printf("%s,%zu,%zu,%d,%.3f,%.3f,%.2fx\n", + operation_name((RowOp)op), rows, cols, threads, before, after, + after > 0.0 ? before / after : 0.0); + } + } + sm_destroy(bias); + sm_destroy(mat); + } + omp_set_num_threads(max_threads); + return 0; +} \ No newline at end of file diff --git a/app/matrix/tests/test_dms.c b/app/matrix/tests/test_dms.c index a19bee8..daad405 100644 --- a/app/matrix/tests/test_dms.c +++ b/app/matrix/tests/test_dms.c @@ -257,9 +257,9 @@ void test_dms_get_col(void) { TEST_ASSERT_EQUAL(1, m2->cols); TEST_ASSERT_EQUAL(3, m2->nnz); TEST_ASSERT_EQUAL(3, m2->capacity); - TEST_ASSERT_EQUAL(4, m2->values[0]); + TEST_ASSERT_EQUAL(2, m2->values[0]); TEST_ASSERT_EQUAL(5, m2->values[1]); - TEST_ASSERT_EQUAL(6, m2->values[2]); + TEST_ASSERT_EQUAL(8, m2->values[2]); dms_destroy(m); dms_destroy(m2); } diff --git a/app/matrix/tests/test_sm_nn.c b/app/matrix/tests/test_sm_nn.c new file mode 100644 index 0000000..ff58fa1 --- /dev/null +++ b/app/matrix/tests/test_sm_nn.c @@ -0,0 +1,519 @@ +#include "sm.h" +#include "sm_kernels.h" + +#include +#include +#include +#include +#include + +#define UNITY_INCLUDE_FLOAT +#define UNITY_FLOAT_PRECISION 6 +#define TEST_CASE(...) +#include "unity.h" + +static bool near_value(float actual, float expected) { + return fabsf(actual - expected) <= 1.0e-6f + 1.0e-5f * fabsf(expected); +} + +static float reference_gelu_tanh(float x) { + const float k = 0.7978845608f; + return 0.5f * x * (1.0f + tanhf(k * (x + 0.044715f * x * x * x))); +} + +static float reference_gelu_erf(float x) { + return 0.5f * x * (1.0f + erff(x * 0.7071067812f)); +} + +static float reference_sigmoid(float x) { + return 1.0f / (1.0f + expf(-x)); +} + +static float reference_activation(SmActivation activation, float x) { + switch (activation) { + case SM_ACT_NONE: return x; + case SM_ACT_RELU: return x > 0.0f ? x : 0.0f; + case SM_ACT_SIGMOID: return 1.0f / (1.0f + expf(-x)); + case SM_ACT_TANH: return tanhf(x); + } + return NAN; +} + +void setUp(void) {} +void tearDown(void) {} + +void test_sm_activation_kernels_match_scalar_reference_sizes(void) { + const size_t sizes[] = {1, 7, 1000, 100000}; + for (size_t s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) { + const size_t n = sizes[s]; + FloatMatrix *mat = sm_create(1, n); + TEST_ASSERT_NOT_NULL(mat); + uint32_t random_state = 0x12345678u; + for (size_t i = 0; i < n; ++i) { + random_state = random_state * 1664525u + 1013904223u; + mat->values[i] = ((float)(random_state >> 8) / 16777216.0f - 0.5f) * 8.0f; + } + + FloatMatrix *actual = sm_clone(mat); + TEST_ASSERT_TRUE(sm_apply_sigmoid(actual)); + for (size_t i = 0; i < n; ++i) { + const float x = mat->values[i]; + TEST_ASSERT_TRUE(near_value(actual->values[i], + 1.0f / (1.0f + expf(-x)))); + } + sm_destroy(actual); + + actual = sm_clone(mat); + TEST_ASSERT_TRUE(sm_apply_tanh(actual)); + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_TRUE(near_value(actual->values[i], tanhf(mat->values[i]))); + } + sm_destroy(actual); + + actual = sm_clone(mat); + TEST_ASSERT_TRUE(sm_apply_gelu_tanh(actual)); + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_TRUE( + near_value(actual->values[i], reference_gelu_tanh(mat->values[i]))); + } + sm_destroy(actual); + + actual = sm_clone(mat); + TEST_ASSERT_TRUE(sm_apply_gelu_erf(actual)); + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_TRUE( + near_value(actual->values[i], reference_gelu_erf(mat->values[i]))); + } + sm_destroy(actual); + sm_destroy(mat); + } +} + +void test_sm_activation_kernels_reject_empty_buffers_and_propagate_nan(void) { + float value = NAN; + TEST_ASSERT_FALSE(sm_kernel_apply_sigmoid(NULL, 1)); + TEST_ASSERT_FALSE(sm_kernel_apply_sigmoid(&value, 0)); + + FloatMatrix *mat = sm_create(1, 1); + TEST_ASSERT_NOT_NULL(mat); + mat->values[0] = NAN; + TEST_ASSERT_TRUE(sm_apply_sigmoid(mat)); + TEST_ASSERT_TRUE(isnan(mat->values[0])); + mat->values[0] = NAN; + TEST_ASSERT_TRUE(sm_apply_tanh(mat)); + TEST_ASSERT_TRUE(isnan(mat->values[0])); + mat->values[0] = NAN; + TEST_ASSERT_TRUE(sm_apply_gelu_tanh(mat)); + TEST_ASSERT_TRUE(isnan(mat->values[0])); + mat->values[0] = NAN; + TEST_ASSERT_TRUE(sm_apply_gelu_erf(mat)); + TEST_ASSERT_TRUE(isnan(mat->values[0])); + sm_destroy(mat); + + float nan_value = NAN; + const float zero_bias = 0.0f; + uint32_t initial_nan_bits = 0; + memcpy(&initial_nan_bits, &nan_value, sizeof(initial_nan_bits)); + TEST_ASSERT_EQUAL_UINT32(0x7f800000u, + initial_nan_bits & 0x7f800000u); + TEST_ASSERT_TRUE(sm_kernel_add_bias_activation( + &nan_value, &zero_bias, 1, 1, 1, SM_ACT_RELU)); + uint32_t nan_bits = 0; + memcpy(&nan_bits, &nan_value, sizeof(nan_bits)); + TEST_ASSERT_EQUAL_UINT32(0x7f800000u, nan_bits & 0x7f800000u); + TEST_ASSERT_TRUE((nan_bits & 0x007fffffu) != 0); +} + +void test_sm_softmax_is_stable_for_large_values_and_propagates_nan(void) { + FloatMatrix *mat = sm_create_with_values( + 2, 3, (float[]){1000.0f, 999.0f, -1000.0f, -1000.0f, -1001.0f, + -1002.0f}); + TEST_ASSERT_NOT_NULL(mat); + TEST_ASSERT_TRUE(sm_apply_softmax(mat)); + for (size_t row = 0; row < 2; ++row) { + float sum = 0.0f; + for (size_t col = 0; col < 3; ++col) { + const float v = mat->values[row * 3 + col]; + TEST_ASSERT_TRUE(isfinite(v)); + sum += v; + } + TEST_ASSERT_TRUE(near_value(sum, 1.0f)); + } + sm_destroy(mat); + + mat = sm_create_with_values(1, 3, (float[]){0.0f, NAN, 1.0f}); + TEST_ASSERT_NOT_NULL(mat); + TEST_ASSERT_TRUE(sm_apply_softmax(mat)); + TEST_ASSERT_TRUE(isnan(mat->values[0])); + TEST_ASSERT_TRUE(isnan(mat->values[1])); + TEST_ASSERT_TRUE(isnan(mat->values[2])); + sm_destroy(mat); +} + +void test_sm_backward_kernels_match_numerical_derivatives(void) { + const float inputs[] = {-1.7f, -0.25f, 0.8f, 2.1f}; + const size_t n = sizeof(inputs) / sizeof(inputs[0]); + const float original_grads[] = {0.5f, -1.0f, 2.0f, 3.0f}; + const float h = 1.0e-3f; + + FloatMatrix *x = sm_create_with_values(1, n, (float *)inputs); + FloatMatrix *activation = sm_clone(x); + FloatMatrix *grad = sm_create_with_values(1, n, (float *)original_grads); + TEST_ASSERT_NOT_NULL(x); + TEST_ASSERT_NOT_NULL(activation); + TEST_ASSERT_NOT_NULL(grad); + TEST_ASSERT_TRUE(sm_apply_sigmoid(activation)); + TEST_ASSERT_TRUE(sm_apply_sigmoid_backward(activation, grad)); + for (size_t i = 0; i < n; ++i) { + const float numeric = + (reference_sigmoid(inputs[i] + h) - + reference_sigmoid(inputs[i] - h)) / + (2.0f * h) * original_grads[i]; + TEST_ASSERT_FLOAT_WITHIN(2.0e-4f, numeric, grad->values[i]); + } + sm_destroy(activation); + sm_destroy(grad); + + activation = sm_clone(x); + grad = sm_create_with_values(1, n, (float *)original_grads); + TEST_ASSERT_TRUE(sm_apply_tanh(activation)); + TEST_ASSERT_TRUE(sm_apply_tanh_backward(activation, grad)); + for (size_t i = 0; i < n; ++i) { + const float numeric = + (tanhf(inputs[i] + h) - tanhf(inputs[i] - h)) / + (2.0f * h) * original_grads[i]; + TEST_ASSERT_FLOAT_WITHIN(2.0e-4f, numeric, grad->values[i]); + } + sm_destroy(activation); + sm_destroy(grad); + + for (int variant = 0; variant < 2; ++variant) { + grad = sm_create_with_values(1, n, (float *)original_grads); + TEST_ASSERT_NOT_NULL(grad); + bool ok = variant == 0 ? sm_apply_gelu_tanh_backward(x, grad) + : sm_apply_gelu_erf_backward(x, grad); + TEST_ASSERT_TRUE(ok); + for (size_t i = 0; i < n; ++i) { + const float xp = inputs[i] + h; + const float xm = inputs[i] - h; + const float yp = variant == 0 ? reference_gelu_tanh(xp) + : reference_gelu_erf(xp); + const float ym = variant == 0 ? reference_gelu_tanh(xm) + : reference_gelu_erf(xm); + const float numeric = (yp - ym) / (2.0f * h) * original_grads[i]; + TEST_ASSERT_FLOAT_WITHIN(5.0e-4f, numeric, grad->values[i]); + } + sm_destroy(grad); + } + sm_destroy(x); +} + +void test_sm_relu_backward_matches_finite_difference(void) { + const float inputs[] = {-2.0f, -0.01f, 0.02f, 3.0f}; + const float original_grad[] = {1.0f, -2.0f, 0.5f, 3.0f}; + FloatMatrix *activation = sm_create_with_values(1, 4, (float *)inputs); + FloatMatrix *grad = sm_create_with_values(1, 4, (float *)original_grad); + TEST_ASSERT_NOT_NULL(activation); + TEST_ASSERT_NOT_NULL(grad); + TEST_ASSERT_TRUE(sm_apply_relu(activation)); + TEST_ASSERT_TRUE(sm_apply_relu_backward(activation, grad)); + const float h = 1.0e-3f; + for (size_t i = 0; i < 4; ++i) { + const float plus = fmaxf(0.0f, inputs[i] + h); + const float minus = fmaxf(0.0f, inputs[i] - h); + const float numeric = (plus - minus) / (2.0f * h) * original_grad[i]; + TEST_ASSERT_FLOAT_WITHIN(5.0e-4f, numeric, grad->values[i]); + } + sm_destroy(grad); + sm_destroy(activation); +} + +void test_sm_elementwise_activation_results_are_thread_count_independent(void) { + FloatMatrix *input = sm_create(1, 100000); + FloatMatrix *one_thread = sm_create(1, 100000); + FloatMatrix *many_threads = sm_create(1, 100000); + TEST_ASSERT_NOT_NULL(input); + TEST_ASSERT_NOT_NULL(one_thread); + TEST_ASSERT_NOT_NULL(many_threads); + for (size_t i = 0; i < input->cols; ++i) { + input->values[i] = sinf((float)i * 0.003f); + } + memcpy(one_thread->values, input->values, input->cols * sizeof(float)); + memcpy(many_threads->values, input->values, input->cols * sizeof(float)); + + const int previous_threads = omp_get_max_threads(); + omp_set_dynamic(0); + for (int activation = SM_ACT_RELU; activation <= SM_ACT_TANH; ++activation) { + memcpy(one_thread->values, input->values, input->cols * sizeof(float)); + memcpy(many_threads->values, input->values, input->cols * sizeof(float)); + omp_set_num_threads(1); + TEST_ASSERT_TRUE(sm_kernel_apply_activation(one_thread->values, + one_thread->cols, + (SmActivation)activation)); + omp_set_num_threads(8); + TEST_ASSERT_TRUE(sm_kernel_apply_activation(many_threads->values, + many_threads->cols, + (SmActivation)activation)); + TEST_ASSERT_EQUAL(0, memcmp(one_thread->values, many_threads->values, + input->cols * sizeof(float))); + } + omp_set_num_threads(1); + TEST_ASSERT_TRUE(sm_apply_gelu_tanh(one_thread)); + omp_set_num_threads(8); + TEST_ASSERT_TRUE(sm_apply_gelu_tanh(many_threads)); + TEST_ASSERT_EQUAL(0, memcmp(one_thread->values, many_threads->values, + input->cols * sizeof(float))); + omp_set_num_threads(previous_threads); + + sm_destroy(input); + sm_destroy(one_thread); + sm_destroy(many_threads); +} + +void test_sm_gemm_bias_act_matches_reference_with_transposes_and_bias_shapes(void) { + FloatMatrix *a = sm_create_with_values( + 3, 2, (float[]){1.0f, -2.0f, 0.5f, 3.0f, -1.0f, 2.0f}); + FloatMatrix *b = sm_create_with_values( + 4, 3, (float[]){1.0f, 2.0f, -1.0f, -2.0f, 0.5f, 1.0f, + 3.0f, -1.0f, 2.0f, 0.25f, 1.5f, -0.5f}); + FloatMatrix *bias = sm_create_with_values( + 1, 4, (float[]){0.25f, -0.5f, 1.0f, -1.0f}); + FloatMatrix *full_bias = sm_create_with_values( + 2, 4, (float[]){0.25f, -0.5f, 1.0f, -1.0f, + -0.25f, 0.5f, -1.0f, 1.0f}); + FloatMatrix *out = sm_create(2, 4); + TEST_ASSERT_NOT_NULL(a); + TEST_ASSERT_NOT_NULL(b); + TEST_ASSERT_NOT_NULL(bias); + TEST_ASSERT_NOT_NULL(full_bias); + TEST_ASSERT_NOT_NULL(out); + + for (int act = SM_ACT_NONE; act <= SM_ACT_TANH; ++act) { + const SmActivation activation = (SmActivation)act; + TEST_ASSERT_TRUE(sm_gemm_bias_act(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + bias, activation)); + for (size_t row = 0; row < out->rows; ++row) { + for (size_t col = 0; col < out->cols; ++col) { + float sum = 0.0f; + for (size_t k = 0; k < 3; ++k) { + sum += a->values[k * a->cols + row] * + b->values[col * b->cols + k]; + } + const float expected = reference_activation( + activation, sum + bias->values[col]); + TEST_ASSERT_TRUE(near_value(out->values[row * out->cols + col], + expected)); + } + } + } + + TEST_ASSERT_TRUE(sm_gemm_bias_act(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + full_bias, SM_ACT_RELU)); + float expected_relu[8]; + memcpy(expected_relu, out->values, sizeof(expected_relu)); + TEST_ASSERT_TRUE(sm_gemm_bias_relu(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + full_bias)); + for (size_t i = 0; i < out->rows * out->cols; ++i) { + TEST_ASSERT_TRUE(out->values[i] >= 0.0f); + TEST_ASSERT_TRUE(near_value(out->values[i], expected_relu[i])); + } + TEST_ASSERT_TRUE(sm_gemm_bias_act(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + NULL, SM_ACT_TANH)); + for (size_t row = 0; row < out->rows; ++row) { + for (size_t col = 0; col < out->cols; ++col) { + float sum = 0.0f; + for (size_t k = 0; k < 3; ++k) { + sum += a->values[k * a->cols + row] * b->values[col * b->cols + k]; + } + TEST_ASSERT_TRUE(near_value(out->values[row * out->cols + col], + tanhf(sum))); + } + } + TEST_ASSERT_FALSE(sm_gemm_bias_act(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + a, SM_ACT_RELU)); + TEST_ASSERT_FALSE(sm_gemm_bias_act(out, a, SM_TRANSPOSE, b, SM_TRANSPOSE, + NULL, (SmActivation)99)); + + sm_destroy(out); + sm_destroy(full_bias); + sm_destroy(bias); + sm_destroy(b); + sm_destroy(a); +} + +void test_sm_row_helpers_and_losses_match_nm_lib_semantics(void) { + FloatMatrix *mat = sm_create_with_values( + 3, 4, (float[]){1.0f, 5.0f, 5.0f, 0.0f, + -1.0f, -2.0f, -3.0f, -4.0f, + 2.0f, 2.0f, 3.0f, 3.0f}); + FloatMatrix *row = sm_create_with_values( + 1, 4, (float[]){10.0f, -1.0f, -1.0f, 2.0f}); + TEST_ASSERT_NOT_NULL(mat); + TEST_ASSERT_NOT_NULL(row); + TEST_ASSERT_TRUE(sm_add_rowwise(mat, row)); + const float expected_values[] = {11.0f, 4.0f, 4.0f, 2.0f, + 9.0f, -3.0f, -4.0f, -2.0f, + 12.0f, 1.0f, 2.0f, 5.0f}; + for (size_t i = 0; i < 12; ++i) { + TEST_ASSERT_TRUE(near_value(mat->values[i], expected_values[i])); + } + + FloatMatrix *sum = sm_sum_rows(mat); + FloatMatrix *argmax = sm_argmax_rows(mat); + TEST_ASSERT_NOT_NULL(sum); + TEST_ASSERT_NOT_NULL(argmax); + const float expected_sum[] = {32.0f, 2.0f, 2.0f, 5.0f}; + for (size_t col = 0; col < 4; ++col) { + TEST_ASSERT_TRUE(near_value(sum->values[col], expected_sum[col])); + } + TEST_ASSERT_EQUAL_FLOAT(1.0f, argmax->values[0]); + TEST_ASSERT_EQUAL_FLOAT(0.0f, argmax->values[1]); + TEST_ASSERT_EQUAL_FLOAT(0.0f, argmax->values[2]); + + FloatMatrix *predicted = sm_create_with_values( + 2, 2, (float[]){0.5f, 0.5f, 0.25f, 0.75f}); + FloatMatrix *target = sm_create_with_values( + 2, 2, (float[]){1.0f, 0.0f, 0.0f, 1.0f}); + FloatMatrix *grad = sm_create(2, 2); + TEST_ASSERT_NOT_NULL(predicted); + TEST_ASSERT_NOT_NULL(target); + TEST_ASSERT_NOT_NULL(grad); + TEST_ASSERT_TRUE(near_value(sm_loss_mse(predicted, target), 0.15625f)); + TEST_ASSERT_TRUE(near_value( + sm_loss_cross_entropy(predicted, target), + (-logf(0.5f) - logf(0.75f)) / 2.0f)); + TEST_ASSERT_TRUE(sm_softmax_ce_grad(predicted, target, grad)); + TEST_ASSERT_TRUE(near_value(grad->values[0], -0.5f)); + TEST_ASSERT_TRUE(near_value(grad->values[1], 0.5f)); + TEST_ASSERT_TRUE(near_value(grad->values[2], 0.25f)); + TEST_ASSERT_TRUE(near_value(grad->values[3], -0.25f)); + + TEST_ASSERT_FALSE(sm_add_rowwise(mat, predicted)); + TEST_ASSERT_EQUAL_FLOAT(-1.0f, sm_loss_mse(predicted, row)); + TEST_ASSERT_FALSE(sm_softmax_ce_grad(predicted, target, row)); + + sm_destroy(grad); + sm_destroy(target); + sm_destroy(predicted); + sm_destroy(argmax); + sm_destroy(sum); + sm_destroy(row); + sm_destroy(mat); +} + +void test_sm_rowwise_add_handles_aliased_bias_and_sum_is_thread_deterministic(void) { + FloatMatrix *aliased = sm_create_with_values( + 2, 3, (float[]){1.0f, 2.0f, 3.0f, 10.0f, 20.0f, 30.0f}); + TEST_ASSERT_NOT_NULL(aliased); + FloatMatrix row_view = { + .rows = 1, + .cols = 3, + .capacity = 3, + .values = aliased->values, + }; + TEST_ASSERT_TRUE(sm_add_rowwise(aliased, &row_view)); + const float expected[] = {2.0f, 4.0f, 6.0f, 11.0f, 22.0f, 33.0f}; + for (size_t i = 0; i < 6; ++i) { + TEST_ASSERT_TRUE(near_value(aliased->values[i], expected[i])); + } + sm_destroy(aliased); + + FloatMatrix *many_rows = sm_create(4096, 7); + TEST_ASSERT_NOT_NULL(many_rows); + for (size_t i = 0; i < many_rows->rows * many_rows->cols; ++i) { + many_rows->values[i] = (float)((i * 29u) % 101u) * 0.01f - 0.5f; + } + const int previous_threads = omp_get_max_threads(); + omp_set_dynamic(0); + omp_set_num_threads(1); + FloatMatrix *one_thread = sm_sum_rows(many_rows); + omp_set_num_threads(8); + FloatMatrix *eight_threads = sm_sum_rows(many_rows); + TEST_ASSERT_NOT_NULL(one_thread); + TEST_ASSERT_NOT_NULL(eight_threads); + TEST_ASSERT_EQUAL(0, memcmp(one_thread->values, eight_threads->values, + many_rows->cols * sizeof(float))); + omp_set_num_threads(previous_threads); + + sm_destroy(eight_threads); + sm_destroy(one_thread); + sm_destroy(many_rows); +} + +void test_sm_row_kernels_and_losses_match_scalar_references_sizes(void) { + const size_t widths[] = {1, 7, 1000, 100000}; + for (size_t s = 0; s < sizeof(widths) / sizeof(widths[0]); ++s) { + const size_t rows = 4; + const size_t cols = widths[s]; + const size_t count = rows * cols; + FloatMatrix *mat = sm_create(rows, cols); + FloatMatrix *bias = sm_create(1, cols); + FloatMatrix *target = sm_create(rows, cols); + TEST_ASSERT_NOT_NULL(mat); + TEST_ASSERT_NOT_NULL(bias); + TEST_ASSERT_NOT_NULL(target); + for (size_t col = 0; col < cols; ++col) { + bias->values[col] = 0.25f * cosf((float)col * 0.017f); + } + for (size_t i = 0; i < count; ++i) { + mat->values[i] = sinf((float)i * 0.011f); + target->values[i] = (float)((i * 13u) % 19u) / 20.0f; + } + + TEST_ASSERT_TRUE(sm_add_rowwise(mat, bias)); + for (size_t row = 0; row < rows; ++row) { + for (size_t col = 0; col < cols; ++col) { + const float original = sinf((float)(row * cols + col) * 0.011f); + TEST_ASSERT_TRUE(near_value(mat->values[row * cols + col], + original + bias->values[col])); + } + } + + FloatMatrix *sum = sm_sum_rows(mat); + FloatMatrix *argmax = sm_argmax_rows(mat); + TEST_ASSERT_NOT_NULL(sum); + TEST_ASSERT_NOT_NULL(argmax); + for (size_t col = 0; col < cols; ++col) { + float expected_sum = 0.0f; + for (size_t row = 0; row < rows; ++row) { + expected_sum += mat->values[row * cols + col]; + } + TEST_ASSERT_EQUAL_FLOAT(expected_sum, sum->values[col]); + } + for (size_t row = 0; row < rows; ++row) { + size_t best = 0; + for (size_t col = 1; col < cols; ++col) { + if (mat->values[row * cols + col] > mat->values[row * cols + best]) { + best = col; + } + } + TEST_ASSERT_EQUAL_FLOAT((float)best, argmax->values[row]); + } + + float expected_mse = 0.0f; + float expected_ce = 0.0f; + for (size_t i = 0; i < count; ++i) { + const float difference = mat->values[i] - target->values[i]; + expected_mse += difference * difference; + const float probability = fmaxf(mat->values[i], 1.0e-7f); + expected_ce += -target->values[i] * logf(probability); + } + expected_mse /= (float)count; + expected_ce /= (float)rows; + TEST_ASSERT_EQUAL_FLOAT(expected_mse, sm_loss_mse(mat, target)); + TEST_ASSERT_EQUAL_FLOAT(expected_ce, sm_loss_cross_entropy(mat, target)); + + sm_destroy(argmax); + sm_destroy(sum); + sm_destroy(target); + sm_destroy(bias); + sm_destroy(mat); + } + + FloatMatrix empty = {0}; + TEST_ASSERT_FALSE(sm_add_rowwise(&empty, &empty)); + TEST_ASSERT_NULL(sm_sum_rows(&empty)); + TEST_ASSERT_NULL(sm_argmax_rows(&empty)); + TEST_ASSERT_EQUAL_FLOAT(-1.0f, sm_loss_mse(&empty, &empty)); + TEST_ASSERT_EQUAL_FLOAT(-1.0f, sm_loss_cross_entropy(&empty, &empty)); +} \ No newline at end of file diff --git a/app/tensor/BUILD b/app/tensor/BUILD index 833f2bc..33eb15b 100644 --- a/app/tensor/BUILD +++ b/app/tensor/BUILD @@ -79,7 +79,7 @@ mps_linkopts = [ default_deps = [ "@openmp//:libomp", - "@log", + "//third_party/log.c:log", ] clang_doc_target( @@ -132,12 +132,14 @@ cc_library( "src/core/st.c", "src/core/st_convert.c", "src/core/st_dtype.c", + "src/core/st_f16.c", "src/core/st_shape_ops.c", "src/core/st_vector_bridge.c", ], hdrs = [ "include/st.h", "include/st_convert.h", + "include/st_f16.h", "include/st_shape_ops.h", "include/st_vector_bridge.h", "src/core/st_bf16_utils.h", @@ -155,10 +157,10 @@ cc_library( target_compatible_with = ["@platforms//os:macos"], visibility = ["//app/tensor:__pkg__"], deps = default_deps + [ + ":st_runtime", "//app/matrix:sm", "//app/vector:sv", "//app/vector:vv", - ":st_runtime", ], ) @@ -220,14 +222,43 @@ cc_library( ], ) +cc_library( + name = "st_ops_interp", + srcs = [ + "src/ops/st_interp.c", + ], + hdrs = [ + "include/st_interp.h", + ], + copts = default_copts + perf_copts + package_copts + openmp_copts, + includes = [ + "include", + "src/backend", + "src/backend/mps", + "src/core", + "src/ops", + "src/runtime", + ], + local_defines = ["ST_INTERNAL_TENSOR_LAYOUT"], + visibility = ["//app/tensor:__pkg__"], + deps = default_deps + [ + ":st_backend", + ":st_core", + ":st_runtime", + "//app/matrix:sm", + ], +) + cc_library( name = "st", hdrs = [ "include/st.h", "include/st_batchnorm.h", "include/st_conv.h", - "include/st_pool.h", "include/st_convert.h", + "include/st_f16.h", + "include/st_interp.h", + "include/st_pool.h", "include/st_shape_ops.h", "include/st_vector_bridge.h", ], @@ -246,11 +277,18 @@ cc_library( deps = default_deps + [ ":st_backend", ":st_core", + ":st_ops_interp", ":st_ops_nn", ":st_runtime", ], ) +filegroup( + name = "tensor_header", + srcs = glob(["include/*.h"]), + visibility = ["//visibility:public"], +) + cc_library( name = "st_internal_headers", hdrs = [ @@ -287,9 +325,9 @@ objc_library( "include/st.h", "include/st_conv.h", "include/st_pool.h", - "src/backend/st_backend.h", "src/backend/mps/st_mps.h", "src/backend/mps/st_stream_mps.h", + "src/backend/st_backend.h", "src/core/st_dtype.h", "src/runtime/st_buffer.h", "src/runtime/st_buffer_metal.h", @@ -366,6 +404,98 @@ unity_test( ], ) +unity_test( + name = "test_st_nn", + srcs = ["tests/test_st_nn.c"], + my_config = ["tests/my_config.yml"], + tags = ["smoke"], + deps = [ + ":st", + ":st_internal_headers", + ], +) + +cc_binary( + name = "bench_nn_activations", + srcs = ["tests/bench_nn_activations.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + "//app/matrix:sm", + ], +) + +unity_test( + name = "test_st_interp", + srcs = ["tests/test_st_interp.c"], + my_config = ["tests/my_config.yml"], + tags = ["smoke"], + deps = [ + ":st", + ":st_internal_headers", + ], +) + +cc_binary( + name = "bench_st_binary", + srcs = ["tests/bench_st_binary.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + "//app/matrix:sm", + ], +) + +cc_binary( + name = "bench_st_norm_reduce", + srcs = ["tests/bench_st_norm_reduce.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + "//app/matrix:sm", + ], +) + +cc_binary( + name = "bench_st_matmul", + srcs = ["tests/bench_st_matmul.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + "//app/matrix:sm", + ], +) + +cc_binary( + name = "bench_st_gather", + srcs = ["tests/bench_st_gather.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + "//app/matrix:sm", + ], +) + +unity_test( + name = "test_st_f16", + srcs = ["tests/test_st_f16.c"], + my_config = ["tests/my_config.yml"], + tags = ["smoke"], + deps = [ + ":st", + ":st_internal_headers", + ], +) + +cc_binary( + name = "bench_st_f16", + srcs = ["tests/bench_st_f16.c"], + target_compatible_with = ["@platforms//os:macos"], + deps = [ + ":st", + ], +) + unity_test( name = "test_st_batchnorm", srcs = ["tests/test_st_batchnorm.c"], diff --git a/app/tensor/include/st.h b/app/tensor/include/st.h index fc5a99c..8ae358c 100644 --- a/app/tensor/include/st.h +++ b/app/tensor/include/st.h @@ -284,6 +284,27 @@ bool st_apply_relu(FloatTensor *t); /** @brief Apply ReLU backward mask in-place to @p grad. */ bool st_apply_relu_backward(const FloatTensor *activation, FloatTensor *grad); +/** @brief Apply sigmoid in-place to every tensor element. */ +bool st_apply_sigmoid(FloatTensor *t); +/** @brief Apply hyperbolic tangent in-place to every tensor element. */ +bool st_apply_tanh(FloatTensor *t); +/** @brief Apply tanh-approximation GELU in-place. */ +bool st_apply_gelu_tanh(FloatTensor *t); +/** @brief Apply erf-form GELU in-place. */ +bool st_apply_gelu_erf(FloatTensor *t); +/** @brief Apply softmax independently over the last tensor axis. */ +bool st_apply_softmax(FloatTensor *t); + +/** @brief Multiply @p grad in-place by the sigmoid derivative at @p activation. */ +bool st_apply_sigmoid_backward(const FloatTensor *activation, + FloatTensor *grad); +/** @brief Multiply @p grad in-place by the tanh derivative at @p activation. */ +bool st_apply_tanh_backward(const FloatTensor *activation, FloatTensor *grad); +/** @brief Multiply @p grad in-place by the tanh-GELU derivative at @p input. */ +bool st_apply_gelu_tanh_backward(const FloatTensor *input, FloatTensor *grad); +/** @brief Multiply @p grad in-place by the erf-GELU derivative at @p input. */ +bool st_apply_gelu_erf_backward(const FloatTensor *input, FloatTensor *grad); + // ---- Reduction ---- /** diff --git a/app/tensor/include/st_f16.h b/app/tensor/include/st_f16.h new file mode 100644 index 0000000..5a9da22 --- /dev/null +++ b/app/tensor/include/st_f16.h @@ -0,0 +1,72 @@ +/** + * @file st_f16.h + * @brief Public API for IEEE binary16 (half precision) conversion. + * + * The scalar functions reproduce nm_lib's nm_halfprec.h bit for bit: + * round-to-nearest-even, exact subnormals, overflow to infinity, NaN kept + * quiet (payload dropped, sign preserved), and the sign of ±0 preserved. + * The bulk functions use NEON FCVT where available; NaN results are + * corrected to the reference bit pattern because the hardware keeps the + * payload. + */ + +/* + * Copyright (c) 2026 @tragisch + * SPDX-License-Identifier: MIT + * + * This file is part of a project licensed under the MIT License. + * See the LICENSE file in the root directory for details. + */ + +#ifndef ST_F16_H +#define ST_F16_H + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * @brief Convert one f32 value to IEEE binary16 bits. + * @param value Input value. + * @return binary16 bit pattern (round to nearest even, overflow to + * infinity, subnormals exact, NaN quiet with sign preserved). + */ +uint16_t st_f32_to_f16(float value); + +/** + * @brief Convert IEEE binary16 bits to one f32 value. + * @param bits binary16 bit pattern. + * @return Exact f32 value (all binary16 values are exactly representable). + */ +float st_f16_to_f32(uint16_t bits); + +/** + * @brief Convert a contiguous f32 buffer to binary16 bits. + * @param src Source buffer with @p n values. + * @param dst Destination buffer with @p n uint16 slots. + * @param n Number of elements. + * + * @details Bit-identical to @ref st_f32_to_f16 applied per element; uses + * NEON `vcvt_f16_f32` with a NaN correction when available. + */ +void st_f32_to_f16_bulk(const float *src, uint16_t *dst, size_t n); + +/** + * @brief Convert a contiguous binary16 buffer to f32. + * @param src Source buffer with @p n uint16 slots. + * @param dst Destination buffer with @p n values. + * @param n Number of elements. + * + * @details Bit-identical to @ref st_f16_to_f32 applied per element; uses + * NEON `vcvt_f32_f16` when available. + */ +void st_f16_to_f32_bulk(const uint16_t *src, float *dst, size_t n); + +#ifdef __cplusplus +} +#endif + +#endif // ST_F16_H diff --git a/app/tensor/include/st_interp.h b/app/tensor/include/st_interp.h new file mode 100644 index 0000000..8b1c2e3 --- /dev/null +++ b/app/tensor/include/st_interp.h @@ -0,0 +1,244 @@ +/** + * @file st_interp.h + * @brief Public API for graph-interpreter tensor kernels. + * + * These kernels mirror the scalar reference loops of the nm_lib graph + * interpreter (broadcast binary ops, LayerNorm, ReduceMax, batched matmul, + * gather, and materializing permute) so the interpreter can reuse them + * instead of its own loops. All functions require contiguous f32 tensors + * (views are rejected, matching the Phase 1 activation kernels); bf16 + * tensors are promoted to f32 for the computation and rounded back where + * the destination is bf16. + * + * Aliasing policy for the write-in variants: an output may alias an input + * only where the individual kernel documents it (element-wise aligned + * operands for @ref st_binary_broadcast_into, x for @ref st_layernorm); + * every other output must not overlap any input buffer. + */ + +/* + * Copyright (c) 2026 @tragisch + * SPDX-License-Identifier: MIT + * + * This file is part of a project licensed under the MIT License. + * See the LICENSE file in the root directory for details. + */ + +#ifndef ST_INTERP_H +#define ST_INTERP_H + +#include +#include + +#include "st.h" + +/** @brief Element-wise binary operations for the broadcast kernels. */ +typedef enum StBinOp { + ST_BIN_ADD = 0, /* out = a + b */ + ST_BIN_SUB = 1, /* out = a - b */ + ST_BIN_MUL = 2, /* out = a * b */ + ST_BIN_DIV = 3, /* out = a / b (IEEE values for division by zero) */ +} StBinOp; + +/** + * @brief Apply a binary op with NumPy-style broadcasting over any axes. + * @param op Operation to apply. + * @param a Left input tensor (contiguous, f32 or bf16). + * @param b Right input tensor (contiguous, f32 or bf16). + * @return New contiguous f32 tensor on success, or NULL on invalid input, + * non-broadcastable shapes, or allocation failure. + * + * @details Fast paths (matching the nm_lib reference) are used when both + * shapes are equal, when one side is a scalar, or when one operand + * is a shape suffix of the output (bias/scale vectors); everything + * else runs a strided walk with vectorized row kernels. Division + * by zero produces the usual IEEE values. The result is always + * f32; call @ref st_to_bf16 to round-trip bf16 inputs. + */ +FloatTensor *st_binary_broadcast(StBinOp op, const FloatTensor *a, + const FloatTensor *b); + +/** + * @brief Apply a binary op with broadcasting into a prepared output tensor. + * @param op Operation to apply. + * @param a Left input tensor (contiguous, f32 or bf16). + * @param b Right input tensor (contiguous, f32 or bf16). + * @param out Output tensor; contiguous with the broadcast shape of @p a and + * @p b, f32 or bf16. May alias @p a or @p b when that operand's + * shape equals the output shape (the row kernels read and write + * the same element-wise position, so in-place accumulation like + * `a += bias` is safe); it must not partially overlap any operand + * with a different shape. + * @retval true Success. + * @retval false Invalid input or shape mismatch; nothing is written. + * + * @details Companion of @ref st_binary_broadcast for callers that reuse a + * pooled output buffer. When @p out is bf16 the f32 result is + * rounded back to bf16 with the st conversion helpers. + */ +bool st_binary_broadcast_into(StBinOp op, const FloatTensor *a, + const FloatTensor *b, FloatTensor *out); + +/** + * @brief Layer-normalize over the last @p norm_rank axes. + * @param x Input tensor (contiguous, f32 or bf16). + * @param gamma Optional per-element scale of the normalized group; NULL or a + * contiguous tensor with `group` elements (f32 or bf16). + * @param beta Optional per-element shift; NULL or like @p gamma. + * @param norm_rank Number of trailing axes to normalize over, `[1, ndim]`. + * @param eps Variance epsilon added before the inverse square root. + * @param out Output tensor, contiguous with the same shape as @p x; f32 or + * bf16. May alias @p x (in-place normalization per group). + * @retval true Success. + * @retval false Invalid input; nothing is written. + * + * @details Matches the nm_lib reference: mean via eight partial sums, then + * the centered values, the variance over the centered values with + * eight partial sums, and `1/sqrtf(var + eps)`, so results are + * bit-identical regardless of the OpenMP thread count. bf16 inputs + * are promoted to f32; a bf16 output is rounded back. + */ +bool st_layernorm(const FloatTensor *x, const FloatTensor *gamma, + const FloatTensor *beta, size_t norm_rank, float eps, + FloatTensor *out); + +/** + * @brief Maximum over the axes selected in @p axes_mask. + * @param x Input tensor (contiguous, f32 or bf16). + * @param axes_mask Bitmask; axis `i` is reduced when bit `i` is set. Bits + * at or above the tensor rank are rejected. + * @param keepdims Keep reduced axes as size-1 axes in the output. + * @return New contiguous f32 tensor on success, or NULL on error. + * + * @details NaN handling deviates deliberately from the nm_lib scalar loop + * (which silently drops NaNs under `x > max`): if any element of a + * reduction group is NaN, the output slot becomes the bit pattern + * of the first NaN in scan order, so errors are not swallowed. + * Finite results and -INFINITY empty regions follow the reference. + */ +FloatTensor *st_reduce_max(const FloatTensor *x, uint32_t axes_mask, + bool keepdims); + +/** + * @brief Maximum over selected axes into a prepared output tensor. + * @param x Input tensor (contiguous, f32 or bf16). + * @param axes_mask Bitmask of axes to reduce; see @ref st_reduce_max. + * @param keepdims Keep reduced axes as size-1 axes in the output. + * @param out Output tensor with the reduced shape, f32 or bf16. Must not + * alias @p x (an empty mask copies; use @ref st_reduce_max instead + * for in-place results). + * @retval true Success. + * @retval false Invalid input or shape mismatch; nothing is written. + */ +bool st_reduce_max_into(const FloatTensor *x, uint32_t axes_mask, + bool keepdims, FloatTensor *out); + +/** + * @brief Batched matrix product with broadcast over the leading axes. + * @param a Left tensor, rank 2 to 4, contiguous, f32 or bf16; the last two + * axes are `[M, K]`. + * @param b Right tensor, rank 2 to 4, contiguous, f32 or bf16; the last two + * axes are `[K, N]`. + * @param c Output tensor with the broadcast batch shape plus `[M, N]`, + * contiguous, f32 or bf16. Must not alias @p a or @p b (the GEMM + * backend reads inputs while accumulating). + * @retval true Success. + * @retval false Invalid input or shape mismatch; nothing is written. + * + * @details Matches nm_lib's run_matmul: when @p b is a single `[K, N]` + * matrix, the batches of @p a stack into the rows of one GEMM; + * otherwise every batch index maps onto @p a and @p b via NumPy + * broadcasting of the leading axes and each batch runs through + * the same GEMM backend as @ref sm_gemm (Accelerate/OpenBLAS, + * MPS through the runtime backend). Small per-batch GEMMs are + * distributed over OpenMP only above a measured threshold so the + * BLAS threads are not oversubscribed. Accumulation order follows + * the backend, so results are compared with a relative tolerance, + * not bit-exactly. + */ +bool st_matmul_batched(const FloatTensor *a, const FloatTensor *b, + FloatTensor *c); + +/** + * @brief Gather slices along one axis, indexed by int64 values. + * @param table Source tensor, contiguous, f32 or bf16. + * @param indices Index array of length @p count; negative indices count + * from the end of @p axis. + * @param count Number of indices; must be greater than zero. + * @param axis Gather axis, below the tensor rank. + * @param out Output tensor, contiguous, same dtype as @p table, shape equal + * to the table shape with @p axis replaced by @p count. Must not + * alias @p table. + * @retval true Success. + * @retval false Invalid input or an index out of range; nothing is written. + * + * @details Indices are validated before any copy, so an out-of-range index + * fails without partial output (unlike the nm_lib reference, which + * copies until it hits the bad index). Slices are copied as + * contiguous blocks with memcpy. Index tensors of higher rank are + * covered by flattening them, matching the reference's + * [outer, count, inner] layout. + */ +bool st_gather(const FloatTensor *table, const int64_t *indices, size_t count, + size_t axis, FloatTensor *out); + +/** + * @brief Gather for raw buffers of any element size. + * @param table Source buffer, row-major, shape @p table_shape. + * @param table_ndim Rank of @p table, `[1, 8]`. + * @param table_shape Shape of @p table, all dimensions non-zero. + * @param indices Index array of length @p count, negative counts from the + * end of @p axis (see @ref st_gather). + * @param count Number of indices, greater than zero. + * @param axis Gather axis, below @p table_ndim. + * @param elem_bytes Element size in bytes (4 for f32/bf16-packed, 8 for + * int64, 1 for bool). + * @param out Output buffer with `outer * count * inner` elements. + * @retval true Success. + * @retval false Invalid input or an index out of range; nothing is written. + * + * @details nm_lib gathers int64 and bool buffers too; the tensor API only + * has float dtypes, so this raw variant carries the element size. + */ +bool st_gather_bytes(const void *table, size_t table_ndim, + const size_t *table_shape, const int64_t *indices, + size_t count, size_t axis, size_t elem_bytes, void *out); + +/** + * @brief Materialize a permuted copy into a prepared output tensor. + * @param src Source tensor, contiguous, f32 or bf16. + * @param perm Permutation where output axis `i` reads input axis + * `perm[i]` (same convention as @ref st_permute and nm_lib's + * run_transpose). + * @param out Output tensor, contiguous, same dtype as @p src, shape equal + * to the permuted source shape. Must not alias @p src (including + * identity permutations, which would copy onto itself). + * @retval true Success. + * @retval false Invalid input; nothing is written. + * + * @details @ref st_permute only creates views; the interpreter writes into + * pooled buffers, so it needs this materializing variant. + */ +bool st_permute_copy_into(const FloatTensor *src, const size_t *perm, + FloatTensor *out); + +/** + * @brief Materializing permute for raw buffers of any element size. + * @param src Source buffer, row-major, shape @p shape. + * @param ndim Rank, `[1, 8]`. + * @param shape Source shape, all dimensions non-zero. + * @param perm Permutation array where output axis `i` reads input axis + * `perm[i]`. + * @param elem_bytes Element size in bytes. + * @param out Output buffer with the permuted shape's element count. + * @retval true Success. + * @retval false Invalid input. + * + * @details Covers nm_lib's run_transpose for int64 and bool buffers, which + * the float-only tensor API cannot express. + */ +bool st_permute_copy_into_bytes(const void *src, size_t ndim, + const size_t *shape, const size_t *perm, + size_t elem_bytes, void *out); + +#endif // ST_INTERP_H diff --git a/app/tensor/src/core/st.c b/app/tensor/src/core/st.c index 6329698..eecd420 100644 --- a/app/tensor/src/core/st.c +++ b/app/tensor/src/core/st.c @@ -56,6 +56,7 @@ #include "st_dtype.h" #include "st_gpu_guard.h" #include "sm.h" +#include "sm_kernels.h" #include "st_bf16_utils.h" #include @@ -965,6 +966,135 @@ bool st_apply_relu_backward(const FloatTensor *activation, FloatTensor *grad) { return true; } +static bool st_kernel_tensor_valid(const FloatTensor *t) { + if (t == NULL || t->values == NULL || t->numel == 0 || t->ndim == 0 || + t->ndim > ST_MAX_DIMS || !st_is_contiguous(t)) { + return false; + } + size_t expected_numel = 0; + return st_numel_from_shape(t->ndim, t->shape, &expected_numel) && + expected_numel == t->numel; +} + +static bool st_kernel_tensors_compatible(const FloatTensor *a, + const FloatTensor *b) { + return st_kernel_tensor_valid(a) && st_kernel_tensor_valid(b) && + a->ndim == b->ndim && a->numel == b->numel && + memcmp(a->shape, b->shape, a->ndim * sizeof(size_t)) == 0; +} + +bool st_apply_sigmoid(FloatTensor *t) { + if (!st_kernel_tensor_valid(t)) { + return false; + } + ST_ASSERT_NOT_PENDING(t); + ST_INPLACE_UNARY_BF16_PROMOTE(t, st_apply_sigmoid); + return sm_kernel_apply_sigmoid(t->values, t->numel); +} + +bool st_apply_tanh(FloatTensor *t) { + if (!st_kernel_tensor_valid(t)) { + return false; + } + ST_ASSERT_NOT_PENDING(t); + ST_INPLACE_UNARY_BF16_PROMOTE(t, st_apply_tanh); + return sm_kernel_apply_tanh(t->values, t->numel); +} + +bool st_apply_gelu_tanh(FloatTensor *t) { + if (!st_kernel_tensor_valid(t)) { + return false; + } + ST_ASSERT_NOT_PENDING(t); + ST_INPLACE_UNARY_BF16_PROMOTE(t, st_apply_gelu_tanh); + return sm_kernel_apply_gelu_tanh(t->values, t->numel); +} + +bool st_apply_gelu_erf(FloatTensor *t) { + if (!st_kernel_tensor_valid(t)) { + return false; + } + ST_ASSERT_NOT_PENDING(t); + ST_INPLACE_UNARY_BF16_PROMOTE(t, st_apply_gelu_erf); + return sm_kernel_apply_gelu_erf(t->values, t->numel); +} + +bool st_apply_softmax(FloatTensor *t) { + if (!st_kernel_tensor_valid(t)) { + return false; + } + ST_ASSERT_NOT_PENDING(t); + ST_INPLACE_UNARY_BF16_PROMOTE(t, st_apply_softmax); + const size_t cols = t->shape[t->ndim - 1]; + return sm_kernel_apply_softmax(t->values, t->numel / cols, cols); +} + +static bool st_apply_sigmoid_backward_promote(FloatTensor *grad, + const FloatTensor *activation) { + return st_apply_sigmoid_backward(activation, grad); +} + +static bool st_apply_tanh_backward_promote(FloatTensor *grad, + const FloatTensor *activation) { + return st_apply_tanh_backward(activation, grad); +} + +static bool st_apply_gelu_tanh_backward_promote(FloatTensor *grad, + const FloatTensor *input) { + return st_apply_gelu_tanh_backward(input, grad); +} + +static bool st_apply_gelu_erf_backward_promote(FloatTensor *grad, + const FloatTensor *input) { + return st_apply_gelu_erf_backward(input, grad); +} + +bool st_apply_sigmoid_backward(const FloatTensor *activation, + FloatTensor *grad) { + if (!st_kernel_tensors_compatible(activation, grad)) { + return false; + } + ST_ASSERT_NOT_PENDING(activation); + ST_ASSERT_NOT_PENDING(grad); + ST_INPLACE_BINARY_BF16_PROMOTE(grad, activation, + st_apply_sigmoid_backward_promote); + return sm_kernel_sigmoid_backward(activation->values, grad->values, + grad->numel); +} + +bool st_apply_tanh_backward(const FloatTensor *activation, FloatTensor *grad) { + if (!st_kernel_tensors_compatible(activation, grad)) { + return false; + } + ST_ASSERT_NOT_PENDING(activation); + ST_ASSERT_NOT_PENDING(grad); + ST_INPLACE_BINARY_BF16_PROMOTE(grad, activation, + st_apply_tanh_backward_promote); + return sm_kernel_tanh_backward(activation->values, grad->values, grad->numel); +} + +bool st_apply_gelu_tanh_backward(const FloatTensor *input, FloatTensor *grad) { + if (!st_kernel_tensors_compatible(input, grad)) { + return false; + } + ST_ASSERT_NOT_PENDING(input); + ST_ASSERT_NOT_PENDING(grad); + ST_INPLACE_BINARY_BF16_PROMOTE(grad, input, + st_apply_gelu_tanh_backward_promote); + return sm_kernel_gelu_tanh_backward(input->values, grad->values, grad->numel); +} + +bool st_apply_gelu_erf_backward(const FloatTensor *input, FloatTensor *grad) { + if (!st_kernel_tensors_compatible(input, grad)) { + return false; + } + ST_ASSERT_NOT_PENDING(input); + ST_ASSERT_NOT_PENDING(grad); + ST_INPLACE_BINARY_BF16_PROMOTE(grad, input, + st_apply_gelu_erf_backward_promote); + return sm_kernel_gelu_erf_backward(input->values, grad->values, grad->numel); +} + /* ---- Reduction: sum over axes ---- */ FloatTensor *st_sum_axes(const FloatTensor *t, const size_t *axes, diff --git a/app/tensor/src/core/st_f16.c b/app/tensor/src/core/st_f16.c new file mode 100644 index 0000000..ff34e47 --- /dev/null +++ b/app/tensor/src/core/st_f16.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2026 @tragisch + * SPDX-License-Identifier: MIT + * + * This file is part of a project licensed under the MIT License. + * See the LICENSE file in the root directory for details. + */ + +/* IEEE binary16 conversion, bit-identical to nm_lib's nm_halfprec.h + * (round to nearest even, subnormals, overflow to inf, quiet NaN). + * + * The scalar paths are the reference algorithm; the bulk paths use NEON + * FCVT where available. For f32 -> f16 the hardware NaN result keeps the + * payload while the reference canonicalizes to sign | 0x7e00, so the bulk + * path blends the reference bits into NaN lanes. */ + +#include "st_f16.h" + +#include + +#if defined(__ARM_NEON) || defined(__aarch64__) +#include +#define ST_F16_HAVE_NEON 1 +#endif + +uint16_t st_f32_to_f16(float value) { + uint32_t bits = 0; + memcpy(&bits, &value, sizeof(bits)); + const uint16_t sign = (uint16_t)((bits >> 16) & 0x8000u); + const int32_t exponent = (int32_t)((bits >> 23) & 0xffu); + uint32_t mantissa = bits & 0x007fffffu; + if (exponent == 0xff) { + return (uint16_t)(sign | 0x7c00u | (mantissa ? 0x0200u : 0u)); + } + const int32_t half_exponent = exponent - 127 + 15; + if (half_exponent >= 0x1f) { + return (uint16_t)(sign | 0x7c00u); + } + if (half_exponent <= 0) { + if (half_exponent < -10) { + return sign; + } + const uint32_t full = mantissa | 0x00800000u; + const uint32_t shift = (uint32_t)(14 - half_exponent); + uint32_t half = full >> shift; + const uint32_t remainder = full & ((1u << shift) - 1u); + const uint32_t halfway = 1u << (shift - 1u); + if (remainder > halfway || (remainder == halfway && (half & 1u))) { + half += 1u; + } + return (uint16_t)(sign | half); + } + uint32_t half = ((uint32_t)half_exponent << 10) | (mantissa >> 13); + const uint32_t remainder = mantissa & 0x1fffu; + if (remainder > 0x1000u || (remainder == 0x1000u && (half & 1u))) { + half += 1u; + } + return (uint16_t)(sign | half); +} + +float st_f16_to_f32(uint16_t bits) { + const uint32_t sign = (uint32_t)(bits & 0x8000u) << 16; + uint32_t exponent = (bits >> 10) & 0x1fu; + uint32_t mantissa = bits & 0x03ffu; + uint32_t out; + if (exponent == 0 && mantissa == 0) { + out = sign; + } else if (exponent == 0) { + int32_t e = 127 - 15 + 1; + while ((mantissa & 0x0400u) == 0) { + mantissa <<= 1; + --e; + } + out = sign | ((uint32_t)e << 23) | ((mantissa & 0x03ffu) << 13); + } else if (exponent == 0x1f) { + out = sign | 0x7f800000u | (mantissa ? 0x00400000u | (mantissa << 13) : 0u); + } else { + out = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13); + } + float value; + memcpy(&value, &out, sizeof(value)); + return value; +} + +void st_f32_to_f16_bulk(const float *src, uint16_t *dst, size_t n) { +#if defined(ST_F16_HAVE_NEON) + size_t i = 0; + for (; i + 4 <= n; i += 4) { + const float32x4_t value = vld1q_f32(src + i); + const uint32x4_t bits = vreinterpretq_u32_f32(value); + /* Hardware conversion; the reference NaN pattern differs (the + * hardware keeps the payload, the reference drops it), so NaN lanes + * are blended with sign | 0x7e00. */ + const float16x4_t converted = vcvt_f16_f32(value); + const uint32x4_t abs_bits = vandq_u32(bits, vdupq_n_u32(0x7fffffffu)); + const uint32x4_t is_nan = + vcgtq_u32(abs_bits, vdupq_n_u32(0x7f800000u)); + const uint32x4_t top = vshrq_n_u32(bits, 16); + const uint16x4_t ref_nan = vorr_u16( + vand_u16(vmovn_u32(top), vdup_n_u16(0x8000u)), vdup_n_u16(0x7e00u)); + const uint16x4_t mask = vreinterpret_u16_u32(vmovn_u32(is_nan)); + const uint16x4_t result = vbsl_u16( + mask, ref_nan, vreinterpret_u16_f16(converted)); + vst1_u16(dst + i, result); + } + for (; i < n; ++i) { + dst[i] = st_f32_to_f16(src[i]); + } +#else + for (size_t i = 0; i < n; ++i) { + dst[i] = st_f32_to_f16(src[i]); + } +#endif +} + +void st_f16_to_f32_bulk(const uint16_t *src, float *dst, size_t n) { +#if defined(ST_F16_HAVE_NEON) + size_t i = 0; + for (; i + 4 <= n; i += 4) { + const float16x4_t value = vld1_u16(src + i); + vst1q_f32(dst + i, vcvt_f32_f16(value)); + } + for (; i < n; ++i) { + dst[i] = st_f16_to_f32(src[i]); + } +#else + for (size_t i = 0; i < n; ++i) { + dst[i] = st_f16_to_f32(src[i]); + } +#endif +} diff --git a/app/tensor/src/ops/st_interp.c b/app/tensor/src/ops/st_interp.c new file mode 100644 index 0000000..b254ea2 --- /dev/null +++ b/app/tensor/src/ops/st_interp.c @@ -0,0 +1,1559 @@ +/* + * Copyright (c) 2026 @tragisch + * SPDX-License-Identifier: MIT + * + * This file is part of a project licensed under the MIT License. + * See the LICENSE file in the root directory for details. + */ + +/* Graph-interpreter tensor kernels: broadcast binary operations. + * + * The reference semantics are the scalar loops of nm_lib's graph + * interpreter (run_binary in nm_graph_interp.c): NumPy broadcasting over + * any axes, fast paths for equal shapes, scalars, and shape suffixes, and + * a strided walk with vectorized row kernels for everything else. + */ + +#include "st_interp.h" + +#include +#include +#include +#include + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpedantic" +#endif +#include +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#if defined(USE_ACCELERATE) +#include +#endif + +#include "sm.h" +#include "st_bf16_utils.h" +#include "st_gpu_guard.h" + +#if defined(__ARM_NEON) || defined(__aarch64__) +#include +#define ST_INTERP_HAVE_NEON 1 +#endif + +/* OpenMP is only used above this many output elements. bench_st_binary's + * thread sweep on this 4-performance-core machine shows the fork/join cost + * regressing small and mid inputs (0.15x-0.9x below ~1M elements) and only + * marginal gains above ~2M, so the threshold keeps realistic transformer + * tensor sizes on the sequential path (same rationale as sv_optim). */ +#define ST_BIN_OMP_THRESHOLD 2097152u + +/* ---- Walk machinery (adapted from nm_lib's interpreter) ---------------- * + * + * Walks a row-major index space of rank ST_WALK_DIMS and keeps up to three + * operand offsets in step, each with its own strides. walk_finish drops + * dimensions of size one and merges dimensions every operand reads + * contiguously, so the kernels loop over rows of inner elements. */ + +enum { ST_WALK_DIMS = ST_MAX_DIMS }; +enum { ST_WALK_OPERANDS = 3 }; + +typedef struct { + size_t d[ST_WALK_DIMS]; + size_t c[ST_WALK_DIMS]; + ptrdiff_t stride[ST_WALK_OPERANDS][ST_WALK_DIMS]; + ptrdiff_t offset[ST_WALK_OPERANDS]; + size_t operands; + size_t rank; +} StWalk; + +/* Left-pads `ndim/shape` with ones to rank ST_WALK_DIMS. */ +static void st_walk_pad(size_t ndim, const size_t *shape, size_t out[ST_WALK_DIMS]) { + const size_t offset = ST_WALK_DIMS - ndim; + for (size_t i = 0; i < ST_WALK_DIMS; ++i) { + out[i] = i < offset ? 1 : shape[i - offset]; + } +} + +/* Element strides of `ndim/shape` padded to rank ST_WALK_DIMS, zero along + * the dimensions that broadcast to `d`. */ +static void st_walk_broadcast_strides(size_t ndim, const size_t *shape, + const size_t d[ST_WALK_DIMS], + ptrdiff_t out[ST_WALK_DIMS]) { + size_t padded[ST_WALK_DIMS]; + st_walk_pad(ndim, shape, padded); + ptrdiff_t running = 1; + for (size_t i = ST_WALK_DIMS; i-- > 0;) { + out[i] = padded[i] == 1 && d[i] != 1 ? 0 : running; + running *= (ptrdiff_t)padded[i]; + } +} + +static void st_walk_init(StWalk *w, size_t ndim, const size_t *shape, + size_t operands) { + memset(w, 0, sizeof(*w)); + st_walk_pad(ndim, shape, w->d); + w->operands = operands; +} + +static void st_walk_finish(StWalk *w) { + size_t rank = 0; + for (size_t i = 0; i < ST_WALK_DIMS; ++i) { + if (w->d[i] == 1) { + continue; + } + bool merge = rank > 0; + for (size_t j = 0; merge && j < w->operands; ++j) { + merge = w->stride[j][rank - 1] == w->stride[j][i] * (ptrdiff_t)w->d[i]; + } + if (merge) { + w->d[rank - 1] *= w->d[i]; + for (size_t j = 0; j < w->operands; ++j) { + w->stride[j][rank - 1] = w->stride[j][i]; + } + continue; + } + w->d[rank] = w->d[i]; + for (size_t j = 0; j < w->operands; ++j) { + w->stride[j][rank] = w->stride[j][i]; + } + rank++; + } + if (rank == 0) { + w->d[0] = 1; + for (size_t j = 0; j < w->operands; ++j) { + w->stride[j][0] = 0; + } + rank = 1; + } + w->rank = rank; +} + +static size_t st_walk_inner(const StWalk *w) { return w->d[w->rank - 1]; } + +static size_t st_walk_rows(const StWalk *w) { + size_t rows = 1; + for (size_t i = 0; i + 1 < w->rank; ++i) { + rows *= w->d[i]; + } + return rows; +} + +/* Operand offsets at the start of walk row `row` (fixed radix order, the + * last axis is the fastest). Used once per parallel block; within a block + * the walk advances incrementally. */ +static void st_walk_row_offsets(const StWalk *w, size_t row, + ptrdiff_t *offsets) { + size_t rest = row; + for (size_t j = 0; j < w->operands; ++j) { + offsets[j] = 0; + } + for (size_t k = w->rank - 1; k-- > 0;) { + const size_t digit = rest % w->d[k]; + rest /= w->d[k]; + for (size_t j = 0; j < w->operands; ++j) { + offsets[j] += (ptrdiff_t)digit * w->stride[j][k]; + } + } +} + +/* Row coordinates of `row`, so walk_next_row can advance from it. */ +static void st_walk_row_digits(const StWalk *w, size_t row, size_t *digits) { + size_t rest = row; + for (size_t k = 0; k < ST_WALK_DIMS; ++k) { + digits[k] = 0; + } + for (size_t k = w->rank - 1; k-- > 0;) { + digits[k] = rest % w->d[k]; + rest /= w->d[k]; + } +} + +/* Advance to the next walk row, odometer style (last axis fastest). */ +static void st_walk_next_row(StWalk *w) { + for (size_t k = w->rank - 1; k-- > 0;) { + for (size_t j = 0; j < w->operands; ++j) { + w->offset[j] += w->stride[j][k]; + } + if (++w->c[k] < w->d[k]) { + return; + } + for (size_t j = 0; j < w->operands; ++j) { + w->offset[j] -= w->stride[j][k] * (ptrdiff_t)w->d[k]; + } + w->c[k] = 0; + } +} + +/* ---- Shared input checks ---------------------------------------------- */ + +#ifndef ST_LOG_ERROR +#define ST_LOG_ERROR(fmt, ...) \ + fprintf(stderr, "[st][error] " fmt "\n", ##__VA_ARGS__) +#endif + +static bool st_interp_tensor_valid(const FloatTensor *t) { + if (t == NULL || t->values == NULL || t->numel == 0 || t->ndim == 0 || + t->ndim > ST_MAX_DIMS || !st_is_contiguous(t)) { + return false; + } + size_t expected_numel = 0; + return st_numel_from_shape(t->ndim, t->shape, &expected_numel) && + expected_numel == t->numel; +} + +/* ---- Broadcast helpers ------------------------------------------------- */ + +/* NumPy-style broadcast of two shapes; right-aligned, size 1 expands. */ +static bool st_interp_broadcast_shape(size_t a_ndim, const size_t *a_dims, + size_t b_ndim, const size_t *b_dims, + size_t *out_ndim, size_t *out_dims) { + const size_t ndim = a_ndim > b_ndim ? a_ndim : b_ndim; + for (size_t i = 0; i < ndim; ++i) { + const size_t da = i < ndim - a_ndim ? 1 : a_dims[i - (ndim - a_ndim)]; + const size_t db = i < ndim - b_ndim ? 1 : b_dims[i - (ndim - b_ndim)]; + if (da != db && da != 1 && db != 1) { + return false; + } + out_dims[i] = da == 1 ? db : da; + } + *out_ndim = ndim; + return true; +} + +static size_t st_interp_shape_numel(size_t ndim, const size_t *dims) { + size_t numel = 1; + for (size_t i = 0; i < ndim; ++i) { + numel *= dims[i]; + } + return numel; +} + +/* ---- Binary row kernels ------------------------------------------------- * + * + * The operation is hoisted out of the loop so the inner loop vectorizes. + * `sa`/`sb` select contiguous (a[i]) or broadcast (a[0]) operand access. + * OpenMP is applied by the callers over rows/blocks, never nested. */ + +static void st_bin_row(StBinOp op, const float *a, const float *b, float *out, + size_t n) { +#if defined(USE_ACCELERATE) + /* Division stays on the C path: vDSP_vdiv is not bit-identical to IEEE + * scalar division (1 ulp differences), and the binary kernels promise + * bit-exact results. Add/sub/mul are single IEEE ops and exact in vDSP. */ + if (n >= 32 && op != ST_BIN_DIV) { + switch (op) { + case ST_BIN_ADD: + vDSP_vadd(a, 1, b, 1, out, 1, (vDSP_Length)n); + return; + case ST_BIN_SUB: + vDSP_vsub(a, 1, b, 1, out, 1, (vDSP_Length)n); + return; + case ST_BIN_MUL: + vDSP_vmul(a, 1, b, 1, out, 1, (vDSP_Length)n); + return; + default: + break; + } + } +#endif + switch (op) { + case ST_BIN_ADD: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] + b[i]; + } + return; + case ST_BIN_SUB: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] - b[i]; + } + return; + case ST_BIN_MUL: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] * b[i]; + } + return; + case ST_BIN_DIV: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] / b[i]; + } + return; + } +} + +/* Division by a loop-invariant scalar, IEEE-exact per element. + * + * -ffast-math lets the compiler rewrite `a[i] / c` into `a[i] * (1/c)` with a + * once-rounded reciprocal (double rounding, 1 ulp off), even through NEON + * splat intrinsics. clang: `#pragma float_control(precise, on)` disables the + * transform for this function while keeping -O3; gcc: the no-fast-math + * optimize attribute, same pattern as the NaN check in sm_kernels.c. */ +#if defined(__clang__) +#pragma float_control(push) +#pragma float_control(precise, on) +#elif defined(__GNUC__) +__attribute__((optimize("no-fast-math"))) +#endif +static void st_div_scalar_b_exact(const float *a, const float *b, float *out, + size_t n) { + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] / b[0]; + } +} +#if defined(__clang__) +#pragma float_control(pop) +#endif + +static void st_bin_row_scalar_b(StBinOp op, const float *a, const float *b, + float *out, size_t n) { + switch (op) { + case ST_BIN_ADD: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] + b[0]; + } + return; + case ST_BIN_SUB: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] - b[0]; + } + return; + case ST_BIN_MUL: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[i] * b[0]; + } + return; + case ST_BIN_DIV: + /* Exact division: see st_div_scalar_b_exact. */ + st_div_scalar_b_exact(a, b, out, n); + return; + } +} + +static void st_bin_row_scalar_a(StBinOp op, const float *a, const float *b, + float *out, size_t n) { + switch (op) { + case ST_BIN_ADD: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[0] + b[i]; + } + return; + case ST_BIN_SUB: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[0] - b[i]; + } + return; + case ST_BIN_MUL: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[0] * b[i]; + } + return; + case ST_BIN_DIV: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[0] / b[i]; + } + return; + } +} + +static void st_bin_row_strided(StBinOp op, const float *a, ptrdiff_t ta, + const float *b, ptrdiff_t tb, float *out, + size_t n) { + switch (op) { + case ST_BIN_ADD: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[(ptrdiff_t)i * ta] + b[(ptrdiff_t)i * tb]; + } + return; + case ST_BIN_SUB: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[(ptrdiff_t)i * ta] - b[(ptrdiff_t)i * tb]; + } + return; + case ST_BIN_MUL: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[(ptrdiff_t)i * ta] * b[(ptrdiff_t)i * tb]; + } + return; + case ST_BIN_DIV: +#pragma omp simd + for (size_t i = 0; i < n; ++i) { + out[i] = a[(ptrdiff_t)i * ta] / b[(ptrdiff_t)i * tb]; + } + return; + } +} + +/* ---- Binary core on raw f32 buffers ------------------------------------ */ + +static bool st_binary_f32(StBinOp op, const float *a, size_t a_ndim, + const size_t *a_dims, const float *b, size_t b_ndim, + const size_t *b_dims, float *out, size_t o_ndim, + const size_t *o_dims) { + const size_t n = st_interp_shape_numel(o_ndim, o_dims); + const size_t na = st_interp_shape_numel(a_ndim, a_dims); + const size_t nb = st_interp_shape_numel(b_ndim, b_dims); + + const bool use_omp = n > ST_BIN_OMP_THRESHOLD && omp_get_max_threads() > 1; + + if (na == n && nb == n) { + if (use_omp) { +#pragma omp parallel + { + const size_t threads = (size_t)omp_get_num_threads(); + const size_t thread = (size_t)omp_get_thread_num(); + const size_t first = n * thread / threads; + const size_t last = n * (thread + 1) / threads; + if (last > first) { + st_bin_row(op, a + first, b + first, out + first, last - first); + } + } + } else { + st_bin_row(op, a, b, out, n); + } + return true; + } + if (na == n && nb == 1) { + if (use_omp) { +#pragma omp parallel + { + const size_t threads = (size_t)omp_get_num_threads(); + const size_t thread = (size_t)omp_get_thread_num(); + const size_t first = n * thread / threads; + const size_t last = n * (thread + 1) / threads; + if (last > first) { + st_bin_row_scalar_b(op, a + first, b, out + first, last - first); + } + } + } else { + st_bin_row_scalar_b(op, a, b, out, n); + } + return true; + } + if (na == n && nb > 0 && n % nb == 0 && b_ndim <= o_ndim) { + /* b broadcasts along leading dimensions only (bias, scale): b's shape + * is a suffix of the output shape. */ + bool suffix = true; + for (size_t i = 0; i < b_ndim; ++i) { + suffix = suffix && b_dims[i] == o_dims[o_ndim - b_ndim + i]; + } + if (suffix) { + const size_t rows = n / nb; + if (use_omp) { +#pragma omp parallel for schedule(static) + for (size_t r = 0; r < rows; ++r) { + st_bin_row(op, a + r * nb, b, out + r * nb, nb); + } + } else { + for (size_t r = 0; r < rows; ++r) { + st_bin_row(op, a + r * nb, b, out + r * nb, nb); + } + } + return true; + } + } + + /* General case: strided walk with row kernels. */ + StWalk w; + st_walk_init(&w, o_ndim, o_dims, 2); + st_walk_broadcast_strides(a_ndim, a_dims, w.d, w.stride[0]); + st_walk_broadcast_strides(b_ndim, b_dims, w.d, w.stride[1]); + st_walk_finish(&w); + const size_t inner = st_walk_inner(&w); + const size_t rows = st_walk_rows(&w); + const ptrdiff_t ta = w.stride[0][w.rank - 1]; + const ptrdiff_t tb = w.stride[1][w.rank - 1]; + if (use_omp && rows > 1) { + /* Parallel blocks; each block walks incrementally from its first row. */ + const size_t row_block = rows / 8 + 1; +#pragma omp parallel for schedule(static) + for (size_t block = 0; block < rows; block += row_block) { + StWalk local = w; + st_walk_row_digits(&w, block, local.c); + st_walk_row_offsets(&w, block, local.offset); + const size_t block_end = + block + row_block < rows ? block + row_block : rows; + float *po = out + block * inner; + for (size_t r = block; r < block_end; + ++r, st_walk_next_row(&local), po += inner) { + const float *pa = a + local.offset[0]; + const float *pb = b + local.offset[1]; + if (ta == 1 && tb == 1) { + st_bin_row(op, pa, pb, po, inner); + } else if (ta == 1 && tb == 0) { + st_bin_row_scalar_b(op, pa, pb, po, inner); + } else if (ta == 0 && tb == 1) { + st_bin_row_scalar_a(op, pa, pb, po, inner); + } else { + st_bin_row_strided(op, pa, ta, pb, tb, po, inner); + } + } + } + return true; + } + for (size_t r = 0; r < rows; ++r, st_walk_next_row(&w), out += inner) { + const float *pa = a + w.offset[0]; + const float *pb = b + w.offset[1]; + if (ta == 1 && tb == 1) { + st_bin_row(op, pa, pb, out, inner); + } else if (ta == 1 && tb == 0) { + st_bin_row_scalar_b(op, pa, pb, out, inner); + } else if (ta == 0 && tb == 1) { + st_bin_row_scalar_a(op, pa, pb, out, inner); + } else { + st_bin_row_strided(op, pa, ta, pb, tb, out, inner); + } + } + return true; +} + +/* ---- Public API -------------------------------------------------------- */ + +bool st_binary_broadcast_into(StBinOp op, const FloatTensor *a, + const FloatTensor *b, FloatTensor *out) { + if (op != ST_BIN_ADD && op != ST_BIN_SUB && op != ST_BIN_MUL && + op != ST_BIN_DIV) { + ST_LOG_ERROR("st_binary_broadcast_into: unknown op %d.", (int)op); + return false; + } + if (!st_interp_tensor_valid(a) || !st_interp_tensor_valid(b) || + !st_interp_tensor_valid(out)) { + ST_LOG_ERROR("st_binary_broadcast_into: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(a); + ST_ASSERT_NOT_PENDING(b); + ST_ASSERT_NOT_PENDING(out); + + size_t broadcast_ndim = 0; + size_t broadcast_dims[ST_MAX_DIMS]; + if (!st_interp_broadcast_shape(a->ndim, a->shape, b->ndim, b->shape, + &broadcast_ndim, broadcast_dims)) { + ST_LOG_ERROR("st_binary_broadcast_into: shapes do not broadcast."); + return false; + } + if (broadcast_ndim != out->ndim || + memcmp(broadcast_dims, out->shape, out->ndim * sizeof(size_t)) != 0) { + ST_LOG_ERROR("st_binary_broadcast_into: output shape mismatch."); + return false; + } + + /* bf16 operands promote to f32; a bf16 output is rounded back at the end. */ + FloatTensor *a_f32 = a->dtype == ST_DTYPE_BF16 ? st_to_f32(a) : NULL; + FloatTensor *b_f32 = b->dtype == ST_DTYPE_BF16 ? st_to_f32(b) : NULL; + if ((a->dtype == ST_DTYPE_BF16 && a_f32 == NULL) || + (b->dtype == ST_DTYPE_BF16 && b_f32 == NULL)) { + st_destroy(a_f32); + st_destroy(b_f32); + return false; + } + const float *a_values = a_f32 ? a_f32->values : a->values; + const float *b_values = b_f32 ? b_f32->values : b->values; + + FloatTensor *out_f32 = NULL; + float *out_values = out->values; + if (out->dtype == ST_DTYPE_BF16) { + out_f32 = st_create(out->ndim, out->shape); + if (out_f32 == NULL) { + st_destroy(a_f32); + st_destroy(b_f32); + return false; + } + out_values = out_f32->values; + } + + const bool ok = st_binary_f32(op, a_values, a->ndim, a->shape, b_values, + b->ndim, b->shape, out_values, out->ndim, + out->shape); + if (ok && out_f32 != NULL) { + st_f32_to_bf16_bulk(out_f32->values, (uint16_t *)out->values, + out->numel); + } + st_destroy(out_f32); + st_destroy(a_f32); + st_destroy(b_f32); + return ok; +} + +FloatTensor *st_binary_broadcast(StBinOp op, const FloatTensor *a, + const FloatTensor *b) { + if (op != ST_BIN_ADD && op != ST_BIN_SUB && op != ST_BIN_MUL && + op != ST_BIN_DIV) { + return NULL; + } + if (!st_interp_tensor_valid(a) || !st_interp_tensor_valid(b)) { + ST_LOG_ERROR("st_binary_broadcast: invalid tensor input."); + return NULL; + } + ST_ASSERT_NOT_PENDING(a); + ST_ASSERT_NOT_PENDING(b); + + size_t out_ndim = 0; + size_t out_dims[ST_MAX_DIMS]; + if (!st_interp_broadcast_shape(a->ndim, a->shape, b->ndim, b->shape, + &out_ndim, out_dims)) { + ST_LOG_ERROR("st_binary_broadcast: shapes do not broadcast."); + return NULL; + } + FloatTensor *out = st_create(out_ndim, out_dims); + if (out == NULL) { + return NULL; + } + if (!st_binary_broadcast_into(op, a, b, out)) { + st_destroy(out); + return NULL; + } + return out; +} + +/* ---- LayerNorm ---------------------------------------------------------- */ + +/* OpenMP over groups only above this many input elements; see + * bench_st_norm_reduce for the measured threshold. */ +#define ST_LAYERNORM_OMP_THRESHOLD 262144u + +/* Bit-level finiteness check (-ffast-math breaks isnan/isfinite), same + * pattern as sv_optim.c. */ +static bool st_interp_float_finite(float value) { + uint32_t bits = 0; + memcpy(&bits, &value, sizeof(bits)); + return (bits & 0x7f800000u) != 0x7f800000u; +} + +/* Sum of `n` floats with eight partial sums, so the loop vectorizes and the + * result is independent of the OpenMP thread count (nm_lib's sum8). */ +static float st_interp_sum8(const float *x, size_t n) { + float lanes[8] = {0.0f}; + size_t j = 0; + for (; j + 8 <= n; j += 8) { + for (size_t l = 0; l < 8; ++l) { + lanes[l] += x[j + l]; + } + } + float sum = 0.0f; + for (; j < n; ++j) { + sum += x[j]; + } + for (size_t l = 0; l < 8; ++l) { + sum += lanes[l]; + } + return sum; +} + +/* One normalization group, verbatim run_layernorm from nm_lib: mean via + * sum8, centered values into y, variance via eight partial sums, then + * scale and bias. Safe in-place when in == out. The test compares against + * this algorithm with a tolerance relative to the intermediate magnitude + * (see test_st_interp.c): the library's -ffast-math reassociation rounds + * each operation differently, and result-relative tolerances blow up on + * outputs where the scaled value cancels against beta. */ +static void st_layernorm_group(const float *in, const float *scale, + const float *bias, size_t group, float eps, + float *out) { + const float mean = st_interp_sum8(in, group) / (float)group; + for (size_t j = 0; j < group; ++j) { + out[j] = in[j] - mean; + } + float lanes[8] = {0.0f}; + size_t j = 0; + for (; j + 8 <= group; j += 8) { + for (size_t l = 0; l < 8; ++l) { + lanes[l] += out[j + l] * out[j + l]; + } + } + float variance = 0.0f; + for (; j < group; ++j) { + variance += out[j] * out[j]; + } + for (size_t l = 0; l < 8; ++l) { + variance += lanes[l]; + } + variance /= (float)group; + const float inverse = 1.0f / sqrtf(variance + eps); + if (scale != NULL && bias != NULL) { +#pragma omp simd + for (size_t i = 0; i < group; ++i) { + out[i] = out[i] * inverse * scale[i] + bias[i]; + } + } else { + for (size_t i = 0; i < group; ++i) { + float value = out[i] * inverse; + if (scale != NULL) { + value *= scale[i]; + } + if (bias != NULL) { + value += bias[i]; + } + out[i] = value; + } + } +} + +static bool st_layernorm_f32(const float *in, const float *scale, + const float *bias, size_t group, size_t groups, + float eps, float *out) { + const size_t total = groups * group; + if (total > ST_LAYERNORM_OMP_THRESHOLD && omp_get_max_threads() > 1) { +#pragma omp parallel for schedule(static) + for (size_t g = 0; g < groups; ++g) { + st_layernorm_group(in + g * group, scale, bias, group, eps, + out + g * group); + } + } else { + for (size_t g = 0; g < groups; ++g) { + st_layernorm_group(in + g * group, scale, bias, group, eps, + out + g * group); + } + } + return true; +} + +bool st_layernorm(const FloatTensor *x, const FloatTensor *gamma, + const FloatTensor *beta, size_t norm_rank, float eps, + FloatTensor *out) { + if (!st_interp_tensor_valid(x) || !st_interp_tensor_valid(out)) { + ST_LOG_ERROR("st_layernorm: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(x); + ST_ASSERT_NOT_PENDING(out); + if (out->dtype != ST_DTYPE_F32 && out->dtype != ST_DTYPE_BF16) { + return false; + } + if (out->ndim != x->ndim || + memcmp(out->shape, x->shape, x->ndim * sizeof(size_t)) != 0) { + ST_LOG_ERROR("st_layernorm: output shape mismatch."); + return false; + } + if (norm_rank == 0 || norm_rank > x->ndim) { + ST_LOG_ERROR("st_layernorm: norm_rank out of range."); + return false; + } + if (!st_interp_float_finite(eps) || eps < 0.0f) { + ST_LOG_ERROR("st_layernorm: eps must be finite and non-negative."); + return false; + } + + size_t group = 1; + for (size_t i = x->ndim - norm_rank; i < x->ndim; ++i) { + group *= x->shape[i]; + } + const size_t groups = x->numel / group; + + /* gamma/beta are optional; when present they must cover one group. */ + const FloatTensor *scale = gamma; + const FloatTensor *shift = beta; + if (scale != NULL && !st_interp_tensor_valid(scale)) { + ST_LOG_ERROR("st_layernorm: invalid gamma tensor."); + return false; + } + if (shift != NULL && !st_interp_tensor_valid(shift)) { + ST_LOG_ERROR("st_layernorm: invalid beta tensor."); + return false; + } + if (scale != NULL && scale->numel != group) { + ST_LOG_ERROR("st_layernorm: gamma must have one element per group."); + return false; + } + if (shift != NULL && shift->numel != group) { + ST_LOG_ERROR("st_layernorm: beta must have one element per group."); + return false; + } + if (scale != NULL) { + ST_ASSERT_NOT_PENDING(scale); + } + if (shift != NULL) { + ST_ASSERT_NOT_PENDING(shift); + } + + /* bf16 promotes to f32; a bf16 output is rounded back at the end. */ + FloatTensor *x_f32 = x->dtype == ST_DTYPE_BF16 ? st_to_f32(x) : NULL; + FloatTensor *gamma_f32 = + scale != NULL && scale->dtype == ST_DTYPE_BF16 ? st_to_f32(scale) : NULL; + FloatTensor *beta_f32 = + shift != NULL && shift->dtype == ST_DTYPE_BF16 ? st_to_f32(shift) : NULL; + if ((x->dtype == ST_DTYPE_BF16 && x_f32 == NULL) || + (scale != NULL && scale->dtype == ST_DTYPE_BF16 && gamma_f32 == NULL) || + (shift != NULL && shift->dtype == ST_DTYPE_BF16 && beta_f32 == NULL)) { + st_destroy(beta_f32); + st_destroy(gamma_f32); + st_destroy(x_f32); + return false; + } + const float *in_values = x_f32 ? x_f32->values : x->values; + const float *scale_values = + scale == NULL ? NULL : (gamma_f32 ? gamma_f32->values : scale->values); + const float *shift_values = + shift == NULL ? NULL : (beta_f32 ? beta_f32->values : shift->values); + + FloatTensor *out_f32 = NULL; + float *out_values = out->values; + if (out->dtype == ST_DTYPE_BF16) { + out_f32 = st_create(out->ndim, out->shape); + if (out_f32 == NULL) { + st_destroy(beta_f32); + st_destroy(gamma_f32); + st_destroy(x_f32); + return false; + } + out_values = out_f32->values; + } + + const bool ok = st_layernorm_f32(in_values, scale_values, shift_values, + group, groups, eps, out_values); + if (ok && out_f32 != NULL) { + st_f32_to_bf16_bulk(out_f32->values, (uint16_t *)out->values, + out->numel); + } + st_destroy(out_f32); + st_destroy(beta_f32); + st_destroy(gamma_f32); + st_destroy(x_f32); + return ok; +} + +/* ---- Reduce max ---------------------------------------------------------- */ + +/* Parallel over output slots only above this many input elements. + * bench_st_norm_reduce measured no gain at 131k elements on this + * 4-performance-core machine (fork/join cost of a one-op-per-element fold), + * so realistic sizes stay sequential, like the Phase 1 optimizer kernels. */ +#define ST_REDUCE_OMP_THRESHOLD 1048576u + +/* Horizontal max over the eight fold lanes. noinline forces the vectorizer + * to load all eight lanes from memory: when some lanes stay in registers, + * clang pads the horizontal fmaxnm with -FLT_MAX (under -ffinite-math-only + * it assumes nothing can be smaller), which would beat real -INFINITY + * inputs. */ +__attribute__((noinline)) static float st_max8_lanes(const float *lanes) { + float best = lanes[0]; + for (size_t l = 1; l < 8; ++l) { + best = best > lanes[l] ? best : lanes[l]; + } + return best; +} + +/* Fold one contiguous run: eight independent lanes track the maximum (the + * OpenMP max reduction is not usable here because its private lanes start + * at -FLT_MAX, which would swallow all -INFINITY inputs), and an integer + * side channel records whether NaN bits appeared; a NaN rescan then + * returns the first NaN in scan order. Bit checks on the integer pattern + * stay meaningful under -ffast-math (no float comparisons on NaN). */ +static float st_reduce_max_run(const float *x, size_t n) { + float lanes[8]; + for (size_t l = 0; l < 8; ++l) { + lanes[l] = -INFINITY; + } + uint32_t any_nan = 0; + size_t i = 0; + for (; i + 8 <= n; i += 8) { + for (size_t l = 0; l < 8; ++l) { + uint32_t bits = 0; + memcpy(&bits, &x[i + l], sizeof(bits)); + any_nan |= (((bits & 0x7f800000u) == 0x7f800000u) & + ((bits & 0x007fffffu) != 0u)) + ? 1u + : 0u; + lanes[l] = lanes[l] > x[i + l] ? lanes[l] : x[i + l]; + } + } + for (; i < n; ++i) { + uint32_t bits = 0; + memcpy(&bits, &x[i], sizeof(bits)); + any_nan |= (((bits & 0x7f800000u) == 0x7f800000u) & + ((bits & 0x007fffffu) != 0u)) + ? 1u + : 0u; + lanes[0] = lanes[0] > x[i] ? lanes[0] : x[i]; + } + const float best = st_max8_lanes(lanes); + if (any_nan != 0u) { + for (size_t j = 0; j < n; ++j) { + uint32_t bits = 0; + memcpy(&bits, &x[j], sizeof(bits)); + if (((bits & 0x7f800000u) == 0x7f800000u) && + ((bits & 0x007fffffu) != 0u)) { + return x[j]; + } + } + } + return best; +} + +/* Vectorizable NaN detection over a contiguous buffer: the fold kernels + * then only need the careful path when a NaN is actually present. */ +static bool st_interp_any_nan(const float *in, size_t n) { + uint32_t any_nan = 0; +#pragma omp simd reduction(| : any_nan) + for (size_t i = 0; i < n; ++i) { + uint32_t bits = 0; + memcpy(&bits, &in[i], sizeof(bits)); + any_nan |= (((bits & 0x7f800000u) == 0x7f800000u) & + ((bits & 0x007fffffu) != 0u)) + ? 1u + : 0u; + } + return any_nan != 0u; +} + +/* One careful walk row for the NaN path: the first NaN per slot in scan + * order sticks, later elements leave it in place. */ +static void st_reduce_max_walk_row_careful(const float *x, float *y, + ptrdiff_t to, size_t inner) { + if (to == 0) { + for (size_t i = 0; i < inner; ++i) { + uint32_t bits = 0; + memcpy(&bits, &x[i], sizeof(bits)); + if (((bits & 0x7f800000u) == 0x7f800000u) && + ((bits & 0x007fffffu) != 0u)) { + *y = x[i]; + return; + } + if (x[i] > *y) { + *y = x[i]; + } + } + return; + } + for (size_t i = 0; i < inner; ++i) { + float *slot = y + (ptrdiff_t)i * to; + uint32_t slot_bits = 0; + memcpy(&slot_bits, slot, sizeof(slot_bits)); + if (((slot_bits & 0x7f800000u) == 0x7f800000u) && + ((slot_bits & 0x007fffffu) != 0u)) { + continue; /* the first NaN already won */ + } + uint32_t bits = 0; + memcpy(&bits, &x[i], sizeof(bits)); + if (((bits & 0x7f800000u) == 0x7f800000u) && + ((bits & 0x007fffffu) != 0u)) { + *slot = x[i]; + } else if (x[i] > *slot) { + *slot = x[i]; + } + } +} + +/* Sequential nm_lib-style walk for masks that reduce non-trailing axes: the + * input is visited in memory order and folds into output slots that stand + * still along reduced axes. Without NaNs this is the plain reference fold; + * a single vectorized pre-scan decides, and the careful walk runs only + * when a NaN is present. */ +static void st_reduce_max_walk(const float *in, size_t ndim, + const size_t *dims, uint32_t mask, + size_t in_numel, float *out, + size_t out_numel) { + size_t kept_dims[ST_MAX_DIMS]; + for (size_t a = 0; a < ndim; ++a) { + kept_dims[a] = ((mask >> a) & 1u) != 0u ? 1 : dims[a]; + } + StWalk w; + st_walk_init(&w, ndim, dims, 2); + st_walk_broadcast_strides(ndim, kept_dims, w.d, w.stride[0]); + st_walk_broadcast_strides(ndim, dims, w.d, w.stride[1]); + st_walk_finish(&w); + const size_t inner = st_walk_inner(&w); + const size_t rows = st_walk_rows(&w); + const ptrdiff_t to = w.stride[0][w.rank - 1]; + + if (st_interp_any_nan(in, in_numel)) { + for (size_t i = 0; i < out_numel; ++i) { + out[i] = -INFINITY; + } + for (size_t r = 0; r < rows; ++r, st_walk_next_row(&w)) { + st_reduce_max_walk_row_careful(in + w.offset[1], out + w.offset[0], to, + inner); + } + return; + } + + for (size_t i = 0; i < out_numel; ++i) { + out[i] = -INFINITY; + } + for (size_t r = 0; r < rows; ++r, st_walk_next_row(&w)) { + const float *x = in + w.offset[1]; + float *y = out + w.offset[0]; + if (to == 0) { + float max = *y; + for (size_t i = 0; i < inner; ++i) { + max = x[i] > max ? x[i] : max; + } + *y = max; + } else { + for (size_t i = 0; i < inner; ++i) { + float *slot = y + (ptrdiff_t)i * to; + *slot = x[i] > *slot ? x[i] : *slot; + } + } + } +} + +static void st_reduce_max_f32(const float *in, size_t ndim, + const size_t *dims, uint32_t mask, float *out, + size_t out_numel) { + if (mask == 0u) { + memcpy(out, in, out_numel * sizeof(float)); + return; + } + + size_t reduced = 0; + size_t in_numel = 1; + for (size_t a = 0; a < ndim; ++a) { + reduced += (mask >> a) & 1u; + in_numel *= dims[a]; + } + const uint32_t suffix_mask = + (uint32_t)(((1u << reduced) - 1u) << (ndim - reduced)); + if (mask != suffix_mask) { + /* Non-trailing reduced axes: strided access pattern, sequential walk. */ + st_reduce_max_walk(in, ndim, dims, mask, in_numel, out, out_numel); + return; + } + + /* Trailing reduced axes: every output slot is one contiguous run, and the + * slot base is simply the slot index times the run length. */ + size_t chunk = 1; + for (size_t a = ndim - reduced; a < ndim; ++a) { + chunk *= dims[a]; + } + const size_t slots = out_numel; + const bool use_omp = slots > 1 && slots * chunk > ST_REDUCE_OMP_THRESHOLD && + omp_get_max_threads() > 1; + if (use_omp) { +#pragma omp parallel for schedule(static) + for (size_t slot = 0; slot < slots; ++slot) { + out[slot] = st_reduce_max_run(in + slot * chunk, chunk); + } + return; + } + for (size_t slot = 0; slot < slots; ++slot) { + out[slot] = st_reduce_max_run(in + slot * chunk, chunk); + } +} + +bool st_reduce_max_into(const FloatTensor *x, uint32_t axes_mask, + bool keepdims, FloatTensor *out) { + if (!st_interp_tensor_valid(x) || !st_interp_tensor_valid(out)) { + ST_LOG_ERROR("st_reduce_max_into: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(x); + ST_ASSERT_NOT_PENDING(out); + if (x->ndim > 32 || (axes_mask >> x->ndim) != 0u) { + ST_LOG_ERROR("st_reduce_max_into: axes_mask out of range."); + return false; + } + + size_t expected[ST_MAX_DIMS]; + size_t expected_ndim = 0; + const uint32_t reduced = + axes_mask & ((x->ndim >= 32) ? 0xffffffffu : ((1u << x->ndim) - 1u)); + for (size_t a = 0; a < x->ndim; ++a) { + const bool is_reduced = ((reduced >> a) & 1u) != 0u; + if (is_reduced && keepdims) { + expected[expected_ndim++] = 1; + } else if (!is_reduced) { + expected[expected_ndim++] = x->shape[a]; + } + } + const bool shape_ok = + (expected_ndim == out->ndim && + memcmp(expected, out->shape, expected_ndim * sizeof(size_t)) == 0) || + /* Reducing every axis yields a scalar; tensors keep it as [1]. */ + (expected_ndim == 0 && out->ndim == 1 && out->shape[0] == 1); + if (!shape_ok) { + ST_LOG_ERROR("st_reduce_max_into: output shape mismatch."); + return false; + } + + /* bf16 promotes to f32; a bf16 output is rounded back at the end. */ + FloatTensor *x_f32 = x->dtype == ST_DTYPE_BF16 ? st_to_f32(x) : NULL; + if (x->dtype == ST_DTYPE_BF16 && x_f32 == NULL) { + return false; + } + const float *in_values = x_f32 ? x_f32->values : x->values; + + FloatTensor *out_f32 = NULL; + float *out_values = out->values; + if (out->dtype == ST_DTYPE_BF16) { + out_f32 = st_create(out->ndim, out->shape); + if (out_f32 == NULL) { + st_destroy(x_f32); + return false; + } + out_values = out_f32->values; + } + + st_reduce_max_f32(in_values, x->ndim, x->shape, axes_mask, out_values, + out->numel); + if (out_f32 != NULL) { + st_f32_to_bf16_bulk(out_f32->values, (uint16_t *)out->values, + out->numel); + } + st_destroy(out_f32); + st_destroy(x_f32); + return true; +} + +FloatTensor *st_reduce_max(const FloatTensor *x, uint32_t axes_mask, + bool keepdims) { + if (!st_interp_tensor_valid(x)) { + ST_LOG_ERROR("st_reduce_max: invalid tensor input."); + return NULL; + } + ST_ASSERT_NOT_PENDING(x); + if (x->ndim > 32 || (axes_mask >> x->ndim) != 0u) { + return NULL; + } + + size_t out_dims[ST_MAX_DIMS]; + size_t out_ndim = 0; + for (size_t a = 0; a < x->ndim; ++a) { + const bool is_reduced = ((axes_mask >> a) & 1u) != 0u; + if (is_reduced && keepdims) { + out_dims[out_ndim++] = 1; + } else if (!is_reduced) { + out_dims[out_ndim++] = x->shape[a]; + } + } + if (out_ndim == 0) { + /* All axes reduced: a scalar stays a [1] tensor. */ + out_dims[out_ndim++] = 1; + } + FloatTensor *out = st_create(out_ndim, out_dims); + if (out == NULL) { + return NULL; + } + if (!st_reduce_max_into(x, axes_mask, keepdims, out)) { + st_destroy(out); + return NULL; + } + return out; +} + +/* ---- Batched matmul ------------------------------------------------------ */ + +/* OpenMP distributes batch GEMMs only when there are many batches and each + * product is small enough that the BLAS backend stays single-threaded; + * bench_st_matmul measured the cutover (oversubscribing the BLAS threads + * regresses up to 2x). */ +#define ST_MATMUL_OMP_MIN_BATCHES 8u +#define ST_MATMUL_OMP_MAX_GEMM 262144u + +/* One GEMM on raw buffers through the sm_gemm backend (like nm_lib's gemm). */ +static bool st_matmul_gemm(const float *a, const float *b, float *c, size_t m, + size_t k, size_t n) { + FloatMatrix ma = {.rows = m, .cols = k, .capacity = m * k, + .values = (float *)a}; + FloatMatrix mb = {.rows = k, .cols = n, .capacity = k * n, + .values = (float *)b}; + FloatMatrix mc = {.rows = m, .cols = n, .capacity = m * n, .values = c}; + return sm_gemm(&mc, 1.0f, &ma, SM_NO_TRANSPOSE, &mb, SM_NO_TRANSPOSE, 0.0f); +} + +static bool st_matmul_batched_f32(const float *a, size_t a_ndim, + const size_t *a_dims, const float *b, + size_t b_ndim, const size_t *b_dims, + float *c, size_t c_ndim, + const size_t *c_dims) { + const size_t m = a_dims[a_ndim - 2]; + const size_t k = a_dims[a_ndim - 1]; + const size_t n = b_dims[b_ndim - 1]; + + size_t b_numel = 1; + for (size_t i = 0; i < b_ndim; ++i) { + b_numel *= b_dims[i]; + } + if (b_numel == k * n) { + /* b is one matrix shared by every batch: a's batches stack into the + * rows of one product (nm_lib's projection fast path). */ + size_t a_numel = 1; + for (size_t i = 0; i < a_ndim; ++i) { + a_numel *= a_dims[i]; + } + return st_matmul_gemm(a, b, c, a_numel / k, k, n); + } + + /* Batch broadcast over the leading axes (nm_lib's padded decomposition). */ + const size_t batch_a = a_ndim - 2; + const size_t batch_b = b_ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + size_t da[ST_MAX_DIMS] = {0}; + size_t db[ST_MAX_DIMS] = {0}; + size_t d[ST_MAX_DIMS] = {0}; + for (size_t i = 0; i < batch_rank; ++i) { + da[i] = i < batch_rank - batch_a ? 1 : a_dims[i - (batch_rank - batch_a)]; + db[i] = i < batch_rank - batch_b ? 1 : b_dims[i - (batch_rank - batch_b)]; + if (da[i] != db[i] && da[i] != 1 && db[i] != 1) { + return false; + } + d[i] = da[i] == 1 ? db[i] : da[i]; + } + size_t batches = 1; + for (size_t i = 0; i < batch_rank; ++i) { + batches *= d[i]; + } + + const size_t gemm_work = m * k * n; + const bool use_omp = + batches >= ST_MATMUL_OMP_MIN_BATCHES && gemm_work <= ST_MATMUL_OMP_MAX_GEMM && + omp_get_max_threads() > 1; + if (use_omp) { +#pragma omp parallel for schedule(static) + for (size_t index = 0; index < batches; ++index) { + size_t rest = index; + size_t offset_a = 0; + size_t offset_b = 0; + size_t scale_a = 1; + size_t scale_b = 1; + for (size_t i = batch_rank; i-- > 0;) { + const size_t coordinate = rest % d[i]; + rest /= d[i]; + offset_a += (da[i] == 1 ? 0 : coordinate) * scale_a; + offset_b += (db[i] == 1 ? 0 : coordinate) * scale_b; + scale_a *= da[i]; + scale_b *= db[i]; + } + st_matmul_gemm(a + offset_a * m * k, b + offset_b * k * n, + c + index * m * n, m, k, n); + } + return true; + } + for (size_t index = 0; index < batches; ++index) { + size_t rest = index; + size_t offset_a = 0; + size_t offset_b = 0; + size_t scale_a = 1; + size_t scale_b = 1; + for (size_t i = batch_rank; i-- > 0;) { + const size_t coordinate = rest % d[i]; + rest /= d[i]; + offset_a += (da[i] == 1 ? 0 : coordinate) * scale_a; + offset_b += (db[i] == 1 ? 0 : coordinate) * scale_b; + scale_a *= da[i]; + scale_b *= db[i]; + } + if (!st_matmul_gemm(a + offset_a * m * k, b + offset_b * k * n, + c + index * m * n, m, k, n)) { + return false; + } + } + return true; +} + +bool st_matmul_batched(const FloatTensor *a, const FloatTensor *b, + FloatTensor *c) { + if (!st_interp_tensor_valid(a) || !st_interp_tensor_valid(b) || + !st_interp_tensor_valid(c)) { + ST_LOG_ERROR("st_matmul_batched: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(a); + ST_ASSERT_NOT_PENDING(b); + ST_ASSERT_NOT_PENDING(c); + if (a->ndim < 2 || b->ndim < 2) { + ST_LOG_ERROR("st_matmul_batched: operands need rank >= 2."); + return false; + } + const size_t m = a->shape[a->ndim - 2]; + const size_t k = a->shape[a->ndim - 1]; + const size_t n = b->shape[b->ndim - 1]; + if (b->shape[b->ndim - 2] != k) { + ST_LOG_ERROR("st_matmul_batched: inner dimensions mismatch."); + return false; + } + + /* Expected output shape: broadcast batch shape plus [M, N]. */ + size_t out_dims[ST_MAX_DIMS]; + const size_t batch_a = a->ndim - 2; + const size_t batch_b = b->ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + if (batch_rank + 2 > ST_MAX_DIMS) { + return false; + } + for (size_t i = 0; i < batch_rank; ++i) { + const size_t da = i < batch_rank - batch_a ? 1 : a->shape[i - (batch_rank - batch_a)]; + const size_t db = i < batch_rank - batch_b ? 1 : b->shape[i - (batch_rank - batch_b)]; + if (da != db && da != 1 && db != 1) { + ST_LOG_ERROR("st_matmul_batched: batch shapes do not broadcast."); + return false; + } + out_dims[i] = da == 1 ? db : da; + } + out_dims[batch_rank] = m; + out_dims[batch_rank + 1] = n; + const size_t out_ndim = batch_rank + 2; + if (c->ndim != out_ndim || + memcmp(c->shape, out_dims, out_ndim * sizeof(size_t)) != 0) { + ST_LOG_ERROR("st_matmul_batched: output shape mismatch."); + return false; + } + + /* bf16 promotes to f32; a bf16 output is rounded back at the end. */ + FloatTensor *a_f32 = a->dtype == ST_DTYPE_BF16 ? st_to_f32(a) : NULL; + FloatTensor *b_f32 = b->dtype == ST_DTYPE_BF16 ? st_to_f32(b) : NULL; + if ((a->dtype == ST_DTYPE_BF16 && a_f32 == NULL) || + (b->dtype == ST_DTYPE_BF16 && b_f32 == NULL)) { + st_destroy(b_f32); + st_destroy(a_f32); + return false; + } + const float *a_values = a_f32 ? a_f32->values : a->values; + const float *b_values = b_f32 ? b_f32->values : b->values; + + FloatTensor *c_f32 = NULL; + float *c_values = c->values; + if (c->dtype == ST_DTYPE_BF16) { + c_f32 = st_create(c->ndim, c->shape); + if (c_f32 == NULL) { + st_destroy(b_f32); + st_destroy(a_f32); + return false; + } + c_values = c_f32->values; + } + + const bool ok = st_matmul_batched_f32(a_values, a->ndim, a->shape, b_values, + b->ndim, b->shape, c_values, c->ndim, + c->shape); + if (ok && c_f32 != NULL) { + st_f32_to_bf16_bulk(c_f32->values, (uint16_t *)c->values, c->numel); + } + st_destroy(c_f32); + st_destroy(b_f32); + st_destroy(a_f32); + return ok; +} + +/* ---- Gather --------------------------------------------------------------- */ + +/* OpenMP over the outer x count copy blocks only above this many copied + * elements: bench_st_gather measured a 2x regression at 1M copied elements + * (memcpy-bound work plus per-block index math), so realistic gathers stay + * sequential. */ +#define ST_GATHER_OMP_THRESHOLD 4194304u + +bool st_gather_bytes(const void *table, size_t table_ndim, + const size_t *table_shape, const int64_t *indices, + size_t count, size_t axis, size_t elem_bytes, void *out) { + if (table == NULL || table_shape == NULL || indices == NULL || out == NULL || + table_ndim == 0 || table_ndim > ST_MAX_DIMS || count == 0 || + axis >= table_ndim || elem_bytes == 0) { + ST_LOG_ERROR("st_gather_bytes: invalid input."); + return false; + } + size_t outer = 1; + size_t dim = table_shape[axis]; + size_t inner = 1; + for (size_t i = 0; i < table_ndim; ++i) { + if (table_shape[i] == 0 || table_shape[i] > INT64_MAX) { + ST_LOG_ERROR("st_gather_bytes: invalid shape."); + return false; + } + if (i < axis) { + if (outer > SIZE_MAX / table_shape[i]) { + return false; + } + outer *= table_shape[i]; + } else if (i > axis) { + if (inner > SIZE_MAX / table_shape[i]) { + return false; + } + inner *= table_shape[i]; + } + } + if (inner > SIZE_MAX / elem_bytes) { + return false; + } + const size_t bytes = inner * elem_bytes; + if (dim > SIZE_MAX / bytes || count > SIZE_MAX / bytes) { + return false; + } + const size_t row_bytes = dim * bytes; + const size_t total = count * bytes; + if (row_bytes > SIZE_MAX / outer || total > SIZE_MAX / outer) { + return false; + } + + /* Validate every index before writing anything, so an out-of-range index + * fails without partial output (nm_lib copies until it hits the bad one). */ + for (size_t j = 0; j < count; ++j) { + const int64_t index = indices[j] < 0 ? indices[j] + (int64_t)dim : indices[j]; + if (index < 0 || index >= (int64_t)dim) { + ST_LOG_ERROR("st_gather_bytes: index out of range."); + return false; + } + } + + const uint8_t *src = table; + uint8_t *dst = out; + const size_t blocks = outer * count; + const size_t copied = blocks * bytes; + if (copied > ST_GATHER_OMP_THRESHOLD && omp_get_max_threads() > 1) { +#pragma omp parallel for schedule(static) + for (size_t block = 0; block < blocks; ++block) { + const size_t o = block / count; + const size_t j = block % count; + const int64_t index = + indices[j] < 0 ? indices[j] + (int64_t)dim : indices[j]; + memcpy(dst + block * bytes, + src + (o * dim + (size_t)index) * bytes, bytes); + } + return true; + } + for (size_t o = 0; o < outer; ++o) { + for (size_t j = 0; j < count; ++j) { + const int64_t index = + indices[j] < 0 ? indices[j] + (int64_t)dim : indices[j]; + memcpy(dst, src + (o * dim + (size_t)index) * bytes, bytes); + dst += bytes; + } + } + return true; +} + +bool st_gather(const FloatTensor *table, const int64_t *indices, size_t count, + size_t axis, FloatTensor *out) { + if (!st_interp_tensor_valid(table) || !st_interp_tensor_valid(out)) { + ST_LOG_ERROR("st_gather: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(table); + ST_ASSERT_NOT_PENDING(out); + if (axis >= table->ndim) { + ST_LOG_ERROR("st_gather: axis out of range."); + return false; + } + if (table->dtype != out->dtype) { + ST_LOG_ERROR("st_gather: dtype mismatch."); + return false; + } + if (out->ndim != table->ndim || + memcmp(out->shape, table->shape, axis * sizeof(size_t)) != 0 || + out->shape[axis] != count || + memcmp(out->shape + axis + 1, table->shape + axis + 1, + (table->ndim - axis - 1) * sizeof(size_t)) != 0) { + ST_LOG_ERROR("st_gather: output shape mismatch."); + return false; + } + return st_gather_bytes(table->values, table->ndim, table->shape, indices, + count, axis, st_dtype_size(table->dtype), out->values); +} + +/* ---- Materializing permute ------------------------------------------------ */ + +bool st_permute_copy_into_bytes(const void *src, size_t ndim, + const size_t *shape, const size_t *perm, + size_t elem_bytes, void *out) { + if (src == NULL || shape == NULL || perm == NULL || out == NULL || + ndim == 0 || ndim > ST_MAX_DIMS || elem_bytes == 0) { + ST_LOG_ERROR("st_permute_copy_into_bytes: invalid input."); + return false; + } + bool seen[ST_MAX_DIMS] = {false}; + size_t in_stride[ST_MAX_DIMS]; + size_t running = 1; + for (size_t i = ndim; i-- > 0;) { + if (shape[i] == 0 || running > (size_t)PTRDIFF_MAX / shape[i]) { + ST_LOG_ERROR("st_permute_copy_into_bytes: invalid shape."); + return false; + } + in_stride[i] = running; + running *= shape[i]; + } + if (running > SIZE_MAX / elem_bytes) { + return false; + } + for (size_t i = 0; i < ndim; ++i) { + if (perm[i] >= ndim || seen[perm[i]]) { + ST_LOG_ERROR("st_permute_copy_into_bytes: perm is not a permutation."); + return false; + } + seen[perm[i]] = true; + } + + /* Walk the output space (the permuted shape); output axis i steps + * through the input with in_stride[perm[i]] (nm_lib's run_transpose + + * walk_copy). */ + size_t out_shape[ST_MAX_DIMS]; + for (size_t i = 0; i < ndim; ++i) { + out_shape[i] = shape[perm[i]]; + } + StWalk w; + st_walk_init(&w, ndim, out_shape, 1); + { + const size_t offset = ST_WALK_DIMS - ndim; + for (size_t i = 0; i < ndim; ++i) { + w.stride[0][offset + i] = (ptrdiff_t)in_stride[perm[i]]; + } + } + st_walk_finish(&w); + const size_t inner = st_walk_inner(&w); + const size_t rows = st_walk_rows(&w); + const ptrdiff_t step = w.stride[0][w.rank - 1]; + uint8_t *dst = out; + for (size_t r = 0; r < rows; ++r, st_walk_next_row(&w), dst += inner * elem_bytes) { + const uint8_t *from = (const uint8_t *)src + + (size_t)w.offset[0] * elem_bytes; + if (step == 1) { + memcpy(dst, from, inner * elem_bytes); + } else if (elem_bytes == 4) { + for (size_t i = 0; i < inner; ++i) { + memcpy(dst + i * 4, from + (size_t)((ptrdiff_t)i * step) * 4, 4); + } + } else if (elem_bytes == 8) { + for (size_t i = 0; i < inner; ++i) { + memcpy(dst + i * 8, from + (size_t)((ptrdiff_t)i * step) * 8, 8); + } + } else { + for (size_t i = 0; i < inner; ++i) { + memcpy(dst + i * elem_bytes, + from + (size_t)((ptrdiff_t)i * step) * elem_bytes, elem_bytes); + } + } + } + return true; +} + +bool st_permute_copy_into(const FloatTensor *src, const size_t *perm, + FloatTensor *out) { + if (!st_interp_tensor_valid(src) || !st_interp_tensor_valid(out)) { + ST_LOG_ERROR("st_permute_copy_into: invalid tensor input."); + return false; + } + ST_ASSERT_NOT_PENDING(src); + ST_ASSERT_NOT_PENDING(out); + if (perm == NULL || src->dtype != out->dtype || out->ndim != src->ndim) { + ST_LOG_ERROR("st_permute_copy_into: invalid input."); + return false; + } + bool seen[ST_MAX_DIMS] = {false}; + for (size_t i = 0; i < src->ndim; ++i) { + if (perm[i] >= src->ndim || seen[perm[i]]) { + ST_LOG_ERROR("st_permute_copy_into: perm is not a permutation."); + return false; + } + seen[perm[i]] = true; + if (out->shape[i] != src->shape[perm[i]]) { + ST_LOG_ERROR("st_permute_copy_into: output shape mismatch."); + return false; + } + } + return st_permute_copy_into_bytes(src->values, src->ndim, src->shape, perm, + st_dtype_size(src->dtype), out->values); +} diff --git a/app/tensor/tests/bench_nn_activations.c b/app/tensor/tests/bench_nn_activations.c new file mode 100644 index 0000000..7683174 --- /dev/null +++ b/app/tensor/tests/bench_nn_activations.c @@ -0,0 +1,234 @@ +/* + * Median benchmark for scalar NN activations versus mMatrix sm/st kernels. + * Run: bazel run //app/tensor:bench_nn_activations + */ + +#include "sm.h" +#include "st.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +typedef enum BenchActivation { + BENCH_SIGMOID, + BENCH_TANH, + BENCH_GELU_TANH, + BENCH_GELU_ERF, + BENCH_SOFTMAX, +} BenchActivation; + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static float scalar_gelu_tanh(float x) { + return 0.5f * x * + (1.0f + tanhf(0.7978845608f * (x + 0.044715f * x * x * x))); +} + +static float scalar_apply(BenchActivation activation, float x) { + switch (activation) { + case BENCH_SIGMOID: + return 1.0f / (1.0f + expf(-x)); + case BENCH_TANH: + return tanhf(x); + case BENCH_GELU_TANH: + return scalar_gelu_tanh(x); + case BENCH_GELU_ERF: + return 0.5f * x * (1.0f + erff(x * 0.7071067812f)); + case BENCH_SOFTMAX: + break; + } + return x; +} + +static const char *activation_name(BenchActivation activation) { + switch (activation) { + case BENCH_SIGMOID: return "sigmoid"; + case BENCH_TANH: return "tanh"; + case BENCH_GELU_TANH: return "gelu_tanh"; + case BENCH_GELU_ERF: return "gelu_erf"; + case BENCH_SOFTMAX: return "softmax"; + } + return "unknown"; +} + +static void scalar_softmax(float *values, size_t rows, size_t cols) { + for (size_t row = 0; row < rows; ++row) { + float *x = values + row * cols; + float maximum = x[0]; + for (size_t col = 1; col < cols; ++col) { + if (x[col] > maximum) maximum = x[col]; + } + float sum = 0.0f; + for (size_t col = 0; col < cols; ++col) { + x[col] = expf(x[col] - maximum); + sum += x[col]; + } + for (size_t col = 0; col < cols; ++col) x[col] /= sum; + } +} + +static bool apply_sm(FloatMatrix *mat, BenchActivation activation) { + switch (activation) { + case BENCH_SIGMOID: return sm_apply_sigmoid(mat); + case BENCH_TANH: return sm_apply_tanh(mat); + case BENCH_GELU_TANH: return sm_apply_gelu_tanh(mat); + case BENCH_GELU_ERF: return sm_apply_gelu_erf(mat); + case BENCH_SOFTMAX: return sm_apply_softmax(mat); + } + return false; +} + +static bool apply_st(FloatTensor *tensor, BenchActivation activation) { + switch (activation) { + case BENCH_SIGMOID: return st_apply_sigmoid(tensor); + case BENCH_TANH: return st_apply_tanh(tensor); + case BENCH_GELU_TANH: return st_apply_gelu_tanh(tensor); + case BENCH_GELU_ERF: return st_apply_gelu_erf(tensor); + case BENCH_SOFTMAX: return st_apply_softmax(tensor); + } + return false; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 256) return 4096; + if (n <= 1024) return 2048; + if (n <= 10000) return 256; + if (n <= 100000) return 32; + return 4; +} + +static double measure_scalar(const float *input, float *work, size_t rows, + size_t cols, BenchActivation activation, + size_t repetitions) { + enum { SAMPLES = 9 }; + double samples[SAMPLES]; + const size_t n = rows * cols; + for (size_t s = 0; s < SAMPLES; ++s) { + memcpy(work, input, n * sizeof(float)); + const uint64_t start = now_ns(); + for (size_t r = 0; r < repetitions; ++r) { + if (activation == BENCH_SOFTMAX) { + scalar_softmax(work, rows, cols); + } else { + for (size_t i = 0; i < n; ++i) { + work[i] = scalar_apply(activation, work[i]); + } + } + } + samples[s] = (double)(now_ns() - start) / 1000.0 / (double)repetitions; + } + return median(samples, SAMPLES); +} + +static double measure_sm(const float *input, FloatMatrix *work, + BenchActivation activation, size_t repetitions) { + enum { SAMPLES = 9 }; + double samples[SAMPLES]; + const size_t n = work->rows * work->cols; + for (size_t s = 0; s < SAMPLES; ++s) { + memcpy(work->values, input, n * sizeof(float)); + const uint64_t start = now_ns(); + for (size_t r = 0; r < repetitions; ++r) { + if (!apply_sm(work, activation)) return -1.0; + } + samples[s] = (double)(now_ns() - start) / 1000.0 / (double)repetitions; + } + return median(samples, SAMPLES); +} + +static double measure_st(const float *input, FloatTensor *work, + BenchActivation activation, size_t repetitions) { + enum { SAMPLES = 9 }; + double samples[SAMPLES]; + const size_t n = work->numel; + for (size_t s = 0; s < SAMPLES; ++s) { + memcpy(work->values, input, n * sizeof(float)); + const uint64_t start = now_ns(); + for (size_t r = 0; r < repetitions; ++r) { + if (!apply_st(work, activation)) return -1.0; + } + samples[s] = (double)(now_ns() - start) / 1000.0 / (double)repetitions; + } + return median(samples, SAMPLES); +} + +int main(void) { + const size_t sizes[] = {256, 1024, 10000, 100000, 1000000}; + const BenchActivation activations[] = { + BENCH_SIGMOID, BENCH_TANH, BENCH_GELU_TANH, BENCH_GELU_ERF, + BENCH_SOFTMAX}; + const int default_threads = omp_get_max_threads(); + omp_set_dynamic(0); + + printf("activation,n,threads,scalar_us,sm_us,st_us,sm_speedup,st_speedup\n"); + for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); ++si) { + const size_t n = sizes[si]; + const size_t cols = n / 4; + const size_t rows = n / cols; + float *input = (float *)malloc(n * sizeof(float)); + float *scalar_work = (float *)malloc(n * sizeof(float)); + FloatMatrix *sm = sm_create(rows, cols); + const size_t shape[2] = {rows, cols}; + FloatTensor *st = st_create(2, shape); + if (!input || !scalar_work || !sm || !st) { + fprintf(stderr, "allocation failed for n=%zu\n", n); + free(input); + free(scalar_work); + sm_destroy(sm); + st_destroy(st); + return 1; + } + for (size_t i = 0; i < n; ++i) { + input[i] = sinf((float)i * 0.013f) * 4.0f; + } + + const int thread_counts[2] = {1, default_threads}; + for (size_t ti = 0; ti < 2; ++ti) { + const int threads = thread_counts[ti]; + if (ti == 1 && threads == thread_counts[0]) continue; + omp_set_num_threads(threads); + for (size_t ai = 0; ai < sizeof(activations) / sizeof(activations[0]); + ++ai) { + const BenchActivation activation = activations[ai]; + const size_t repetitions = repetitions_for(n); + const double before = measure_scalar(input, scalar_work, rows, cols, + activation, repetitions); + const double sm_us = measure_sm(input, sm, activation, repetitions); + const double st_us = measure_st(input, st, activation, repetitions); + printf("%s,%zu,%d,%.3f,%.3f,%.3f,%.2fx,%.2fx\n", + activation_name(activation), n, threads, before, sm_us, st_us, + sm_us > 0.0 ? before / sm_us : 0.0, + st_us > 0.0 ? before / st_us : 0.0); + } + } + sm_destroy(sm); + st_destroy(st); + free(input); + free(scalar_work); + } + omp_set_num_threads(default_threads); + return 0; +} \ No newline at end of file diff --git a/app/tensor/tests/bench_st_binary.c b/app/tensor/tests/bench_st_binary.c new file mode 100644 index 0000000..7495829 --- /dev/null +++ b/app/tensor/tests/bench_st_binary.c @@ -0,0 +1,374 @@ +/* + * Median benchmark for broadcast binary ops: nm_lib's run_binary loops + * (before) versus the st_binary_broadcast kernels (after). + * Run: bazel run //app/tensor:bench_st_binary --config=benchmark + */ + +#include "st.h" +#include "st_interp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BENCH_DIMS 8 + +/* ---- nm_lib reference (nm_graph_interp.c, run_binary) ------------------- */ + +typedef struct { + size_t d[BENCH_DIMS]; + size_t c[BENCH_DIMS]; + ptrdiff_t stride[2][BENCH_DIMS]; + ptrdiff_t offset[2]; + size_t rank; +} NmWalk; + +static void nm_pad_dims(size_t ndim, const size_t *shape, size_t out[BENCH_DIMS]) { + const size_t offset = BENCH_DIMS - ndim; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + out[i] = i < offset ? 1 : shape[i - offset]; + } +} + +static void nm_broadcast_strides(size_t ndim, const size_t *shape, + const size_t d[BENCH_DIMS], + ptrdiff_t out[BENCH_DIMS]) { + size_t padded[BENCH_DIMS]; + nm_pad_dims(ndim, shape, padded); + ptrdiff_t running = 1; + for (size_t i = BENCH_DIMS; i-- > 0;) { + out[i] = padded[i] == 1 && d[i] != 1 ? 0 : running; + running *= (ptrdiff_t)padded[i]; + } +} + +static void nm_walk_init(NmWalk *w, size_t ndim, const size_t *shape) { + memset(w, 0, sizeof(*w)); + nm_pad_dims(ndim, shape, w->d); +} + +static void nm_walk_finish(NmWalk *w) { + size_t rank = 0; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + if (w->d[i] == 1) { + continue; + } + bool merge = rank > 0 && w->stride[0][rank - 1] == + w->stride[0][i] * (ptrdiff_t)w->d[i] && + w->stride[1][rank - 1] == + w->stride[1][i] * (ptrdiff_t)w->d[i]; + if (merge) { + w->d[rank - 1] *= w->d[i]; + w->stride[0][rank - 1] = w->stride[0][i]; + w->stride[1][rank - 1] = w->stride[1][i]; + continue; + } + w->d[rank] = w->d[i]; + w->stride[0][rank] = w->stride[0][i]; + w->stride[1][rank] = w->stride[1][i]; + rank++; + } + if (rank == 0) { + w->d[0] = 1; + w->stride[0][0] = 0; + w->stride[1][0] = 0; + rank = 1; + } + w->rank = rank; +} + +static void nm_walk_next_row(NmWalk *w) { + for (size_t k = w->rank - 1; k-- > 0;) { + w->offset[0] += w->stride[0][k]; + w->offset[1] += w->stride[1][k]; + if (++w->c[k] < w->d[k]) { + return; + } + w->offset[0] -= w->stride[0][k] * (ptrdiff_t)w->d[k]; + w->offset[1] -= w->stride[1][k] * (ptrdiff_t)w->d[k]; + w->c[k] = 0; + } +} + +static float nm_apply_binary(StBinOp op, float a, float b) { + switch (op) { + case ST_BIN_ADD: + return a + b; + case ST_BIN_SUB: + return a - b; + case ST_BIN_MUL: + return a * b; + default: + return a / b; + } +} + +static void nm_binary_row(StBinOp op, const float *a, const float *b, + float *out, size_t n) { + for (size_t i = 0; i < n; ++i) { + out[i] = nm_apply_binary(op, a[i], b[i]); + } +} + +static void nm_binary_row_scalar_b(StBinOp op, const float *a, const float *b, + float *out, size_t n) { + for (size_t i = 0; i < n; ++i) { + out[i] = nm_apply_binary(op, a[i], b[0]); + } +} + +static void nm_binary_row_scalar_a(StBinOp op, const float *a, const float *b, + float *out, size_t n) { + for (size_t i = 0; i < n; ++i) { + out[i] = nm_apply_binary(op, a[0], b[i]); + } +} + +static void nm_run_binary(StBinOp op, const float *a, size_t a_ndim, + const size_t *a_dims, const float *b, size_t b_ndim, + const size_t *b_dims, float *out, size_t o_ndim, + const size_t *o_dims) { + size_t n = 1, na = 1, nb = 1; + for (size_t i = 0; i < o_ndim; ++i) n *= o_dims[i]; + for (size_t i = 0; i < a_ndim; ++i) na *= a_dims[i]; + for (size_t i = 0; i < b_ndim; ++i) nb *= b_dims[i]; + if (na == n && nb == n) { + nm_binary_row(op, a, b, out, n); + return; + } + if (na == n && nb == 1) { + nm_binary_row_scalar_b(op, a, b, out, n); + return; + } + if (na == n && nb > 0 && n % nb == 0 && b_ndim <= o_ndim) { + bool suffix = true; + for (size_t i = 0; i < b_ndim; ++i) { + suffix = suffix && b_dims[i] == o_dims[o_ndim - b_ndim + i]; + } + if (suffix) { + for (size_t i = 0; i < n; i += nb) { + nm_binary_row(op, a + i, b, out + i, nb); + } + return; + } + } + NmWalk w; + nm_walk_init(&w, o_ndim, o_dims); + nm_broadcast_strides(a_ndim, a_dims, w.d, w.stride[0]); + nm_broadcast_strides(b_ndim, b_dims, w.d, w.stride[1]); + nm_walk_finish(&w); + const size_t inner = w.d[w.rank - 1]; + size_t rows = 1; + for (size_t i = 0; i + 1 < w.rank; ++i) { + rows *= w.d[i]; + } + const ptrdiff_t ta = w.stride[0][w.rank - 1]; + const ptrdiff_t tb = w.stride[1][w.rank - 1]; + for (size_t r = 0; r < rows; ++r, nm_walk_next_row(&w), out += inner) { + const float *pa = a + w.offset[0]; + const float *pb = b + w.offset[1]; + if (ta == 1 && tb == 1) { + nm_binary_row(op, pa, pb, out, inner); + } else if (ta == 1 && tb == 0) { + nm_binary_row_scalar_b(op, pa, pb, out, inner); + } else if (ta == 0 && tb == 1) { + nm_binary_row_scalar_a(op, pa, pb, out, inner); + } else { + for (size_t i = 0; i < inner; ++i) { + out[i] = nm_apply_binary(op, pa[(ptrdiff_t)i * ta], + pb[(ptrdiff_t)i * tb]); + } + } + } +} + +/* ---- Benchmark driver --------------------------------------------------- */ + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 4096) return 2000; + if (n <= 65536) return 400; + if (n <= 1048576) return 40; + return 8; +} + +typedef struct { + const char *name; + StBinOp op; + size_t a_ndim; + size_t a_dims[ST_MAX_DIMS]; + size_t b_ndim; + size_t b_dims[ST_MAX_DIMS]; +} BenchCase; + +static FloatTensor *make_input(size_t ndim, const size_t *shape) { + FloatTensor *t = st_create(ndim, shape); + if (t == NULL) { + exit(1); + } + float *values = st_tensor_mutable_data(t); + for (size_t i = 0; i < st_tensor_numel(t); ++i) { + values[i] = sinf((float)i * 0.7351f) * 1.5f + 0.5f; + } + return t; +} + +static void run_case(const BenchCase *bench) { + FloatTensor *a = make_input(bench->a_ndim, bench->a_dims); + FloatTensor *b = make_input(bench->b_ndim, bench->b_dims); + size_t o_ndim = bench->a_ndim > bench->b_ndim ? bench->a_ndim : bench->b_ndim; + size_t o_dims[ST_MAX_DIMS]; + size_t n = 1; + for (size_t i = 0; i < o_ndim; ++i) { + const size_t da = + i < o_ndim - bench->a_ndim ? 1 : bench->a_dims[i - (o_ndim - bench->a_ndim)]; + const size_t db = + i < o_ndim - bench->b_ndim ? 1 : bench->b_dims[i - (o_ndim - bench->b_ndim)]; + o_dims[i] = da == 1 ? db : da; + n *= o_dims[i]; + } + FloatTensor *out = st_create(o_ndim, o_dims); + if (out == NULL) { + exit(1); + } + + const size_t reps = repetitions_for(n); + double before_samples[9]; + double after_samples[9]; + for (size_t s = 0; s < 9; ++s) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + nm_run_binary(bench->op, st_tensor_data(a), bench->a_ndim, bench->a_dims, + st_tensor_data(b), bench->b_ndim, bench->b_dims, + st_tensor_mutable_data(out), o_ndim, o_dims); + } + before_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + + start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + if (!st_binary_broadcast_into(bench->op, a, b, out)) { + fprintf(stderr, "kernel failed\n"); + exit(1); + } + } + after_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double before = median(before_samples, 9); + const double after = median(after_samples, 9); + printf("| %s | %s | %zu | %.3f | %.3f | %.2fx |\n", bench->name, + omp_get_max_threads() == 1 ? "1" : "default", n, before, after, + before / after); + + st_destroy(out); + st_destroy(b); + st_destroy(a); +} + +/* Threshold sweep: API time at one versus default threads across sizes. */ +static void sweep_thread_threshold(void) { + static const size_t sizes[] = {4096, 16384, 65536, 262144, + 1048576, 2097152, 4194304, 8388608}; + const int default_threads = omp_get_max_threads(); + printf("\nThread threshold sweep (st_binary_broadcast_into, µs/call)\n"); + printf("| Elements | Path | 1 thread | default (%d) | parallel gain |\n", default_threads); + printf("| ---: | --- | ---: | ---: | ---: |\n"); + for (size_t s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) { + const size_t n = sizes[s]; + const size_t shape[1] = {n}; + FloatTensor *a = make_input(1, shape); + for (int path = 0; path < 2; ++path) { + FloatTensor *b; + const char *label; + if (path == 0) { + b = make_input(1, shape); + label = "same-shape"; + } else { + b = make_input(1, (size_t[1]){1}); + label = "scalar"; + } + FloatTensor *out = st_create(1, shape); + const size_t reps = repetitions_for(n); + double one_samples[9]; + double many_samples[9]; + omp_set_num_threads(1); + for (size_t s2 = 0; s2 < 9; ++s2) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + st_binary_broadcast_into(ST_BIN_ADD, a, b, out); + } + one_samples[s2] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + omp_set_num_threads(default_threads); + for (size_t s2 = 0; s2 < 9; ++s2) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + st_binary_broadcast_into(ST_BIN_ADD, a, b, out); + } + many_samples[s2] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double one = median(one_samples, 9); + const double many = median(many_samples, 9); + printf("| %zu | %s | %.3f | %.3f | %.2fx |\n", n, label, one, many, + one / many); + st_destroy(out); + st_destroy(b); + } + st_destroy(a); + } +} + +int main(void) { + printf("| Case | Threads | Elements | nm_lib before (µs) | st_binary (µs) | Speedup |\n"); + printf("| --- | --- | ---: | ---: | ---: | ---: |\n"); + + const int default_threads = omp_get_max_threads(); + const BenchCase cases[] = { + {"same-shape add [8,256,512]", ST_BIN_ADD, 3, {8, 256, 512}, 3, {8, 256, 512}}, + {"same-shape add [1,16,64]", ST_BIN_ADD, 3, {1, 16, 64}, 3, {1, 16, 64}}, + {"bias add [4,128,256]+[256]", ST_BIN_ADD, 3, {4, 128, 256}, 1, {256}}, + {"bias add [4,512,768]+[768]", ST_BIN_ADD, 3, {4, 512, 768}, 1, {768}}, + {"bias add [1,16,64]+[64]", ST_BIN_ADD, 3, {1, 16, 64}, 1, {64}}, + {"attn mask add (walk) [2,4,128,128]+[1,1,128,128]", ST_BIN_ADD, 4, {2, 4, 128, 128}, 4, {1, 1, 128, 128}}, + {"attn mask add (suffix) [2,4,128,128]+[128,128]", ST_BIN_ADD, 4, {2, 4, 128, 128}, 2, {128, 128}}, + {"attn mask add (walk) [1,4,16,16]+[1,1,16,16]", ST_BIN_ADD, 4, {1, 4, 16, 16}, 3, {1, 1, 16, 16}}, + {"scale mul [4,128,256]*[1]", ST_BIN_MUL, 3, {4, 128, 256}, 1, {1}}, + {"scale mul (mid) [4,1,256]*[1,128,1]", ST_BIN_MUL, 3, {4, 1, 256}, 3, {1, 128, 1}}, + {"suffix div [4,128,256]/[256]", ST_BIN_DIV, 3, {4, 128, 256}, 1, {256}}, + }; + const size_t count = sizeof(cases) / sizeof(cases[0]); + + omp_set_num_threads(default_threads); + for (size_t c = 0; c < count; ++c) { + run_case(&cases[c]); + } + omp_set_num_threads(1); + for (size_t c = 0; c < count; ++c) { + run_case(&cases[c]); + } + omp_set_num_threads(default_threads); + sweep_thread_threshold(); + return 0; +} diff --git a/app/tensor/tests/bench_st_f16.c b/app/tensor/tests/bench_st_f16.c new file mode 100644 index 0000000..703fddf --- /dev/null +++ b/app/tensor/tests/bench_st_f16.c @@ -0,0 +1,122 @@ +/* + * Median benchmark for f16 conversion: nm_lib's per-element conversion + * loop (before) versus the bulk NEON kernels (after). + * Run: bazel run //app/tensor:bench_st_f16 --config=benchmark + */ + +#include "st_f16.h" + +#include +#include +#include +#include +#include +#include +#include + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 4096) return 2000; + if (n <= 65536) return 400; + if (n <= 1048576) return 40; + return 8; +} + +typedef struct { + float *f32; + uint16_t *f16; + size_t n; +} Ctx; + +static void f32_to_f16_before(void *context) { + Ctx *c = context; + /* nm_lib's loop: one nm_f32_to_f16 call per element. */ + for (size_t i = 0; i < c->n; ++i) { + c->f16[i] = st_f32_to_f16(c->f32[i]); + } +} + +static void f32_to_f16_after(void *context) { + Ctx *c = context; + st_f32_to_f16_bulk(c->f32, c->f16, c->n); +} + +static void f16_to_f32_before(void *context) { + Ctx *c = context; + for (size_t i = 0; i < c->n; ++i) { + c->f32[i] = st_f16_to_f32(c->f16[i]); + } +} + +static void f16_to_f32_after(void *context) { + Ctx *c = context; + st_f16_to_f32_bulk(c->f16, c->f32, c->n); +} + +static void measure(const char *name, size_t n, void (*before)(void *), + void (*after)(void *), Ctx *ctx) { + const size_t reps = repetitions_for(n); + double before_samples[9]; + double after_samples[9]; + for (size_t s = 0; s < 9; ++s) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + before(ctx); + } + before_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + after(ctx); + } + after_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double b = median(before_samples, 9); + const double a = median(after_samples, 9); + printf("| %s | %zu | %.3f | %.3f | %.2fx |\n", name, n, b, a, b / a); +} + +int main(void) { + printf("| Case | Elements | scalar loop (µs) | bulk (µs) | Speedup |\n"); + printf("| --- | ---: | ---: | ---: | ---: |\n"); + static const size_t sizes[] = {4096, 65536, 262144, 1048576}; + Ctx ctx; + ctx.n = sizes[sizeof(sizes) / sizeof(sizes[0]) - 1]; + ctx.f32 = malloc(ctx.n * sizeof(float)); + ctx.f16 = malloc(ctx.n * sizeof(uint16_t)); + if (ctx.f32 == NULL || ctx.f16 == NULL) { + return 1; + } + for (size_t i = 0; i < ctx.n; ++i) { + ctx.f32[i] = sinf((float)i * 0.7351f) * 1.5f + 0.5f; + ctx.f16[i] = st_f32_to_f16(ctx.f32[i]); + } + /* The conversion is single-threaded by design (NEON, no OpenMP). */ + for (size_t s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) { + ctx.n = sizes[s]; + measure("f32 -> f16", sizes[s], f32_to_f16_before, f32_to_f16_after, + &ctx); + measure("f16 -> f32", sizes[s], f16_to_f32_before, f16_to_f32_after, &ctx); + } + (void)omp_get_max_threads; + free(ctx.f16); + free(ctx.f32); + return 0; +} diff --git a/app/tensor/tests/bench_st_gather.c b/app/tensor/tests/bench_st_gather.c new file mode 100644 index 0000000..7211df7 --- /dev/null +++ b/app/tensor/tests/bench_st_gather.c @@ -0,0 +1,308 @@ +/* + * Median benchmark for gather and materializing permute: nm_lib's + * run_gather/run_transpose loops (before) versus the st_interp kernels. + * Run: bazel run //app/tensor:bench_st_gather --config=benchmark + */ + +#include "st.h" +#include "st_interp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BENCH_DIMS 8 + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 65536) return 400; + if (n <= 1048576) return 60; + return 12; +} + +static FloatTensor *make_input(size_t ndim, const size_t *shape) { + FloatTensor *t = st_create(ndim, shape); + if (t == NULL) { + exit(1); + } + float *values = st_tensor_mutable_data(t); + for (size_t i = 0; i < st_tensor_numel(t); ++i) { + values[i] = sinf((float)i * 0.7351f) * 1.5f + 0.5f; + } + return t; +} + +/* nm_lib's run_gather (without its partial-write failure behavior). */ +static void nm_run_gather(const void *data, size_t ndim, const size_t *dims, + size_t axis, const int64_t *indices, size_t count, + size_t elem, void *out) { + size_t outer = 1; + size_t inner = 1; + for (size_t i = 0; i < axis; ++i) { + outer *= dims[i]; + } + for (size_t i = axis + 1; i < ndim; ++i) { + inner *= dims[i]; + } + const int64_t dim = (int64_t)dims[axis]; + const size_t bytes = inner * elem; + uint8_t *dst = out; + for (size_t o = 0; o < outer; ++o) { + for (size_t j = 0; j < count; ++j) { + const int64_t index = indices[j] < 0 ? indices[j] + dim : indices[j]; + memcpy(dst, + (const uint8_t *)data + (o * (size_t)dim + (size_t)index) * bytes, + bytes); + dst += bytes; + } + } +} + +/* nm_lib's run_transpose + walk_copy. */ +static void nm_run_transpose(const void *in, size_t ndim, const size_t *dims, + const size_t *perm, size_t elem, void *out) { + size_t out_dims[BENCH_DIMS]; + size_t in_stride[BENCH_DIMS]; + size_t running = 1; + for (size_t i = ndim; i-- > 0;) { + in_stride[i] = running; + running *= dims[i]; + } + for (size_t i = 0; i < ndim; ++i) { + out_dims[i] = dims[perm[i]]; + } + /* Walk rows of the output space. */ + size_t d[BENCH_DIMS]; + size_t c[BENCH_DIMS] = {0}; + ptrdiff_t stride[BENCH_DIMS]; + ptrdiff_t offset = 0; + const size_t pad = BENCH_DIMS - ndim; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + d[i] = i < pad ? 1 : out_dims[i - pad]; + } + for (size_t i = 0; i < BENCH_DIMS; ++i) { + stride[i] = i < pad ? 0 : (ptrdiff_t)in_stride[perm[i - pad]]; + } + size_t rank = 0; + size_t wd[BENCH_DIMS]; + ptrdiff_t wstride[BENCH_DIMS]; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + if (d[i] == 1) { + continue; + } + const bool merge = + rank > 0 && wstride[rank - 1] == stride[i] * (ptrdiff_t)d[i]; + if (merge) { + wd[rank - 1] *= d[i]; + wstride[rank - 1] = stride[i]; + continue; + } + wd[rank] = d[i]; + wstride[rank] = stride[i]; + rank++; + } + if (rank == 0) { + wd[0] = 1; + wstride[0] = 0; + rank = 1; + } + const size_t inner = wd[rank - 1]; + size_t rows = 1; + for (size_t i = 0; i + 1 < rank; ++i) { + rows *= wd[i]; + } + const ptrdiff_t step = wstride[rank - 1]; + uint8_t *dst = out; + for (size_t r = 0; r < rows; ++r) { + const uint8_t *from = (const uint8_t *)in + (size_t)offset * elem; + if (step == 1) { + memcpy(dst, from, inner * elem); + } else { + for (size_t i = 0; i < inner; ++i) { + memcpy(dst + i * elem, + from + (size_t)((ptrdiff_t)i * step) * elem, elem); + } + } + dst += inner * elem; + for (size_t k = rank - 1; k-- > 0;) { + offset += wstride[k]; + if (++c[k] < wd[k]) { + break; + } + offset -= wstride[k] * (ptrdiff_t)wd[k]; + c[k] = 0; + } + } +} + +static void measure(const char *name, size_t elements, void (*fn)(void *), + void (*before)(void *), void (*after)(void *), + void *context) { + const size_t reps = repetitions_for(elements); + double before_samples[9]; + double after_samples[9]; + for (size_t s = 0; s < 9; ++s) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + before(context); + } + before_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + after(context); + } + after_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double b = median(before_samples, 9); + const double a = median(after_samples, 9); + printf("| %s | %s | %.3f | %.3f | %.2fx |\n", name, + omp_get_max_threads() == 1 ? "1" : "default", b, a, b / a); + (void)fn; +} + +typedef struct { + FloatTensor *table; + FloatTensor *out; + int64_t *indices; + size_t count; + size_t axis; +} GatherCtx; + +static void gather_before(void *ctx) { + GatherCtx *c = ctx; + nm_run_gather(st_tensor_data(c->table), c->table->ndim, c->table->shape, + c->axis, c->indices, c->count, sizeof(float), + st_tensor_mutable_data(c->out)); +} + +static void gather_after(void *ctx) { + GatherCtx *c = ctx; + if (!st_gather(c->table, c->indices, c->count, c->axis, c->out)) { + fprintf(stderr, "st_gather failed\n"); + exit(1); + } +} + +static void run_gather_case(const char *name, size_t ndim, const size_t *dims, + size_t axis, size_t count) { + GatherCtx ctx; + ctx.table = make_input(ndim, dims); + ctx.count = count; + ctx.axis = axis; + ctx.indices = malloc(count * sizeof(int64_t)); + if (ctx.indices == NULL) { + exit(1); + } + for (size_t i = 0; i < count; ++i) { + ctx.indices[i] = (int64_t)((i * 7919) % dims[axis]); + } + size_t out_dims[ST_MAX_DIMS]; + memcpy(out_dims, dims, ndim * sizeof(size_t)); + out_dims[axis] = count; + ctx.out = st_create(ndim, out_dims); + if (ctx.out == NULL) { + exit(1); + } + const size_t elements = st_tensor_numel(ctx.out); + measure(name, elements, NULL, gather_before, gather_after, &ctx); + st_destroy(ctx.out); + free(ctx.indices); + st_destroy(ctx.table); +} + +typedef struct { + FloatTensor *src; + FloatTensor *out; + const size_t *perm; +} PermuteCtx; + +static void permute_before(void *ctx) { + PermuteCtx *c = ctx; + nm_run_transpose(st_tensor_data(c->src), c->src->ndim, c->src->shape, + c->perm, sizeof(float), st_tensor_mutable_data(c->out)); +} + +static void permute_after(void *ctx) { + PermuteCtx *c = ctx; + if (!st_permute_copy_into(c->src, c->perm, c->out)) { + fprintf(stderr, "st_permute_copy_into failed\n"); + exit(1); + } +} + +static void run_permute_case(const char *name, size_t ndim, + const size_t *dims, const size_t *perm) { + PermuteCtx ctx; + ctx.src = make_input(ndim, dims); + ctx.perm = perm; + size_t out_dims[ST_MAX_DIMS]; + for (size_t i = 0; i < ndim; ++i) { + out_dims[i] = dims[perm[i]]; + } + ctx.out = st_create(ndim, out_dims); + if (ctx.out == NULL) { + exit(1); + } + measure(name, st_tensor_numel(ctx.src), NULL, permute_before, permute_after, + &ctx); + st_destroy(ctx.out); + st_destroy(ctx.src); +} + +int main(void) { + printf("| Case | Threads | nm_lib before (µs) | st kernel (µs) | Speedup |\n"); + printf("| --- | --- | ---: | ---: | ---: |\n"); + const int default_threads = omp_get_max_threads(); + + const size_t perm_attn[4] = {0, 2, 1, 3}; + const size_t perm_last[4] = {0, 1, 3, 2}; + const size_t perm_small[4] = {0, 2, 1, 3}; + + omp_set_num_threads(default_threads); + run_gather_case("gather [30000,256] axis 0, count 128", 2, + (size_t[2]){30000, 256}, 0, 128); + run_gather_case("gather [30000,64] axis 0, count 16", 2, + (size_t[2]){30000, 64}, 0, 16); + run_gather_case("gather [2,30000,1024] axis 1, count 512", 3, + (size_t[3]){2, 30000, 1024}, 1, 512); + run_gather_case("gather [100,7,3] axis 1, count 7", 3, + (size_t[3]){100, 7, 3}, 1, 7); + run_permute_case("permute [2,4,128,64] -> [2,128,4,64]", 4, + (size_t[4]){2, 4, 128, 64}, perm_attn); + run_permute_case("permute [2,4,128,64] -> [2,4,64,128]", 4, + (size_t[4]){2, 4, 128, 64}, perm_last); + run_permute_case("permute [1,4,16,16] -> [1,16,4,16]", 4, + (size_t[4]){1, 4, 16, 16}, perm_small); + + omp_set_num_threads(1); + run_gather_case("gather [30000,256] axis 0, count 128", 2, + (size_t[2]){30000, 256}, 0, 128); + run_gather_case("gather [2,30000,1024] axis 1, count 512", 3, + (size_t[3]){2, 30000, 1024}, 1, 512); + run_permute_case("permute [2,4,128,64] -> [2,128,4,64]", 4, + (size_t[4]){2, 4, 128, 64}, perm_attn); + return 0; +} diff --git a/app/tensor/tests/bench_st_matmul.c b/app/tensor/tests/bench_st_matmul.c new file mode 100644 index 0000000..b7a34f7 --- /dev/null +++ b/app/tensor/tests/bench_st_matmul.c @@ -0,0 +1,215 @@ +/* + * Median benchmark for batched matmul: nm_lib's run_matmul loop (before) + * versus st_matmul_batched (after). + * Run: bazel run //app/tensor:bench_st_matmul --config=benchmark + */ + +#include "sm.h" +#include "st.h" +#include "st_interp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BENCH_DIMS 8 + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 65536) return 400; + if (n <= 4194304) return 40; + return 8; +} + +static FloatTensor *make_input(size_t ndim, const size_t *shape) { + FloatTensor *t = st_create(ndim, shape); + if (t == NULL) { + exit(1); + } + float *values = st_tensor_mutable_data(t); + for (size_t i = 0; i < st_tensor_numel(t); ++i) { + values[i] = sinf((float)i * 0.7351f) * 1.5f + 0.5f; + } + return t; +} + +/* nm_lib's run_matmul: sequential sm_gemm per batch, with the single-matrix + * stacking fast path. */ +static bool nm_gemm(const float *a, const float *b, float *c, size_t m, + size_t k, size_t n) { + FloatMatrix ma = {.rows = m, .cols = k, .capacity = m * k, .values = (float *)a}; + FloatMatrix mb = {.rows = k, .cols = n, .capacity = k * n, .values = (float *)b}; + FloatMatrix mc = {.rows = m, .cols = n, .capacity = m * n, .values = c}; + return sm_gemm(&mc, 1.0f, &ma, SM_NO_TRANSPOSE, &mb, SM_NO_TRANSPOSE, 0.0f); +} + +static void nm_run_matmul(const float *a, size_t a_ndim, const size_t *a_dims, + const float *b, size_t b_ndim, const size_t *b_dims, + float *out) { + const size_t m = a_dims[a_ndim - 2]; + const size_t k = a_dims[a_ndim - 1]; + const size_t n = b_dims[b_ndim - 1]; + size_t b_numel = 1; + for (size_t i = 0; i < b_ndim; ++i) { + b_numel *= b_dims[i]; + } + if (b_numel == k * n) { + size_t a_numel = 1; + for (size_t i = 0; i < a_ndim; ++i) { + a_numel *= a_dims[i]; + } + nm_gemm(a, b, out, a_numel / k, k, n); + return; + } + const size_t batch_a = a_ndim - 2; + const size_t batch_b = b_ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + size_t da[BENCH_DIMS] = {0}; + size_t db[BENCH_DIMS] = {0}; + size_t d[BENCH_DIMS] = {0}; + for (size_t i = 0; i < batch_rank; ++i) { + da[i] = i < batch_rank - batch_a ? 1 : a_dims[i - (batch_rank - batch_a)]; + db[i] = i < batch_rank - batch_b ? 1 : b_dims[i - (batch_rank - batch_b)]; + d[i] = da[i] == 1 ? db[i] : da[i]; + } + size_t batches = 1; + for (size_t i = 0; i < batch_rank; ++i) { + batches *= d[i]; + } + for (size_t index = 0; index < batches; ++index) { + size_t rest = index; + size_t offset_a = 0; + size_t offset_b = 0; + size_t scale_a = 1; + size_t scale_b = 1; + for (size_t i = batch_rank; i-- > 0;) { + const size_t coordinate = rest % d[i]; + rest /= d[i]; + offset_a += (da[i] == 1 ? 0 : coordinate) * scale_a; + offset_b += (db[i] == 1 ? 0 : coordinate) * scale_b; + scale_a *= da[i]; + scale_b *= db[i]; + } + nm_gemm(a + offset_a * m * k, b + offset_b * k * n, out + index * m * n, m, + k, n); + } +} + +typedef struct { + const char *name; + size_t a_ndim; + size_t a_dims[ST_MAX_DIMS]; + size_t b_ndim; + size_t b_dims[ST_MAX_DIMS]; +} MatmulCase; + +static void run_case(const MatmulCase *bench) { + FloatTensor *a = make_input(bench->a_ndim, bench->a_dims); + FloatTensor *b = make_input(bench->b_ndim, bench->b_dims); + const size_t m = bench->a_dims[bench->a_ndim - 2]; + const size_t n = bench->b_dims[bench->b_ndim - 1]; + const size_t batch_a = bench->a_ndim - 2; + const size_t batch_b = bench->b_ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + size_t out_dims[ST_MAX_DIMS]; + size_t out_ndim = 0; + size_t batches = 1; + for (size_t i = 0; i < batch_rank; ++i) { + const size_t da = + i < batch_rank - batch_a ? 1 : bench->a_dims[i - (batch_rank - batch_a)]; + const size_t db = + i < batch_rank - batch_b ? 1 : bench->b_dims[i - (batch_rank - batch_b)]; + const size_t d = da == 1 ? db : da; + out_dims[out_ndim++] = d; + batches *= d; + } + out_dims[out_ndim++] = m; + out_dims[out_ndim++] = n; + FloatTensor *c = st_create(out_ndim, out_dims); + if (c == NULL) { + exit(1); + } + + const size_t work = st_tensor_numel(a) + st_tensor_numel(b) + + st_tensor_numel(c); + const size_t reps = repetitions_for(work); + double before_samples[9]; + double after_samples[9]; + for (size_t s = 0; s < 9; ++s) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + nm_run_matmul(st_tensor_data(a), bench->a_ndim, bench->a_dims, + st_tensor_data(b), bench->b_ndim, bench->b_dims, + st_tensor_mutable_data(c)); + } + before_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + + start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + if (!st_matmul_batched(a, b, c)) { + fprintf(stderr, "st_matmul_batched failed\n"); + exit(1); + } + } + after_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double before = median(before_samples, 9); + const double after = median(after_samples, 9); + printf("| %s | %s | %zu | %zu | %.3f | %.3f | %.2fx |\n", bench->name, + omp_get_max_threads() == 1 ? "1" : "default", batches, m * m * n, + before, after, before / after); + + st_destroy(c); + st_destroy(b); + st_destroy(a); +} + +int main(void) { + printf("| Case | Threads | Batches | GEMM MxKxN | nm_lib before (µs) | st_matmul_batched (µs) | Speedup |\n"); + printf("| --- | --- | ---: | ---: | ---: | ---: | ---: |\n"); + const int default_threads = omp_get_max_threads(); + + const MatmulCase cases[] = { + {"proj stacked [4,128,256]x[256,512]", 3, {4, 128, 256}, 2, {256, 512}}, + {"proj stacked [8,256,512]x[512,512]", 3, {8, 256, 512}, 2, {512, 512}}, + {"proj stacked [1,16,64]x[64,64]", 3, {1, 16, 64}, 2, {64, 64}}, + {"batched [2,4,128,64]x[2,4,64,128] (attn scores)", 4, {2, 4, 128, 64}, 4, {2, 4, 64, 128}}, + {"batched [1,4,16,64]x[1,4,64,16] (small, many)", 4, {1, 4, 16, 64}, 4, {1, 4, 64, 16}}, + {"batched [2,4,128,64]x[2,1,64,128] (shared head)", 4, {2, 4, 128, 64}, 4, {2, 1, 64, 128}}, + {"batched [4,12,64,64]x[4,12,64,64]", 4, {4, 12, 64, 64}, 4, {4, 12, 64, 64}}, + }; + const size_t count = sizeof(cases) / sizeof(cases[0]); + + omp_set_num_threads(default_threads); + for (size_t i = 0; i < count; ++i) { + run_case(&cases[i]); + } + omp_set_num_threads(1); + for (size_t i = 0; i < count; ++i) { + run_case(&cases[i]); + } + return 0; +} diff --git a/app/tensor/tests/bench_st_norm_reduce.c b/app/tensor/tests/bench_st_norm_reduce.c new file mode 100644 index 0000000..abfb955 --- /dev/null +++ b/app/tensor/tests/bench_st_norm_reduce.c @@ -0,0 +1,429 @@ +/* + * Median benchmark for LayerNorm and ReduceMax: nm_lib's interpreter loops + * (before) versus the st_interp kernels (after). + * Run: bazel run //app/tensor:bench_st_norm_reduce --config=benchmark + */ + +#include "st.h" +#include "st_interp.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#define BENCH_DIMS 8 + +/* ---- Timing helpers ----------------------------------------------------- */ + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions_for(size_t n) { + if (n <= 4096) return 2000; + if (n <= 65536) return 400; + if (n <= 1048576) return 40; + return 8; +} + +static FloatTensor *make_input(size_t ndim, const size_t *shape) { + FloatTensor *t = st_create(ndim, shape); + if (t == NULL) { + exit(1); + } + float *values = st_tensor_mutable_data(t); + for (size_t i = 0; i < st_tensor_numel(t); ++i) { + values[i] = sinf((float)i * 0.7351f) * 1.5f + 0.5f; + } + return t; +} + +static void measure(const char *name, size_t elements, size_t out_elements, + void (*before)(void *), void (*after)(void *), + void *context) { + const size_t reps = repetitions_for(elements); + double before_samples[9]; + double after_samples[9]; + for (size_t s = 0; s < 9; ++s) { + uint64_t start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + before(context); + } + before_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + + start = now_ns(); + for (size_t r = 0; r < reps; ++r) { + after(context); + } + after_samples[s] = (double)(now_ns() - start) / (double)reps / 1000.0; + } + const double b = median(before_samples, 9); + const double a = median(after_samples, 9); + printf("| %s | %s | %zu | %.3f | %.3f | %.2fx |\n", name, + omp_get_max_threads() == 1 ? "1" : "default", elements, b, a, b / a); + (void)out_elements; +} + +/* ---- nm_lib references --------------------------------------------------- */ + +/* nm_lib's sum8 (nm_graph_interp.c). */ +static float nm_sum8(const float *x, size_t n) { + float lanes[8] = {0.0f}; + size_t j = 0; + for (; j + 8 <= n; j += 8) { + for (size_t l = 0; l < 8; ++l) { + lanes[l] += x[j + l]; + } + } + float sum = 0.0f; + for (; j < n; ++j) { + sum += x[j]; + } + for (size_t l = 0; l < 8; ++l) { + sum += lanes[l]; + } + return sum; +} + +/* nm_lib's run_layernorm. */ +static void nm_run_layernorm(const float *in, size_t ndim, + const size_t *dims, size_t norm_rank, + const float *scale, const float *bias, + float eps, float *out) { + size_t group = 1; + for (size_t i = ndim - norm_rank; i < ndim; ++i) { + group *= dims[i]; + } + size_t numel = 1; + for (size_t i = 0; i < ndim; ++i) { + numel *= dims[i]; + } + const size_t groups = numel / group; + for (size_t g = 0; g < groups; ++g) { + const float *x = in + g * group; + float *y = out + g * group; + const float mean = nm_sum8(x, group) / (float)group; + for (size_t j = 0; j < group; ++j) { + y[j] = x[j] - mean; + } + float lanes[8] = {0.0f}; + size_t j = 0; + for (; j + 8 <= group; j += 8) { + for (size_t l = 0; l < 8; ++l) { + lanes[l] += y[j + l] * y[j + l]; + } + } + float variance = 0.0f; + for (; j < group; ++j) { + variance += y[j] * y[j]; + } + for (size_t l = 0; l < 8; ++l) { + variance += lanes[l]; + } + variance /= (float)group; + const float inverse = 1.0f / sqrtf(variance + eps); + if (scale && bias) { + for (size_t i = 0; i < group; ++i) { + y[i] = y[i] * inverse * scale[i] + bias[i]; + } + } else { + for (size_t i = 0; i < group; ++i) { + float value = y[i] * inverse; + if (scale) { + value *= scale[i]; + } + if (bias) { + value += bias[i]; + } + y[i] = value; + } + } + } +} + +/* nm_lib's run_reducemax: walk the input, the output offset stands still + * along reduced axes. */ +typedef struct { + size_t d[BENCH_DIMS]; + size_t c[BENCH_DIMS]; + ptrdiff_t stride[2][BENCH_DIMS]; + ptrdiff_t offset[2]; + size_t rank; +} NmWalk; + +static void nm_pad_dims(size_t ndim, const size_t *shape, size_t out[BENCH_DIMS]) { + const size_t offset = BENCH_DIMS - ndim; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + out[i] = i < offset ? 1 : shape[i - offset]; + } +} + +static void nm_broadcast_strides(size_t ndim, const size_t *shape, + const size_t d[BENCH_DIMS], + ptrdiff_t out[BENCH_DIMS]) { + size_t padded[BENCH_DIMS]; + nm_pad_dims(ndim, shape, padded); + ptrdiff_t running = 1; + for (size_t i = BENCH_DIMS; i-- > 0;) { + out[i] = padded[i] == 1 && d[i] != 1 ? 0 : running; + running *= (ptrdiff_t)padded[i]; + } +} + +static void nm_walk_init(NmWalk *w, size_t ndim, const size_t *shape) { + memset(w, 0, sizeof(*w)); + nm_pad_dims(ndim, shape, w->d); +} + +static void nm_walk_finish(NmWalk *w) { + size_t rank = 0; + for (size_t i = 0; i < BENCH_DIMS; ++i) { + if (w->d[i] == 1) { + continue; + } + const bool merge = + rank > 0 && w->stride[0][rank - 1] == + w->stride[0][i] * (ptrdiff_t)w->d[i] && + w->stride[1][rank - 1] == w->stride[1][i] * (ptrdiff_t)w->d[i]; + if (merge) { + w->d[rank - 1] *= w->d[i]; + w->stride[0][rank - 1] = w->stride[0][i]; + w->stride[1][rank - 1] = w->stride[1][i]; + continue; + } + w->d[rank] = w->d[i]; + w->stride[0][rank] = w->stride[0][i]; + w->stride[1][rank] = w->stride[1][i]; + rank++; + } + if (rank == 0) { + w->d[0] = 1; + w->stride[0][0] = 0; + w->stride[1][0] = 0; + rank = 1; + } + w->rank = rank; +} + +static void nm_walk_next_row(NmWalk *w) { + for (size_t k = w->rank - 1; k-- > 0;) { + w->offset[0] += w->stride[0][k]; + w->offset[1] += w->stride[1][k]; + if (++w->c[k] < w->d[k]) { + return; + } + w->offset[0] -= w->stride[0][k] * (ptrdiff_t)w->d[k]; + w->offset[1] -= w->stride[1][k] * (ptrdiff_t)w->d[k]; + w->c[k] = 0; + } +} + +static void nm_run_reducemax(const float *in, size_t ndim, + const size_t *dims, uint32_t mask, float *out, + size_t out_numel) { + size_t kept[BENCH_DIMS]; + for (size_t i = 0; i < ndim; ++i) { + kept[i] = (mask >> i) & 1u ? 1 : dims[i]; + } + NmWalk w; + nm_walk_init(&w, ndim, dims); + nm_broadcast_strides(ndim, kept, w.d, w.stride[0]); + nm_broadcast_strides(ndim, dims, w.d, w.stride[1]); + nm_walk_finish(&w); + for (size_t i = 0; i < out_numel; ++i) { + out[i] = -INFINITY; + } + const size_t inner = w.d[w.rank - 1]; + size_t rows = 1; + for (size_t i = 0; i + 1 < w.rank; ++i) { + rows *= w.d[i]; + } + const ptrdiff_t to = w.stride[0][w.rank - 1]; + for (size_t r = 0; r < rows; ++r, nm_walk_next_row(&w)) { + const float *x = in + w.offset[1]; + float *y = out + w.offset[0]; + if (to == 0) { + float max = *y; + for (size_t i = 0; i < inner; ++i) { + max = x[i] > max ? x[i] : max; + } + *y = max; + } else { + for (size_t i = 0; i < inner; ++i) { + float *slot = y + (ptrdiff_t)i * to; + *slot = x[i] > *slot ? x[i] : *slot; + } + } + } +} + +/* ---- Benchmark contexts -------------------------------------------------- */ + +typedef struct { + FloatTensor *x; + FloatTensor *gamma; + FloatTensor *beta; + FloatTensor *out; + size_t norm_rank; + float eps; +} LayernormCtx; + +static void layernorm_before(void *ctx) { + LayernormCtx *c = ctx; + nm_run_layernorm(st_tensor_data(c->x), c->x->ndim, c->x->shape, + c->norm_rank, st_tensor_data(c->gamma), + st_tensor_data(c->beta), c->eps, + st_tensor_mutable_data(c->out)); +} + +static void layernorm_after(void *ctx) { + LayernormCtx *c = ctx; + if (!st_layernorm(c->x, c->gamma, c->beta, c->norm_rank, c->eps, c->out)) { + fprintf(stderr, "st_layernorm failed\n"); + exit(1); + } +} + +typedef struct { + FloatTensor *x; + FloatTensor *out; + uint32_t mask; + bool keepdims; +} ReduceCtx; + +static void reduce_before(void *ctx) { + ReduceCtx *c = ctx; + nm_run_reducemax(st_tensor_data(c->x), c->x->ndim, c->x->shape, c->mask, + st_tensor_mutable_data(c->out), c->out->numel); +} + +static void reduce_after(void *ctx) { + ReduceCtx *c = ctx; + if (!st_reduce_max_into(c->x, c->mask, c->keepdims, c->out)) { + fprintf(stderr, "st_reduce_max_into failed\n"); + exit(1); + } +} + +static void run_layernorm_case(const char *name, size_t ndim, + const size_t *dims, size_t norm_rank) { + size_t elements = 1; + for (size_t i = 0; i < ndim; ++i) { + elements *= dims[i]; + } + size_t group = 1; + for (size_t i = ndim - norm_rank; i < ndim; ++i) { + group *= dims[i]; + } + LayernormCtx ctx; + ctx.x = make_input(ndim, dims); + ctx.gamma = make_input(1, (size_t[1]){group}); + ctx.beta = make_input(1, (size_t[1]){group}); + ctx.out = st_create(ndim, dims); + ctx.norm_rank = norm_rank; + ctx.eps = 1e-5f; + if (ctx.out == NULL) { + exit(1); + } + measure(name, elements, elements, layernorm_before, layernorm_after, &ctx); + st_destroy(ctx.out); + st_destroy(ctx.beta); + st_destroy(ctx.gamma); + st_destroy(ctx.x); +} + +static void run_reduce_case(const char *name, size_t ndim, const size_t *dims, + uint32_t mask) { + size_t elements = 1; + size_t out_elements = 1; + for (size_t i = 0; i < ndim; ++i) { + elements *= dims[i]; + if (!((mask >> i) & 1u)) { + out_elements *= dims[i]; + } + } + size_t out_dims[ST_MAX_DIMS]; + size_t out_ndim = 0; + for (size_t i = 0; i < ndim; ++i) { + if ((mask >> i) & 1u) { + out_dims[out_ndim++] = 1; + } else { + out_dims[out_ndim++] = dims[i]; + } + } + ReduceCtx ctx; + ctx.x = make_input(ndim, dims); + ctx.out = st_create(out_ndim, out_dims); + ctx.mask = mask; + ctx.keepdims = true; + if (ctx.out == NULL) { + exit(1); + } + measure(name, elements, out_elements, reduce_before, reduce_after, &ctx); + st_destroy(ctx.out); + st_destroy(ctx.x); +} + +int main(void) { + printf("| Case | Threads | Elements | nm_lib before (µs) | st kernel (µs) | Speedup |\n"); + printf("| --- | --- | ---: | ---: | ---: | ---: |\n"); + const int default_threads = omp_get_max_threads(); + + omp_set_num_threads(default_threads); + run_layernorm_case("layernorm [1,16,64]", 3, (size_t[3]){1, 16, 64}, 1); + run_layernorm_case("layernorm [2,256,256]", 3, (size_t[3]){2, 256, 256}, 1); + run_layernorm_case("layernorm [4,256,256]", 3, (size_t[3]){4, 256, 256}, 1); + run_layernorm_case("layernorm [4,128,256]", 3, (size_t[3]){4, 128, 256}, 1); + run_layernorm_case("layernorm [8,256,512]", 3, (size_t[3]){8, 256, 512}, 1); + run_layernorm_case("layernorm [4,512,768]", 3, (size_t[3]){4, 512, 768}, 1); + run_layernorm_case("layernorm [16,65,32] rank 2", 3, + (size_t[3]){16, 65, 32}, 2); + run_reduce_case("reduce max [2,4,128,128] axis 3", 4, + (size_t[4]){2, 4, 128, 128}, 0x8); + run_reduce_case("reduce max [1,4,16,16] axis 3", 4, + (size_t[4]){1, 4, 16, 16}, 0x8); + run_reduce_case("reduce max [1,4,128,128] axis 3", 4, + (size_t[4]){1, 4, 128, 128}, 0x8); + run_reduce_case("reduce max [4,128,256] axis 2", 3, + (size_t[3]){4, 128, 256}, 0x4); + run_reduce_case("reduce max [4,128,256] axis 1", 3, + (size_t[3]){4, 128, 256}, 0x2); + run_reduce_case("reduce max [2,1000] axis 0", 2, (size_t[2]){2, 1000}, 0x1); + + omp_set_num_threads(1); + run_layernorm_case("layernorm [1,16,64]", 3, (size_t[3]){1, 16, 64}, 1); + run_layernorm_case("layernorm [2,256,256]", 3, (size_t[3]){2, 256, 256}, 1); + run_layernorm_case("layernorm [4,256,256]", 3, (size_t[3]){4, 256, 256}, 1); + run_layernorm_case("layernorm [4,128,256]", 3, (size_t[3]){4, 128, 256}, 1); + run_layernorm_case("layernorm [8,256,512]", 3, (size_t[3]){8, 256, 512}, 1); + run_layernorm_case("layernorm [4,512,768]", 3, (size_t[3]){4, 512, 768}, 1); + run_reduce_case("reduce max [2,4,128,128] axis 3", 4, + (size_t[4]){2, 4, 128, 128}, 0x8); + run_reduce_case("reduce max [1,4,128,128] axis 3", 4, + (size_t[4]){1, 4, 128, 128}, 0x8); + run_reduce_case("reduce max [4,128,256] axis 2", 3, + (size_t[3]){4, 128, 256}, 0x4); + run_reduce_case("reduce max [4,128,256] axis 1", 3, + (size_t[3]){4, 128, 256}, 0x2); + return 0; +} diff --git a/app/tensor/tests/test_st_f16.c b/app/tensor/tests/test_st_f16.c new file mode 100644 index 0000000..af1debc --- /dev/null +++ b/app/tensor/tests/test_st_f16.c @@ -0,0 +1,310 @@ +/* Bit-exactness tests for the f16 conversion API against a copy of nm_lib's + * nm_halfprec.h reference. */ + +#include "st_f16.h" + +#include +#include +#include +#include +#include + +#define UNITY_INCLUDE_FLOAT +#define TEST_CASE(...) +#include "unity.h" + +void setUp(void) {} +void tearDown(void) {} + +/* ---- Reference (verbatim copy of nm_halfprec.h) ------------------------- */ + +static uint32_t nm_bits_of(float value) { + uint32_t bits; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +static float nm_float_of(uint32_t bits) { + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +static uint16_t nm_f32_to_f16(float value) { + uint32_t bits = nm_bits_of(value); + uint16_t sign = (uint16_t)((bits >> 16) & 0x8000U); + int32_t exponent = (int32_t)((bits >> 23) & 0xffU); + uint32_t mantissa = bits & 0x007fffffU; + if (exponent == 0xffU) + return (uint16_t)(sign | 0x7c00U | (mantissa ? 0x0200U : 0U)); + int32_t half_exponent = exponent - 127 + 15; + if (half_exponent >= 0x1f) + return (uint16_t)(sign | 0x7c00U); + if (half_exponent <= 0) { + if (half_exponent < -10) + return sign; + uint32_t full = mantissa | 0x00800000U; + uint32_t shift = (uint32_t)(14 - half_exponent); + uint32_t half = full >> shift; + uint32_t remainder = full & ((1U << shift) - 1U); + uint32_t halfway = 1U << (shift - 1U); + if (remainder > halfway || (remainder == halfway && (half & 1U))) + half += 1U; + return (uint16_t)(sign | half); + } + uint32_t half = ((uint32_t)half_exponent << 10) | (mantissa >> 13); + uint32_t remainder = mantissa & 0x1fffU; + if (remainder > 0x1000U || (remainder == 0x1000U && (half & 1U))) + half += 1U; + return (uint16_t)(sign | half); +} + +static float nm_f16_to_f32(uint16_t bits) { + uint32_t sign = (uint32_t)(bits & 0x8000U) << 16; + uint32_t exponent = (bits >> 10) & 0x1fU; + uint32_t mantissa = bits & 0x03ffU; + uint32_t out; + if (exponent == 0 && mantissa == 0) { + out = sign; + } else if (exponent == 0) { + int32_t e = 127 - 15 + 1; + while (!(mantissa & 0x0400U)) { + mantissa <<= 1; + --e; + } + out = sign | ((uint32_t)e << 23) | ((mantissa & 0x03ffU) << 13); + } else if (exponent == 0x1f) { + out = sign | 0x7f800000U | (mantissa ? 0x00400000U | (mantissa << 13) : 0U); + } else { + out = sign | ((exponent + 127 - 15) << 23) | (mantissa << 13); + } + return nm_float_of(out); +} + +static float float_of_bits(uint32_t bits) { + float value; + memcpy(&value, &bits, sizeof(value)); + return value; +} + +/* ---- Scalar vs reference ------------------------------------------------- */ + +void test_st_f16_scalar_matches_reference_on_edge_table(void) { + /* +/-0, smallest subnormal, largest finite, overflow, +/-inf, NaN + * payloads, and halfway rounding cases. */ + static const uint32_t f32_patterns[] = { + 0x00000000u, 0x80000000u, /* +/-0 */ + 0x33ff0000u, 0xb3ff0000u, /* near smallest subnormal */ + 0x387f0000u, 0x38800000u, /* 1.0/2 boundary */ + 0x477fe000u, 0x477ff000u, /* near largest finite */ + 0x477ff800u, 0x47800000u, 0x47800800u, /* overflow boundary */ + 0x7f800000u, 0xff800000u, /* +/-inf */ + 0x7f800001u, 0xff800001u, /* signaling NaN */ + 0x7fc00000u, 0xffc00000u, /* quiet NaN */ + 0x7f801000u, 0x7f987a56u, 0x7fffffffu, /* NaN payloads */ + 0x3f000000u, 0x40490fdbu, 0xc0490fdbu, /* 0.5, pi, -pi */ + 0x3f800100u, 0x3f800200u, /* halfway mantissas */ + 0x35000000u, 0x34000000u, 0x33000000u, /* subnormal region */ + 0x4b7fffffu, 0x4b000001u, /* rounding edges */ + }; + for (size_t i = 0; i < sizeof(f32_patterns) / sizeof(f32_patterns[0]); ++i) { + const float value = float_of_bits(f32_patterns[i]); + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(value), st_f32_to_f16(value)); + } + /* Halfway rounding: remainder exactly 0x1000, odd vs even half. */ + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(float_of_bits(0x40000100u)), + st_f32_to_f16(float_of_bits(0x40000100u))); + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(float_of_bits(0x40006100u)), + st_f32_to_f16(float_of_bits(0x40006100u))); +} + +void test_st_f16_scalar_matches_reference_on_all_f16_roundtrips(void) { + /* Every f16 pattern through f32 and back must reproduce the reference + * exactly (non-NaN patterns also round-trip to themselves). */ + for (uint32_t bits = 0; bits <= 0xffffu; ++bits) { + const uint16_t h = (uint16_t)bits; + const float f32 = st_f16_to_f32(h); + TEST_ASSERT_EQUAL_UINT32(nm_bits_of(nm_f16_to_f32(h)), nm_bits_of(f32)); + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(f32), st_f32_to_f16(f32)); + const bool is_nan = (h & 0x7c00u) == 0x7c00u && (h & 0x03ffu) != 0u; + if (!is_nan) { + TEST_ASSERT_EQUAL_UINT16(h, st_f32_to_f16(f32)); + } + } +} + +void test_st_f16_scalar_matches_reference_on_f32_sweep(void) { + /* Strided sweep over all 2^32 f32 bit patterns plus a dense sweep of + * the conversion-critical exponent range (subnormal and overflow + * boundaries of binary16). */ + for (uint64_t bits = 0; bits <= 0xffffffffu; bits += 0x0009e37u) { + const float value = float_of_bits((uint32_t)bits); + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(value), st_f32_to_f16(value)); + } + for (uint32_t exponent = 100; exponent <= 160; ++exponent) { + for (uint32_t mantissa = 0; mantissa <= 0x007fffffu; mantissa += 0x00049249u) { + const uint32_t bits = (exponent << 23) | mantissa; + const float value = float_of_bits(bits); + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(value), st_f32_to_f16(value)); + } + } +} + +/* ---- Bulk vs scalar ------------------------------------------------------- */ + +void test_st_f16_bulk_matches_scalar_for_short_lengths(void) { + float src[40]; + uint16_t dst[40]; + for (size_t n = 0; n <= 33; ++n) { + for (size_t i = 0; i < 40; ++i) { + src[i] = (float)((int)i - 20) * 0.6180339887f + (float)(i % 7) * 1e-4f; + if (i % 11 == 0) { + src[i] = 0.0f; + } + } + st_f32_to_f16_bulk(src, dst, n); + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_EQUAL_UINT16(st_f32_to_f16(src[i]), dst[i]); + } + } +} + +void test_st_f16_bulk_f32_to_f16_matches_scalar_including_nan(void) { + /* The NEON path must blend the reference NaN pattern; include NaN + * payloads, inf, subnormals, and overflow values. */ + float src[1024]; + uint16_t dst[1024]; + uint16_t ref[1024]; + size_t n = 0; + static const uint32_t patterns[] = { + 0x7f800001u, 0x7fc00001u, 0x7fffffffu, 0xff801234u, 0x7f987654u, + 0x00000001u, 0x80000001u, 0x33ff0000u, 0x477ff800u, 0x47800000u, + }; + for (size_t p = 0; p < sizeof(patterns) / sizeof(patterns[0]); ++p) { + src[n++] = float_of_bits(patterns[p]); + src[n++] = -float_of_bits(patterns[p]); + } + for (uint32_t bits = 0x33800000u; bits <= 0x48000000u; bits += 0x001ab551u) { + src[n++] = float_of_bits(bits); + if (n >= 1000) { + break; + } + } + while (n < 1024) { + src[n++] = (float)n * 0.0009765625f; + } + + st_f32_to_f16_bulk(src, dst, n); + for (size_t i = 0; i < n; ++i) { + ref[i] = st_f32_to_f16(src[i]); + } + TEST_ASSERT_EQUAL_MEMORY(ref, dst, n * sizeof(uint16_t)); + /* Also against the copied reference. */ + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(src[i]), dst[i]); + } +} + +void test_st_f16_bulk_f16_to_f32_matches_scalar_all_patterns(void) { + /* All 65536 patterns, converted in one bulk call in chunks that cross + * the 4-lane blocks and tails. */ + uint16_t src[65536]; + for (uint32_t bits = 0; bits <= 0xffffu; ++bits) { + src[bits] = (uint16_t)bits; + } + float dst[65536]; + st_f16_to_f32_bulk(src, dst, 65536); + for (uint32_t bits = 0; bits <= 0xffffu; ++bits) { + TEST_ASSERT_EQUAL_UINT32(nm_bits_of(st_f16_to_f32((uint16_t)bits)), + nm_bits_of(dst[bits])); + } + + /* Short lengths including tails. */ + for (size_t n = 0; n <= 33; ++n) { + float short_dst[40]; + st_f16_to_f32_bulk(src, short_dst, n); + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_EQUAL_UINT32(nm_bits_of(st_f16_to_f32(src[i])), + nm_bits_of(short_dst[i])); + } + } +} + +void test_st_f16_nan_policy_is_documented_behavior(void) { + /* NaN stays NaN, quiet bit set, sign preserved, payload dropped. */ + TEST_ASSERT_EQUAL_UINT16(0x7e00u, st_f32_to_f16(float_of_bits(0x7f800001u))); + TEST_ASSERT_EQUAL_UINT16(0xfe00u, st_f32_to_f16(float_of_bits(0xff801234u))); + TEST_ASSERT_EQUAL_UINT16(0x7c00u, st_f32_to_f16(INFINITY)); + TEST_ASSERT_EQUAL_UINT16(0xfc00u, st_f32_to_f16(-INFINITY)); + /* Sign of +/-0 survives, including underflow to zero. */ + TEST_ASSERT_EQUAL_UINT16(0x0000u, st_f32_to_f16(0.0f)); + TEST_ASSERT_EQUAL_UINT16(0x8000u, st_f32_to_f16(-0.0f)); + TEST_ASSERT_EQUAL_UINT16(0x8000u, st_f32_to_f16(-1e-30f)); + /* Overflow goes to infinity. */ + TEST_ASSERT_EQUAL_UINT16(0x7c00u, st_f32_to_f16(1e30f)); + TEST_ASSERT_EQUAL_UINT16(0xfc00u, st_f32_to_f16(-1e30f)); + /* Smallest subnormal round-trips exactly. */ + const float smallest = st_f16_to_f32(0x0001u); + TEST_ASSERT_EQUAL_UINT16(0x0001u, st_f32_to_f16(smallest)); + /* Largest finite stays finite. */ + const float largest = st_f16_to_f32(0x7bffu); + TEST_ASSERT_EQUAL_UINT16(0x7bffu, st_f32_to_f16(largest)); +} + +void test_st_f16_bulk_subnormals_survive_fast_math(void) { + /* Flush-to-zero would zero the subnormal outputs of the NEON path; this + * builds f32 values around every subnormal half, at and between the + * rounding boundaries, and requires bit-exact bulk == scalar. */ + float src[8192]; + uint16_t dst[8192]; + uint16_t ref[8192]; + size_t n = 0; + for (uint32_t h = 0x0001u; h <= 0x03ffu && n + 5 <= 8192; h += 1u) { + /* f32 values whose halves round around h: exact value and the + * values just below/at/above the halfway points. */ + const uint32_t base = h << 13; + static const uint32_t offsets[] = {0x0000u, 0x0fffu, 0x1000u, + 0x1001u, 0x1fffu}; + for (size_t k = 0; k < 5; ++k) { + src[n++] = float_of_bits(base + offsets[k]); + } + } + /* Denormal-region f32 inputs that round to zero / smallest subnormal. */ + static const uint32_t extra[] = { + 0x33000000u, 0x33fffffdu, 0x34000000u, 0x337fffffu, 0x33800000u, + 0x33ffffffu, 0x33fff800u, 0x33fff7ffu, 0x33fff000u, 0x33efffffu, + }; + for (size_t k = 0; k < sizeof(extra) / sizeof(extra[0]); ++k) { + src[n++] = float_of_bits(extra[k]); + } + TEST_ASSERT_TRUE(n > 0); + + st_f32_to_f16_bulk(src, dst, n); + for (size_t i = 0; i < n; ++i) { + ref[i] = st_f32_to_f16(src[i]); + } + TEST_ASSERT_EQUAL_MEMORY(ref, dst, n * sizeof(uint16_t)); + /* And against the copied nm_lib reference. */ + for (size_t i = 0; i < n; ++i) { + TEST_ASSERT_EQUAL_UINT16(nm_f32_to_f16(src[i]), dst[i]); + } + + /* Full roundtrip through both bulk kernels for all 65536 patterns: + * h -> f32 -> h' must be the identity for non-NaN patterns (NaN + * payloads canonicalize, matching the scalar functions). */ + static uint16_t patterns[65536]; + static float expanded[65536]; + static uint16_t back[65536]; + for (uint32_t bits = 0; bits <= 0xffffu; ++bits) { + patterns[bits] = (uint16_t)bits; + } + st_f16_to_f32_bulk(patterns, expanded, 65536); + st_f32_to_f16_bulk(expanded, back, 65536); + for (uint32_t bits = 0; bits <= 0xffffu; ++bits) { + TEST_ASSERT_EQUAL_UINT16(st_f32_to_f16(expanded[bits]), back[bits]); + const bool is_nan = (bits & 0x7c00u) == 0x7c00u && (bits & 0x03ffu) != 0u; + if (!is_nan) { + TEST_ASSERT_EQUAL_UINT16((uint16_t)bits, back[bits]); + } + } +} diff --git a/app/tensor/tests/test_st_interp.c b/app/tensor/tests/test_st_interp.c new file mode 100644 index 0000000..1470609 --- /dev/null +++ b/app/tensor/tests/test_st_interp.c @@ -0,0 +1,1533 @@ +/* Tests for the graph-interpreter tensor kernels (st_interp.h). */ + +#include "st.h" +#include "st_interp.h" +#include "st_shape_ops.h" + +#include +#include +#include +#include +#include +#include + +#include "st_bf16_utils.h" + +#define UNITY_INCLUDE_FLOAT +#define UNITY_FLOAT_PRECISION 6 +#define TEST_CASE(...) +#include "unity.h" + +void setUp(void) {} +void tearDown(void) {} + +/* ---- Helpers ----------------------------------------------------------- */ + +static float apply_ref_op(StBinOp op, float a, float b) { + switch (op) { + case ST_BIN_ADD: + return a + b; + case ST_BIN_SUB: + return a - b; + case ST_BIN_MUL: + return a * b; + default: + return a / b; + } +} + +static float sample_value(size_t seed) { + /* Deterministic mix of magnitudes and signs; some values are zero so + * division produces IEEE infinities. */ + if (seed % 11 == 0) { + return 0.0f; + } + const float x = (float)((int)(seed % 97) - 48) * 0.137f; + return sinf((float)seed * 1.7f) * (x == 0.0f ? 1.0f : x); +} + +static FloatTensor *make_tensor(size_t ndim, const size_t *shape) { + FloatTensor *t = st_create(ndim, shape); + TEST_ASSERT_NOT_NULL(t); + for (size_t i = 0; i < t->numel; ++i) { + t->values[i] = sample_value(i); + } + return t; +} + +/* Naive scalar reference: NumPy broadcast, one element at a time. */ +static void ref_binary(StBinOp op, const FloatTensor *a, const FloatTensor *b, + float *out, size_t o_ndim, const size_t *o_dims) { + size_t o_numel = 1; + for (size_t i = 0; i < o_ndim; ++i) { + o_numel *= o_dims[i]; + } + size_t index[ST_MAX_DIMS] = {0}; + for (size_t flat = 0; flat < o_numel; ++flat) { + size_t ia = 0, ib = 0; + size_t stride_a = 1, stride_b = 1; + for (size_t d = o_ndim; d-- > 0;) { + const size_t da = d < o_ndim - a->ndim ? 1 : a->shape[d - (o_ndim - a->ndim)]; + const size_t db = d < o_ndim - b->ndim ? 1 : b->shape[d - (o_ndim - b->ndim)]; + ia += index[d] % da * stride_a; + ib += index[d] % db * stride_b; + stride_a *= da; + stride_b *= db; + } + out[flat] = apply_ref_op(op, a->values[ia], b->values[ib]); + for (size_t d = o_ndim; d-- > 0;) { + if (++index[d] < o_dims[d]) { + break; + } + index[d] = 0; + } + } +} + +static void check_binary_case(StBinOp op, const FloatTensor *a, + const FloatTensor *b) { + size_t o_ndim = 0; + size_t o_dims[ST_MAX_DIMS]; + const size_t ndim = a->ndim > b->ndim ? a->ndim : b->ndim; + for (size_t i = 0; i < ndim; ++i) { + const size_t da = i < ndim - a->ndim ? 1 : a->shape[i - (ndim - a->ndim)]; + const size_t db = i < ndim - b->ndim ? 1 : b->shape[i - (ndim - b->ndim)]; + TEST_ASSERT_TRUE(da == db || da == 1 || db == 1); + o_dims[i] = da == 1 ? db : da; + } + o_ndim = ndim; + size_t o_numel = 1; + for (size_t i = 0; i < o_ndim; ++i) { + o_numel *= o_dims[i]; + } + + float *expected = malloc(o_numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_binary(op, a, b, expected, o_ndim, o_dims); + + FloatTensor *out = st_binary_broadcast(op, a, b); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_UINT(o_ndim, out->ndim); + for (size_t i = 0; i < o_ndim; ++i) { + TEST_ASSERT_EQUAL_UINT(o_dims[i], out->shape[i]); + } + TEST_ASSERT_EQUAL_UINT(ST_DTYPE_F32, out->dtype); + TEST_ASSERT_EQUAL_MEMORY(expected, out->values, o_numel * sizeof(float)); + + /* The _into variant writes the same bits into a prepared output. */ + FloatTensor *into = st_create(o_ndim, o_dims); + TEST_ASSERT_NOT_NULL(into); + for (size_t i = 0; i < into->numel; ++i) { + into->values[i] = -12345.0f; + } + TEST_ASSERT_TRUE(st_binary_broadcast_into(op, a, b, into)); + TEST_ASSERT_EQUAL_MEMORY(expected, into->values, o_numel * sizeof(float)); + + free(expected); + st_destroy(into); + st_destroy(out); +} + +/* ---- Shape coverage ---------------------------------------------------- */ + +void test_st_binary_broadcast_covers_shapes_and_ops(void) { + static const struct { + size_t ndim_a; + size_t a[ST_MAX_DIMS]; + size_t ndim_b; + size_t b[ST_MAX_DIMS]; + } cases[] = { + {.ndim_a = 4, .a = {2, 3, 4, 5}, .ndim_b = 4, .b = {2, 3, 4, 5}}, + {.ndim_a = 1, .a = {1}, .ndim_b = 2, .b = {3, 4}}, + {.ndim_a = 2, .a = {3, 4}, .ndim_b = 1, .b = {1}}, + {.ndim_a = 3, .a = {2, 3, 7}, .ndim_b = 1, .b = {7}}, + {.ndim_a = 4, .a = {2, 3, 4, 5}, .ndim_b = 2, .b = {4, 5}}, + {.ndim_a = 4, .a = {2, 3, 4, 5}, .ndim_b = 1, .b = {5}}, + {.ndim_a = 3, .a = {2, 1, 4}, .ndim_b = 3, .b = {2, 3, 1}}, + {.ndim_a = 3, .a = {2, 1, 4}, .ndim_b = 3, .b = {1, 3, 1}}, + {.ndim_a = 3, .a = {1, 3, 1}, .ndim_b = 3, .b = {2, 1, 5}}, + {.ndim_a = 1, .a = {7}, .ndim_b = 2, .b = {2, 7}}, + {.ndim_a = 3, .a = {2, 1, 5}, .ndim_b = 3, .b = {2, 3, 1}}, + {.ndim_a = 4, .a = {2, 2, 4, 4}, .ndim_b = 3, .b = {1, 1, 4}}, + {.ndim_a = 4, .a = {1, 1, 1, 1}, .ndim_b = 2, .b = {2, 3}}, + }; + const StBinOp ops[] = {ST_BIN_ADD, ST_BIN_SUB, ST_BIN_MUL, ST_BIN_DIV}; + + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); ++c) { + FloatTensor *a = make_tensor(cases[c].ndim_a, cases[c].a); + FloatTensor *b = make_tensor(cases[c].ndim_b, cases[c].b); + for (size_t o = 0; o < sizeof(ops) / sizeof(ops[0]); ++o) { + check_binary_case(ops[o], a, b); + } + st_destroy(a); + st_destroy(b); + } +} + +void test_st_binary_broadcast_covers_sizes(void) { + static const size_t sizes[] = {1, 7, 999, 1000, 100001}; + for (size_t s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) { + const size_t shape_a[1] = {sizes[s]}; + const size_t shape_b[1] = {sizes[s]}; + FloatTensor *a = make_tensor(1, shape_a); + FloatTensor *b = make_tensor(1, shape_b); + check_binary_case(ST_BIN_ADD, a, b); + check_binary_case(ST_BIN_DIV, a, b); + st_destroy(a); + st_destroy(b); + } + + /* Zero-sized shapes are rejected at creation and by the kernels. */ + TEST_ASSERT_NULL(st_create(1, (size_t[]){0})); +} + +void test_st_binary_broadcast_scalar_sizes_with_broadcast(void) { + /* Broadcast against a scalar with odd lengths (SIMD tails). */ + static const size_t sizes[] = {1, 7, 1000, 100003}; + for (size_t s = 0; s < sizeof(sizes) / sizeof(sizes[0]); ++s) { + const size_t shape_a[1] = {sizes[s]}; + const size_t shape_one[1] = {1}; + FloatTensor *a = make_tensor(1, shape_a); + FloatTensor *b = make_tensor(1, shape_one); + check_binary_case(ST_BIN_MUL, a, b); + check_binary_case(ST_BIN_SUB, b, a); + st_destroy(a); + st_destroy(b); + } +} + +/* ---- Error cases ------------------------------------------------------- */ + +void test_st_binary_broadcast_rejects_bad_input(void) { + const size_t shape[2] = {2, 3}; + const size_t other[2] = {3, 4}; + FloatTensor *a = make_tensor(2, shape); + FloatTensor *b = make_tensor(2, other); + + /* Shapes that do not broadcast. */ + TEST_ASSERT_NULL(st_binary_broadcast(ST_BIN_ADD, a, b)); + TEST_ASSERT_NULL(st_binary_broadcast(ST_BIN_ADD, b, a)); + + /* NULL inputs. */ + TEST_ASSERT_NULL(st_binary_broadcast(ST_BIN_ADD, NULL, a)); + TEST_ASSERT_NULL(st_binary_broadcast(ST_BIN_ADD, a, NULL)); + + /* Invalid op. */ + TEST_ASSERT_NULL(st_binary_broadcast((StBinOp)77, a, a)); + + /* _into: wrong output shape, NULL, invalid op; nothing is written. */ + FloatTensor *out = st_create(2, other); + TEST_ASSERT_NOT_NULL(out); + for (size_t i = 0; i < out->numel; ++i) { + out->values[i] = 42.0f; + } + TEST_ASSERT_FALSE(st_binary_broadcast_into(ST_BIN_ADD, a, b, out)); + TEST_ASSERT_FALSE(st_binary_broadcast_into(ST_BIN_ADD, NULL, b, out)); + TEST_ASSERT_FALSE(st_binary_broadcast_into(ST_BIN_ADD, a, b, NULL)); + TEST_ASSERT_FALSE(st_binary_broadcast_into((StBinOp)77, a, a, out)); + for (size_t i = 0; i < out->numel; ++i) { + TEST_ASSERT_EQUAL_FLOAT(42.0f, out->values[i]); + } + + /* Non-contiguous tensors are rejected. */ + FloatTensor *base = make_tensor(2, (size_t[]){2, 3}); + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *view = st_view(base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_NULL(st_binary_broadcast(ST_BIN_ADD, view, view)); + TEST_ASSERT_FALSE(st_binary_broadcast_into(ST_BIN_ADD, view, view, out)); + + st_destroy(view); + st_destroy(base); + st_destroy(out); + st_destroy(b); + st_destroy(a); +} + +/* ---- Determinism and bf16 ---------------------------------------------- */ + +void test_st_binary_broadcast_thread_count_bit_identical(void) { + /* Above the OpenMP threshold the row split must not change any bit. */ + const size_t shape_a[3] = {32, 1025, 64}; /* > 2M elements, odd middle */ + const size_t shape_b[3] = {32, 1, 64}; + FloatTensor *a = make_tensor(3, shape_a); + FloatTensor *b = make_tensor(3, shape_b); + + FloatTensor *one = NULL; + FloatTensor *eight = NULL; + omp_set_num_threads(1); + one = st_binary_broadcast(ST_BIN_ADD, a, b); + TEST_ASSERT_NOT_NULL(one); + omp_set_num_threads(8); + eight = st_binary_broadcast(ST_BIN_ADD, a, b); + TEST_ASSERT_NOT_NULL(eight); + TEST_ASSERT_EQUAL_MEMORY(one->values, eight->values, + one->numel * sizeof(float)); + + st_destroy(eight); + st_destroy(one); + st_destroy(b); + st_destroy(a); +} + +void test_st_binary_broadcast_promotes_bf16(void) { + const size_t shape[2] = {3, 7}; + FloatTensor *a_f32 = make_tensor(2, shape); + FloatTensor *b_f32 = make_tensor(2, shape); + FloatTensor *a = st_to_bf16(a_f32); + FloatTensor *b = st_to_bf16(b_f32); + TEST_ASSERT_NOT_NULL(a); + TEST_ASSERT_NOT_NULL(b); + + /* bf16 inputs promote to f32; the result equals the f32 computation on + * the promoted (quantized) inputs. */ + FloatTensor *a_promoted = st_to_f32(a); + FloatTensor *b_promoted = st_to_f32(b); + TEST_ASSERT_NOT_NULL(a_promoted); + TEST_ASSERT_NOT_NULL(b_promoted); + float *expected = malloc(a_promoted->numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_binary(ST_BIN_ADD, a_promoted, b_promoted, expected, 2, shape); + + FloatTensor *out = st_binary_broadcast(ST_BIN_ADD, a, b); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_UINT(ST_DTYPE_F32, out->dtype); + TEST_ASSERT_EQUAL_MEMORY(expected, out->values, out->numel * sizeof(float)); + + /* bf16 output is rounded back with the st conversion helpers. */ + FloatTensor *out_bf16 = st_create_bf16(2, shape); + TEST_ASSERT_NOT_NULL(out_bf16); + TEST_ASSERT_TRUE(st_binary_broadcast_into(ST_BIN_ADD, a_promoted, + b_promoted, out_bf16)); + uint16_t *expected_bf16 = malloc(out->numel * sizeof(uint16_t)); + TEST_ASSERT_NOT_NULL(expected_bf16); + st_f32_to_bf16_bulk(expected, expected_bf16, out->numel); + TEST_ASSERT_EQUAL_MEMORY(expected_bf16, out_bf16->values, + out->numel * sizeof(uint16_t)); + + free(expected_bf16); + free(expected); + st_destroy(out_bf16); + st_destroy(out); + st_destroy(b_promoted); + st_destroy(a_promoted); + st_destroy(b); + st_destroy(a); + st_destroy(b_f32); + st_destroy(a_f32); +} + +/* ---- LayerNorm ---------------------------------------------------------- */ + +/* Tolerance relative to the reference INTERMEDIATE magnitude: the + * library's -ffast-math build rounds the apply chain differently from this + * -O0 reference (reassociation and FMA), each difference bounded by an ulp + * of the operands. A result-relative tolerance blows up on outputs where + * the scaled value cancels against beta, so the reference also reports + * |t| + |b| (t = y * inv * scale) and the check scales with that. */ +static bool near_layernorm_sum8(float actual, float expected, float bound) { + return fabsf(actual - expected) <= 1.0e-6f + 1.0e-5f * bound; +} + +/* nm_lib's run_layernorm verbatim (sum8 mean, eight-lane variance), so the + * kernel is checked against the reference semantics itself. `bound` + * (optional) receives |y * inv * scale| + |beta| per element for the + * intermediate-magnitude tolerance. */ +static void ref_layernorm_sum8(const float *in, const float *gamma, + const float *beta, size_t group, size_t groups, + float eps, float *out, float *bound) { + for (size_t g = 0; g < groups; ++g) { + const float *x = in + g * group; + float *y = out + g * group; + float lanes[8] = {0.0f}; + size_t j = 0; + for (; j + 8 <= group; j += 8) { + for (size_t l = 0; l < 8; ++l) { + lanes[l] += x[j + l]; + } + } + float mean = 0.0f; + for (; j < group; ++j) { + mean += x[j]; + } + for (size_t l = 0; l < 8; ++l) { + mean += lanes[l]; + } + mean /= (float)group; + for (size_t i = 0; i < group; ++i) { + y[i] = x[i] - mean; + } + float var_lanes[8] = {0.0f}; + j = 0; + for (; j + 8 <= group; j += 8) { + for (size_t l = 0; l < 8; ++l) { + var_lanes[l] += y[j + l] * y[j + l]; + } + } + float variance = 0.0f; + for (; j < group; ++j) { + variance += y[j] * y[j]; + } + for (size_t l = 0; l < 8; ++l) { + variance += var_lanes[l]; + } + variance /= (float)group; + const float inverse = 1.0f / sqrtf(variance + eps); + for (size_t i = 0; i < group; ++i) { + float t = y[i] * inverse; + if (gamma != NULL) { + t *= gamma[i]; + } + if (beta != NULL) { + y[i] = t + beta[i]; + } else { + y[i] = t; + } + if (bound != NULL) { + bound[g * group + i] = + fabsf(t) + (beta != NULL ? fabsf(beta[i]) : 0.0f); + } + } + } +} + +static void check_layernorm_case(const FloatTensor *x, const FloatTensor *gamma, + const FloatTensor *beta, size_t norm_rank) { + size_t group = 1; + for (size_t i = x->ndim - norm_rank; i < x->ndim; ++i) { + group *= x->shape[i]; + } + const size_t groups = x->numel / group; + + /* nm_lib reference semantics (sum8), tolerance relative to the + * intermediate magnitude (see near_layernorm_sum8). */ + float *expected_sum8 = malloc(x->numel * sizeof(float)); + float *bound = malloc(x->numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected_sum8); + TEST_ASSERT_NOT_NULL(bound); + ref_layernorm_sum8(x->values, gamma ? gamma->values : NULL, + beta ? beta->values : NULL, group, groups, 1.0e-5f, + expected_sum8, bound); + + FloatTensor *out = st_clone(x); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_TRUE(st_layernorm(x, gamma, beta, norm_rank, 1.0e-5f, out)); + for (size_t i = 0; i < x->numel; ++i) { + TEST_ASSERT_TRUE( + near_layernorm_sum8(out->values[i], expected_sum8[i], bound[i])); + } + /* In-place normalization must produce the same values. */ + FloatTensor *inplace = st_clone(x); + TEST_ASSERT_NOT_NULL(inplace); + TEST_ASSERT_TRUE(st_layernorm(inplace, gamma, beta, norm_rank, 1.0e-5f, inplace)); + for (size_t i = 0; i < x->numel; ++i) { + TEST_ASSERT_TRUE(near_layernorm_sum8(inplace->values[i], + expected_sum8[i], bound[i])); + } + + free(bound); + free(expected_sum8); + st_destroy(inplace); + st_destroy(out); +} + +static size_t group_numel(const FloatTensor *x, size_t norm_rank) { + size_t group = 1; + for (size_t i = x->ndim - norm_rank; i < x->ndim; ++i) { + group *= x->shape[i]; + } + return group; +} + +void test_st_layernorm_matches_reference(void) { + struct { + size_t ndim; + size_t shape[ST_MAX_DIMS]; + size_t norm_rank; + } cases[] = { + {1, {7}, 1}, + {2, {3, 7}, 1}, + {2, {97, 103}, 1}, + {4, {2, 3, 4, 5}, 1}, + {4, {2, 3, 4, 5}, 2}, + {4, {2, 3, 4, 5}, 3}, + {3, {1, 16, 64}, 1}, + {3, {16, 65, 32}, 2}, + {1, {100001}, 1}, + {3, {4, 128, 256}, 1}, + {1, {1}, 1}, + }; + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); ++c) { + FloatTensor *x = make_tensor(cases[c].ndim, cases[c].shape); + const size_t group = group_numel(x, cases[c].norm_rank); + FloatTensor *gamma_full = make_tensor(1, (size_t[1]){group}); + FloatTensor *beta_full = make_tensor(1, (size_t[1]){group}); + + check_layernorm_case(x, NULL, NULL, cases[c].norm_rank); + check_layernorm_case(x, gamma_full, NULL, cases[c].norm_rank); + check_layernorm_case(x, NULL, beta_full, cases[c].norm_rank); + check_layernorm_case(x, gamma_full, beta_full, cases[c].norm_rank); + + st_destroy(beta_full); + st_destroy(gamma_full); + st_destroy(x); + } +} + +void test_st_layernorm_rejects_bad_input(void) { + const size_t shape[2] = {3, 7}; + FloatTensor *x = make_tensor(2, shape); + FloatTensor *out = make_tensor(2, shape); + const size_t group_shape[1] = {7}; + FloatTensor *gamma = make_tensor(1, group_shape); + const size_t wrong[1] = {6}; + FloatTensor *wrong_gamma = make_tensor(1, wrong); + const size_t other_shape[2] = {3, 6}; + FloatTensor *wrong_out = make_tensor(2, other_shape); + FloatTensor *view_base = make_tensor(2, (size_t[]){2, 3}); + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *view = st_view(view_base, 2, view_shape, view_strides, 0); + + TEST_ASSERT_FALSE(st_layernorm(x, gamma, NULL, 0, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, gamma, NULL, 3, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, NULL, 1, -1.0f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, NULL, 1, NAN, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, NULL, 1, 1e-5f, wrong_out)); + TEST_ASSERT_FALSE(st_layernorm(x, wrong_gamma, NULL, 1, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, wrong_gamma, 1, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(NULL, NULL, NULL, 1, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, NULL, 1, 1e-5f, NULL)); + TEST_ASSERT_FALSE(st_layernorm(view, NULL, NULL, 1, 1e-5f, out)); + TEST_ASSERT_FALSE(st_layernorm(x, NULL, NULL, 1, 1e-5f, view)); + + st_destroy(view); + st_destroy(view_base); + st_destroy(wrong_out); + st_destroy(wrong_gamma); + st_destroy(gamma); + st_destroy(out); + st_destroy(x); +} + +void test_st_layernorm_thread_count_bit_identical(void) { + /* Above the parallel threshold; the eight-partial-sum order must keep + * results bit-identical across thread counts. */ + const size_t shape[3] = {32, 129, 64}; /* 264k elements */ + FloatTensor *x = make_tensor(3, shape); + const size_t group_shape[1] = {64}; + FloatTensor *gamma = make_tensor(1, group_shape); + FloatTensor *beta = make_tensor(1, group_shape); + + FloatTensor *one = st_clone(x); + FloatTensor *eight = st_clone(x); + omp_set_num_threads(1); + TEST_ASSERT_TRUE(st_layernorm(x, gamma, beta, 1, 1e-5f, one)); + omp_set_num_threads(8); + TEST_ASSERT_TRUE(st_layernorm(x, gamma, beta, 1, 1e-5f, eight)); + TEST_ASSERT_EQUAL_MEMORY(one->values, eight->values, + one->numel * sizeof(float)); + + st_destroy(eight); + st_destroy(one); + st_destroy(beta); + st_destroy(gamma); + st_destroy(x); +} + +void test_st_layernorm_promotes_bf16(void) { + const size_t shape[2] = {3, 7}; + FloatTensor *x_f32 = make_tensor(2, shape); + const size_t group_shape[1] = {7}; + FloatTensor *gamma_f32 = make_tensor(1, group_shape); + FloatTensor *beta_f32 = make_tensor(1, group_shape); + FloatTensor *x = st_to_bf16(x_f32); + FloatTensor *gamma = st_to_bf16(gamma_f32); + FloatTensor *beta = st_to_bf16(beta_f32); + TEST_ASSERT_NOT_NULL(x); + TEST_ASSERT_NOT_NULL(gamma); + TEST_ASSERT_NOT_NULL(beta); + + /* Reference on the promoted (quantized) inputs, then round to bf16. */ + FloatTensor *x_promoted = st_to_f32(x); + FloatTensor *gamma_promoted = st_to_f32(gamma); + FloatTensor *beta_promoted = st_to_f32(beta); + float *expected = malloc(x->numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_layernorm_sum8(x_promoted->values, gamma_promoted->values, + beta_promoted->values, 7, 3, 1e-5f, expected, NULL); + uint16_t *expected_bf16 = malloc(x->numel * sizeof(uint16_t)); + TEST_ASSERT_NOT_NULL(expected_bf16); + st_f32_to_bf16_bulk(expected, expected_bf16, x->numel); + + FloatTensor *out = st_create_bf16(2, shape); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_TRUE(st_layernorm(x, gamma, beta, 1, 1e-5f, out)); + TEST_ASSERT_EQUAL_MEMORY(expected_bf16, out->values, + x->numel * sizeof(uint16_t)); + + free(expected_bf16); + free(expected); + st_destroy(out); + st_destroy(beta_promoted); + st_destroy(gamma_promoted); + st_destroy(x_promoted); + st_destroy(beta); + st_destroy(gamma); + st_destroy(x); + st_destroy(beta_f32); + st_destroy(gamma_f32); + st_destroy(x_f32); +} + +/* ---- Reduce max ---------------------------------------------------------- */ + +static uint32_t float_bits_of(float value) { + uint32_t bits = 0; + memcpy(&bits, &value, sizeof(bits)); + return bits; +} + +/* Naive reference with the documented NaN behavior: the first NaN in scan + * order wins, otherwise the finite maximum with -INFINITY start. */ +static float ref_reduce_slot(const float *x, size_t n) { + float best = -INFINITY; + for (size_t i = 0; i < n; ++i) { + const uint32_t bits = float_bits_of(x[i]); + if ((bits & 0x7f800000u) == 0x7f800000u && (bits & 0x007fffffu) != 0) { + return x[i]; + } + if (x[i] > best) { + best = x[i]; + } + } + return best; +} + +/* Flat walk over the input in memory order; the first NaN of a slot in + * scan order sticks, otherwise the finite maximum with -INFINITY start. + * This matches the kernel's documented scan order. */ +static void ref_reduce_max(const FloatTensor *x, uint32_t mask, float *out, + size_t out_numel) { + for (size_t i = 0; i < out_numel; ++i) { + out[i] = -INFINITY; + } + size_t index[ST_MAX_DIMS] = {0}; + for (size_t flat = 0; flat < x->numel; ++flat) { + size_t slot = 0; + size_t slot_stride = 1; + for (size_t a = x->ndim; a-- > 0;) { + if ((mask >> a) & 1u) { + continue; + } + slot += index[a] * slot_stride; + slot_stride *= x->shape[a]; + } + const uint32_t out_bits = float_bits_of(out[slot]); + const bool out_is_nan = (out_bits & 0x7f800000u) == 0x7f800000u && + (out_bits & 0x007fffffu) != 0; + if (out_is_nan) { + continue; /* the first NaN already won */ + } + const float value = x->values[flat]; + const uint32_t bits = float_bits_of(value); + if ((bits & 0x7f800000u) == 0x7f800000u && (bits & 0x007fffffu) != 0) { + out[slot] = value; /* first NaN in scan order */ + } else if (value > out[slot]) { + out[slot] = value; + } + for (size_t a = x->ndim; a-- > 0;) { + if (++index[a] < x->shape[a]) { + break; + } + index[a] = 0; + } + } +} + +static void check_reduce_case(const FloatTensor *x, uint32_t mask, + bool keepdims) { + FloatTensor *actual = st_reduce_max(x, mask, keepdims); + TEST_ASSERT_NOT_NULL(actual); + size_t expected_numel = 1; + for (size_t a = 0; a < x->ndim; ++a) { + if ((mask >> a) & 1u) { + if (keepdims) { + /* still a 1-axis */ + } + } else { + expected_numel *= x->shape[a]; + } + } + TEST_ASSERT_EQUAL_UINT(expected_numel, actual->numel); + float *expected = malloc(expected_numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_reduce_max(x, mask, expected, expected_numel); + TEST_ASSERT_EQUAL_MEMORY(expected, actual->values, + expected_numel * sizeof(float)); + + /* _into with keepdims shape must produce the same bits. */ + size_t into_dims[ST_MAX_DIMS]; + size_t into_ndim = 0; + for (size_t a = 0; a < x->ndim; ++a) { + if ((mask >> a) & 1u) { + if (keepdims) { + into_dims[into_ndim++] = 1; + } + } else { + into_dims[into_ndim++] = x->shape[a]; + } + } + FloatTensor *into = st_create(into_ndim == 0 ? 1 : into_ndim, + into_ndim == 0 ? (size_t[1]){1} : into_dims); + TEST_ASSERT_NOT_NULL(into); + TEST_ASSERT_TRUE(st_reduce_max_into(x, mask, keepdims, into)); + TEST_ASSERT_EQUAL_MEMORY(expected, into->values, + expected_numel * sizeof(float)); + + free(expected); + st_destroy(into); + st_destroy(actual); +} + +void test_st_reduce_max_matches_reference(void) { + struct { + size_t ndim; + size_t shape[ST_MAX_DIMS]; + uint32_t mask; + } cases[] = { + {1, {7}, 0x1}, + {1, {1000}, 0x1}, + {1, {100001}, 0x1}, + {2, {2, 3}, 0x1}, + {2, {2, 3}, 0x2}, + {2, {2, 3}, 0x3}, + {2, {97, 103}, 0x2}, + {3, {2, 3, 4}, 0x1}, + {3, {2, 3, 4}, 0x2}, + {3, {2, 3, 4}, 0x4}, + {3, {2, 3, 4}, 0x5}, + {3, {2, 3, 4}, 0x7}, + {4, {2, 3, 4, 5}, 0x8}, + {4, {2, 3, 4, 5}, 0x3}, + {4, {2, 3, 4, 5}, 0xf}, + {4, {2, 4, 129, 128}, 0x8}, + {3, {4, 128, 256}, 0x4}, + {3, {4, 128, 256}, 0x2}, + }; + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); ++c) { + FloatTensor *x = make_tensor(cases[c].ndim, cases[c].shape); + check_reduce_case(x, cases[c].mask, false); + check_reduce_case(x, cases[c].mask, true); + st_destroy(x); + } +} + +void test_st_reduce_max_empty_mask_copies_input(void) { + const size_t shape[2] = {3, 5}; + FloatTensor *x = make_tensor(2, shape); + FloatTensor *out = st_reduce_max(x, 0, false); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_UINT(2, out->ndim); + TEST_ASSERT_EQUAL_MEMORY(x->values, out->values, x->numel * sizeof(float)); + st_destroy(out); + st_destroy(x); +} + +void test_st_reduce_max_propagates_nan(void) { + /* Documented behavior: a NaN in a reduction group makes the output slot + * the bit pattern of the first NaN in scan order; other slots stay + * finite. */ + const size_t shape[2] = {3, 4}; + FloatTensor *x = make_tensor(2, shape); + const uint32_t nan_bits_a = 0x7fc00001u; + const uint32_t nan_bits_b = 0x7f800100u; + float nan_a, nan_b; + memcpy(&nan_a, &nan_bits_a, sizeof(nan_a)); + memcpy(&nan_b, &nan_bits_b, sizeof(nan_b)); + x->values[1 * 4 + 2] = nan_a; /* slot 1 */ + x->values[2 * 4 + 0] = nan_b; /* slot 2 */ + x->values[2 * 4 + 3] = nan_a; /* slot 2, later NaN: must not win */ + + FloatTensor *out = st_reduce_max(x, 0x2, false); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_UINT(3, out->numel); + TEST_ASSERT_EQUAL_UINT32(nan_bits_a, float_bits_of(out->values[1])); + TEST_ASSERT_EQUAL_UINT32(nan_bits_b, float_bits_of(out->values[2])); + TEST_ASSERT_EQUAL_FLOAT(3.0f, out->values[0]); + + st_destroy(out); + st_destroy(x); +} + +void test_st_reduce_max_keeps_infinity(void) { + const size_t shape[2] = {2, 3}; + FloatTensor *x = make_tensor(2, shape); + for (size_t i = 0; i < x->numel; ++i) { + x->values[i] = -INFINITY; + } + x->values[4] = INFINITY; + FloatTensor *out = st_reduce_max(x, 0x2, false); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_FLOAT(-INFINITY, out->values[0]); + TEST_ASSERT_EQUAL_FLOAT(INFINITY, out->values[1]); + st_destroy(out); + st_destroy(x); +} + +void test_st_reduce_max_rejects_bad_input(void) { + const size_t shape[2] = {2, 3}; + FloatTensor *x = make_tensor(2, shape); + const size_t wrong_shape[2] = {3, 2}; + FloatTensor *wrong = make_tensor(2, wrong_shape); + + /* Mask bit at or above the rank. */ + TEST_ASSERT_NULL(st_reduce_max(x, 0x4, false)); + TEST_ASSERT_NULL(st_reduce_max(x, 0x8, true)); + TEST_ASSERT_NULL(st_reduce_max(NULL, 0x1, false)); + TEST_ASSERT_FALSE(st_reduce_max_into(x, 0x4, false, wrong)); + TEST_ASSERT_FALSE(st_reduce_max_into(x, 0x1, false, NULL)); + TEST_ASSERT_FALSE(st_reduce_max_into(NULL, 0x1, false, wrong)); + + /* Non-contiguous input is rejected. */ + FloatTensor *base = make_tensor(2, (size_t[]){2, 3}); + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *view = st_view(base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_NULL(st_reduce_max(view, 0x1, false)); + st_destroy(view); + st_destroy(base); + + st_destroy(wrong); + st_destroy(x); +} + +void test_st_reduce_max_thread_count_bit_identical(void) { + /* Above the parallel threshold; slots are independent so results must be + * bit-identical across thread counts. */ + const size_t shape[4] = {2, 4, 1025, 128}; /* > 1M input, 8200 slots */ + FloatTensor *x = make_tensor(4, shape); + omp_set_num_threads(1); + FloatTensor *one = st_reduce_max(x, 0x8, false); + omp_set_num_threads(8); + FloatTensor *eight = st_reduce_max(x, 0x8, false); + TEST_ASSERT_NOT_NULL(one); + TEST_ASSERT_NOT_NULL(eight); + TEST_ASSERT_EQUAL_MEMORY(one->values, eight->values, + one->numel * sizeof(float)); + st_destroy(eight); + st_destroy(one); + st_destroy(x); +} + +void test_st_reduce_max_promotes_bf16(void) { + const size_t shape[2] = {3, 7}; + FloatTensor *x_f32 = make_tensor(2, shape); + FloatTensor *x = st_to_bf16(x_f32); + TEST_ASSERT_NOT_NULL(x); + FloatTensor *promoted = st_to_f32(x); + TEST_ASSERT_NOT_NULL(promoted); + + float *expected = malloc(3 * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + for (size_t slot = 0; slot < 3; ++slot) { + expected[slot] = ref_reduce_slot(promoted->values + slot * 7, 7); + } + FloatTensor *out = st_reduce_max(x, 0x2, false); + TEST_ASSERT_NOT_NULL(out); + TEST_ASSERT_EQUAL_UINT(ST_DTYPE_F32, out->dtype); + TEST_ASSERT_EQUAL_MEMORY(expected, out->values, 3 * sizeof(float)); + + FloatTensor *out_bf16 = st_create_bf16(1, (size_t[1]){3}); + TEST_ASSERT_NOT_NULL(out_bf16); + TEST_ASSERT_TRUE(st_reduce_max_into(x, 0x2, false, out_bf16)); + FloatTensor *back = st_to_f32(out_bf16); + TEST_ASSERT_NOT_NULL(back); + for (size_t slot = 0; slot < 3; ++slot) { + const uint16_t expected_bf16 = st_f32_to_bf16(expected[slot]); + const uint16_t actual_bf16 = ((const uint16_t *)out_bf16->values)[slot]; + TEST_ASSERT_EQUAL_UINT16(expected_bf16, actual_bf16); + } + + free(expected); + st_destroy(back); + st_destroy(out_bf16); + st_destroy(out); + st_destroy(promoted); + st_destroy(x); + st_destroy(x_f32); +} + +/* ---- Batched matmul ------------------------------------------------------ */ + +/* Naive batched matmul reference. */ +static void ref_matmul(const FloatTensor *a, const FloatTensor *b, + FloatTensor *c) { + const size_t m = a->shape[a->ndim - 2]; + const size_t k = a->shape[a->ndim - 1]; + const size_t n = b->shape[b->ndim - 1]; + const size_t batch_a = a->ndim - 2; + const size_t batch_b = b->ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + size_t d[ST_MAX_DIMS]; + size_t da[ST_MAX_DIMS]; + size_t db[ST_MAX_DIMS]; + for (size_t i = 0; i < batch_rank; ++i) { + da[i] = i < batch_rank - batch_a ? 1 : a->shape[i - (batch_rank - batch_a)]; + db[i] = i < batch_rank - batch_b ? 1 : b->shape[i - (batch_rank - batch_b)]; + d[i] = da[i] == 1 ? db[i] : da[i]; + } + size_t batches = 1; + for (size_t i = 0; i < batch_rank; ++i) { + batches *= d[i]; + } + const bool single_b = b->numel == k * n; + for (size_t index = 0; index < batches; ++index) { + size_t rest = index; + size_t offset_a = 0; + size_t offset_b = 0; + size_t scale_a = 1; + size_t scale_b = 1; + for (size_t i = batch_rank; i-- > 0;) { + const size_t coordinate = rest % d[i]; + rest /= d[i]; + offset_a += (da[i] == 1 ? 0 : coordinate) * scale_a; + offset_b += (db[i] == 1 ? 0 : coordinate) * scale_b; + scale_a *= da[i]; + scale_b *= db[i]; + } + const float *pa = a->values + offset_a * m * k; + const float *pb = b->values + (single_b ? 0 : offset_b * k * n); + float *pc = c->values + index * m * n; + for (size_t i = 0; i < m; ++i) { + for (size_t j = 0; j < n; ++j) { + float sum = 0.0f; + for (size_t p = 0; p < k; ++p) { + sum += pa[i * k + p] * pb[p * n + j]; + } + pc[i * n + j] = sum; + } + } + } +} + +static bool near_matmul(float actual, float expected) { + /* BLAS accumulation order differs from the naive reference. */ + return fabsf(actual - expected) <= 1.0e-4f * (1.0f + fabsf(expected)); +} + +static void check_matmul_case(const FloatTensor *a, const FloatTensor *b) { + const size_t m = a->shape[a->ndim - 2]; + const size_t k = a->shape[a->ndim - 1]; + const size_t n = b->shape[b->ndim - 1]; + const size_t batch_a = a->ndim - 2; + const size_t batch_b = b->ndim - 2; + const size_t batch_rank = batch_a > batch_b ? batch_a : batch_b; + size_t out_dims[ST_MAX_DIMS]; + size_t out_ndim = 0; + for (size_t i = 0; i < batch_rank; ++i) { + const size_t da = + i < batch_rank - batch_a ? 1 : a->shape[i - (batch_rank - batch_a)]; + const size_t db = + i < batch_rank - batch_b ? 1 : b->shape[i - (batch_rank - batch_b)]; + out_dims[out_ndim++] = da == 1 ? db : da; + } + out_dims[out_ndim++] = m; + out_dims[out_ndim++] = n; + + FloatTensor *c = st_create(out_ndim, out_dims); + TEST_ASSERT_NOT_NULL(c); + FloatTensor *expected = st_create(out_ndim, out_dims); + TEST_ASSERT_NOT_NULL(expected); + ref_matmul(a, b, expected); + + TEST_ASSERT_TRUE(st_matmul_batched(a, b, c)); + for (size_t i = 0; i < c->numel; ++i) { + TEST_ASSERT_TRUE(near_matmul(c->values[i], expected->values[i])); + } + + st_destroy(expected); + st_destroy(c); +} + +void test_st_matmul_batched_matches_reference(void) { + static const struct { + size_t ndim_a; + size_t a[ST_MAX_DIMS]; + size_t ndim_b; + size_t b[ST_MAX_DIMS]; + } cases[] = { + {.ndim_a = 2, .a = {7, 5}, .ndim_b = 2, .b = {5, 3}}, + {.ndim_a = 2, .a = {1, 1}, .ndim_b = 2, .b = {1, 1}}, + {.ndim_a = 2, .a = {16, 65}, .ndim_b = 2, .b = {65, 32}}, + {.ndim_a = 3, .a = {3, 4, 5}, .ndim_b = 2, .b = {5, 7}}, + {.ndim_a = 3, .a = {2, 3, 7}, .ndim_b = 3, .b = {2, 7, 5}}, + {.ndim_a = 3, .a = {2, 3, 7}, .ndim_b = 2, .b = {7, 5}}, + {.ndim_a = 4, .a = {2, 3, 4, 5}, .ndim_b = 4, .b = {2, 3, 5, 7}}, + {.ndim_a = 4, .a = {2, 3, 4, 5}, .ndim_b = 4, .b = {2, 1, 5, 7}}, + {.ndim_a = 4, .a = {2, 1, 4, 5}, .ndim_b = 4, .b = {2, 3, 5, 7}}, + {.ndim_a = 4, .a = {2, 1, 4, 5}, .ndim_b = 3, .b = {2, 5, 7}}, + {.ndim_a = 3, .a = {1, 16, 64}, .ndim_b = 2, .b = {64, 64}}, + {.ndim_a = 3, .a = {4, 128, 256}, .ndim_b = 2, .b = {256, 64}}, + {.ndim_a = 4, .a = {2, 4, 128, 64}, .ndim_b = 4, .b = {2, 4, 64, 128}}, + {.ndim_a = 4, .a = {1, 4, 16, 64}, .ndim_b = 4, .b = {1, 4, 64, 16}}, + {.ndim_a = 2, .a = {97, 3}, .ndim_b = 2, .b = {3, 101}}, + }; + for (size_t c = 0; c < sizeof(cases) / sizeof(cases[0]); ++c) { + FloatTensor *a = make_tensor(cases[c].ndim_a, cases[c].a); + FloatTensor *b = make_tensor(cases[c].ndim_b, cases[c].b); + check_matmul_case(a, b); + st_destroy(b); + st_destroy(a); + } +} + +void test_st_matmul_batched_rejects_bad_input(void) { + FloatTensor *a = make_tensor(2, (size_t[2]){3, 5}); + FloatTensor *bad_k = make_tensor(2, (size_t[2]){4, 7}); + FloatTensor *rank1 = make_tensor(1, (size_t[1]){5}); + FloatTensor *c = make_tensor(2, (size_t[2]){3, 7}); + FloatTensor *wrong_c = make_tensor(2, (size_t[2]){7, 3}); + const size_t b_shape[2] = {5, 7}; + + TEST_ASSERT_FALSE(st_matmul_batched(a, bad_k, c)); + TEST_ASSERT_FALSE(st_matmul_batched(rank1, rank1, c)); + TEST_ASSERT_FALSE(st_matmul_batched(a, rank1, c)); + TEST_ASSERT_FALSE(st_matmul_batched(a, a, wrong_c)); + TEST_ASSERT_FALSE(st_matmul_batched(NULL, a, c)); + TEST_ASSERT_FALSE(st_matmul_batched(a, NULL, c)); + TEST_ASSERT_FALSE(st_matmul_batched(a, a, NULL)); + + /* Nothing written on failure. */ + for (size_t i = 0; i < wrong_c->numel; ++i) { + wrong_c->values[i] = 42.0f; + } + TEST_ASSERT_FALSE(st_matmul_batched(a, bad_k, c) == false && false); + TEST_ASSERT_FALSE(st_matmul_batched(a, bad_k, wrong_c)); + for (size_t i = 0; i < wrong_c->numel; ++i) { + TEST_ASSERT_EQUAL_FLOAT(42.0f, wrong_c->values[i]); + } + + /* Non-contiguous operands are rejected. */ + FloatTensor *base = make_tensor(2, (size_t[2]){2, 3}); + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *view = st_view(base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_FALSE(st_matmul_batched(view, view, c)); + st_destroy(view); + st_destroy(base); + + st_destroy(wrong_c); + st_destroy(c); + st_destroy(bad_k); + st_destroy(a); + (void)b_shape; +} + +void test_st_matmul_batched_thread_count_within_tolerance(void) { + /* MatMul is the one interpreter kernel that is NOT bit-identical across + * thread counts: the BLAS backend's accumulation order may differ, so + * the comparison uses the documented 1e-4 relative tolerance instead. */ + /* Eight batches, each below the 262144-work parallel GEMM ceiling. */ + const size_t a_shape[4] = {2, 4, 63, 32}; + const size_t b_shape[4] = {2, 4, 32, 129}; + FloatTensor *a = make_tensor(4, a_shape); + FloatTensor *b = make_tensor(4, b_shape); + const size_t c_shape[4] = {2, 4, 63, 129}; + FloatTensor *one = st_create(4, c_shape); + FloatTensor *eight = st_create(4, c_shape); + TEST_ASSERT_NOT_NULL(one); + TEST_ASSERT_NOT_NULL(eight); + + omp_set_num_threads(1); + TEST_ASSERT_TRUE(st_matmul_batched(a, b, one)); + omp_set_num_threads(8); + TEST_ASSERT_TRUE(st_matmul_batched(a, b, eight)); + for (size_t i = 0; i < one->numel; ++i) { + TEST_ASSERT_TRUE(near_matmul(one->values[i], eight->values[i])); + } + + st_destroy(eight); + st_destroy(one); + st_destroy(b); + st_destroy(a); +} + +void test_st_matmul_batched_promotes_bf16(void) { + const size_t a_shape[3] = {2, 3, 5}; + const size_t b_shape[2] = {5, 7}; + FloatTensor *a_f32 = make_tensor(3, a_shape); + FloatTensor *b_f32 = make_tensor(2, b_shape); + FloatTensor *a = st_to_bf16(a_f32); + FloatTensor *b = st_to_bf16(b_f32); + TEST_ASSERT_NOT_NULL(a); + TEST_ASSERT_NOT_NULL(b); + FloatTensor *a_promoted = st_to_f32(a); + FloatTensor *b_promoted = st_to_f32(b); + TEST_ASSERT_NOT_NULL(a_promoted); + TEST_ASSERT_NOT_NULL(b_promoted); + + /* Reference on the promoted inputs. */ + const size_t c_shape[3] = {2, 3, 7}; + FloatTensor *expected = st_create(3, c_shape); + TEST_ASSERT_NOT_NULL(expected); + ref_matmul(a_promoted, b_promoted, expected); + + /* f32 output from bf16 operands. */ + FloatTensor *c = st_create(3, c_shape); + TEST_ASSERT_NOT_NULL(c); + TEST_ASSERT_TRUE(st_matmul_batched(a, b, c)); + for (size_t i = 0; i < c->numel; ++i) { + TEST_ASSERT_TRUE(near_matmul(c->values[i], expected->values[i])); + } + + /* bf16 output is rounded back. */ + FloatTensor *c_bf16 = st_create_bf16(3, c_shape); + TEST_ASSERT_NOT_NULL(c_bf16); + TEST_ASSERT_TRUE(st_matmul_batched(a_promoted, b_promoted, c_bf16)); + for (size_t i = 0; i < c->numel; ++i) { + const uint16_t expected_bf16 = st_f32_to_bf16(expected->values[i]); + const uint16_t actual_bf16 = ((const uint16_t *)c_bf16->values)[i]; + /* bf16 rounds to 8 mantissa bits; equal bits or equal promoted values + * within the matmul tolerance both pass. */ + if (expected_bf16 != actual_bf16) { + const float e = st_bf16_to_f32(expected_bf16); + const float g = st_bf16_to_f32(actual_bf16); + TEST_ASSERT_TRUE(fabsf(e - g) <= 1.0e-2f * (1.0f + fabsf(e))); + } + } + + st_destroy(c_bf16); + st_destroy(c); + st_destroy(expected); + st_destroy(b_promoted); + st_destroy(a_promoted); + st_destroy(b); + st_destroy(a); + st_destroy(b_f32); + st_destroy(a_f32); +} + +/* ---- Gather -------------------------------------------------------------- */ + +/* Naive gather reference: out[o, j, inner] = table[o, indices[j], inner]. */ +static void ref_gather(const FloatTensor *table, const int64_t *indices, + size_t count, size_t axis, float *out) { + size_t outer = 1; + size_t inner = 1; + for (size_t i = 0; i < axis; ++i) { + outer *= table->shape[i]; + } + for (size_t i = axis + 1; i < table->ndim; ++i) { + inner *= table->shape[i]; + } + const size_t dim = table->shape[axis]; + for (size_t o = 0; o < outer; ++o) { + for (size_t j = 0; j < count; ++j) { + const int64_t index = indices[j] < 0 ? indices[j] + (int64_t)dim : indices[j]; + memcpy(out + (o * count + j) * inner, + table->values + (o * dim + (size_t)index) * inner, + inner * sizeof(float)); + } + } +} + +static void check_gather_case(const FloatTensor *table, const int64_t *indices, + size_t count, size_t axis) { + size_t out_dims[ST_MAX_DIMS]; + memcpy(out_dims, table->shape, table->ndim * sizeof(size_t)); + out_dims[axis] = count; + FloatTensor *out = st_create(table->ndim, out_dims); + TEST_ASSERT_NOT_NULL(out); + float *expected = malloc(st_tensor_numel(out) * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_gather(table, indices, count, axis, expected); + + TEST_ASSERT_TRUE(st_gather(table, indices, count, axis, out)); + TEST_ASSERT_EQUAL_MEMORY(expected, out->values, + st_tensor_numel(out) * sizeof(float)); + free(expected); + st_destroy(out); +} + +void test_st_gather_matches_reference(void) { + const size_t table_shapes[][4] = { + {7}, {16, 7}, {100, 7, 3}, {2, 3, 5, 7}, {30000, 16}, + }; + const size_t ranks[] = {1, 2, 3, 4, 2}; + for (size_t c = 0; c < sizeof(ranks) / sizeof(ranks[0]); ++c) { + FloatTensor *table = make_tensor(ranks[c], table_shapes[c]); + for (size_t axis = 0; axis < ranks[c]; ++axis) { + const int64_t dim = (int64_t)table_shapes[c][axis]; + const int64_t half = dim / 2; + const int64_t indices[] = {0, 1, -1, half, -half - 1, dim - 1, -dim}; + const size_t count = sizeof(indices) / sizeof(indices[0]); + check_gather_case(table, indices, count, axis); + } + st_destroy(table); + } + + /* Larger count on a realistic embedding table. */ + FloatTensor *table = make_tensor(2, (size_t[2]){30000, 64}); + int64_t indices[512]; + for (size_t i = 0; i < 512; ++i) { + indices[i] = (int64_t)((i * 7919) % 30000); + if (i % 3 == 0) { + indices[i] -= 30000; /* negative form */ + } + } + check_gather_case(table, indices, 512, 0); + st_destroy(table); +} + +void test_st_gather_rejects_bad_input(void) { + FloatTensor *table = make_tensor(2, (size_t[2]){4, 3}); + const size_t out_shape[2] = {2, 3}; + FloatTensor *out = make_tensor(2, out_shape); + const int64_t ok_indices[2] = {0, 1}; + const int64_t bad_high[2] = {0, 4}; + const int64_t bad_low[2] = {0, -5}; + + /* Out-of-range indices fail without writing. */ + for (size_t i = 0; i < out->numel; ++i) { + out->values[i] = 42.0f; + } + TEST_ASSERT_FALSE(st_gather(table, bad_high, 2, 0, out)); + TEST_ASSERT_FALSE(st_gather(table, bad_low, 2, 0, out)); + for (size_t i = 0; i < out->numel; ++i) { + TEST_ASSERT_EQUAL_FLOAT(42.0f, out->values[i]); + } + + /* count == 0, NULL, wrong axis, wrong shape, non-contiguous. */ + TEST_ASSERT_FALSE(st_gather(table, ok_indices, 0, 0, out)); + TEST_ASSERT_FALSE(st_gather(NULL, ok_indices, 2, 0, out)); + TEST_ASSERT_FALSE(st_gather(table, NULL, 2, 0, out)); + TEST_ASSERT_FALSE(st_gather(table, ok_indices, 2, 2, out)); + TEST_ASSERT_FALSE(st_gather(table, ok_indices, 2, 1, out)); + FloatTensor *view_base = make_tensor(2, (size_t[2]){4, 4}); + const size_t view_shape[2] = {4, 2}; + const ptrdiff_t view_strides[2] = {4, 1}; + FloatTensor *view = st_view(view_base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_FALSE(st_gather(view, ok_indices, 2, 0, out)); + st_destroy(view); + st_destroy(view_base); + + /* Valid call still works after the failures. */ + TEST_ASSERT_TRUE(st_gather(table, ok_indices, 2, 0, out)); + + st_destroy(out); + st_destroy(table); +} + +void test_st_gather_bytes_handles_i64_and_bool(void) { + /* int64 table, gather along axis 0. */ + const size_t shape[2] = {5, 3}; + int64_t table[15]; + for (size_t i = 0; i < 15; ++i) { + table[i] = (int64_t)(i * i) - 20; + } + const int64_t indices[4] = {4, -5, 2, 0}; + int64_t out[12]; + TEST_ASSERT_TRUE(st_gather_bytes(table, 2, shape, indices, 4, 0, + sizeof(int64_t), out)); + for (size_t j = 0; j < 4; ++j) { + const int64_t row = indices[j] < 0 ? indices[j] + 5 : indices[j]; + for (size_t e = 0; e < 3; ++e) { + TEST_ASSERT_EQUAL_INT64(table[row * 3 + (int64_t)e], out[j * 3 + e]); + } + } + + /* bool table, gather along axis 1. */ + const size_t bool_shape[2] = {2, 5}; + uint8_t bools[10]; + for (size_t i = 0; i < 10; ++i) { + bools[i] = (uint8_t)(i % 2); + } + const int64_t bool_indices[3] = {1, -1, 3}; + uint8_t bool_out[6]; + TEST_ASSERT_TRUE(st_gather_bytes(bools, 2, bool_shape, bool_indices, 3, 1, 1, + bool_out)); + const size_t picked[3] = {1, 4, 3}; + for (size_t o = 0; o < 2; ++o) { + for (size_t j = 0; j < 3; ++j) { + TEST_ASSERT_EQUAL_UINT8(bools[o * 5 + picked[j]], bool_out[o * 3 + j]); + } + } + + TEST_ASSERT_FALSE(st_gather_bytes(table, 2, shape, indices, 4, 2, 8, out)); + TEST_ASSERT_FALSE(st_gather_bytes(table, 2, shape, indices, 0, 0, 8, out)); + TEST_ASSERT_FALSE(st_gather_bytes(NULL, 2, shape, indices, 4, 0, 8, out)); +} + +void test_st_gather_copies_bf16_bits(void) { + const size_t shape[2] = {8, 5}; + FloatTensor *table_f32 = make_tensor(2, shape); + FloatTensor *table = st_to_bf16(table_f32); + TEST_ASSERT_NOT_NULL(table); + const size_t out_shape[2] = {3, 5}; + FloatTensor *out = st_create_bf16(2, out_shape); + TEST_ASSERT_NOT_NULL(out); + const int64_t indices[3] = {7, 0, -4}; + TEST_ASSERT_TRUE(st_gather(table, indices, 3, 0, out)); + const uint16_t *table_bf16 = (const uint16_t *)table->values; + const uint16_t *out_bf16 = (const uint16_t *)out->values; + const size_t picked[3] = {7, 0, 4}; + for (size_t j = 0; j < 3; ++j) { + for (size_t e = 0; e < 5; ++e) { + TEST_ASSERT_EQUAL_UINT16(table_bf16[picked[j] * 5 + e], + out_bf16[j * 5 + e]); + } + } + st_destroy(out); + st_destroy(table); + st_destroy(table_f32); +} + +/* ---- Materializing permute ------------------------------------------------ */ + +static void ref_permute_copy(const FloatTensor *src, const size_t *perm, + float *out, size_t out_numel) { + size_t index[ST_MAX_DIMS] = {0}; + for (size_t flat = 0; flat < out_numel; ++flat) { + size_t src_index[ST_MAX_DIMS]; + for (size_t d = 0; d < src->ndim; ++d) { + src_index[perm[d]] = index[d]; + } + size_t src_flat = 0; + for (size_t d = 0; d < src->ndim; ++d) { + src_flat = src_flat * src->shape[d] + src_index[d]; + } + out[flat] = src->values[src_flat]; + for (size_t d = src->ndim; d-- > 0;) { + if (++index[d] < src->shape[perm[d]]) { + break; + } + index[d] = 0; + } + } +} + +static void check_permute_case(const FloatTensor *src, const size_t *perm) { + size_t out_dims[ST_MAX_DIMS]; + for (size_t i = 0; i < src->ndim; ++i) { + out_dims[i] = src->shape[perm[i]]; + } + FloatTensor *out = st_create(src->ndim, out_dims); + TEST_ASSERT_NOT_NULL(out); + float *expected = malloc(src->numel * sizeof(float)); + TEST_ASSERT_NOT_NULL(expected); + ref_permute_copy(src, perm, expected, src->numel); + + TEST_ASSERT_TRUE(st_permute_copy_into(src, perm, out)); + TEST_ASSERT_EQUAL_MEMORY(expected, out->values, src->numel * sizeof(float)); + + /* The view variant agrees on values. */ + FloatTensor *view = st_permute(src, perm); + TEST_ASSERT_NOT_NULL(view); + FloatTensor *materialized = st_to_f32(view); + TEST_ASSERT_NOT_NULL(materialized); + /* st_permute is a view; st_to_f32 materializes it. */ + TEST_ASSERT_EQUAL_MEMORY(expected, materialized->values, + src->numel * sizeof(float)); + + free(expected); + st_destroy(materialized); + st_destroy(view); + st_destroy(out); +} + +void test_st_permute_copy_into_matches_reference(void) { + const size_t shape4[4] = {2, 3, 4, 5}; + FloatTensor *x4 = make_tensor(4, shape4); + check_permute_case(x4, (size_t[4]){0, 2, 1, 3}); + check_permute_case(x4, (size_t[4]){0, 2, 3, 1}); + check_permute_case(x4, (size_t[4]){3, 2, 1, 0}); + check_permute_case(x4, (size_t[4]){1, 0, 3, 2}); + st_destroy(x4); + + FloatTensor *x2 = make_tensor(2, (size_t[2]){97, 65}); + check_permute_case(x2, (size_t[2]){1, 0}); + st_destroy(x2); + + FloatTensor *x1 = make_tensor(1, (size_t[1]){7}); + check_permute_case(x1, (size_t[1]){0}); + st_destroy(x1); + + /* Attention-style [B, H, T, dk] -> [B, T, H, dk]. */ + FloatTensor *attn = make_tensor(4, (size_t[4]){2, 4, 128, 64}); + check_permute_case(attn, (size_t[4]){0, 2, 1, 3}); + st_destroy(attn); +} + +void test_st_permute_copy_into_rejects_bad_input(void) { + FloatTensor *src = make_tensor(3, (size_t[3]){2, 3, 4}); + FloatTensor *out = make_tensor(3, (size_t[3]){3, 2, 4}); + FloatTensor *wrong = make_tensor(3, (size_t[3]){2, 3, 4}); + + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[3]){0, 1, 3}, out)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[3]){0, 1, 1}, out)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, NULL, out)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[3]){1, 0, 2}, wrong)); + TEST_ASSERT_FALSE(st_permute_copy_into(NULL, (size_t[3]){1, 0, 2}, out)); + + TEST_ASSERT_TRUE(st_permute_copy_into(src, (size_t[3]){1, 0, 2}, out)); + + /* Non-contiguous tensors are rejected (views must be materialized + * first, e.g. via st_to_f32 or the copy kernels themselves). */ + FloatTensor *view_base = make_tensor(2, (size_t[2]){2, 3}); + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *view = st_view(view_base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_FALSE(st_permute_copy_into(view, (size_t[2]){1, 0}, out)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[3]){1, 0, 2}, view)); + st_destroy(view); + st_destroy(view_base); + + st_destroy(wrong); + st_destroy(out); + st_destroy(src); +} + +void test_st_permute_copy_into_bytes_handles_i64(void) { + const size_t shape[3] = {2, 3, 4}; + int64_t src[24]; + for (size_t i = 0; i < 24; ++i) { + src[i] = (int64_t)i; + } + int64_t out[24]; + const size_t perm[3] = {1, 2, 0}; + TEST_ASSERT_TRUE( + st_permute_copy_into_bytes(src, 3, shape, perm, sizeof(int64_t), out)); + /* out[i1][i2][i0] = src[i0][i1][i2] */ + for (size_t i1 = 0; i1 < 3; ++i1) { + for (size_t i2 = 0; i2 < 4; ++i2) { + for (size_t i0 = 0; i0 < 2; ++i0) { + TEST_ASSERT_EQUAL_INT64(src[i0 * 12 + i1 * 4 + i2], + out[i1 * 8 + i2 * 2 + i0]); + } + } + } + TEST_ASSERT_FALSE(st_permute_copy_into_bytes(src, 3, shape, + (size_t[3]){0, 0, 1}, 8, out)); +} + +void test_st_copy_bytes_rejects_overflow_without_writing(void) { + const uint8_t src[4] = {1, 2, 3, 4}; + uint8_t out[4] = {9, 9, 9, 9}; + const uint8_t expected[4] = {9, 9, 9, 9}; + const int64_t index = 0; + const size_t gather_shape[3] = {2, SIZE_MAX / 2 + 1, 2}; + TEST_ASSERT_FALSE(st_gather_bytes(src, 3, gather_shape, &index, 1, 0, 1, out)); + TEST_ASSERT_FALSE(st_gather_bytes(src, 2, (size_t[2]){2, 2}, &index, + SIZE_MAX / 2 + 1, 1, 1, out)); + TEST_ASSERT_FALSE(st_gather_bytes(src, 1, (size_t[1]){2}, &index, 1, 0, + SIZE_MAX, out)); + const size_t shape[2] = {SIZE_MAX / 2 + 1, 2}; + const size_t perm[2] = {0, 1}; + TEST_ASSERT_FALSE(st_permute_copy_into_bytes(src, 2, shape, perm, 1, out)); + TEST_ASSERT_FALSE(st_permute_copy_into_bytes( + src, 1, (size_t[1]){2}, (size_t[1]){0}, SIZE_MAX / 2 + 1, out)); + TEST_ASSERT_EQUAL_MEMORY(expected, out, sizeof(out)); +} + +void test_st_permute_copy_into_rejects_out_of_range_axis(void) { + FloatTensor *src = make_tensor(2, (size_t[2]){2, 3}); + FloatTensor *out = make_tensor(2, (size_t[2]){3, 2}); + float before[6]; + memcpy(before, out->values, sizeof(before)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[2]){SIZE_MAX, 0}, out)); + TEST_ASSERT_FALSE(st_permute_copy_into(src, (size_t[2]){0, 0}, out)); + TEST_ASSERT_EQUAL_MEMORY(before, out->values, sizeof(before)); + st_destroy(out); + st_destroy(src); +} + +void test_st_permute_copy_bytes_preserves_unaligned_payloads(void) { + const uint32_t bits[6] = {0x7fc12345u, 0x80000000u, 0xffffffffu, + 0x12345678u, 0u, 0x7f800001u}; + uint8_t src[sizeof(bits) + 1]; + uint8_t out[sizeof(bits) + 1]; + memcpy(src + 1, bits, sizeof(bits)); + TEST_ASSERT_TRUE(st_permute_copy_into_bytes( + src + 1, 2, (size_t[2]){2, 3}, (size_t[2]){1, 0}, 4, out + 1)); + for (size_t i = 0; i < 3; ++i) { + for (size_t j = 0; j < 2; ++j) { + TEST_ASSERT_EQUAL_MEMORY(bits + j * 3 + i, out + 1 + (i * 2 + j) * 4, 4); + } + } + const uint64_t wide[6] = {0, UINT64_MAX, UINT64_C(0x7ff0000000000001), + UINT64_C(0x8000000000000000), 123, 456}; + uint8_t src_wide[sizeof(wide) + 1]; + uint8_t out_wide[sizeof(wide) + 1]; + memcpy(src_wide + 1, wide, sizeof(wide)); + TEST_ASSERT_TRUE(st_permute_copy_into_bytes( + src_wide + 1, 2, (size_t[2]){2, 3}, (size_t[2]){1, 0}, 8, out_wide + 1)); + for (size_t i = 0; i < 3; ++i) { + for (size_t j = 0; j < 2; ++j) { + TEST_ASSERT_EQUAL_MEMORY(wide + j * 3 + i, + out_wide + 1 + (i * 2 + j) * 8, 8); + } + } +} + +/* ---- Aliasing policy (in-place) ------------------------------------------ */ + +void test_st_binary_broadcast_into_allows_elementwise_inplace(void) { + /* Documented aliasing rule: out may alias an operand whose shape equals + * out's shape; the row kernels read and write the same element-wise + * position. Partial overlap with a differently shaped operand stays + * forbidden. */ + const size_t shape[2] = {3, 7}; + FloatTensor *a = make_tensor(2, shape); + FloatTensor *b = make_tensor(2, shape); + float *expected = malloc(30 * sizeof(float)); /* largest case below */ + TEST_ASSERT_NOT_NULL(expected); + for (size_t i = 0; i < a->numel; ++i) { + expected[i] = a->values[i] + b->values[i]; + } + TEST_ASSERT_TRUE(st_binary_broadcast_into(ST_BIN_ADD, a, b, a)); + TEST_ASSERT_EQUAL_MEMORY(expected, a->values, a->numel * sizeof(float)); + + /* Suffix bias path, in-place accumulation like `x += bias`. */ + const size_t x_shape[3] = {2, 3, 5}; + const size_t bias_shape[1] = {5}; + FloatTensor *x = make_tensor(3, x_shape); + FloatTensor *bias = make_tensor(1, bias_shape); + for (size_t i = 0; i < x->numel; ++i) { + expected[i] = x->values[i] + bias->values[i % 5]; + } + TEST_ASSERT_TRUE(st_binary_broadcast_into(ST_BIN_ADD, x, bias, x)); + TEST_ASSERT_EQUAL_MEMORY(expected, x->values, x->numel * sizeof(float)); + + /* General walk path with a mid-broadcast operand, in-place. */ + const size_t mid_shape[3] = {1, 3, 1}; + FloatTensor *mid = make_tensor(3, mid_shape); + for (size_t i = 0; i < x->numel; ++i) { + expected[i] = x->values[i] * mid->values[(i / 5) % 3]; + } + TEST_ASSERT_TRUE(st_binary_broadcast_into(ST_BIN_MUL, x, mid, x)); + TEST_ASSERT_EQUAL_MEMORY(expected, x->values, x->numel * sizeof(float)); + + /* out aliases the full-size right operand. */ + FloatTensor *c = make_tensor(2, shape); + FloatTensor *d = make_tensor(2, shape); + for (size_t i = 0; i < c->numel; ++i) { + expected[i] = c->values[i] - d->values[i]; + } + TEST_ASSERT_TRUE(st_binary_broadcast_into(ST_BIN_SUB, c, d, d)); + TEST_ASSERT_EQUAL_MEMORY(expected, d->values, d->numel * sizeof(float)); + + free(expected); + st_destroy(d); + st_destroy(c); + st_destroy(mid); + st_destroy(bias); + st_destroy(x); + st_destroy(b); + st_destroy(a); +} diff --git a/app/tensor/tests/test_st_nn.c b/app/tensor/tests/test_st_nn.c new file mode 100644 index 0000000..8d79c4e --- /dev/null +++ b/app/tensor/tests/test_st_nn.c @@ -0,0 +1,146 @@ +#include "st.h" + +#include +#include + +#define UNITY_INCLUDE_FLOAT +#define UNITY_FLOAT_PRECISION 6 +#define TEST_CASE(...) +#include "unity.h" + +static bool near_value(float actual, float expected) { + return fabsf(actual - expected) <= 1.0e-6f + 1.0e-5f * fabsf(expected); +} + +static bool near_bf16(float actual, float expected) { + return fabsf(actual - expected) <= 1.0e-2f + 1.0e-2f * fabsf(expected); +} + +void setUp(void) {} +void tearDown(void) {} + +void test_st_activation_kernels_match_matrix_and_scalar_reference(void) { + const size_t shape[2] = {2, 7}; + FloatTensor *tensor = st_create(2, shape); + TEST_ASSERT_NOT_NULL(tensor); + for (size_t i = 0; i < tensor->numel; ++i) { + tensor->values[i] = ((float)i - 6.0f) * 0.35f; + } + + FloatTensor *actual = st_clone(tensor); + TEST_ASSERT_TRUE(st_apply_sigmoid(actual)); + for (size_t i = 0; i < tensor->numel; ++i) { + TEST_ASSERT_TRUE(near_value(actual->values[i], + 1.0f / (1.0f + expf(-tensor->values[i])))); + } + st_destroy(actual); + + actual = st_clone(tensor); + TEST_ASSERT_TRUE(st_apply_tanh(actual)); + for (size_t i = 0; i < tensor->numel; ++i) { + TEST_ASSERT_TRUE(near_value(actual->values[i], tanhf(tensor->values[i]))); + } + st_destroy(actual); + + actual = st_clone(tensor); + TEST_ASSERT_TRUE(st_apply_gelu_tanh(actual)); + for (size_t i = 0; i < tensor->numel; ++i) { + const float x = tensor->values[i]; + const float expected = + 0.5f * x * (1.0f + tanhf(0.7978845608f * + (x + 0.044715f * x * x * x))); + TEST_ASSERT_TRUE(near_value(actual->values[i], expected)); + } + st_destroy(actual); + + TEST_ASSERT_TRUE(st_apply_softmax(tensor)); + for (size_t row = 0; row < shape[0]; ++row) { + float sum = 0.0f; + for (size_t col = 0; col < shape[1]; ++col) { + sum += tensor->values[row * shape[1] + col]; + } + TEST_ASSERT_TRUE(near_value(sum, 1.0f)); + } + st_destroy(tensor); +} + +void test_st_activation_kernels_promote_bf16_and_reject_views(void) { + const size_t shape[2] = {1, 7}; + FloatTensor *f32 = st_create(2, shape); + TEST_ASSERT_NOT_NULL(f32); + for (size_t i = 0; i < f32->numel; ++i) { + f32->values[i] = ((float)i - 3.0f) * 0.5f; + } + FloatTensor *bf16 = st_to_bf16(f32); + TEST_ASSERT_NOT_NULL(bf16); + FloatTensor *quantized_input = st_to_f32(bf16); + TEST_ASSERT_NOT_NULL(quantized_input); + TEST_ASSERT_TRUE(st_apply_gelu_erf(bf16)); + FloatTensor *result = st_to_f32(bf16); + TEST_ASSERT_NOT_NULL(result); + for (size_t i = 0; i < result->numel; ++i) { + const float x = quantized_input->values[i]; + TEST_ASSERT_TRUE(near_bf16( + result->values[i], 0.5f * x * (1.0f + erff(x * 0.7071067812f)))); + } + + const size_t view_shape[2] = {2, 2}; + const ptrdiff_t view_strides[2] = {3, 1}; + FloatTensor *base = st_create(2, (size_t[]){2, 3}); + TEST_ASSERT_NOT_NULL(base); + FloatTensor *view = st_view(base, 2, view_shape, view_strides, 0); + TEST_ASSERT_NOT_NULL(view); + TEST_ASSERT_FALSE(st_apply_sigmoid(view)); + + st_destroy(view); + st_destroy(base); + st_destroy(result); + st_destroy(quantized_input); + st_destroy(bf16); + st_destroy(f32); +} + +void test_st_activation_backward_promotes_bf16_and_checks_shapes(void) { + const size_t shape[1] = {7}; + FloatTensor *activation_f32 = st_create(1, shape); + FloatTensor *grad_f32 = st_create(1, shape); + TEST_ASSERT_NOT_NULL(activation_f32); + TEST_ASSERT_NOT_NULL(grad_f32); + for (size_t i = 0; i < shape[0]; ++i) { + activation_f32->values[i] = -1.0f + (float)i * 0.3f; + grad_f32->values[i] = 1.0f + (float)i; + } + FloatTensor *activation = st_to_bf16(activation_f32); + FloatTensor *grad = st_to_bf16(grad_f32); + TEST_ASSERT_NOT_NULL(activation); + TEST_ASSERT_NOT_NULL(grad); + FloatTensor *activation_before = st_to_f32(activation); + FloatTensor *grad_before = st_to_f32(grad); + TEST_ASSERT_NOT_NULL(activation_before); + TEST_ASSERT_NOT_NULL(grad_before); + TEST_ASSERT_TRUE(st_apply_tanh_backward(activation, grad)); + FloatTensor *activation_back = st_to_f32(activation); + FloatTensor *grad_back = st_to_f32(grad); + TEST_ASSERT_NOT_NULL(activation_back); + TEST_ASSERT_NOT_NULL(grad_back); + for (size_t i = 0; i < shape[0]; ++i) { + const float value = activation_before->values[i]; + const float expected = grad_before->values[i] * (1.0f - value * value); + TEST_ASSERT_FLOAT_WITHIN(0.02f, expected, grad_back->values[i]); + } + + const size_t wrong_shape[1] = {6}; + FloatTensor *wrong = st_create(1, wrong_shape); + TEST_ASSERT_NOT_NULL(wrong); + TEST_ASSERT_FALSE(st_apply_sigmoid_backward(activation, wrong)); + + st_destroy(wrong); + st_destroy(grad_back); + st_destroy(activation_back); + st_destroy(grad_before); + st_destroy(activation_before); + st_destroy(grad); + st_destroy(activation); + st_destroy(grad_f32); + st_destroy(activation_f32); +} \ No newline at end of file diff --git a/app/vector/BUILD b/app/vector/BUILD index d81cb83..1447bdf 100644 --- a/app/vector/BUILD +++ b/app/vector/BUILD @@ -86,7 +86,6 @@ perf_copts = select({ "-fomit-frame-pointer", "-march=native", "-mtune=native", - "-flto", "-fno-math-errno", "-fno-trapping-math", "-falign-functions=32", @@ -130,7 +129,7 @@ accelerate_linkopts = select({ openblas_linkopts = [] default_deps = [ - "@log", + "//third_party/log.c:log", "@openmp//:libomp", ] @@ -159,6 +158,22 @@ cc_library( }), ) +cc_library( + name = "sv_optim", + srcs = ["src/sv_optim.c"], + hdrs = ["include/sv_optim.h"], + copts = default_copts + perf_copts + openmp_copts, + includes = ["include"], + linkopts = openmp_linkopts, + target_compatible_with = select({ + ":backend_accelerate": ["@platforms//os:macos"], + ":backend_openblas": ["@platforms//os:linux"], + "//conditions:default": [], + }), + visibility = ["//visibility:public"], + deps = default_deps, +) + cc_library( name = "dv", srcs = ["src/dv.c"], @@ -272,6 +287,7 @@ cc_library( ":dv", ":dvs", ":sv", + ":sv_optim", ":vec3", ":vector_bridge", ":vv", @@ -286,6 +302,14 @@ unity_test( deps = [":sv"], ) +unity_test( + name = "test_sv_optim", + srcs = ["tests/test_sv_optim.c"], + my_config = ["tests/my_config.yml"], + tags = ["smoke"], + deps = [":sv_optim"], +) + unity_test( name = "test_dv", srcs = ["tests/test_dv.c"], @@ -347,3 +371,16 @@ cc_binary( ":sv", ], ) + +cc_binary( + name = "bench_sv_optim", + srcs = ["tests/bench_sv_optim.c"], + copts = default_copts + perf_copts + package_copts + openmp_copts, + linkopts = openmp_linkopts + accelerate_linkopts + openblas_linkopts, + target_compatible_with = select({ + ":backend_accelerate": ["@platforms//os:macos"], + ":backend_openblas": ["@platforms//os:linux"], + "//conditions:default": [], + }), + deps = [":sv_optim"], +) diff --git a/app/vector/include/sv_optim.h b/app/vector/include/sv_optim.h new file mode 100644 index 0000000..101696f --- /dev/null +++ b/app/vector/include/sv_optim.h @@ -0,0 +1,90 @@ +/** + * @file sv_optim.h + * @brief Optimizer kernels for caller-owned flat float buffers. + */ + +#ifndef SV_OPTIM_H +#define SV_OPTIM_H + +#include +#include +#include + +/** + * @brief Apply an SGD parameter update. + * @param params Mutable parameter buffer. + * @param grads Gradient buffer. + * @param n Number of elements. + * @param learning_rate Positive learning rate. + * @param grad_scale Scale applied to each gradient before the update. + * @param weight_decay Decoupled weight decay; zero disables it. + * @return true on success, false on invalid arguments. + */ +bool sv_optim_sgd(float *params, const float *grads, size_t n, + float learning_rate, float grad_scale, float weight_decay); + +/** + * @brief Apply a momentum update using caller-owned velocity storage. + * @param params Mutable parameter buffer. + * @param grads Gradient buffer. + * @param momentum_buffer Mutable momentum buffer of length @p n. + * @param n Number of elements. + * @param learning_rate Positive learning rate. + * @param momentum Momentum coefficient in `[0, 1)`. + * @param grad_scale Scale applied to each gradient before the update. + * @param weight_decay Decoupled weight decay; zero disables it. + * @return true on success, false on invalid arguments. + */ +bool sv_optim_momentum(float *params, const float *grads, + float *momentum_buffer, size_t n, float learning_rate, + float momentum, float grad_scale, float weight_decay); + +/** + * @brief Apply RMSprop using caller-owned squared-gradient storage. + * @param params Mutable parameter buffer. + * @param grads Gradient buffer. + * @param squared_grad_buffer Mutable second-moment buffer of length @p n. + * @param n Number of elements. + * @param learning_rate Positive learning rate. + * @param beta2 RMS decay coefficient in `(0, 1)`. + * @param epsilon Positive denominator stabilizer. + * @param grad_scale Scale applied to each gradient before the update. + * @param weight_decay Decoupled weight decay; zero disables it. + * @return true on success, false on invalid arguments. + */ +bool sv_optim_rmsprop(float *params, const float *grads, + float *squared_grad_buffer, size_t n, + float learning_rate, float beta2, float epsilon, + float grad_scale, float weight_decay); + +/** + * @brief Apply Adam using caller-owned first- and second-moment storage. + * @param params Mutable parameter buffer. + * @param grads Gradient buffer. + * @param first_moment Mutable first-moment buffer of length @p n. + * @param second_moment Mutable second-moment buffer of length @p n. + * @param n Number of elements. + * @param learning_rate Positive learning rate. + * @param beta1 First-moment decay coefficient in `(0, 1)`. + * @param beta2 Second-moment decay coefficient in `(0, 1)`. + * @param epsilon Positive denominator stabilizer. + * @param step One-based optimizer step for bias correction. + * @param grad_scale Scale applied to each gradient before the update. + * @param weight_decay Decoupled weight decay; zero disables it. + * @return true on success, false on invalid arguments. + */ +bool sv_optim_adam(float *params, const float *grads, float *first_moment, + float *second_moment, size_t n, float learning_rate, + float beta1, float beta2, float epsilon, uint64_t step, + float grad_scale, float weight_decay); + +/** + * @brief Clip gradients to an L2 norm and return the pre-clip norm. + * @param grads Mutable gradient buffer; scaled only when norm exceeds limit. + * @param n Number of elements. + * @param max_norm Non-negative maximum L2 norm. + * @return Original L2 norm, 0 for an empty buffer, or -1 for invalid arguments. + */ +float sv_clip_norm(float *grads, size_t n, float max_norm); + +#endif // SV_OPTIM_H \ No newline at end of file diff --git a/app/vector/src/sv_optim.c b/app/vector/src/sv_optim.c new file mode 100644 index 0000000..b3c8a20 --- /dev/null +++ b/app/vector/src/sv_optim.c @@ -0,0 +1,164 @@ +#include "sv_optim.h" + +#include +#include +#include + +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wpedantic" +#endif +#include +#if defined(__clang__) +#pragma clang diagnostic pop +#endif + +#define SV_OPTIM_OMP_THRESHOLD 1048576u +#define SV_OPTIM_USE_OMP(N) \ + ((N) > SV_OPTIM_OMP_THRESHOLD && omp_get_max_threads() > 1) + +static bool sv_float_is_finite(float value) { + uint32_t bits = 0; + memcpy(&bits, &value, sizeof(bits)); + return (bits & 0x7f800000u) != 0x7f800000u; +} + +#define SV_OPTIM_LOOP(N, ...) \ + do { \ + if (SV_OPTIM_USE_OMP(N)) { \ + _Pragma("omp parallel for simd schedule(static)") \ + for (size_t i = 0; i < (N); ++i) { \ + __VA_ARGS__ \ + } \ + } else { \ + _Pragma("omp simd") \ + for (size_t i = 0; i < (N); ++i) { \ + __VA_ARGS__ \ + } \ + } \ + } while (0) + +static bool sv_optim_valid_base(const float *params, const float *grads, + size_t n, float learning_rate, + float grad_scale, float weight_decay) { + return params != NULL && grads != NULL && n > 0 && + sv_float_is_finite(learning_rate) && learning_rate > 0.0f && + sv_float_is_finite(weight_decay) && weight_decay >= 0.0f && + sv_float_is_finite(grad_scale); +} + +bool sv_optim_sgd(float *params, const float *grads, size_t n, + float learning_rate, float grad_scale, float weight_decay) { + if (!sv_optim_valid_base(params, grads, n, learning_rate, grad_scale, + weight_decay)) { + return false; + } + + SV_OPTIM_LOOP(n, + float parameter = params[i] - learning_rate * weight_decay * params[i]; + parameter -= learning_rate * (grads[i] * grad_scale); + params[i] = parameter; + ); + return true; +} + +bool sv_optim_momentum(float *params, const float *grads, + float *momentum_buffer, size_t n, float learning_rate, + float momentum, float grad_scale, float weight_decay) { + if (!sv_optim_valid_base(params, grads, n, learning_rate, grad_scale, + weight_decay) || + momentum_buffer == NULL || !sv_float_is_finite(momentum) || + momentum < 0.0f || momentum >= 1.0f) { + return false; + } + + SV_OPTIM_LOOP(n, + const float gradient = grads[i] * grad_scale; + const float velocity = momentum * momentum_buffer[i] + gradient; + momentum_buffer[i] = velocity; + float parameter = params[i] - learning_rate * weight_decay * params[i]; + parameter -= learning_rate * velocity; + params[i] = parameter; + ); + return true; +} + +bool sv_optim_rmsprop(float *params, const float *grads, + float *squared_grad_buffer, size_t n, + float learning_rate, float beta2, float epsilon, + float grad_scale, float weight_decay) { + if (!sv_optim_valid_base(params, grads, n, learning_rate, grad_scale, + weight_decay) || + squared_grad_buffer == NULL || !sv_float_is_finite(beta2) || + !sv_float_is_finite(epsilon) || beta2 <= 0.0f || beta2 >= 1.0f || + epsilon <= 0.0f) { + return false; + } + + SV_OPTIM_LOOP(n, + const float gradient = grads[i] * grad_scale; + const float square = beta2 * squared_grad_buffer[i] + + (1.0f - beta2) * gradient * gradient; + squared_grad_buffer[i] = square; + float parameter = params[i] - learning_rate * weight_decay * params[i]; + parameter -= learning_rate * gradient / (sqrtf(square) + epsilon); + params[i] = parameter; + ); + return true; +} + +bool sv_optim_adam(float *params, const float *grads, float *first_moment, + float *second_moment, size_t n, float learning_rate, + float beta1, float beta2, float epsilon, uint64_t step, + float grad_scale, float weight_decay) { + if (!sv_optim_valid_base(params, grads, n, learning_rate, grad_scale, + weight_decay) || + first_moment == NULL || second_moment == NULL || + !sv_float_is_finite(beta1) || !sv_float_is_finite(beta2) || + !sv_float_is_finite(epsilon) || beta1 <= 0.0f || beta1 >= 1.0f || + beta2 <= 0.0f || beta2 >= 1.0f || epsilon <= 0.0f || step == 0) { + return false; + } + + const float inverse_bias1 = 1.0f / (1.0f - powf(beta1, (float)step)); + const float inverse_bias2 = 1.0f / (1.0f - powf(beta2, (float)step)); + SV_OPTIM_LOOP(n, + const float gradient = grads[i] * grad_scale; + const float first = beta1 * first_moment[i] + (1.0f - beta1) * gradient; + const float second = beta2 * second_moment[i] + + (1.0f - beta2) * gradient * gradient; + first_moment[i] = first; + second_moment[i] = second; + const float corrected_first = first * inverse_bias1; + const float corrected_second = second * inverse_bias2; + float parameter = params[i] - learning_rate * weight_decay * params[i]; + parameter -= learning_rate * corrected_first / + (sqrtf(corrected_second) + epsilon); + params[i] = parameter; + ); + return true; +} + +float sv_clip_norm(float *grads, size_t n, float max_norm) { + if (!sv_float_is_finite(max_norm) || max_norm < 0.0f || + (n > 0 && grads == NULL)) { + return -1.0f; + } + if (n == 0) { + return 0.0f; + } + + double square_sum = 0.0; + for (size_t i = 0; i < n; ++i) { + const double value = grads[i]; + square_sum += value * value; + } + const double norm = sqrt(square_sum); + if (norm > (double)max_norm && norm > 0.0) { + const float scale = (float)((double)max_norm / norm); + SV_OPTIM_LOOP(n, + grads[i] *= scale; + ); + } + return (float)norm; +} \ No newline at end of file diff --git a/app/vector/tests/bench_sv_optim.c b/app/vector/tests/bench_sv_optim.c new file mode 100644 index 0000000..ecbe667 --- /dev/null +++ b/app/vector/tests/bench_sv_optim.c @@ -0,0 +1,308 @@ +/* + * Scalar reference versus sv_optim kernels, including the 16,384-element + * parallelization threshold. + * Run: bazel run //app/vector:bench_sv_optim + */ + +#include "sv_optim.h" + +#include +#include +#include +#include +#include +#include +#include + +typedef enum BenchOptimizer { + OPT_SGD, + OPT_MOMENTUM, + OPT_RMSPROP, + OPT_ADAM, + OPT_CLIP, +} BenchOptimizer; + +typedef struct BenchBuffers { + float *params; + float *grads; + float *first; + float *second; + float *initial_params; + float *initial_grads; + float *initial_first; + float *initial_second; + size_t n; +} BenchBuffers; + +static uint64_t now_ns(void) { + struct timespec ts; + clock_gettime(CLOCK_MONOTONIC, &ts); + return (uint64_t)ts.tv_sec * 1000000000ull + (uint64_t)ts.tv_nsec; +} + +static double median(double *samples, size_t count) { + for (size_t i = 1; i < count; ++i) { + const double value = samples[i]; + size_t j = i; + while (j > 0 && samples[j - 1] > value) { + samples[j] = samples[j - 1]; + --j; + } + samples[j] = value; + } + return samples[count / 2]; +} + +static size_t repetitions(size_t n) { + if (n <= 1000) return 32; + if (n <= 10000) return 8; + if (n <= 100000) return 2; + return 1; +} + +static const char *optimizer_name(BenchOptimizer optimizer) { + switch (optimizer) { + case OPT_SGD: return "sgd"; + case OPT_MOMENTUM: return "momentum"; + case OPT_RMSPROP: return "rmsprop"; + case OPT_ADAM: return "adam"; + case OPT_CLIP: return "clip_norm"; + } + return "unknown"; +} + +static void reset(BenchBuffers *b) { + const size_t bytes = b->n * sizeof(float); + memcpy(b->params, b->initial_params, bytes); + memcpy(b->grads, b->initial_grads, bytes); + memcpy(b->first, b->initial_first, bytes); + memcpy(b->second, b->initial_second, bytes); +} + +static void scalar_apply(BenchBuffers *b, BenchOptimizer optimizer) { + const float lr = 0.001f; + const float grad_scale = 0.75f; + const float decay = 1.0e-4f; + const float beta1 = 0.9f; + const float beta2 = 0.999f; + const float epsilon = 1.0e-8f; + const uint64_t step = 5; + const float inv_b1 = 1.0f / (1.0f - powf(beta1, (float)step)); + const float inv_b2 = 1.0f / (1.0f - powf(beta2, (float)step)); + + if (optimizer == OPT_CLIP) { + double square_sum = 0.0; + for (size_t i = 0; i < b->n; ++i) { + const double g = b->grads[i]; + square_sum += g * g; + } + const double norm = sqrt(square_sum); + if (norm > 1.0) { + const float scale = (float)(1.0 / norm); + for (size_t i = 0; i < b->n; ++i) b->grads[i] *= scale; + } + return; + } + + for (size_t i = 0; i < b->n; ++i) { + const float g = b->grads[i] * grad_scale; + const float old_parameter = b->params[i]; + float parameter = old_parameter - lr * decay * old_parameter; + switch (optimizer) { + case OPT_SGD: + parameter -= lr * g; + break; + case OPT_MOMENTUM: + b->first[i] = 0.9f * b->first[i] + g; + parameter -= lr * b->first[i]; + break; + case OPT_RMSPROP: + b->second[i] = beta2 * b->second[i] + (1.0f - beta2) * g * g; + parameter -= lr * g / (sqrtf(b->second[i]) + epsilon); + break; + case OPT_ADAM: + b->first[i] = beta1 * b->first[i] + (1.0f - beta1) * g; + b->second[i] = beta2 * b->second[i] + (1.0f - beta2) * g * g; + parameter -= lr * b->first[i] * inv_b1 / + (sqrtf(b->second[i] * inv_b2) + epsilon); + break; + case OPT_CLIP: + break; + } + b->params[i] = parameter; + } +} + +static void forced_parallel_apply(BenchBuffers *b, BenchOptimizer optimizer) { + const float lr = 0.001f; + const float grad_scale = 0.75f; + const float decay = 1.0e-4f; + const float beta1 = 0.9f; + const float beta2 = 0.999f; + const float epsilon = 1.0e-8f; + const float inv_b1 = 1.0f / (1.0f - powf(beta1, 5.0f)); + const float inv_b2 = 1.0f / (1.0f - powf(beta2, 5.0f)); + + if (optimizer == OPT_CLIP) { + double square_sum = 0.0; + for (size_t i = 0; i < b->n; ++i) { + const double g = b->grads[i]; + square_sum += g * g; + } + const double norm = sqrt(square_sum); + if (norm > 1.0) { + const float scale = (float)(1.0 / norm); +#pragma omp parallel for simd schedule(static) + for (size_t i = 0; i < b->n; ++i) b->grads[i] *= scale; + } + return; + } + +#pragma omp parallel for simd schedule(static) + for (size_t i = 0; i < b->n; ++i) { + const float gradient = b->grads[i] * grad_scale; + const float original = b->params[i]; + float parameter = original - lr * decay * original; + switch (optimizer) { + case OPT_SGD: + parameter -= lr * gradient; + break; + case OPT_MOMENTUM: + b->first[i] = 0.9f * b->first[i] + gradient; + parameter -= lr * b->first[i]; + break; + case OPT_RMSPROP: + b->second[i] = beta2 * b->second[i] + + (1.0f - beta2) * gradient * gradient; + parameter -= lr * gradient / (sqrtf(b->second[i]) + epsilon); + break; + case OPT_ADAM: + b->first[i] = beta1 * b->first[i] + (1.0f - beta1) * gradient; + b->second[i] = beta2 * b->second[i] + + (1.0f - beta2) * gradient * gradient; + parameter -= lr * b->first[i] * inv_b1 / + (sqrtf(b->second[i] * inv_b2) + epsilon); + break; + case OPT_CLIP: + break; + } + b->params[i] = parameter; + } +} + +static bool optimized_apply(BenchBuffers *b, BenchOptimizer optimizer) { + const float lr = 0.001f; + const float grad_scale = 0.75f; + const float decay = 1.0e-4f; + switch (optimizer) { + case OPT_SGD: + return sv_optim_sgd(b->params, b->grads, b->n, lr, grad_scale, decay); + case OPT_MOMENTUM: + return sv_optim_momentum(b->params, b->grads, b->first, b->n, lr, 0.9f, + grad_scale, decay); + case OPT_RMSPROP: + return sv_optim_rmsprop(b->params, b->grads, b->second, b->n, lr, 0.999f, + 1.0e-8f, grad_scale, decay); + case OPT_ADAM: + return sv_optim_adam(b->params, b->grads, b->first, b->second, b->n, lr, + 0.9f, 0.999f, 1.0e-8f, 5, grad_scale, decay); + case OPT_CLIP: + return sv_clip_norm(b->grads, b->n, 1.0f) >= 0.0f; + } + return false; +} + +static double measure(BenchBuffers *buffers, BenchOptimizer optimizer, + int mode, size_t reps) { + double samples[7]; + for (size_t sample = 0; sample < 7; ++sample) { + reset(buffers); + const uint64_t start = now_ns(); + for (size_t i = 0; i < reps; ++i) { + if (mode == 1) { + if (!optimized_apply(buffers, optimizer)) return -1.0; + } else if (mode == 2) { + forced_parallel_apply(buffers, optimizer); + } else { + scalar_apply(buffers, optimizer); + } + } + samples[sample] = (double)(now_ns() - start) / 1000.0 / (double)reps; + } + return median(samples, 7); +} + +static bool allocate_buffers(BenchBuffers *b, size_t n) { + memset(b, 0, sizeof(*b)); + b->n = n; + b->params = (float *)malloc(n * sizeof(float)); + b->grads = (float *)malloc(n * sizeof(float)); + b->first = (float *)malloc(n * sizeof(float)); + b->second = (float *)malloc(n * sizeof(float)); + b->initial_params = (float *)malloc(n * sizeof(float)); + b->initial_grads = (float *)malloc(n * sizeof(float)); + b->initial_first = (float *)malloc(n * sizeof(float)); + b->initial_second = (float *)malloc(n * sizeof(float)); + if (!b->params || !b->grads || !b->first || !b->second || + !b->initial_params || !b->initial_grads || !b->initial_first || + !b->initial_second) { + return false; + } + for (size_t i = 0; i < n; ++i) { + b->initial_params[i] = 0.1f + (float)(i % 19u) * 0.001f; + b->initial_grads[i] = (float)((i * 7u) % 37u) * 0.0001f; + b->initial_first[i] = 0.0f; + b->initial_second[i] = 0.0f; + } + return true; +} + +static void destroy_buffers(BenchBuffers *b) { + free(b->initial_second); + free(b->initial_first); + free(b->initial_grads); + free(b->initial_params); + free(b->second); + free(b->first); + free(b->grads); + free(b->params); +} + +int main(void) { + const size_t sizes[] = {1000, 10000, 100000, 1000000}; + const BenchOptimizer optimizers[] = { + OPT_SGD, OPT_MOMENTUM, OPT_RMSPROP, OPT_ADAM, OPT_CLIP}; + const int max_threads = omp_get_max_threads(); + const int thread_counts[] = {1, max_threads}; + omp_set_dynamic(0); + + printf("optimizer,n,threads,scalar_us,sv_optim_us,forced_parallel_us," + "api_speedup,parallel_speedup\n"); + for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); ++si) { + BenchBuffers buffers; + if (!allocate_buffers(&buffers, sizes[si])) { + fprintf(stderr, "allocation failed for n=%zu\n", sizes[si]); + destroy_buffers(&buffers); + return 1; + } + for (size_t ti = 0; ti < 2; ++ti) { + const int threads = thread_counts[ti]; + if (ti == 1 && threads == thread_counts[0]) continue; + omp_set_num_threads(threads); + for (size_t oi = 0; oi < sizeof(optimizers) / sizeof(optimizers[0]); + ++oi) { + const size_t reps = repetitions(buffers.n); + const double before = measure(&buffers, optimizers[oi], 0, reps); + const double after = measure(&buffers, optimizers[oi], 1, reps); + const double parallel = measure(&buffers, optimizers[oi], 2, reps); + printf("%s,%zu,%d,%.3f,%.3f,%.3f,%.2fx,%.2fx\n", + optimizer_name(optimizers[oi]), buffers.n, threads, before, + after, parallel, after > 0.0 ? before / after : 0.0, + parallel > 0.0 ? before / parallel : 0.0); + } + } + destroy_buffers(&buffers); + } + omp_set_num_threads(max_threads); + return 0; +} \ No newline at end of file diff --git a/app/vector/tests/test_sv_optim.c b/app/vector/tests/test_sv_optim.c new file mode 100644 index 0000000..763003e --- /dev/null +++ b/app/vector/tests/test_sv_optim.c @@ -0,0 +1,190 @@ +#include "sv_optim.h" + +#include +#include +#include +#include +#include + +#define UNITY_INCLUDE_FLOAT +#define UNITY_FLOAT_PRECISION 6 +#define TEST_CASE(...) +#include "unity.h" + +static bool near_value(float actual, float expected) { + return fabsf(actual - expected) <= 1.0e-6f * fmaxf(1.0f, fabsf(expected)); +} + +void setUp(void) {} +void tearDown(void) {} + +void test_sv_optimizers_match_scalar_reference_sizes(void) { + const size_t sizes[] = {1, 7, 1000, 100000}; + const float learning_rate = 0.025f; + const float grad_scale = 0.4f; + const float weight_decay = 0.003f; + const float beta1 = 0.9f; + const float beta2 = 0.99f; + const float epsilon = 1.0e-7f; + const uint64_t step = 7; + + for (size_t si = 0; si < sizeof(sizes) / sizeof(sizes[0]); ++si) { + const size_t n = sizes[si]; + float *params = (float *)malloc(n * sizeof(float)); + float *grads = (float *)malloc(n * sizeof(float)); + float *state1 = (float *)malloc(n * sizeof(float)); + float *state2 = (float *)malloc(n * sizeof(float)); + TEST_ASSERT_NOT_NULL(params); + TEST_ASSERT_NOT_NULL(grads); + TEST_ASSERT_NOT_NULL(state1); + TEST_ASSERT_NOT_NULL(state2); + + for (size_t i = 0; i < n; ++i) { + params[i] = 0.25f + (float)(i % 23u) * 0.01f; + grads[i] = ((float)(i % 19u) - 9.0f) * 0.003f; + state1[i] = 0.02f; + state2[i] = 0.03f; + } + + for (size_t i = 0; i < n; ++i) { + const float gradient = grads[i] * grad_scale; + const float expected = params[i] - learning_rate * weight_decay * params[i] - + learning_rate * gradient; + params[i] = 0.25f + (float)(i % 23u) * 0.01f; + TEST_ASSERT_TRUE(sv_optim_sgd(params + i, grads + i, 1, learning_rate, + grad_scale, weight_decay)); + TEST_ASSERT_TRUE(near_value(params[i], expected)); + } + + for (size_t i = 0; i < n; ++i) { + params[i] = 0.25f + (float)(i % 23u) * 0.01f; + state1[i] = 0.02f; + } + TEST_ASSERT_TRUE(sv_optim_momentum(params, grads, state1, n, + learning_rate, 0.8f, grad_scale, + weight_decay)); + for (size_t i = 0; i < n; ++i) { + const float gradient = grads[i] * grad_scale; + const float velocity = 0.8f * 0.02f + gradient; + const float original = 0.25f + (float)(i % 23u) * 0.01f; + const float expected = original - learning_rate * weight_decay * original - + learning_rate * velocity; + TEST_ASSERT_TRUE(near_value(state1[i], velocity)); + TEST_ASSERT_TRUE(near_value(params[i], expected)); + } + + for (size_t i = 0; i < n; ++i) { + params[i] = 0.25f + (float)(i % 23u) * 0.01f; + state2[i] = 0.03f; + } + TEST_ASSERT_TRUE(sv_optim_rmsprop(params, grads, state2, n, learning_rate, + beta2, epsilon, grad_scale, weight_decay)); + for (size_t i = 0; i < n; ++i) { + const float gradient = grads[i] * grad_scale; + const float square = beta2 * 0.03f + (1.0f - beta2) * gradient * gradient; + const float original = 0.25f + (float)(i % 23u) * 0.01f; + const float expected = original - learning_rate * weight_decay * original - + learning_rate * gradient / (sqrtf(square) + epsilon); + TEST_ASSERT_TRUE(near_value(state2[i], square)); + TEST_ASSERT_TRUE(near_value(params[i], expected)); + } + + for (size_t i = 0; i < n; ++i) { + params[i] = 0.25f + (float)(i % 23u) * 0.01f; + state1[i] = 0.02f; + state2[i] = 0.03f; + } + TEST_ASSERT_TRUE(sv_optim_adam(params, grads, state1, state2, n, + learning_rate, beta1, beta2, epsilon, step, + grad_scale, weight_decay)); + const float inverse_bias1 = 1.0f / (1.0f - powf(beta1, (float)step)); + const float inverse_bias2 = 1.0f / (1.0f - powf(beta2, (float)step)); + for (size_t i = 0; i < n; ++i) { + const float gradient = grads[i] * grad_scale; + const float first = beta1 * 0.02f + (1.0f - beta1) * gradient; + const float second = beta2 * 0.03f + (1.0f - beta2) * gradient * gradient; + const float original = 0.25f + (float)(i % 23u) * 0.01f; + const float expected = original - learning_rate * weight_decay * original - + learning_rate * first * inverse_bias1 / + (sqrtf(second * inverse_bias2) + epsilon); + TEST_ASSERT_TRUE(near_value(state1[i], first)); + TEST_ASSERT_TRUE(near_value(state2[i], second)); + TEST_ASSERT_TRUE(near_value(params[i], expected)); + } + + free(state2); + free(state1); + free(grads); + free(params); + } +} + +void test_sv_optimizers_validate_inputs_and_empty_lengths(void) { + float value = 1.0f; + TEST_ASSERT_FALSE(sv_optim_sgd(NULL, &value, 1, 0.1f, 1.0f, 0.0f)); + TEST_ASSERT_FALSE(sv_optim_sgd(&value, &value, 0, 0.1f, 1.0f, 0.0f)); + TEST_ASSERT_FALSE(sv_optim_momentum(&value, &value, NULL, 1, 0.1f, 0.9f, + 1.0f, 0.0f)); + TEST_ASSERT_FALSE(sv_optim_rmsprop(&value, &value, &value, 1, 0.1f, 1.0f, + 1.0e-8f, 1.0f, 0.0f)); + TEST_ASSERT_FALSE(sv_optim_adam(&value, &value, &value, &value, 1, 0.1f, + 0.9f, 0.999f, 1.0e-8f, 0, 1.0f, 0.0f)); + TEST_ASSERT_EQUAL_FLOAT(0.0f, sv_clip_norm(NULL, 0, 1.0f)); + TEST_ASSERT_FALSE(sv_optim_sgd(&value, &value, 1, NAN, 1.0f, 0.0f)); + TEST_ASSERT_EQUAL_FLOAT(-1.0f, sv_clip_norm(&value, 1, NAN)); +} + +void test_sv_clip_norm_returns_old_norm_and_scales_only_when_needed(void) { + float gradients[] = {6.0f, 8.0f}; + TEST_ASSERT_TRUE(near_value(sv_clip_norm(gradients, 2, 5.0f), 10.0f)); + TEST_ASSERT_TRUE(near_value(gradients[0], 3.0f)); + TEST_ASSERT_TRUE(near_value(gradients[1], 4.0f)); + + float below_limit[] = {3.0f, 4.0f}; + TEST_ASSERT_TRUE(near_value(sv_clip_norm(below_limit, 2, 5.0f), 5.0f)); + TEST_ASSERT_EQUAL_FLOAT(3.0f, below_limit[0]); + TEST_ASSERT_EQUAL_FLOAT(4.0f, below_limit[1]); +} + +void test_sv_adam_is_bitwise_independent_of_openmp_thread_count(void) { + const size_t n = 100000; + float *params1 = (float *)malloc(n * sizeof(float)); + float *params8 = (float *)malloc(n * sizeof(float)); + float *grads = (float *)malloc(n * sizeof(float)); + float *m1 = (float *)calloc(n, sizeof(float)); + float *m8 = (float *)calloc(n, sizeof(float)); + float *v1 = (float *)calloc(n, sizeof(float)); + float *v8 = (float *)calloc(n, sizeof(float)); + TEST_ASSERT_NOT_NULL(params1); + TEST_ASSERT_NOT_NULL(params8); + TEST_ASSERT_NOT_NULL(grads); + TEST_ASSERT_NOT_NULL(m1); + TEST_ASSERT_NOT_NULL(m8); + TEST_ASSERT_NOT_NULL(v1); + TEST_ASSERT_NOT_NULL(v8); + for (size_t i = 0; i < n; ++i) { + params1[i] = params8[i] = 0.1f + (float)(i % 17u) * 0.01f; + grads[i] = (float)((i * 7u) % 31u) * 0.001f; + } + + const int previous_threads = omp_get_max_threads(); + omp_set_dynamic(0); + omp_set_num_threads(1); + TEST_ASSERT_TRUE(sv_optim_adam(params1, grads, m1, v1, n, 0.001f, 0.9f, + 0.999f, 1.0e-8f, 3, 0.75f, 0.0f)); + omp_set_num_threads(8); + TEST_ASSERT_TRUE(sv_optim_adam(params8, grads, m8, v8, n, 0.001f, 0.9f, + 0.999f, 1.0e-8f, 3, 0.75f, 0.0f)); + TEST_ASSERT_EQUAL(0, memcmp(params1, params8, n * sizeof(float))); + TEST_ASSERT_EQUAL(0, memcmp(m1, m8, n * sizeof(float))); + TEST_ASSERT_EQUAL(0, memcmp(v1, v8, n * sizeof(float))); + omp_set_num_threads(previous_threads); + + free(v8); + free(v1); + free(m8); + free(m1); + free(grads); + free(params8); + free(params1); +} \ No newline at end of file diff --git a/doc/api/matrix/m_convert.md b/doc/api/matrix/m_convert.md index 7cb2490..6e280db 100644 --- a/doc/api/matrix/m_convert.md +++ b/doc/api/matrix/m_convert.md @@ -1,6 +1,6 @@ -# `m_convert.h` – Matrix/Tensor Conversion API +# `m_convert.h` – Dense/Sparse Matrix Conversion API -Public conversion API between dense/sparse matrix and tensor types. +Public conversion API between dense and sparse matrix types. ## Functions @@ -51,27 +51,3 @@ Parameters: - `src` (`const DoubleMatrix *`): Source double matrix. Returns: Dense float matrix with precision-reduced values, or NULL on allocation failure. - -### `st_from_sm` - -`FloatTensor * st_from_sm(const FloatMatrix *src)` - -Convert dense float matrix to float tensor. - -Parameters: - -- `src` (`const FloatMatrix *`): Source float matrix (shape: rows x cols interpreted as batch 1, rows, cols, 1). - -Returns: Float tensor with matrix reshaped, or NULL on allocation failure. - -### `sm_from_st` - -`sm_from_st(const FloatTensor *src)` - -Convert float tensor to dense float matrix. - -Parameters: - -- `src` (`const FloatTensor *`): Source float tensor (extracted as 2D: flatten/reshape to rows x cols). - -Returns: Dense float matrix, or NULL on allocation failure or invalid tensor shape. diff --git a/doc/api/matrix/sm.md b/doc/api/matrix/sm.md index 9326fe6..d93defd 100644 --- a/doc/api/matrix/sm.md +++ b/doc/api/matrix/sm.md @@ -24,11 +24,23 @@ Values: - `SM_NO_TRANSPOSE` - `SM_TRANSPOSE` +### `SmActivation` + +Activation applied by fused matrix kernels. + +Values: + +- `SM_ACT_NONE` +- `SM_ACT_RELU` +- `SM_ACT_SIGMOID` +- `SM_ACT_TANH` + ### Typedefs - `enum SmBackend` — Available compute backends (runtime selection). - `struct FloatMatrix FloatMatrix` — Dense float matrix with row-major storage: values[i * cols + j]. - `enum SmTranspose` — Transpose mode for BLAS-style operations. +- `enum SmActivation` — Activation applied by fused matrix kernels. ## Functions @@ -490,6 +502,121 @@ Parameters: Returns: Solution vector x, or NULL on allocation/singularity error. +### `sm_apply_relu` + +`bool sm_apply_relu(FloatMatrix *mat)` + +Apply ReLU in-place to every matrix element. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_sigmoid` + +`bool sm_apply_sigmoid(FloatMatrix *mat)` + +Apply sigmoid in-place to every matrix element. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_tanh` + +`bool sm_apply_tanh(FloatMatrix *mat)` + +Apply hyperbolic tangent in-place to every matrix element. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_gelu_tanh` + +`bool sm_apply_gelu_tanh(FloatMatrix *mat)` + +Apply tanh-approximation GELU in-place. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_gelu_erf` + +`bool sm_apply_gelu_erf(FloatMatrix *mat)` + +Apply erf-form GELU in-place. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_softmax` + +`bool sm_apply_softmax(FloatMatrix *mat)` + +Apply numerically stable softmax independently to each row. + +Parameters: + +- `mat` (`FloatMatrix *`) + +### `sm_apply_relu_backward` + +`bool sm_apply_relu_backward(const FloatMatrix *activation, FloatMatrix *grad)` + +Multiply grad in-place by the ReLU mask at activation. + +Parameters: + +- `activation` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + +### `sm_apply_sigmoid_backward` + +`bool sm_apply_sigmoid_backward(const FloatMatrix *activation, FloatMatrix *grad)` + +Multiply grad in-place by the sigmoid derivative at activation. + +Parameters: + +- `activation` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + +### `sm_apply_tanh_backward` + +`bool sm_apply_tanh_backward(const FloatMatrix *activation, FloatMatrix *grad)` + +Multiply grad in-place by the tanh derivative at activation. + +Parameters: + +- `activation` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + +### `sm_apply_gelu_tanh_backward` + +`bool sm_apply_gelu_tanh_backward(const FloatMatrix *input, FloatMatrix *grad)` + +Multiply grad in-place by the tanh-GELU derivative at input. + +Parameters: + +- `input` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + +### `sm_apply_gelu_erf_backward` + +`bool sm_apply_gelu_erf_backward(const FloatMatrix *input, FloatMatrix *grad)` + +Multiply grad in-place by the erf-GELU derivative at input. + +Parameters: + +- `input` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + ### `sm_gemm` `bool sm_gemm(FloatMatrix *C, float alpha, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, float beta)` @@ -508,6 +635,24 @@ Parameters: Returns: true on success, false on shape mismatch or allocation failure. +### `sm_gemm_bias_act` + +`bool sm_gemm_bias_act(FloatMatrix *C, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, const FloatMatrix *bias, SmActivation activation)` + +Fused GEMM + optional row bias + activation. + +Parameters: + +- `C` (`FloatMatrix *`): Output matrix, shape M x N. +- `A` (`const FloatMatrix *`): Left input matrix. +- `trans_a` (`SmTranspose`): Transpose mode for A. +- `B` (`const FloatMatrix *`): Right input matrix. +- `trans_b` (`SmTranspose`): Transpose mode for B. +- `bias` (`const FloatMatrix *`): Bias row (1 x N), per-element bias (M x N), or NULL. +- `activation` (`SmActivation`): Activation to apply after bias. + +Returns: true on success, false on invalid input or incompatible shapes. + ### `sm_gemm_bias_relu` `bool sm_gemm_bias_relu(FloatMatrix *C, const FloatMatrix *A, SmTranspose trans_a, const FloatMatrix *B, SmTranspose trans_b, const FloatMatrix *bias)` @@ -525,6 +670,83 @@ Parameters: Returns: true on success, false on shape/allocation error. +### `sm_add_rowwise` + +`bool sm_add_rowwise(FloatMatrix *mat, const FloatMatrix *row)` + +Add a 1 x cols row to every row of a matrix in place. + +Parameters: + +- `mat` (`FloatMatrix *`) +- `row` (`const FloatMatrix *`) + +Returns: true on success, false on null input or shape mismatch. + +### `sm_sum_rows` + +`sm_sum_rows(const FloatMatrix *mat)` + +Sum matrix rows in fixed row order. + +Parameters: + +- `mat` (`const FloatMatrix *`) + +Returns: New 1 x cols matrix, or NULL on invalid input/allocation failure. + +### `sm_argmax_rows` + +`sm_argmax_rows(const FloatMatrix *mat)` + +Return the first maximal column index for every row as float values. + +Parameters: + +- `mat` (`const FloatMatrix *`) + +Returns: New rows x 1 matrix, or NULL on invalid input/allocation failure. + +### `sm_softmax_ce_grad` + +`bool sm_softmax_ce_grad(const FloatMatrix *predicted, const FloatMatrix *target, FloatMatrix *grad)` + +In-place softmax-cross-entropy gradient predicted - target. + +Parameters: + +- `predicted` (`const FloatMatrix *`) +- `target` (`const FloatMatrix *`) +- `grad` (`FloatMatrix *`) + +Returns: true on success, false on null input or shape mismatch. + +### `sm_loss_mse` + +`float sm_loss_mse(const FloatMatrix *predicted, const FloatMatrix *target)` + +Mean squared error, matching nm_lib reduction semantics. + +Parameters: + +- `predicted` (`const FloatMatrix *`) +- `target` (`const FloatMatrix *`) + +Returns: Loss value, or -1.0f for invalid/empty inputs. + +### `sm_loss_cross_entropy` + +`float sm_loss_cross_entropy(const FloatMatrix *predicted, const FloatMatrix *target)` + +Cross-entropy loss, averaged over rows and clamped at 1e-7. + +Parameters: + +- `predicted` (`const FloatMatrix *`) +- `target` (`const FloatMatrix *`) + +Returns: Loss value, or -1.0f for invalid inputs. + ### `sm_inplace_add` `bool sm_inplace_add(FloatMatrix *mat1, const FloatMatrix *mat2)` diff --git a/doc/api/matrix/sm_mps.md b/doc/api/matrix/sm_mps.md new file mode 100644 index 0000000..afa5c5f --- /dev/null +++ b/doc/api/matrix/sm_mps.md @@ -0,0 +1,270 @@ +# `sm_mps.h` – GPU-resident Float Matrix API + +## Types + +### Typedefs + +- `struct SmMpsMatrix` +- `struct SmMpsStream` +- `struct SmMpsGemmPlan` +- `struct SmMpsCounters SmMpsCounters` + +## Functions + +### `mps_get_shared_device` + +`void * mps_get_shared_device(void)` + +Returns the shared MTLDevice as an opaque pointer, or NULL if unavailable. + +Parameters: + +- `(unnamed)` (`void`) + +### `mps_get_shared_command_queue` + +`void * mps_get_shared_command_queue(void)` + +Returns the shared MTLCommandQueue as an opaque pointer, or NULL if unavailable. + +Parameters: + +- `(unnamed)` (`void`) + +### `sm_mps_get_counters` + +`sm_mps_get_counters(void)` + +Parameters: + +- `(unnamed)` (`void`) + +### `sm_mps_reset_counters` + +`void sm_mps_reset_counters(void)` + +Parameters: + +- `(unnamed)` (`void`) + +### `sm_mps_matrix_create` + +`sm_mps_matrix_create(size_t rows, size_t cols)` + +Parameters: + +- `rows` (`size_t`) +- `cols` (`size_t`) + +### `sm_mps_matrix_destroy` + +`void sm_mps_matrix_destroy(SmMpsMatrix *matrix)` + +Parameters: + +- `matrix` (`SmMpsMatrix *`) + +### `sm_mps_matrix_contents` + +`float * sm_mps_matrix_contents(SmMpsMatrix *matrix)` + +Parameters: + +- `matrix` (`SmMpsMatrix *`) + +### `sm_mps_matrix_const_contents` + +`const float * sm_mps_matrix_const_contents(const SmMpsMatrix *matrix)` + +Parameters: + +- `matrix` (`const SmMpsMatrix *`) + +### `sm_mps_matrix_upload` + +`bool sm_mps_matrix_upload(SmMpsMatrix *matrix, const float *values)` + +Parameters: + +- `matrix` (`SmMpsMatrix *`) +- `values` (`const float *`) + +### `sm_mps_matrix_download` + +`bool sm_mps_matrix_download(const SmMpsMatrix *matrix, float *values)` + +Parameters: + +- `matrix` (`const SmMpsMatrix *`) +- `values` (`float *`) + +### `sm_mps_stream_create` + +`sm_mps_stream_create(void)` + +Parameters: + +- `(unnamed)` (`void`) + +### `sm_mps_stream_commit` + +`bool sm_mps_stream_commit(SmMpsStream *stream)` + +Parameters: + +- `stream` (`SmMpsStream *`) + +### `sm_mps_stream_wait` + +`bool sm_mps_stream_wait(SmMpsStream *stream)` + +Parameters: + +- `stream` (`SmMpsStream *`) + +### `sm_mps_stream_destroy` + +`void sm_mps_stream_destroy(SmMpsStream *stream)` + +Parameters: + +- `stream` (`SmMpsStream *`) + +### `sm_mps_gemm_plan_create` + +`sm_mps_gemm_plan_create(size_t result_rows, size_t result_cols, size_t interior_cols, bool transpose_left, bool transpose_right, float alpha, float beta)` + +Parameters: + +- `result_rows` (`size_t`) +- `result_cols` (`size_t`) +- `interior_cols` (`size_t`) +- `transpose_left` (`bool`) +- `transpose_right` (`bool`) +- `alpha` (`float`) +- `beta` (`float`) + +### `sm_mps_gemm_plan_destroy` + +`void sm_mps_gemm_plan_destroy(SmMpsGemmPlan *plan)` + +Parameters: + +- `plan` (`SmMpsGemmPlan *`) + +### `sm_mps_gemm_plan_encode` + +`bool sm_mps_gemm_plan_encode(SmMpsStream *stream, const SmMpsGemmPlan *plan, SmMpsMatrix *C, const SmMpsMatrix *A, const SmMpsMatrix *B)` + +Parameters: + +- `stream` (`SmMpsStream *`) +- `plan` (`const SmMpsGemmPlan *`) +- `C` (`SmMpsMatrix *`) +- `A` (`const SmMpsMatrix *`) +- `B` (`const SmMpsMatrix *`) + +### `sm_mps_matrix_gemm_async` + +`bool sm_mps_matrix_gemm_async(SmMpsStream *stream, SmMpsMatrix *C, float alpha, const SmMpsMatrix *A, bool transpose_left, const SmMpsMatrix *B, bool transpose_right, float beta)` + +Parameters: + +- `stream` (`SmMpsStream *`) +- `C` (`SmMpsMatrix *`) +- `alpha` (`float`) +- `A` (`const SmMpsMatrix *`) +- `transpose_left` (`bool`) +- `B` (`const SmMpsMatrix *`) +- `transpose_right` (`bool`) +- `beta` (`float`) + +### `sm_mps_matrix_bias_relu_async` + +`bool sm_mps_matrix_bias_relu_async(SmMpsStream *stream, SmMpsMatrix *C, const SmMpsMatrix *bias, bool bias_is_row)` + +Parameters: + +- `stream` (`SmMpsStream *`) +- `C` (`SmMpsMatrix *`) +- `bias` (`const SmMpsMatrix *`) +- `bias_is_row` (`bool`) + +### `sm_mps_matrix_gemm_bias_relu_async` + +`bool sm_mps_matrix_gemm_bias_relu_async(SmMpsStream *stream, SmMpsMatrix *C, float alpha, const SmMpsMatrix *A, bool transpose_left, const SmMpsMatrix *B, bool transpose_right, const SmMpsMatrix *bias, bool bias_is_row)` + +Parameters: + +- `stream` (`SmMpsStream *`) +- `C` (`SmMpsMatrix *`) +- `alpha` (`float`) +- `A` (`const SmMpsMatrix *`) +- `transpose_left` (`bool`) +- `B` (`const SmMpsMatrix *`) +- `transpose_right` (`bool`) +- `bias` (`const SmMpsMatrix *`) +- `bias_is_row` (`bool`) + +### `sm_mps_matrix_gemm_bias_relu_ex` + +`bool sm_mps_matrix_gemm_bias_relu_ex(SmMpsMatrix *C, float alpha, const SmMpsMatrix *A, bool transpose_left, const SmMpsMatrix *B, bool transpose_right, const SmMpsMatrix *bias, bool bias_is_row)` + +Parameters: + +- `C` (`SmMpsMatrix *`) +- `alpha` (`float`) +- `A` (`const SmMpsMatrix *`) +- `transpose_left` (`bool`) +- `B` (`const SmMpsMatrix *`) +- `transpose_right` (`bool`) +- `bias` (`const SmMpsMatrix *`) +- `bias_is_row` (`bool`) + +### `sm_mps_matrix_gemm_ex` + +`bool sm_mps_matrix_gemm_ex(SmMpsMatrix *C, float alpha, const SmMpsMatrix *A, bool transpose_left, const SmMpsMatrix *B, bool transpose_right, float beta)` + +Parameters: + +- `C` (`SmMpsMatrix *`) +- `alpha` (`float`) +- `A` (`const SmMpsMatrix *`) +- `transpose_left` (`bool`) +- `B` (`const SmMpsMatrix *`) +- `transpose_right` (`bool`) +- `beta` (`float`) + +### `mps_matrix_multiply` + +`bool mps_matrix_multiply(const float *mat1, size_t rows1, size_t cols1, const float *mat2, size_t rows2, size_t cols2, float *result)` + +Parameters: + +- `mat1` (`const float *`) +- `rows1` (`size_t`) +- `cols1` (`size_t`) +- `mat2` (`const float *`) +- `rows2` (`size_t`) +- `cols2` (`size_t`) +- `result` (`float *`) + +### `mps_matrix_multiply_ex` + +`bool mps_matrix_multiply_ex(const float *mat1, size_t rows1, size_t cols1, bool transpose_left, const float *mat2, size_t rows2, size_t cols2, bool transpose_right, float alpha, float beta, float *result, size_t result_rows, size_t result_cols)` + +Parameters: + +- `mat1` (`const float *`) +- `rows1` (`size_t`) +- `cols1` (`size_t`) +- `transpose_left` (`bool`) +- `mat2` (`const float *`) +- `rows2` (`size_t`) +- `cols2` (`size_t`) +- `transpose_right` (`bool`) +- `alpha` (`float`) +- `beta` (`float`) +- `result` (`float *`) +- `result_rows` (`size_t`) +- `result_cols` (`size_t`) diff --git a/doc/api/tensor/st.md b/doc/api/tensor/st.md index 153d0b1..60fd061 100644 --- a/doc/api/tensor/st.md +++ b/doc/api/tensor/st.md @@ -268,7 +268,7 @@ Parameters: ### `st_tensor_shape` -`constsize_t* st_tensor_shape(const FloatTensor *tensor)` +`const size_t * st_tensor_shape(const FloatTensor *tensor)` Return shape pointer, or NULL for NULL tensor. @@ -298,7 +298,7 @@ Parameters: ### `st_tensor_data` -`constfloat* st_tensor_data(const FloatTensor *tensor)` +`const float * st_tensor_data(const FloatTensor *tensor)` Return raw const data pointer. @@ -308,7 +308,7 @@ Parameters: ### `st_tensor_mutable_data` -`float* st_tensor_mutable_data(FloatTensor *tensor)` +`float * st_tensor_mutable_data(FloatTensor *tensor)` Return raw mutable data pointer. @@ -392,6 +392,100 @@ Parameters: - `activation` (`const FloatTensor *`) - `grad` (`FloatTensor *`) +### `st_apply_sigmoid` + +`bool st_apply_sigmoid(FloatTensor *t)` + +Apply sigmoid in-place to every tensor element. + +Parameters: + +- `t` (`FloatTensor *`) + +### `st_apply_tanh` + +`bool st_apply_tanh(FloatTensor *t)` + +Apply hyperbolic tangent in-place to every tensor element. + +Parameters: + +- `t` (`FloatTensor *`) + +### `st_apply_gelu_tanh` + +`bool st_apply_gelu_tanh(FloatTensor *t)` + +Apply tanh-approximation GELU in-place. + +Parameters: + +- `t` (`FloatTensor *`) + +### `st_apply_gelu_erf` + +`bool st_apply_gelu_erf(FloatTensor *t)` + +Apply erf-form GELU in-place. + +Parameters: + +- `t` (`FloatTensor *`) + +### `st_apply_softmax` + +`bool st_apply_softmax(FloatTensor *t)` + +Apply softmax independently over the last tensor axis. + +Parameters: + +- `t` (`FloatTensor *`) + +### `st_apply_sigmoid_backward` + +`bool st_apply_sigmoid_backward(const FloatTensor *activation, FloatTensor *grad)` + +Multiply grad in-place by the sigmoid derivative at activation. + +Parameters: + +- `activation` (`const FloatTensor *`) +- `grad` (`FloatTensor *`) + +### `st_apply_tanh_backward` + +`bool st_apply_tanh_backward(const FloatTensor *activation, FloatTensor *grad)` + +Multiply grad in-place by the tanh derivative at activation. + +Parameters: + +- `activation` (`const FloatTensor *`) +- `grad` (`FloatTensor *`) + +### `st_apply_gelu_tanh_backward` + +`bool st_apply_gelu_tanh_backward(const FloatTensor *input, FloatTensor *grad)` + +Multiply grad in-place by the tanh-GELU derivative at input. + +Parameters: + +- `input` (`const FloatTensor *`) +- `grad` (`FloatTensor *`) + +### `st_apply_gelu_erf_backward` + +`bool st_apply_gelu_erf_backward(const FloatTensor *input, FloatTensor *grad)` + +Multiply grad in-place by the erf-GELU derivative at input. + +Parameters: + +- `input` (`const FloatTensor *`) +- `grad` (`FloatTensor *`) + ### `st_sum_axes` `st_sum_axes(const FloatTensor *t, const size_t *axes, size_t num_axes)` diff --git a/doc/api/tensor/st_conv.md b/doc/api/tensor/st_conv.md index da10756..f4b7e03 100644 --- a/doc/api/tensor/st_conv.md +++ b/doc/api/tensor/st_conv.md @@ -189,7 +189,7 @@ Parameters: ### `st_conv2d_last_backend` -`constchar* st_conv2d_last_backend(void)` +`const char * st_conv2d_last_backend(void)` Return backend label used by the latest st_conv2d_nchw call. diff --git a/doc/api/tensor/st_convert.md b/doc/api/tensor/st_convert.md new file mode 100644 index 0000000..89f689a --- /dev/null +++ b/doc/api/tensor/st_convert.md @@ -0,0 +1,29 @@ +# `st_convert.h` – Tensor/Matrix Conversion API + +Conversion helpers between tensor and matrix types. + +## Functions + +### `st_from_sm` + +`st_from_sm(const FloatMatrix *src)` + +Convert dense float matrix to float tensor. + +Parameters: + +- `src` (`const FloatMatrix *`): Source float matrix. + +Returns: Float tensor with shape [rows, cols], or NULL on allocation/input error. + +### `sm_from_st` + +`sm_from_st(const FloatTensor *src)` + +Convert float tensor to dense float matrix. + +Parameters: + +- `src` (`const FloatTensor *`): Source float tensor (2D). + +Returns: Dense float matrix, or NULL on allocation/input error. diff --git a/doc/api/tensor/st_f16.md b/doc/api/tensor/st_f16.md new file mode 100644 index 0000000..c874aa0 --- /dev/null +++ b/doc/api/tensor/st_f16.md @@ -0,0 +1,53 @@ +# `st_f16.h` – IEEE Binary16 Conversion API + +Public API for IEEE binary16 (half precision) conversion. + +## Functions + +### `st_f32_to_f16` + +`uint16_t st_f32_to_f16(float value)` + +Convert one f32 value to IEEE binary16 bits. + +Parameters: + +- `value` (`float`): Input value. + +Returns: binary16 bit pattern (round to nearest even, overflow to infinity, subnormals exact, NaN quiet with sign preserved). + +### `st_f16_to_f32` + +`float st_f16_to_f32(uint16_t bits)` + +Convert IEEE binary16 bits to one f32 value. + +Parameters: + +- `bits` (`uint16_t`): binary16 bit pattern. + +Returns: Exact f32 value (all binary16 values are exactly representable). + +### `st_f32_to_f16_bulk` + +`void st_f32_to_f16_bulk(const float *src, uint16_t *dst, size_t n)` + +Convert a contiguous f32 buffer to binary16 bits. + +Parameters: + +- `src` (`const float *`): Source buffer with n values. +- `dst` (`uint16_t *`): Destination buffer with n uint16 slots. +- `n` (`size_t`): Number of elements. + +### `st_f16_to_f32_bulk` + +`void st_f16_to_f32_bulk(const uint16_t *src, float *dst, size_t n)` + +Convert a contiguous binary16 buffer to f32. + +Parameters: + +- `src` (`const uint16_t *`): Source buffer with n uint16 slots. +- `dst` (`float *`): Destination buffer with n values. +- `n` (`size_t`): Number of elements. diff --git a/doc/api/tensor/st_interp.md b/doc/api/tensor/st_interp.md new file mode 100644 index 0000000..cc8d796 --- /dev/null +++ b/doc/api/tensor/st_interp.md @@ -0,0 +1,161 @@ +# `st_interp.h` – Graph Interpreter Kernel API + +Public API for graph-interpreter tensor kernels. + +## Types + +### `StBinOp` + +Element-wise binary operations for the broadcast kernels. + +Values: + +- `ST_BIN_ADD` +- `ST_BIN_SUB` +- `ST_BIN_MUL` +- `ST_BIN_DIV` + +### Typedefs + +- `enum StBinOp` — Element-wise binary operations for the broadcast kernels. + +## Functions + +### `st_binary_broadcast` + +`st_binary_broadcast(StBinOp op, const FloatTensor *a, const FloatTensor *b)` + +Apply a binary op with NumPy-style broadcasting over any axes. + +Parameters: + +- `op` (`StBinOp`): Operation to apply. +- `a` (`const FloatTensor *`): Left input tensor (contiguous, f32 or bf16). +- `b` (`const FloatTensor *`): Right input tensor (contiguous, f32 or bf16). + +Returns: New contiguous f32 tensor on success, or NULL on invalid input, non-broadcastable shapes, or allocation failure. + +### `st_binary_broadcast_into` + +`bool st_binary_broadcast_into(StBinOp op, const FloatTensor *a, const FloatTensor *b, FloatTensor *out)` + +Apply a binary op with broadcasting into a prepared output tensor. + +Parameters: + +- `op` (`StBinOp`): Operation to apply. +- `a` (`const FloatTensor *`): Left input tensor (contiguous, f32 or bf16). +- `b` (`const FloatTensor *`): Right input tensor (contiguous, f32 or bf16). +- `out` (`FloatTensor *`): Output tensor; contiguous with the broadcast shape of a and b, f32 or bf16. May alias a or b when that operand's shape equals the output shape (the row kernels read and write the same element-wise position, so in-place accumulation like a += bias is safe); it must not partially overlap any operand with a different shape. + +### `st_layernorm` + +`bool st_layernorm(const FloatTensor *x, const FloatTensor *gamma, const FloatTensor *beta, size_t norm_rank, float eps, FloatTensor *out)` + +Layer-normalize over the last norm_rank axes. + +Parameters: + +- `x` (`const FloatTensor *`): Input tensor (contiguous, f32 or bf16). +- `gamma` (`const FloatTensor *`): Optional per-element scale of the normalized group; NULL or a contiguous tensor with group elements (f32 or bf16). +- `beta` (`const FloatTensor *`): Optional per-element shift; NULL or like gamma. +- `norm_rank` (`size_t`): Number of trailing axes to normalize over, [1, ndim]. +- `eps` (`float`): Variance epsilon added before the inverse square root. +- `out` (`FloatTensor *`): Output tensor, contiguous with the same shape as x; f32 or bf16. May alias x (in-place normalization per group). + +### `st_reduce_max` + +`st_reduce_max(const FloatTensor *x, uint32_t axes_mask, bool keepdims)` + +Maximum over the axes selected in axes_mask. + +Parameters: + +- `x` (`const FloatTensor *`): Input tensor (contiguous, f32 or bf16). +- `axes_mask` (`uint32_t`): Bitmask; axis i is reduced when bit i is set. Bits at or above the tensor rank are rejected. +- `keepdims` (`bool`): Keep reduced axes as size-1 axes in the output. + +Returns: New contiguous f32 tensor on success, or NULL on error. + +### `st_reduce_max_into` + +`bool st_reduce_max_into(const FloatTensor *x, uint32_t axes_mask, bool keepdims, FloatTensor *out)` + +Maximum over selected axes into a prepared output tensor. + +Parameters: + +- `x` (`const FloatTensor *`): Input tensor (contiguous, f32 or bf16). +- `axes_mask` (`uint32_t`): Bitmask of axes to reduce; see st_reduce_max. +- `keepdims` (`bool`): Keep reduced axes as size-1 axes in the output. +- `out` (`FloatTensor *`): Output tensor with the reduced shape, f32 or bf16. Must not alias x (an empty mask copies; use st_reduce_max instead for in-place results). + +### `st_matmul_batched` + +`bool st_matmul_batched(const FloatTensor *a, const FloatTensor *b, FloatTensor *c)` + +Batched matrix product with broadcast over the leading axes. + +Parameters: + +- `a` (`const FloatTensor *`): Left tensor, rank 2 to 4, contiguous, f32 or bf16; the last two axes are [M, K]. +- `b` (`const FloatTensor *`): Right tensor, rank 2 to 4, contiguous, f32 or bf16; the last two axes are [K, N]. +- `c` (`FloatTensor *`): Output tensor with the broadcast batch shape plus [M, N], contiguous, f32 or bf16. Must not alias a or b (the GEMM backend reads inputs while accumulating). + +### `st_gather` + +`bool st_gather(const FloatTensor *table, const int64_t *indices, size_t count, size_t axis, FloatTensor *out)` + +Gather slices along one axis, indexed by int64 values. + +Parameters: + +- `table` (`const FloatTensor *`): Source tensor, contiguous, f32 or bf16. +- `indices` (`const int64_t *`): Index array of length count; negative indices count from the end of axis. +- `count` (`size_t`): Number of indices; must be greater than zero. +- `axis` (`size_t`): Gather axis, below the tensor rank. +- `out` (`FloatTensor *`): Output tensor, contiguous, same dtype as table, shape equal to the table shape with axis replaced by count. Must not alias table. + +### `st_gather_bytes` + +`bool st_gather_bytes(const void *table, size_t table_ndim, const size_t *table_shape, const int64_t *indices, size_t count, size_t axis, size_t elem_bytes, void *out)` + +Gather for raw buffers of any element size. + +Parameters: + +- `table` (`const void *`): Source buffer, row-major, shape table_shape. +- `table_ndim` (`size_t`): Rank of table, [1, 8]. +- `table_shape` (`const size_t *`): Shape of table, all dimensions non-zero. +- `indices` (`const int64_t *`): Index array of length count, negative counts from the end of axis (see st_gather). +- `count` (`size_t`): Number of indices, greater than zero. +- `axis` (`size_t`): Gather axis, below table_ndim. +- `elem_bytes` (`size_t`): Element size in bytes (4 for f32/bf16-packed, 8 for int64, 1 for bool). +- `out` (`void *`): Output buffer with outer * count * inner elements. + +### `st_permute_copy_into` + +`bool st_permute_copy_into(const FloatTensor *src, const size_t *perm, FloatTensor *out)` + +Materialize a permuted copy into a prepared output tensor. + +Parameters: + +- `src` (`const FloatTensor *`): Source tensor, contiguous, f32 or bf16. +- `perm` (`const size_t *`): Permutation where output axis i reads input axis perm[i] (same convention as st_permute and nm_lib's run_transpose). +- `out` (`FloatTensor *`): Output tensor, contiguous, same dtype as src, shape equal to the permuted source shape. Must not alias src (including identity permutations, which would copy onto itself). + +### `st_permute_copy_into_bytes` + +`bool st_permute_copy_into_bytes(const void *src, size_t ndim, const size_t *shape, const size_t *perm, size_t elem_bytes, void *out)` + +Materializing permute for raw buffers of any element size. + +Parameters: + +- `src` (`const void *`): Source buffer, row-major, shape shape. +- `ndim` (`size_t`): Rank, [1, 8]. +- `shape` (`const size_t *`): Source shape, all dimensions non-zero. +- `perm` (`const size_t *`): Permutation array where output axis i reads input axis perm[i]. +- `elem_bytes` (`size_t`): Element size in bytes. +- `out` (`void *`): Output buffer with the permuted shape's element count. diff --git a/doc/api/tensor/st_vector_bridge.md b/doc/api/tensor/st_vector_bridge.md new file mode 100644 index 0000000..920e58b --- /dev/null +++ b/doc/api/tensor/st_vector_bridge.md @@ -0,0 +1,21 @@ +# `st_vector_bridge.h` – Tensor/Vector Bridge API + +Tensor-specific bridge helpers for vector views and copies. + +## Functions + +### `st_as_vv_view` + +`FloatVectorView st_as_vv_view(FloatTensor *tensor)` + +Parameters: + +- `tensor` (`FloatTensor *`) + +### `st_to_sv` + +`FloatVector * st_to_sv(const FloatTensor *tensor)` + +Parameters: + +- `tensor` (`const FloatTensor *`) diff --git a/doc/api/vector/sv_optim.md b/doc/api/vector/sv_optim.md new file mode 100644 index 0000000..cc7d86d --- /dev/null +++ b/doc/api/vector/sv_optim.md @@ -0,0 +1,47 @@ +# `sv_optim.h` – Flat-Buffer Optimizers + +The optimizer kernels update caller-owned, contiguous `float` buffers. Momentum +and second-moment buffers are also caller-owned and must be initialized by the +caller (normally to zero). All functions return `false` for invalid pointers, +lengths, or hyperparameters. + +## SGD + +`bool sv_optim_sgd(float *params, const float *grads, size_t n, float learning_rate, float grad_scale, float weight_decay)` + +Updates `params -= learning_rate * (grads * grad_scale)`. Decoupled weight +decay is applied separately as `params -= learning_rate * weight_decay * params`; +pass zero to disable it. + +## Momentum + +`bool sv_optim_momentum(float *params, const float *grads, float *momentum_buffer, size_t n, float learning_rate, float momentum, float grad_scale, float weight_decay)` + +Updates velocity as `momentum * velocity + grads * grad_scale`, then applies the +learning-rate step and optional decoupled weight decay. + +## RMSprop + +`bool sv_optim_rmsprop(float *params, const float *grads, float *squared_grad_buffer, size_t n, float learning_rate, float beta2, float epsilon, float grad_scale, float weight_decay)` + +Updates the squared-gradient average and divides the scaled gradient by +`sqrt(average) + epsilon`. + +## Adam + +`bool sv_optim_adam(float *params, const float *grads, float *first_moment, float *second_moment, size_t n, float learning_rate, float beta1, float beta2, float epsilon, uint64_t step, float grad_scale, float weight_decay)` + +Uses the nm_lib first/second moments and bias corrections based on the one-based +`step`. The denominator is `sqrt(v_hat) + epsilon`. + +## Gradient norm clipping + +`float sv_clip_norm(float *grads, size_t n, float max_norm)` + +Returns the original L2 norm and scales gradients only when that norm exceeds +`max_norm`. An empty buffer returns zero; invalid arguments return `-1.0f`. +Norm accumulation uses a fixed scalar +order; scaling is elementwise. + +The optimizer loops use OpenMP only when `n > 1,048,576`, avoiding parallel-region +overhead for small models. \ No newline at end of file diff --git a/doc/index.md b/doc/index.md index 84785ea..8e84fe9 100644 --- a/doc/index.md +++ b/doc/index.md @@ -9,15 +9,25 @@ This documentation is fully located under `doc/`. - [Convolution](api/tensor/st_conv.md) - [Pooling](api/tensor/st_pool.md) - [Shape Ops](api/tensor/st_shape_ops.md) +- [Graph interpreter kernels](api/tensor/st_interp.md) +- [IEEE binary16 conversions](api/tensor/st_f16.md) +- [Tensor/matrix conversions](api/tensor/st_convert.md) +- [Tensor/vector bridge](api/tensor/st_vector_bridge.md) +- [NN Kernel Migration Map](nm_lib_kernels.md) ## Matrix API - [Dense Double (dm)](api/matrix/dm.md) - [Dense Float (sm)](api/matrix/sm.md) +- [GPU-resident Float (sm_mps)](api/matrix/sm_mps.md) - [Sparse Double (dms)](api/matrix/dms.md) - [I/O (m_io)](api/matrix/m_io.md) - [Conversion (m_convert)](api/matrix/m_convert.md) +## Vector API + +- [Vector optimizers](api/vector/sv_optim.md) + ## Examples - [Tensor NN forward pass](examples/tensor_basic.md) diff --git a/doc/mkdocs.tpl.yaml b/doc/mkdocs.tpl.yaml index d2a321f..91a5df2 100644 --- a/doc/mkdocs.tpl.yaml +++ b/doc/mkdocs.tpl.yaml @@ -10,13 +10,20 @@ nav: - Matrix: - Dense Double (dm): doc/api/matrix/dm.md - Dense Float (sm): doc/api/matrix/sm.md + - GPU-resident Float (sm_mps): doc/api/matrix/sm_mps.md - Sparse Double (dms): doc/api/matrix/dms.md - I/O (m_io): doc/api/matrix/m_io.md - Conversion (m_convert): doc/api/matrix/m_convert.md + - Vector optimizers: doc/api/vector/sv_optim.md - Tensor Core: doc/api/tensor/st.md - BatchNorm: doc/api/tensor/st_batchnorm.md - Convolution: doc/api/tensor/st_conv.md - Pooling: doc/api/tensor/st_pool.md - Shape Ops: doc/api/tensor/st_shape_ops.md + - Interpreter kernels: doc/api/tensor/st_interp.md + - IEEE binary16 conversions: doc/api/tensor/st_f16.md + - Tensor/matrix conversions: doc/api/tensor/st_convert.md + - Tensor/vector bridge: doc/api/tensor/st_vector_bridge.md + - nm_lib kernel migration: doc/nm_lib_kernels.md - Examples: - Tensor NN forward pass: doc/examples/tensor_basic.md diff --git a/doc/nm_lib_kernels.md b/doc/nm_lib_kernels.md new file mode 100644 index 0000000..d150333 --- /dev/null +++ b/doc/nm_lib_kernels.md @@ -0,0 +1,356 @@ +# nm_lib Kernel Migration Map + +This checklist maps scalar kernels in `14_NN/app/nm_lib` to the reusable +mMatrix APIs. Phase 1 covers dense NN kernels; Phase 2 covers the +graph-interpreter tensor operators. + +| nm_lib function / operation | mMatrix function | +| --- | --- | +| `nm_relu` | `sm_apply_relu`, existing `st_apply_relu` | +| Dense GEMM + bias + activation | `sm_gemm_bias_act` | +| `nm_sigmoid` | `sm_apply_sigmoid`, `st_apply_sigmoid` | +| `nm_tanh` | `sm_apply_tanh`, `st_apply_tanh` | +| `nm_softmax` | `sm_apply_softmax`, `st_apply_softmax` | +| Interpreter tanh GELU | `sm_apply_gelu_tanh`, `st_apply_gelu_tanh` | +| Interpreter erf GELU | `sm_apply_gelu_erf`, `st_apply_gelu_erf` | +| `nm_sigmoid_grad` | `sm_apply_sigmoid_backward`, `st_apply_sigmoid_backward` | +| `nm_tanh_grad` | `sm_apply_tanh_backward`, `st_apply_tanh_backward` | +| `nm_relu_grad` | `sm_apply_relu_backward`, existing `st_apply_relu_backward` | +| Interpreter GELU gradient | `sm_apply_gelu_*_backward`, `st_apply_gelu_*_backward` | +| `nm_softmax_ce_grad` | `sm_softmax_ce_grad` | +| `nm_loss_mse` | `sm_loss_mse` | +| `nm_loss_cross_entropy` | `sm_loss_cross_entropy` | +| `nm_add_rowwise` | `sm_add_rowwise` | +| `nm_sum_rows` | `sm_sum_rows` | +| `nm_argmax_rows` | `sm_argmax_rows` | +| `optimizer_apply_array` (SGD) | `sv_optim_sgd` | +| `optimizer_apply_array` (Momentum) | `sv_optim_momentum` | +| `optimizer_apply_array` (RMSprop) | `sv_optim_rmsprop` | +| `optimizer_apply_array` (Adam) | `sv_optim_adam` | +| Gradient L2 clipping | `sv_clip_norm` | +| Interpreter `run_binary` (add/sub/mul/div) | `st_binary_broadcast`, `st_binary_broadcast_into` | +| Interpreter `run_layernorm` | `st_layernorm` | +| Interpreter `run_reducemax` | `st_reduce_max`, `st_reduce_max_into` | +| Interpreter `run_matmul` | `st_matmul_batched` | +| Interpreter `run_gather` | `st_gather`, `st_gather_bytes` | +| Interpreter `run_transpose` | `st_permute_copy_into`, `st_permute_copy_into_bytes` (view: `st_permute`) | +| `run_concat` | `st_concat` (existing, materializing) | +| `run_slice` | not covered; `st_split` only does equal splits, so the reference keeps its clamped start/end/step logic | +| `nm_f32_to_f16` / `nm_f16_to_f32` (nm_halfprec.h) | `st_f32_to_f16`, `st_f16_to_f32`, `st_f32_to_f16_bulk`, `st_f16_to_f32_bulk` | + +## Accuracy and determinism + +Accelerate uses vector vForce math where available. Its results can differ +slightly from scalar libm, so activation comparisons use relative tolerance +$10^{-5}$ plus absolute tolerance $10^{-6}$. The erf GELU uses the standard +erf definition; the tanh GELU uses the common cubic approximation. + +Elementwise work is partitioned by index and is independent of OpenMP thread +count. Softmax and row reductions keep each row/column sum in a fixed order to +avoid thread-count-dependent accumulation. BF16 tensor activation kernels +promote to F32, compute, then round back to BF16. + +The interpreter broadcast kernels (`st_binary_broadcast*`) are bit-exact +against the scalar reference for all four operations, including IEEE results +for division by zero. Two caveats under the vector packages' `-ffast-math`: + +* `vDSP_vadd`/`vsub`/`vmul` are single IEEE operations and used on the + Accelerate path; `vDSP_vdiv` differs from scalar division by 1 ulp, so + division always runs on the C path. +* `-ffast-math` rewrites a division by a loop-invariant scalar into a + multiply with a once-rounded reciprocal (double rounding). The scalar + operand path uses an IEEE-exact division helper compiled with + `#pragma float_control(precise, on)` (clang) or + `__attribute__((optimize("no-fast-math")))` (gcc). + +## Broadcast binary benchmark + +`//app/tensor:bench_st_binary` replicates nm_lib's `run_binary` (walk plus +row kernels) as the `before` baseline and compares medians for transformer +shapes (default OpenMP thread count was 4 on this 4-performance-core Apple +Silicon machine; with the thread threshold below, default and single-thread +runs take the same sequential path): + +| Case | Threads | Elements | nm_lib before | `st_binary_broadcast_into` | Speedup | +| --- | ---: | ---: | ---: | ---: | ---: | +| same-shape add [8,256,512] | 1 | 1,048,576 | 389.500 | 77.025 | 5.06× | +| bias add [4,128,256]+[256] | 1 | 131,072 | 60.150 | 31.550 | 1.91× | +| bias add [4,512,768]+[768] | 1 | 1,572,864 | 1,077.750 | 663.125 | 1.63× | +| attn mask add (walk) [2,4,128,128]+[1,1,128,128] | 1 | 131,072 | 59.975 | 39.200 | 1.53× | +| attn mask add (suffix) [2,4,128,128]+[128,128] | 1 | 131,072 | 70.950 | 37.500 | 1.89× | +| scale mul [4,128,256]*[1] | 1 | 131,072 | 46.375 | 18.925 | 2.45× | +| scale mul (mid) [4,1,256]*[1,128,1] | 1 | 131,072 | 48.975 | 16.800 | 2.92× | +| suffix div [4,128,256]/[256] | 1 | 131,072 | 49.625 | 20.550 | 2.41× | +| attn mask add (walk) [1,4,16,16]+[1,1,16,16] | 1 | 1,024 | 0.628 | 0.760 | 0.83× | + +The 1,024-element walk case is the fixed API overhead (validation and shape +checks) on a sub-microsecond kernel; larger walk inputs are 1.5× or better. + +## f16 conversion + +`//app/tensor:bench_st_f16` compares nm_lib's per-element conversion loop +with the bulk kernels (NEON `vcvt_f16_f32`/`vcvt_f32_f16`, scalar +fallback elsewhere; single-threaded by design): + +| Case | Elements | scalar loop | bulk | Speedup | +| --- | ---: | ---: | ---: | ---: | +| f32 -> f16 | 4,096 | 18.082 | 1.540 | 11.74× | +| f32 -> f16 | 65,536 | 297.582 | 28.727 | 10.36× | +| f32 -> f16 | 1,048,576 | 3,524.700 | 298.575 | 11.81× | +| f16 -> f32 | 4,096 | 11.163 | 0.701 | 15.94× | +| f16 -> f32 | 262,144 | 696.600 | 42.150 | 16.53× | +| f16 -> f32 | 1,048,576 | 3,157.950 | 206.775 | 15.27× | + +### f16 accuracy and determinism + +The scalar converters reproduce nm_lib's nm_halfprec.h bit for bit: +round-to-nearest-even, exact subnormals, overflow to infinity, NaN kept +quiet with the sign preserved (payload dropped), and the sign of ±0 +preserved. The bulk NEON path is bit-identical to the scalar path: for +f32 to f16 the hardware keeps the NaN payload where the reference +canonicalizes to `sign | 0x7e00`, so the bulk kernel blends the +reference pattern into NaN lanes; tests sweep every f16 pattern (all +65,536, both directions), a strided sample of the full f32 pattern +space, the conversion-critical exponent range, an explicit edge table, +and all bulk lengths 0 to 33. A dedicated bulk test converts f32 values +around every subnormal half (including the rounding boundaries) and +round-trips all 65,536 patterns through both bulk kernels, so +flush-to-zero would fail loudly; it was not observed, because FCVT is +exact for subnormals under the packages' `-ffast-math` flags, which do +not change FPCR. Conversion stays a pure `uint16_t` buffer API; no +`StDtype` entry was added because bf16 promotion and buffer layouts +would need wider changes. + +## Gather and materializing permute + +`//app/tensor:bench_st_gather` replicates nm_lib's `run_gather` and +`run_transpose` as the `before` baseline (medians, µs/call). Both kernels +are memcpy-bound, so the comparison is near parity with the fixed API +overhead on small inputs; the gather validates every index before writing, +so out-of-range indices fail without partial output. + +| Case | Threads | nm_lib before | `st` kernel | Speedup | +| --- | ---: | ---: | ---: | ---: | +| gather [30000,256] axis 0, count 128 | default | 18.698 | 22.980 | 0.81× | +| gather [2,30000,1024] axis 1, count 512 | default | 1,857.517 | 1,389.117 | 1.34× | +| gather [30000,64] axis 0, count 16 | default | 0.177 | 0.250 | 0.71× | +| permute [2,4,128,64] -> [2,128,4,64] | default | 17.657 | 21.848 | 0.81× | +| permute [2,4,128,64] -> [2,4,64,128] (strided) | default | 85.198 | 93.487 | 0.91× | +| permute [1,4,16,16] -> [1,16,4,16] | default | 0.438 | 0.725 | 0.60× | + +The gather distributes copy blocks over OpenMP only above 4,194,304 copied +elements (`ST_GATHER_OMP_THRESHOLD`); a trial at 1M regressed 2x on this +machine, so realistic embedding-table gathers stay sequential. The +strided permute rows use typed 4/8-byte loops like the reference's +`walk_copy`; a variable-size `memcpy` per element is 8x slower. + +### Phase 2 shape-op coverage (task: can existing ops replace the reference?) + +| nm_lib function | mMatrix coverage | +| --- | --- | +| `run_transpose` (float tensors) | `st_permute` (view) + `st_permute_copy_into` (materializing, pooled buffers) | +| `run_transpose` (int64/bool buffers) | `st_permute_copy_into_bytes` (raw, element-size parameter) | +| `run_concat` | `st_concat` already materializes and covers it | +| `run_slice` | not covered: `st_split` only splits evenly; the reference's clamped start/end/step slicing (including negative steps) has no mMatrix equivalent and stays in nm_lib | + +## LayerNorm and ReduceMax + +`//app/tensor:bench_st_norm_reduce` replicates nm_lib's `run_layernorm` and +`run_reducemax` as the `before` baseline (medians on the same machine; +default OpenMP thread count 4). `st_layernorm` is the same algorithm as the +reference, so single-threaded it matches nm_lib; above the thread threshold +it parallelizes over groups. `st_reduce_max` folds trailing reductions with +vectorized run kernels; non-trailing masks walk like the reference plus one +vectorized NaN pre-scan. + +| Case | Threads | Elements | nm_lib before | `st` kernel | Speedup | +| --- | ---: | ---: | ---: | ---: | ---: | +| layernorm [4,128,256] | 1 | 131,072 | 83.100 | 81.100 | 1.02× | +| layernorm [8,256,512] | default | 1,048,576 | 634.700 | 224.275 | 2.83× | +| layernorm [4,512,768] | default | 1,572,864 | 970.250 | 378.500 | 2.56× | +| layernorm [16,65,32] rank 2 | default | 33,280 | 20.547 | 21.418 | 0.96× | +| reduce max [2,4,128,128] axis 3 | 1 | 131,072 | 97.775 | 39.300 | 2.49× | +| reduce max [1,4,128,128] axis 3 | 1 | 65,536 | 50.562 | 19.738 | 2.56× | +| reduce max [4,128,256] axis 2 | 1 | 131,072 | 109.825 | 36.850 | 2.98× | +| reduce max [4,128,256] axis 1 (walk) | 1 | 131,072 | 18.250 | 44.725 | 0.41× | +| reduce max [2,1000] axis 0 (walk) | default | 2,000 | 0.270 | 0.719 | 0.38× | + +### Norm and reduce thresholds + +`st_layernorm` parallelizes over groups only above 262,144 input elements +(`ST_LAYERNORM_OMP_THRESHOLD`): at and below 262k the sequential kernel +matches the reference, and at 1M+ the parallel gain is 2.5× or better. +`st_reduce_max` parallelizes over output slots only above 1,048,576 input +elements (`ST_REDUCE_OMP_THRESHOLD`): measurements at 65k and 131k showed +no gain from a team on this machine, so realistic sizes stay sequential +(same rationale as the Phase 1 optimizer threshold). Both should be +retuned on the deployment machine. + +### LayerNorm accuracy and NaN policy in ReduceMax + +`st_layernorm` reproduces nm_lib's sum8 semantics exactly: the mean and the +variance use eight deterministic partial sums, so results are bit-identical +across thread counts; against a naive scalar loop the tolerance is 1e-5 +relative plus 1e-6 absolute (and for groups of eight or fewer elements the +sum8 order is bit-identical to the naive order). + +### Batched matmul + +`st_matmul_batched` routes every product through the same backend as +`sm_gemm`, so compared with nm_lib's `run_matmul` loop (which also calls +`sm_gemm` per batch) the wall time is equivalent by construction; +`//app/tensor:bench_st_matmul` confirms this (medians, µs/call, default +thread count 4): + +| Case | Threads | nm_lib before | `st_matmul_batched` | Speedup | +| --- | ---: | ---: | ---: | ---: | +| proj stacked [4,128,256]x[256,512] | 1 | 192.200 | 199.825 | 0.96× | +| proj stacked [8,256,512]x[512,512] | 1 | 1,923.750 | 1,914.575 | 1.00× | +| attn scores [2,4,128,64]x[2,4,64,128] | default | 45.000 | 45.175 | 1.00× | +| batched [4,12,64,64]x[4,12,64,64] | default | 81.925 | 80.400 | 1.02× | + +The kernel's value is the unified API: NumPy batch broadcast, the +single-matrix stacking fast path, bf16 promote, and shape validation with +no partial writes. Batches are distributed over OpenMP only when there +are at least 8 of them and each GEMM is at most 262,144 elements +(`ST_MATMUL_OMP_MIN_BATCHES`/`ST_MATMUL_OMP_MAX_GEMM`), so BLAS-internal +threads are never oversubscribed. On this machine Accelerate serializes +small GEMMs, so the team measured neutral there; the path exists for +backends that benefit (for example OpenBLAS on Linux) and should be +re-measured on the deployment machine. + +`st_reduce_max` deviates deliberately from the nm_lib reference for NaNs: +the reference's `x > max` fold silently drops NaNs, while `st_reduce_max` +propagates the bit pattern of the first NaN in scan order to the output +slot, so errors are not swallowed (documented in the header and covered by +tests). Finite values, ±INFINITY, and the -INFINITY start value follow the +reference. Under `-ffast-math` the fold therefore cannot use the OpenMP +`max` reduction (its private lanes start at -FLT_MAX and would swallow +-INFINITY) nor a horizontal vector max with compiler-inserted padding +lanes; the eight-lane fold combines through a noinline helper so all eight +lanes come from memory. + +MatMul is the one interpreter kernel that is not bit-exact: the BLAS +accumulation order defines the result, so tests compare against a naive +reference (and across thread counts) with 1e-4 relative tolerance. + +### Binary OpenMP threshold + +The benchmark's thread sweep measures `st_binary_broadcast_into` at one +versus default threads (median µs/call): + +| Elements | Path | 1 thread | default (4) | parallel gain | +| ---: | --- | ---: | ---: | ---: | +| 4,096 | same-shape | 1.409 | 2.350 | 0.60× | +| 16,384 | same-shape | 7.107 | 11.805 | 0.60× | +| 65,536 | same-shape | 5.018 | 4.688 | 1.07× | +| 262,144 | same-shape | 24.525 | 24.625 | 1.00× | +| 1,048,576 | same-shape | 124.300 | 236.050 | 0.53× | +| 2,097,152 | same-shape | 2,280.125 | 2,064.250 | 1.10× | +| 4,194,304 | same-shape | 1,422.750 | 1,280.500 | 1.11× | +| 8,388,608 | same-shape | 2,583.000 | 2,175.875 | 1.19× | +| 4,194,304 | scalar | 1,633.500 | 779.125 | 2.10× | +| 8,388,608 | scalar | 1,850.000 | 1,207.875 | 1.53× | + +Parallel regions regress or stay flat below roughly one million elements +(OpenMP fork/join cost dominates a memory-bound kernel) and only gain +modestly above two million. `ST_BIN_OMP_THRESHOLD` is therefore 2,097,152 +output elements, which keeps every tested transformer shape on the +sequential path; this follows the same rationale as the optimizer +threshold in Phase 1 and should be retuned on the deployment machine. + +## Activation benchmark + +The benchmark compares scalar reference loops (`before`) with the mMatrix +matrix/tensor wrappers (`after`). It prints medians for 256 through 1M elements, +with one thread and the default OpenMP thread count. The table below shows the +`tanh` case from an Apple Silicon Accelerate build (times are median µs/call; +default OpenMP thread count was 4). + +| Elements | Threads | Scalar before | `sm_apply_tanh` | Speedup | `st_apply_tanh` | Speedup | +| ---: | ---: | ---: | ---: | ---: | ---: | ---: | +| 256 | 1 | 2.680 | 0.559 | 4.80× | 0.572 | 4.69× | +| 1,024 | 1 | 9.776 | 1.603 | 6.10× | 1.708 | 5.73× | +| 10,000 | 1 | 114.867 | 19.629 | 5.85× | 22.133 | 5.19× | +| 100,000 | 1 | 1,107.156 | 187.500 | 5.90× | 219.094 | 5.05× | +| 1,000,000 | 1 | 10,862.750 | 1,801.000 | 6.03× | 1,578.750 | 6.88× | +| 256 | 4 | 2.263 | 0.402 | 5.63× | 0.386 | 5.86× | +| 1,024 | 4 | 10.060 | 2.159 | 4.66× | 2.252 | 4.47× | +| 10,000 | 4 | 108.777 | 17.047 | 6.38× | 15.004 | 7.25× | +| 100,000 | 4 | 1,095.938 | 158.938 | 6.90× | 202.344 | 5.42× | +| 1,000,000 | 4 | 11,149.250 | 1,622.000 | 6.87× | 1,501.250 | 7.43× | + +The benchmark executable also reports sigmoid, both GELU variants, and softmax. +Small sub-microsecond results are averaged over repeated calls within each +sample; the displayed values are medians of nine samples. + +## Fused GEMM benchmark + +The fused kernel was compared against GEMM, a row-bias loop, and activation +separately. Entries are median µs/call for ReLU on the same Apple Silicon +Accelerate build. + +| Shape | Threads | Split before | `sm_gemm_bias_act` | Speedup | +| ---: | ---: | ---: | ---: | ---: | +| 32×32 | 1 | 5.910 | 1.220 | 4.84× | +| 64×64 | 1 | 25.500 | 3.708 | 6.88× | +| 128×128 | 1 | 111.500 | 16.167 | 6.90× | +| 256×256 | 1 | 505.000 | 153.000 | 3.30× | +| 32×32 | 4 | 6.950 | 1.220 | 5.70× | +| 64×64 | 4 | 21.917 | 3.750 | 5.84× | +| 128×128 | 4 | 107.167 | 16.500 | 6.49× | +| 256×256 | 4 | 493.000 | 87.000 | 5.67× | + +## Row-helper benchmark + +Median µs/call on the same machine, comparing scalar row loops with mMatrix +helpers (1024×256 and 4096×256 cases): + +| Operation | Shape | Threads | Scalar before | mMatrix after | Speedup | +| --- | ---: | ---: | ---: | ---: | ---: | +| `add_rowwise` | 1024×256 | 1 | 1,422.250 | 56.750 | 25.06× | +| `add_rowwise` | 1024×256 | 4 | 1,374.000 | 60.500 | 22.71× | +| `sum_rows` | 1024×256 | 1 | 1,735.250 | 491.000 | 3.53× | +| `sum_rows` | 1024×256 | 4 | 1,516.250 | 464.250 | 3.27× | +| `argmax_rows` | 1024×256 | 1 | 1,596.250 | 352.250 | 4.53× | +| `argmax_rows` | 1024×256 | 4 | 1,592.750 | 256.250 | 6.22× | +| `add_rowwise` | 4096×256 | 1 | 5,726.750 | 253.500 | 22.59× | +| `add_rowwise` | 4096×256 | 4 | 5,232.750 | 258.000 | 20.28× | +| `sum_rows` | 4096×256 | 1 | 8,220.500 | 3,115.750 | 2.64× | +| `sum_rows` | 4096×256 | 4 | 8,619.500 | 1,383.250 | 6.23× | +| `argmax_rows` | 4096×256 | 1 | 6,444.250 | 1,134.750 | 5.68× | +| `argmax_rows` | 4096×256 | 4 | 6,001.250 | 735.250 | 8.16× | + +## Optimizer parallelization + +The optimizer elementwise loops use OpenMP only for `n > 1,048,576`. Adam bias +correction and all update formulas match `optimizer_apply_array`; optional +decoupled weight decay is disabled when its argument is zero. `sv_clip_norm` +computes its norm in a fixed scalar order and parallelizes only the independent +scaling pass. + +`//app/vector:bench_sv_optim` compares scalar-reference and API medians for +SGD, momentum, RMSprop, Adam, and clipping at 1k, 10k, 100k, and 1M elements, +with one and default OpenMP thread counts. A 16,384 threshold regressed several +optimizers at 100k. A follow-up trial at 262,144 activated a team at 1M but also +regressed most optimizers. The selected threshold is therefore above the +largest measured input, keeping tested models out of parallel regions. The +behavior beyond 1M remains unmeasured and should be retuned on the deployment +machine if larger parameter buffers are common. + +Selected medians from the 262,144-threshold trial at 1M elements +(microseconds/call; default thread count 4). `API` uses the thresholded +optimizer; `forced OMP` is the parallel reference loop: + +| Optimizer | Scalar | API (262k) | Forced OMP | Forced speedup | +| --- | ---: | ---: | ---: | ---: | +| SGD | 121.000 | 124.000 | 110.000 | 1.10× | +| Momentum | 201.000 | 429.000 | 153.000 | 1.31× | +| RMSprop | 220.000 | 266.000 | 156.000 | 1.41× | +| Adam | 467.000 | 408.000 | 316.000 | 1.48× | +| Clip norm | 137.000 | 127.000 | 115.000 | 1.19× | + +The API-level OpenMP path still regressed momentum and RMSprop in this trial, +so the final threshold was raised above 1M. The direct parallel reference shows +potential benefit for later tuning, but does not justify parallelizing the +tested small-network range today. \ No newline at end of file diff --git a/doc/release-0.4.0.md b/doc/release-0.4.0.md new file mode 100644 index 0000000..f8b4b9c --- /dev/null +++ b/doc/release-0.4.0.md @@ -0,0 +1,78 @@ +# Release preparation: 0.4.0 + +Candidate based on `feat/interpreter-kernels`, initially `9955884`; last version tag: `v0.3.1`. +The worktree was clean before this preparation. Validation commits are collected on `codex/release-0.4.0-validation`; no release tag or publication has been created. + +## Publish in the source release + +- Public source, headers, module tests and benchmark sources, BUILD files and helper rules. +- `.bazelversion` (9.3.0), `.bazelignore`, `MODULE.bazel` (0.4.0), and the resolved `MODULE.bazel.lock`. +- README, changelog with API migration notes, documentation sources and generated public API pages. +- `doc/requirements.in` alongside the existing pinned requirements file. +- `share/simple_benchmark/simple_bench.h`: required by the already-published MPS benchmark but previously ignored. +- The MIT project license and upstream license text embedded in vendored `log.c`. + +A source release is the current deliverable. The local installer also collects third-party static libraries; +a redistributable binary SDK requires separate platform/runtime validation and complete third-party license notices. + +## Remove / keep local + +Removed obsolete Bzlmod replacement wrappers for Unity, libomp, pcg, zlib, matio, and SuiteSparse, +the unused local HDF5 repository declaration, stale SuiteSparse installer dependency, +and legacy benchmark targets whose sources were not versioned. Vendored `log.c` is built directly as a workspace target, without a local-repository path assumption. Removed the personal Conda activation `.envrc`. +The shared `.clangd` now contains only public include paths. + +Keep local and ignored: `third_party/hdf5` and other vendor experiments, legacy benchmark sources/results, +Bazel links, caches, `compile_commands.json`, profiles, build output, editor state, and machine-specific `.envrc`. +`.bazelignore` also excludes local BUILD trees: Git ignore rules alone do not stop Bazel's `//...` discovery. +No ignored vendor or experiment directory was deleted from disk. + +## Review findings and repairs + +| Finding | Repair | +| --- | --- | +| Gather/permute raw byte APIs can wrap shape/byte products before copying. Reproduced by a failing overflow regression test. | Check products before multiplication and reject dimensions outside supported offset/index ranges. | +| Thread-comparison tests for binary/reduce/matmul never crossed the actual OpenMP thresholds. | Adjust test shapes to exercise the parallel branches. | +| Tensor permute reads `shape[perm[i]]` before validating the permutation. | Validate bounds and uniqueness before shape access. | +| Raw permute uses typed loads/stores for generic byte buffers. | Use byte copies that preserve payloads and support unaligned buffers. | +| Installer references removed SuiteSparse; unconditionally requires OpenBLAS; omits private/transitive archives and tensors. | Collect archives from the C/C++ link closure, deduplicate files, add platform-specific tensor/MPS artifacts. | +| Installer uses GNU `install -T`, unsupported by macOS. | Use portable explicit file destinations. | +| A fresh source release lacks sources named by legacy benchmark targets. | Keep only the maintained MPS target and publish its helper header. | +| Recursive discovery loads ignored HDF5/experimental BUILD files. | Version an explicit `.bazelignore`; remove unused local dependency wiring. | +| CI points at nonexistent `//tests/...`. | Build/test `//...`, pin Bazel and configure its setup. | +| README incorrectly requires SuiteSparse and implies Linux tensor support; API docs omit recent headers. | Correct dependencies/platforms, regenerate API pages, add migrations. | + +Review focused on the changes since `v0.3.1`, especially the interpreter boundary, removed dependencies, +source-release completeness and installation. This is not exhaustive verification of every existing numerical routine. + +## Validation + +- Full local `bazel test //...`: 32 test targets pass on macOS ARM64; 138 top-level targets analyzed and built, including documentation and installer. +- Overflow regression failed before the fix and passed after it. +- Standard installation executed successfully in a temporary directory, including transitive libraries and public tensor headers. +- Tensor interpreter tests pass with AddressSanitizer and UndefinedBehaviorSanitizer (`halt_on_error=1`). +- Standard and bundle installations both build and run a standalone consumer exercising dense/sparse matrices, SGD, tensor conversion/broadcast, and f16 conversion. +- A separate source-only copy without ignored files or Git metadata builds all targets and passes all 32 test targets using the shared build/test cache. This checks source completeness, not an empty-cache build. + +The workflow uses the documented upstream [checkout](https://github.com/actions/checkout) and [Bazel setup](https://github.com/bazel-contrib/setup-bazel) action versions. + +## Cross-platform CI + +- GitHub workflow `BUILD` was manually disabled and has been re-enabled for validation. +- [Initial run](https://github.com/tragisch/MMatrix/actions/runs/39021224152): macOS passed all 32 test targets and both installation consumers. +- Ubuntu x86-64 exposed a missing link to separately packaged LAPACKE and two failing NN tests (NaN propagation and thread-independent activation results). +- Repairs: include a separate LAPACKE archive when present; compile the shared NN kernel translation unit without fast-math/reassociation or floating-point contraction. +- The CPU-only NN suite also passes locally with `--config=openmp_only` after the repair. +- [Second run](https://github.com/tragisch/MMatrix/actions/runs/39022026793): all 14 Linux-supported test targets pass; 18 macOS-only targets are skipped. The separate installation check then reproduced unreadable LLVM-bitcode archives (`file format not recognized`) with the system linker. +- Remove forced Linux `-flto` so installed archives contain ordinary object code; group separate archives for GNU ld, and select Apple libtool only on macOS. +- Installation consumers now also solve a linear system, exercising LAPACKE from the installed artifacts. +- [Cross-platform run](https://github.com/tragisch/MMatrix/actions/runs/39022691316) for `ebfc694`: macOS passes all 32 test targets; Ubuntu x86-64 passes all 14 supported targets (18 macOS-only targets skipped). Linux standard and bundle installations build and run the system-compiler consumer, including the LAPACKE solver. +- Log inspection found that macOS Bash 3.2 aborted on empty-array expansion under `nounset`, while its EXIT cleanup masked the failure. Remove the empty arrays and require both installation consumers to complete before a successful exit; explicit completion messages make the evidence visible. +- Build caching is enabled. Branch pushes no longer duplicate pull-request validation; pushes to master and pull requests retain CI, with manual dispatch available. The final documentation/trigger update is checked by a single PR run. + +## Before tagging + +- Confirm the latest PR commit retains green macOS/Ubuntu checks before merging. Linux uses system OpenBLAS and LAPACKE. +- Merge the reviewed PR; no release tag or publication has been created by this validation task. +- Existing optimized builds emit compiler warnings (including fast-math/NaN assumptions). Numerical tests pass locally, but cross-toolchain numerical guarantees and GPU performance have not been newly established. +- Tag the reviewed release commit as `v0.4.0` and publish the source release with the changelog. diff --git a/doc/requirements.in b/doc/requirements.in new file mode 100644 index 0000000..9a8a4ca --- /dev/null +++ b/doc/requirements.in @@ -0,0 +1,2 @@ +mkdocs +mkdocs-material diff --git a/share/simple_benchmark/BUILD b/share/simple_benchmark/BUILD index ba0ea7c..86a9842 100644 --- a/share/simple_benchmark/BUILD +++ b/share/simple_benchmark/BUILD @@ -1,78 +1,6 @@ load("@rules_cc//cc:cc_binary.bzl", "cc_binary") load("@rules_cc//cc:cc_library.bzl", "cc_library") -cc_binary( - name = "bench_dm", - srcs = ["bench_dm.c"], - deps = [ - ":simple_bench", - "//app/matrix:dm", - ], -) - -cc_binary( - name = "bench_sm", - srcs = ["bench_sm.c"], - deps = [ - ":simple_bench", - "//app/matrix:sm", - ], -) - -cc_binary( - name = "bench_dms", - srcs = ["bench_dms.c"], - deps = [ - ":simple_bench", - "//app/matrix:dms", - ], -) - -cc_binary( - name = "bench_dm_multiply_number", - srcs = ["bench_dm_multiply_number.c"], - deps = [ - ":simple_bench", - "//app/matrix:dm", - ], -) - -cc_binary( - name = "bench_dm_transpose", - srcs = ["bench_dm_transpose.c"], - deps = [ - ":simple_bench", - "//app/matrix:dm", - ], -) - -cc_binary( - name = "bench_st_conv", - srcs = ["bench_st_conv.c"], - deps = [ - ":simple_bench", - "//app/tensor:st", - ], -) - -cc_binary( - name = "bench_st_pool", - srcs = ["bench_st_pool.c"], - deps = [ - ":simple_bench", - "//app/tensor:st", - ], -) - -cc_binary( - name = "bench_sm_comparison", - srcs = ["bench_sm_comparison.c"], - deps = [ - ":simple_bench", - "//app/matrix:sm", - ], -) - cc_binary( name = "bench_sm_mps", srcs = ["bench_sm_mps.c"], @@ -93,16 +21,6 @@ cc_binary( ], ) -cc_binary( - name = "bench_graphblas", - srcs = ["bench_graphblas.c"], - tags = ["manual"], - deps = [ - ":simple_bench", - "@suitesparse_graphblas_host//:suitesparse", - ], -) - cc_library( name = "simple_bench", srcs = [ diff --git a/share/simple_benchmark/simple_bench.h b/share/simple_benchmark/simple_bench.h new file mode 100644 index 0000000..82d6dd8 --- /dev/null +++ b/share/simple_benchmark/simple_bench.h @@ -0,0 +1,556 @@ +#ifndef SIMPLE_BENCH_H +#define SIMPLE_BENCH_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __MACH__ +#include +#include +#endif + +/***************************** + * Compiler barrier + * + * Prevents the compiler from optimizing away the benchmarked code. + * Usage: BENCH_DO_NOT_OPTIMIZE(result); + *******************************/ + +#define BENCH_DO_NOT_OPTIMIZE(val) \ + __asm__ volatile("" : : "r,m"(val) : "memory") + +#define BENCH_CLOBBER_MEMORY() __asm__ volatile("" ::: "memory") + +/***************************** + * Color codes + *******************************/ + +#define BENCH_BLUE "\x1B[34m" +#define BENCH_RED "\x1B[31m" +#define BENCH_GREEN "\x1B[32m" +#define BENCH_WHITE "\x1B[37m" +#define BENCH_RESET "\x1B[0m" + +/* backward compat */ +#define BLUE BENCH_BLUE +#define RED BENCH_RED +#define WHITE BENCH_WHITE +#define RESET BENCH_RESET + +/***************************** + * Statistics helpers + *******************************/ + +typedef struct { + double min_ns; + double max_ns; + double mean_ns; + double median_ns; + double stddev_ns; + int n; +} bench_stats_t; + +static inline int bench_cmp_ll_(const void* a, const void* b) { + long long va = *(const long long*)a; + long long vb = *(const long long*)b; + return (va > vb) - (va < vb); +} + +/* Compute statistics from an array of nanosecond timings. */ +static inline bench_stats_t bench_compute_stats(long long* samples, int n) { + bench_stats_t s = {0}; + if (n <= 0) return s; + + s.n = n; + + /* sort for median */ + qsort(samples, (size_t)n, sizeof(long long), bench_cmp_ll_); + + s.min_ns = (double)samples[0]; + s.max_ns = (double)samples[n - 1]; + + if (n % 2 == 1) { + s.median_ns = (double)samples[n / 2]; + } else { + s.median_ns = (double)(samples[n / 2 - 1] + samples[n / 2]) / 2.0; + } + + double sum = 0.0; + for (int i = 0; i < n; ++i) sum += (double)samples[i]; + s.mean_ns = sum / n; + + if (n > 1) { + double var = 0.0; + for (int i = 0; i < n; ++i) { + double d = (double)samples[i] - s.mean_ns; + var += d * d; + } + s.stddev_ns = sqrt(var / (n - 1)); + } + + return s; +} + +/***************************** + * Formatting helpers + *******************************/ + +/* Human-readable time format with fractional precision. */ +static inline void format_duration(long long nanoseconds, char* buffer, + size_t buffer_size) { + const double ns = (double)nanoseconds; + const double us = ns / 1e3; + const double ms = ns / 1e6; + const double sec = ns / 1e9; + + if (ns < 1e3) { + snprintf(buffer, buffer_size, "%lld ns", nanoseconds); + } else if (ns < 1e6) { + snprintf(buffer, buffer_size, "%.2f us", us); + } else if (ns < 1e9) { + snprintf(buffer, buffer_size, "%.2f ms", ms); + } else { + long long minutes = (long long)(sec / 60.0); + double remaining_sec = sec - minutes * 60.0; + if (minutes > 0) { + snprintf(buffer, buffer_size, "%lldm %.1fs", minutes, remaining_sec); + } else { + snprintf(buffer, buffer_size, "%.2f s", sec); + } + } +} + +/* Human-readable memory format. */ +static inline void format_memory_usage(long memory_usage, char* buffer, + size_t buffer_size) { + const long KB = 1024; + const long MB = 1024 * KB; + +#ifdef __linux__ + /* On Linux, ru_maxrss is in kilobytes */ + long memory_kilobytes = memory_usage; +#else + /* On BSD and macOS, ru_maxrss is in bytes */ + long memory_kilobytes = memory_usage / KB; +#endif + + if (memory_kilobytes < KB) { + snprintf(buffer, buffer_size, "%ld KB", memory_kilobytes); + } else if (memory_kilobytes < MB) { + snprintf(buffer, buffer_size, "%.2f MB", memory_kilobytes / (double)KB); + } else { + snprintf(buffer, buffer_size, "%.2f GB", memory_kilobytes / (double)MB); + } +} + +/***************************** + * Output helpers + *******************************/ + +static inline void print_benchmark_result(const char* formatted_time, + const long long memory, + const char* fcall, const char* file, + int line) { + char formatted_memory[64]; + format_memory_usage(memory, formatted_memory, sizeof(formatted_memory)); + printf(BENCH_BLUE "ℬ|" BENCH_RESET " Wall-Time: " BENCH_RED + "%s\t" BENCH_RESET "Peak-RSS:" BENCH_RED + " %s\t" BENCH_RESET "| %s in " BENCH_WHITE " %s:%i\n", + formatted_time, formatted_memory, fcall, file, line); +} + +static inline void print_benchmark_stats(const char* label, + const bench_stats_t* s, + const long peak_rss) { + char fmt_min[64], fmt_med[64], fmt_mean[64], fmt_std[64], fmt_mem[64]; + format_duration((long long)s->min_ns, fmt_min, sizeof(fmt_min)); + format_duration((long long)s->median_ns, fmt_med, sizeof(fmt_med)); + format_duration((long long)s->mean_ns, fmt_mean, sizeof(fmt_mean)); + format_duration((long long)s->stddev_ns, fmt_std, sizeof(fmt_std)); + format_memory_usage(peak_rss, fmt_mem, sizeof(fmt_mem)); + + printf(BENCH_BLUE "ℬ|" BENCH_RESET " %s " BENCH_GREEN "[%d runs]" + BENCH_RESET "\n", label, s->n); + printf(" min: " BENCH_RED "%s" BENCH_RESET + " median: " BENCH_RED "%s" BENCH_RESET + " mean: " BENCH_RED "%s" BENCH_RESET + " stddev: %s Peak-RSS: %s\n", + fmt_min, fmt_med, fmt_mean, fmt_std, fmt_mem); +} + +static inline void update_countdown(int remaining) { + printf("\r" BENCH_BLUE "ℬ|" BENCH_RESET " %i ", remaining); + fflush(stdout); +} + +static inline void clear_line(void) { printf("\r\033[K"); } + +/***************************** + * Memory measurement + *******************************/ + +static inline long get_memory_usage(void) { + struct rusage usage; + getrusage(RUSAGE_SELF, &usage); + return usage.ru_maxrss; /* peak RSS (bytes on macOS, KB on Linux) */ +} + +/***************************** + * File output + *******************************/ + +/* Write a benchmark result line to a TSV file. + * Renamed from write_double_to_file for clarity. */ +static inline void bench_write_result(const char* filename, + const char* function_name, + const char* time_usage, + const char* memory_usage, + const char* comment) { + FILE* file = fopen(filename, "a"); + if (file == NULL) { + perror("fopen"); + exit(1); + } + + struct utsname sysinfo; + if (uname(&sysinfo) != 0) { + strcpy(sysinfo.sysname, "UnknownOS"); + } + + char hostname[256]; + if (gethostname(hostname, sizeof(hostname)) != 0) { + strcpy(hostname, "UnknownHost"); + } + + time_t tm = 0; + time(&tm); + char* timestr = ctime(&tm); + /* ctime() appends a newline — strip it */ + if (timestr) { + size_t len = strlen(timestr); + if (len > 0 && timestr[len - 1] == '\n') { + timestr[len - 1] = '\0'; + } + } + + /* Columns: Function | Time | Memory | Comment | OS | Host | Timestamp */ + fprintf(file, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", function_name, time_usage, + memory_usage, comment, sysinfo.sysname, hostname, + timestr ? timestr : "unknown"); + fclose(file); +} + +/* Backward-compatible alias */ +static inline void write_double_to_file(char* filename, + const char* function_name, + const char* time_usage, + const char* memory_usage, + const char* comment) { + bench_write_result(filename, function_name, time_usage, memory_usage, + comment); +} + +/* ---- JSON string escaping (minimal, no alloc) ---- */ +static inline void bench_json_escape_(const char* src, char* dst, + size_t dst_size) { + size_t j = 0; + for (size_t i = 0; src[i] && j + 6 < dst_size; ++i) { + switch (src[i]) { + case '"': dst[j++] = '\\'; dst[j++] = '"'; break; + case '\\': dst[j++] = '\\'; dst[j++] = '\\'; break; + case '\n': dst[j++] = '\\'; dst[j++] = 'n'; break; + case '\r': dst[j++] = '\\'; dst[j++] = 'r'; break; + case '\t': dst[j++] = '\\'; dst[j++] = 't'; break; + default: dst[j++] = src[i]; break; + } + } + dst[j] = '\0'; +} + +/* Write a single benchmark result as one JSON line (.jsonl). + * + * Stores both the raw nanosecond value (for machine processing) and a + * human-readable formatted string. Appends one line per call. + * + * Usage: + * bench_write_jsonl("results.jsonl", + * "dm_multiply (1000x1000)", elapsed_ns, + * peak_rss, "OpenBLAS"); + */ +static inline void bench_write_jsonl(const char* filename, + const char* function_name, + long long wall_ns, + long peak_rss, + const char* comment) { + FILE* file = fopen(filename, "a"); + if (file == NULL) { + perror("fopen"); + exit(1); + } + + struct utsname sysinfo; + if (uname(&sysinfo) != 0) { + strcpy(sysinfo.sysname, "UnknownOS"); + } + + char hostname[256]; + if (gethostname(hostname, sizeof(hostname)) != 0) { + strcpy(hostname, "UnknownHost"); + } + + /* ISO 8601 timestamp */ + time_t tm = 0; + time(&tm); + char iso_ts[64]; + strftime(iso_ts, sizeof(iso_ts), "%Y-%m-%dT%H:%M:%S%z", localtime(&tm)); + + /* Human-readable strings */ + char fmt_time[64], fmt_mem[64]; + format_duration(wall_ns, fmt_time, sizeof(fmt_time)); + format_memory_usage(peak_rss, fmt_mem, sizeof(fmt_mem)); + + /* Escape strings that may contain special characters */ + char esc_func[256], esc_comment[256]; + bench_json_escape_(function_name, esc_func, sizeof(esc_func)); + bench_json_escape_(comment ? comment : "", esc_comment, sizeof(esc_comment)); + + fprintf(file, + "{\"function\":\"%s\"" + ",\"wall_ns\":%lld" + ",\"wall_fmt\":\"%s\"" + ",\"peak_rss\":%ld" + ",\"peak_rss_fmt\":\"%s\"" + ",\"comment\":\"%s\"" + ",\"os\":\"%s\"" + ",\"host\":\"%s\"" + ",\"timestamp\":\"%s\"}\n", + esc_func, wall_ns, fmt_time, peak_rss, fmt_mem, + esc_comment, sysinfo.sysname, hostname, iso_ts); + fclose(file); +} + +/* Write statistics from a BENCH_N run as one JSON line (.jsonl). + * + * Usage: + * bench_write_stats_jsonl("results.jsonl", + * "dm_multiply (1000x1000)", + * &stats, peak_rss, "OpenBLAS"); + */ +static inline void bench_write_stats_jsonl(const char* filename, + const char* function_name, + const bench_stats_t* s, + long peak_rss, + const char* comment) { + FILE* file = fopen(filename, "a"); + if (file == NULL) { + perror("fopen"); + exit(1); + } + + struct utsname sysinfo; + if (uname(&sysinfo) != 0) { + strcpy(sysinfo.sysname, "UnknownOS"); + } + + char hostname[256]; + if (gethostname(hostname, sizeof(hostname)) != 0) { + strcpy(hostname, "UnknownHost"); + } + + char iso_ts[64]; + time_t tm = 0; + time(&tm); + strftime(iso_ts, sizeof(iso_ts), "%Y-%m-%dT%H:%M:%S%z", localtime(&tm)); + + char fmt_mem[64]; + format_memory_usage(peak_rss, fmt_mem, sizeof(fmt_mem)); + + char esc_func[256], esc_comment[256]; + bench_json_escape_(function_name, esc_func, sizeof(esc_func)); + bench_json_escape_(comment ? comment : "", esc_comment, sizeof(esc_comment)); + + fprintf(file, + "{\"function\":\"%s\"" + ",\"iterations\":%d" + ",\"min_ns\":%.0f" + ",\"max_ns\":%.0f" + ",\"mean_ns\":%.0f" + ",\"median_ns\":%.0f" + ",\"stddev_ns\":%.0f" + ",\"peak_rss\":%ld" + ",\"peak_rss_fmt\":\"%s\"" + ",\"comment\":\"%s\"" + ",\"os\":\"%s\"" + ",\"host\":\"%s\"" + ",\"timestamp\":\"%s\"}\n", + esc_func, s->n, s->min_ns, s->max_ns, s->mean_ns, + s->median_ns, s->stddev_ns, peak_rss, fmt_mem, + esc_comment, sysinfo.sysname, hostname, iso_ts); + fclose(file); +} + +/***************************** + * Environment info + *******************************/ + +static inline void print_environment_info(void) { + struct utsname sysinfo; + if (uname(&sysinfo) == 0) { + printf("\r" BENCH_BLUE "ℬ|" BENCH_RESET " OS: %s %s (%s)", + sysinfo.sysname, sysinfo.release, sysinfo.machine); + } + + long nprocs = sysconf(_SC_NPROCESSORS_ONLN); + if (nprocs > 0) { + printf(" | Cores: %ld", nprocs); + } + + long pages = sysconf(_SC_PHYS_PAGES); + long page_size = sysconf(_SC_PAGE_SIZE); + if (pages > 0 && page_size > 0) { + double mem_gb = (pages * page_size) / (1024.0 * 1024.0 * 1024.0); + printf(" | RAM: %.2f GB", mem_gb); + } + + double loadavg[3]; + if (getloadavg(loadavg, 3) == 3) { + printf(" | Load: %.2f / %.2f / %.2f", loadavg[0], loadavg[1], loadavg[2]); + } + + printf("\n"); + fflush(stdout); +} + +/***************************** + * BENCH MACROs + *******************************/ + +/* ---- Single-shot benchmark (wall-time + peak RSS) ---- */ +#define BENCH(FCALL) \ + do { \ + long long bench_elapsed_, bench_rss_; \ + WALLTIME_RSS(FCALL, &bench_elapsed_, &bench_rss_); \ + char bench_fmt_[64]; \ + format_duration(bench_elapsed_, bench_fmt_, sizeof(bench_fmt_)); \ + print_benchmark_result(bench_fmt_, bench_rss_, #FCALL, __FILE__, \ + __LINE__); \ + } while (0) + +/* ---- Multi-iteration benchmark with warmup & statistics ---- * + * + * Usage: + * BENCH_N(5, 2, { + * DoubleMatrix *r = dm_multiply(a, b); + * dm_destroy(r); + * }); + * + * Parameters: + * ITERS - number of timed iterations + * WARMUP - number of warmup iterations (not measured) + * FCALL - code block to benchmark + */ +#define BENCH_N(ITERS, WARMUP, FCALL) \ + do { \ + /* Warmup */ \ + for (int bench_w_ = 0; bench_w_ < (WARMUP); ++bench_w_) { \ + FCALL; \ + } \ + /* Timed iterations */ \ + long long bench_samples_[(ITERS)]; \ + long bench_peak_rss_ = get_memory_usage(); \ + for (int bench_i_ = 0; bench_i_ < (ITERS); ++bench_i_) { \ + update_countdown((ITERS) - bench_i_); \ + struct timespec bench_ts0_, bench_ts1_; \ + clock_gettime(CLOCK_MONOTONIC, &bench_ts0_); \ + FCALL; \ + clock_gettime(CLOCK_MONOTONIC, &bench_ts1_); \ + bench_samples_[bench_i_] = \ + (bench_ts1_.tv_sec - bench_ts0_.tv_sec) * 1000000000LL + \ + (bench_ts1_.tv_nsec - bench_ts0_.tv_nsec); \ + } \ + long bench_peak_rss_after_ = get_memory_usage(); \ + if (bench_peak_rss_after_ > bench_peak_rss_) \ + bench_peak_rss_ = bench_peak_rss_after_; \ + clear_line(); \ + bench_stats_t bench_st_ = bench_compute_stats(bench_samples_, (ITERS));\ + print_benchmark_stats(#FCALL, &bench_st_, bench_peak_rss_); \ + } while (0) + +/* ---- Wall-time expression macro (GCC/Clang extension) ---- * + * + * Returns elapsed nanoseconds as long long. + * Note: uses GCC statement-expression ({...}). + * + * Usage: long long ns = WALLTIME({ do_work(); }); + */ +#define WALLTIME(FCALL) \ + ({ \ + struct timespec bench_s_, bench_e_; \ + long long bench_elapsed_; \ + if (clock_gettime(CLOCK_MONOTONIC, &bench_s_) != 0) { \ + perror("clock_gettime"); \ + bench_elapsed_ = -1; \ + } else { \ + FCALL; \ + BENCH_CLOBBER_MEMORY(); \ + if (clock_gettime(CLOCK_MONOTONIC, &bench_e_) != 0) { \ + perror("clock_gettime"); \ + bench_elapsed_ = -1; \ + } else { \ + bench_elapsed_ = \ + (bench_e_.tv_sec - bench_s_.tv_sec) * 1000000000LL +\ + (bench_e_.tv_nsec - bench_s_.tv_nsec); \ + } \ + } \ + bench_elapsed_; \ + }) + +/* backward-compatible alias */ +#define CPUTIME(FCALL) WALLTIME(FCALL) + +/* ---- Wall-time + RSS measurement ---- */ +#define WALLTIME_RSS(FCALL, TIME_PTR, RSS_PTR) \ + do { \ + struct timespec bench_s_, bench_e_; \ + long bench_rss_before_ = get_memory_usage(); \ + \ + if (clock_gettime(CLOCK_MONOTONIC, &bench_s_) != 0) { \ + perror("clock_gettime"); \ + *(TIME_PTR) = -1; \ + } else { \ + FCALL; \ + BENCH_CLOBBER_MEMORY(); \ + if (clock_gettime(CLOCK_MONOTONIC, &bench_e_) != 0) { \ + perror("clock_gettime"); \ + *(TIME_PTR) = -1; \ + } else { \ + *(TIME_PTR) = \ + (bench_e_.tv_sec - bench_s_.tv_sec) * 1000000000LL + \ + (bench_e_.tv_nsec - bench_s_.tv_nsec); \ + } \ + } \ + long bench_rss_after_ = get_memory_usage(); \ + *(RSS_PTR) = bench_rss_after_ - bench_rss_before_; \ + } while (0) + +/* backward-compatible alias */ +#define CPUTIME_RSS(FCALL, TIME_PTR, RSS_PTR) \ + WALLTIME_RSS(FCALL, TIME_PTR, RSS_PTR) + +/* ---- Peak RSS expression macro ---- */ +#define RSS(FCALL) \ + ({ \ + long bench_mem_b_ = get_memory_usage(); \ + FCALL; \ + long bench_mem_a_ = get_memory_usage(); \ + (bench_mem_a_ - bench_mem_b_); \ + }) + +#endif /* SIMPLE_BENCH_H */ diff --git a/third_party/Unity.BUILD b/third_party/Unity.BUILD deleted file mode 100644 index eb92104..0000000 --- a/third_party/Unity.BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -filegroup( - name = "TestRunnerGenerator", - srcs = ["auto/generate_test_runner.rb"], - visibility = ["//visibility:public"], -) - -filegroup( - name = "HelperScripts", - srcs = glob(["auto/*.rb"]), - visibility = ["//visibility:public"], -) - -cc_library( - name = "Unity", - srcs = [ - "src/unity.c", - ], - hdrs = [ - "src/unity.h", - "src/unity_internals.h", - ], - copts = [ - "-DUNITY_INCLUDE_DOUBLE", - "-DUNITY_USE_COMMAND_LINE_ARGS", - ], - defines = ["UNITY_SUPPORT_64"], - includes = ["src"], - visibility = ["//visibility:public"], -) diff --git a/third_party/libomp.BUILD b/third_party/libomp.BUILD deleted file mode 100644 index 13d5fdd..0000000 --- a/third_party/libomp.BUILD +++ /dev/null @@ -1,31 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -# homebrew installed local libomp -cc_library( - name = "libomp", - srcs = [ - "lib/libomp.a", - ], - hdrs = [ - "include/omp.h", - "include/omp-tools.h", - "include/ompt.h", - ], - includes = [ - "include", - ], - visibility = [ - "//visibility:public", - ], -) - -filegroup( - name = "install_files", - srcs = [ - "lib/libomp.a", - "include/omp.h", - "include/omp-tools.h", - "include/ompt.h", - ], - visibility = ["//visibility:public"], -) diff --git a/third_party/matio/BUILD b/third_party/matio/BUILD deleted file mode 100644 index 7be2bd0..0000000 --- a/third_party/matio/BUILD +++ /dev/null @@ -1,10 +0,0 @@ -exports_files( - [ - "matio.BUILD", - "matioConfig.h", - "matio_pubconf.h", - "matioConfig-Linux.h", - "matio_pubconf-Linux.h", - ], - visibility = ["//visibility:public"], -) diff --git a/third_party/matio/matio.BUILD b/third_party/matio/matio.BUILD deleted file mode 100644 index f105308..0000000 --- a/third_party/matio/matio.BUILD +++ /dev/null @@ -1,60 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -config_setting( - name = "macos_arm64", - constraint_values = [ - "@platforms//os:macos", - "@platforms//cpu:arm64", - ], -) - -config_setting( - name = "linux_x86_64", - constraint_values = [ - "@platforms//os:linux", - "@platforms//cpu:x86_64", - ], -) - -genrule( - name = "matioConfig_h", - srcs = select({ - ":macos_arm64": ["@//third_party/matio:matioConfig.h"], - ":linux_x86_64": ["@//third_party/matio:matioConfig-Linux.h"], - }), - outs = ["matioConfig.h"], - cmd = "cp $< $@", - visibility = ["//visibility:public"], -) - -genrule( - name = "matio_pubconf_h", - srcs = select({ - ":macos_arm64": ["@//third_party/matio:matio_pubconf.h"], - ":linux_x86_64": ["@//third_party/matio:matio_pubconf-Linux.h"], - }), - outs = ["matio_pubconf.h"], - cmd = "cp $< $@", - visibility = ["//visibility:public"], -) - -cc_library( - name = "matio", - srcs = glob(["src/*.c"]) + [ - "matioConfig.h", - "matio_pubconf.h", - ], - hdrs = glob(["src/*.h"]), - copts = [ - "-Wno-unused-but-set-variable", - ], - includes = [ - "src", - ], - visibility = ["//visibility:public"], - deps = [ - ":matioConfig_h", - ":matio_pubconf_h", - "@zlib", - ], -) diff --git a/third_party/matio/matioConfig-Linux.h b/third_party/matio/matioConfig-Linux.h deleted file mode 100644 index c8d7dba..0000000 --- a/third_party/matio/matioConfig-Linux.h +++ /dev/null @@ -1,280 +0,0 @@ -/* src/matioConfig.h. Generated from matioConfig.h.in by configure. */ -/* - * Copyright (c) 2015-2025, The matio contributors - * Copyright (c) 2012-2014, Christopher C. Hulbert - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* Debug enabled */ -/* #undef DEBUG */ - -/* Extended sparse matrix data types */ -#define EXTENDED_SPARSE 1 /**/ - -/* Define to dummy `main' function (if any) required to link to the Fortran - libraries. */ -/* #undef FC_DUMMY_MAIN */ - -/* Define if F77 and FC dummy `main' functions are identical. */ -/* #undef FC_DUMMY_MAIN_EQ_F77 */ - -/* Define to a macro mangling the given C identifier (in lower and upper - case), which must not contain underscores, for linking with Fortran. */ -/* #undef FC_FUNC */ - -/* As FC_FUNC, but for C identifiers containing underscores. */ -/* #undef FC_FUNC_ */ - -/* Define to 1 if you have the `fseeko' function. */ -#define HAVE_FSEEKO 1 - -/* Define to 1 if you have the `ftello' function. */ -#define HAVE_FTELLO 1 - -/* Define to 1 if you have the `fseeko64' function. */ -#define HAVE_FSEEKO64 1 - -/* Define to 1 if you have the `ftello64' function. */ -#define HAVE_FTELLO64 1 - -/* Define to 1 if you have the `_fseeki64' function. */ -/* #undef HAVE__FSEEKI64 */ - -/* Define to 1 if you have the `_ftelli64' function. */ -/* #undef HAVE__FTELLI64 */ - -/* Define if 64-bit file address support in 32-bit OS. */ -#define _FILE_OFFSET_BITS 64 - -/* Define if 64-bit file address support in 32-bit OS. */ -#define _LARGEFILE64_SOURCE 1 - -/* Define to 1 if you have the `asprintf' function. */ -#define HAVE_ASPRINTF 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_FLOAT_H 1 - -/* Have HDF5 */ -#undef HAVE_HDF5 - -/* Define to 1 if the system has the type `intmax_t'. */ -#define HAVE_INTMAX_T 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_INTSAFE_H */ - -/* Define to 1 if you have the `m' library (-lm). */ -#define HAVE_LIBM 1 - -/* Define to 1 if you have the `localeconv' function. */ -#define HAVE_LOCALECONV 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_LOCALE_H 1 - -/* Define to 1 if the system has the type `long double'. */ -#define HAVE_LONG_DOUBLE 1 - -/* Define to 1 if the system has the type `long long int'. */ -#define HAVE_LONG_LONG_INT 1 - -/* Have MAT int16 */ -#define HAVE_MAT_INT16_T /**/ - -/* Have MAT int32 */ -#define HAVE_MAT_INT32_T /**/ - -/* Have MAT int64 */ -#define HAVE_MAT_INT64_T /**/ - -/* Have MAT int8 */ -#define HAVE_MAT_INT8_T /**/ - -/* Have MAT uint16 */ -#define HAVE_MAT_UINT16_T /**/ - -/* Have MAT uint32 */ -#define HAVE_MAT_UINT32_T /**/ - -/* Have MAT uint64 */ -#define HAVE_MAT_UINT64_T /**/ - -/* Have MAT uint8 */ -#define HAVE_MAT_UINT8_T /**/ - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_MEMORY_H */ - -/* Define to 1 if the system has the type `ptrdiff_t'. */ -#define HAVE_PTRDIFF_T 1 - -/* Define to 1 if you have a C99 compliant `snprintf' function. */ -#define HAVE_SNPRINTF 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDARG_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDDEF_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if `decimal_point' is member of `struct lconv'. */ -#define HAVE_STRUCT_LCONV_DECIMAL_POINT 1 - -/* Define to 1 if `thousands_sep' is member of `struct lconv'. */ -#define HAVE_STRUCT_LCONV_THOUSANDS_SEP 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if the system has the type `uintmax_t'. */ -#define HAVE_UINTMAX_T 1 - -/* Define to 1 if the system has the type `uintptr_t'. */ -#define HAVE_UINTPTR_T 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to 1 if the system has the type `unsigned long long int'. */ -#define HAVE_UNSIGNED_LONG_LONG_INT 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_VARARGS_H */ - -/* Define to 1 if you have the `vasprintf' function. */ -#define HAVE_VASPRINTF 1 - -/* Define to 1 if you have the `va_copy' function or macro. */ -#define HAVE_VA_COPY 1 - -/* Define to 1 if you have a C99 compliant `vsnprintf' function. */ -#define HAVE_VSNPRINTF 1 - -/* Have zlib */ -#define HAVE_ZLIB 1 - -/* Define to 1 if you have the `__va_copy' function or macro. */ -/* #undef HAVE___VA_COPY */ - -/* OS is Linux */ -#define LINUX /**/ - -/* Define to the sub-directory in which libtool stores uninstalled libraries. - */ -#define LT_OBJDIR ".libs/" - -/* MAT v7.3 file support */ -/* #undef MAT73 */ - -/* Platform */ -#define MATIO_PLATFORM "x86_64-pc-linux-gnu" - -/* Debug disabled */ -#define NODEBUG /**/ - -/* Fixed types in safe-math.h disabled */ -#define PSNIP_SAFE_NO_FIXED 1 - -/* Name of package */ -#define PACKAGE "matio" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "t-beu@users.sourceforge.net" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "MATIO" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "MATIO 1.5.28" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "matio" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "https://sourceforge.net/projects/matio" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "1.5.28" - -/* The size of `char', as computed by sizeof. */ -#define SIZEOF_CHAR 1 - -/* The size of `double', as computed by sizeof. */ -#define SIZEOF_DOUBLE 8 - -/* The size of `float', as computed by sizeof. */ -#define SIZEOF_FLOAT 4 - -/* The size of `int', as computed by sizeof. */ -#define SIZEOF_INT 4 - -/* The size of `long', as computed by sizeof. */ -#define SIZEOF_LONG 8 - -/* The size of `long long', as computed by sizeof. */ -#define SIZEOF_LONG_LONG 8 - -/* The size of `short', as computed by sizeof. */ -#define SIZEOF_SHORT 2 - -/* The size of `size_t', as computed by sizeof. */ -#define SIZEOF_SIZE_T 8 - -/* The size of `void *', as computed by sizeof. */ -#define SIZEOF_VOID_P 8 - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* OS is Solaris */ -/* #undef SUN */ - -/* Version number of package */ -#define VERSION "1.5.28" - -/* Z prefix */ -/* #undef Z_PREFIX */ diff --git a/third_party/matio/matioConfig.h b/third_party/matio/matioConfig.h deleted file mode 100644 index 83b5da4..0000000 --- a/third_party/matio/matioConfig.h +++ /dev/null @@ -1,277 +0,0 @@ -/* src/matioConfig.h. Generated from matioConfig.h.in by configure. */ -/* - * Copyright (c) 2015-2024, The matio contributors - * Copyright (c) 2012-2014, Christopher C. Hulbert - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - */ - -/* Debug enabled */ -/* #undef DEBUG */ - -/* Extended sparse matrix data types */ -#define EXTENDED_SPARSE 1 /**/ - -/* Define to dummy `main' function (if any) required to link to the Fortran - libraries. */ -/* #undef FC_DUMMY_MAIN */ - -/* Define if F77 and FC dummy `main' functions are identical. */ -/* #undef FC_DUMMY_MAIN_EQ_F77 */ - -/* Define to a macro mangling the given C identifier (in lower and upper - case), which must not contain underscores, for linking with Fortran. */ -/* #undef FC_FUNC */ - -/* As FC_FUNC, but for C identifiers containing underscores. */ -/* #undef FC_FUNC_ */ - -/* Define to 1 if you have the `fseeko' function. */ -#define HAVE_FSEEKO 1 - -/* Define to 1 if you have the `ftello' function. */ -#define HAVE_FTELLO 1 - -/* Define to 1 if you have the `fseeko64' function. */ -/* #undef HAVE_FSEEKO64 */ - -/* Define to 1 if you have the `ftello64' function. */ -/* #undef HAVE_FTELLO64 */ - -/* Define to 1 if you have the `_fseeki64' function. */ -/* #undef HAVE__FSEEKI64 */ - -/* Define to 1 if you have the `_ftelli64' function. */ -/* #undef HAVE__FTELLI64 */ - -/* Define if 64-bit file address support in 32-bit OS. */ -#define _FILE_OFFSET_BITS 64 - -/* Define if 64-bit file address support in 32-bit OS. */ -#define _LARGEFILE64_SOURCE 1 - -/* Define to 1 if you have the `asprintf' function. */ -#define HAVE_ASPRINTF 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_DLFCN_H 1 - -/* Have HDF5 */ -#define HAVE_HDF5 1 - -/* Define to 1 if the system has the type `intmax_t'. */ -#define HAVE_INTMAX_T 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_INTTYPES_H 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_INTSAFE_H */ - -/* Define to 1 if you have the `m' library (-lm). */ -#define HAVE_LIBM 1 - -/* Define to 1 if you have the `localeconv' function. */ -#define HAVE_LOCALECONV 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_LOCALE_H 1 - -/* Define to 1 if the system has the type `long double'. */ -#define HAVE_LONG_DOUBLE 1 - -/* Define to 1 if the system has the type `long long int'. */ -#define HAVE_LONG_LONG_INT 1 - -/* Have MAT int16 */ -#define HAVE_MAT_INT16_T 1 /**/ - -/* Have MAT int32 */ -#define HAVE_MAT_INT32_T 1 /**/ - -/* Have MAT int64 */ -#define HAVE_MAT_INT64_T 1 /**/ - -/* Have MAT int8 */ -#define HAVE_MAT_INT8_T 1 /**/ - -/* Have MAT uint16 */ -#define HAVE_MAT_UINT16_T 1 /**/ - -/* Have MAT uint32 */ -#define HAVE_MAT_UINT32_T 1 /**/ - -/* Have MAT uint64 */ -#define HAVE_MAT_UINT64_T 1 /**/ - -/* Have MAT uint8 */ -#define HAVE_MAT_UINT8_T 1 /**/ - -/* Define to 1 if you have the header file. */ -#define HAVE_MEMORY_H 1 - -/* Define to 1 if the system has the type `ptrdiff_t'. */ -#define HAVE_PTRDIFF_T 1 - -/* Define to 1 if you have a C99 compliant `snprintf' function. */ -#define HAVE_SNPRINTF 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDARG_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDDEF_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STDLIB_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRINGS_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_STRING_H 1 - -/* Define to 1 if `decimal_point' is member of `struct lconv'. */ -#define HAVE_STRUCT_LCONV_DECIMAL_POINT 1 - -/* Define to 1 if `thousands_sep' is member of `struct lconv'. */ -#define HAVE_STRUCT_LCONV_THOUSANDS_SEP 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_STAT_H 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_SYS_TYPES_H 1 - -/* Define to 1 if the system has the type `uintmax_t'. */ -#define HAVE_UINTMAX_T 1 - -/* Define to 1 if the system has the type `uintptr_t'. */ -#define HAVE_UINTPTR_T 1 - -/* Define to 1 if you have the header file. */ -#define HAVE_UNISTD_H 1 - -/* Define to 1 if the system has the type `unsigned long long int'. */ -#define HAVE_UNSIGNED_LONG_LONG_INT 1 - -/* Define to 1 if you have the header file. */ -/* #undef HAVE_VARARGS_H */ - -/* Define to 1 if you have the `vasprintf' function. */ -#define HAVE_VASPRINTF 1 - -/* Define to 1 if you have the `va_copy' function or macro. */ -#define HAVE_VA_COPY 1 - -/* Define to 1 if you have a C99 compliant `vsnprintf' function. */ -#define HAVE_VSNPRINTF 1 - -/* Have zlib */ -#define HAVE_ZLIB 1 - -/* Define to 1 if you have the `__va_copy' function or macro. */ -/* #undef HAVE___VA_COPY */ - -/* OS is Linux */ -/* #undef LINUX */ - -/* Define to the sub-directory in which libtool stores uninstalled libraries. - */ -#define LT_OBJDIR ".libs/" - -/* MAT v7.3 file support */ -#undef MAT73 - -/* Platform */ -#define MATIO_PLATFORM "arm-apple-darwin24.0.0" - -/* Debug disabled */ -#define NODEBUG /**/ - -/* Fixed types in safe-math.h disabled */ -#define PSNIP_SAFE_NO_FIXED 1 - -/* Name of package */ -#define PACKAGE "matio" - -/* Define to the address where bug reports for this package should be sent. */ -#define PACKAGE_BUGREPORT "t-beu@users.sourceforge.net" - -/* Define to the full name of this package. */ -#define PACKAGE_NAME "MATIO" - -/* Define to the full name and version of this package. */ -#define PACKAGE_STRING "MATIO 1.5.27" - -/* Define to the one symbol short name of this package. */ -#define PACKAGE_TARNAME "matio" - -/* Define to the home page for this package. */ -#define PACKAGE_URL "https://sourceforge.net/projects/matio" - -/* Define to the version of this package. */ -#define PACKAGE_VERSION "1.5.27" - -/* The size of `char', as computed by sizeof. */ -#define SIZEOF_CHAR 1 - -/* The size of `double', as computed by sizeof. */ -#define SIZEOF_DOUBLE 8 - -/* The size of `float', as computed by sizeof. */ -#define SIZEOF_FLOAT 4 - -/* The size of `int', as computed by sizeof. */ -#define SIZEOF_INT 4 - -/* The size of `long', as computed by sizeof. */ -#define SIZEOF_LONG 8 - -/* The size of `long long', as computed by sizeof. */ -#define SIZEOF_LONG_LONG 8 - -/* The size of `short', as computed by sizeof. */ -#define SIZEOF_SHORT 2 - -/* The size of `size_t', as computed by sizeof. */ -#define SIZEOF_SIZE_T 8 - -/* The size of `void *', as computed by sizeof. */ -#define SIZEOF_VOID_P 8 - -/* Define to 1 if you have the ANSI C header files. */ -#define STDC_HEADERS 1 - -/* OS is Solaris */ -/* #undef SUN */ - -/* Version number of package */ -#define VERSION "1.5.27" - -/* Z prefix */ -/* #undef Z_PREFIX */ diff --git a/third_party/matio/matio_pubconf-Linux.h b/third_party/matio/matio_pubconf-Linux.h deleted file mode 100644 index 846af69..0000000 --- a/third_party/matio/matio_pubconf-Linux.h +++ /dev/null @@ -1,183 +0,0 @@ -/* src/matio_pubconf.h. Generated from matio_pubconf.h.in by configure. */ -/* - * Copyright (c) 2015-2025, The matio contributors - * Copyright (c) 2010-2014, Christopher C. Hulbert - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef MATIO_PUBCONF_H -#define MATIO_PUBCONF_H 1 - -/* Matio major version number */ -#define MATIO_MAJOR_VERSION 1 - -/* Matio minor version number */ -#define MATIO_MINOR_VERSION 5 - -/* Matio release level number */ -#define MATIO_RELEASE_LEVEL 28 - -/* Matio version number */ -#define MATIO_VERSION 1528 - -/* Matio version string */ -#define MATIO_VERSION_STR "1.5.27" - -/* Default file format */ -#define MAT_FT_DEFAULT MAT_FT_MAT5 - -/* Define to 1 if you have the header file. */ -#define MATIO_HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define MATIO_HAVE_INTTYPES_H 1 - -/* int16 type */ -#define _mat_int16_t int16_t - -/* int32 type */ -#define _mat_int32_t int32_t - -/* int64 type */ -#define _mat_int64_t int64_t - -/* int8 type */ -#define _mat_int8_t int8_t - -/* uint16 type */ -#define _mat_uint16_t uint16_t - -/* uint32 type */ -#define _mat_uint32_t uint32_t - -/* uint64 type */ -#define _mat_uint64_t uint64_t - -/* uint8 type */ -#define _mat_uint8_t uint8_t - -#if MATIO_HAVE_INTTYPES_H -#include -#endif - -#if MATIO_HAVE_STDINT_H -#include -#endif - -#ifdef _mat_int64_t -typedef _mat_int64_t mat_int64_t; -#endif -#ifdef _mat_uint64_t -typedef _mat_uint64_t mat_uint64_t; -#endif -#ifdef _mat_int32_t -typedef _mat_int32_t mat_int32_t; -#endif -#ifdef _mat_uint32_t -typedef _mat_uint32_t mat_uint32_t; -#endif -#ifdef _mat_int16_t -typedef _mat_int16_t mat_int16_t; -#endif -#ifdef _mat_uint16_t -typedef _mat_uint16_t mat_uint16_t; -#endif -#ifdef _mat_int8_t -typedef _mat_int8_t mat_int8_t; -#endif -#ifdef _mat_uint8_t -typedef _mat_uint8_t mat_uint8_t; -#endif - -/* - The following macros handle noreturn attributes according to the - C11/C++11 standard with fallback to GNU, Clang or MSVC extensions if using - an older compiler. -*/ -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -#define MATIO_NORETURN _Noreturn -#define MATIO_NORETURNATTR -#elif __cplusplus >= 201103L -#if (defined(__GNUC__) && __GNUC__ >= 5) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 8) -#define MATIO_NORETURN [[noreturn]] -#define MATIO_NORETURNATTR -#elif (defined(__GNUC__) && __GNUC__ >= 3) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 2 && __GNUC_MINOR__ >= 8) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#elif defined(__GNUC__) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#else -#define MATIO_NORETURN [[noreturn]] -#define MATIO_NORETURNATTR -#endif -#elif defined(__clang__) -#if __has_attribute(noreturn) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#else -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#endif -#elif (defined(__GNUC__) && __GNUC__ >= 3) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 2 && __GNUC_MINOR__ >= 8) || \ - (defined(__SUNPRO_C) && __SUNPRO_C >= 0x5110) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#elif (defined(_MSC_VER) && _MSC_VER >= 1200) || defined(__BORLANDC__) -#define MATIO_NORETURN __declspec(noreturn) -#define MATIO_NORETURNATTR -#else -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#endif - -/* - The following macros handle format attributes for type-checks against a - format string. -*/ - -#if defined(__GNUC__) && __GNUC__ >= 3 -#define MATIO_FORMATATTR_PRINTF1 __attribute__((format(printf, 1, 2))) -#define MATIO_FORMATATTR_PRINTF2 __attribute__((format(printf, 2, 3))) -#define MATIO_FORMATATTR_VPRINTF __attribute__((format(printf, 1, 0))) -#elif defined(__clang__) -#if __has_attribute(format) -#define MATIO_FORMATATTR_PRINTF1 __attribute__((format(printf, 1, 2))) -#define MATIO_FORMATATTR_PRINTF2 __attribute__((format(printf, 2, 3))) -#define MATIO_FORMATATTR_VPRINTF __attribute__((format(printf, 1, 0))) -#else -#define MATIO_FORMATATTR_PRINTF1 -#define MATIO_FORMATATTR_PRINTF2 -#define MATIO_FORMATATTR_VPRINTF -#endif -#else -#define MATIO_FORMATATTR_PRINTF1 -#define MATIO_FORMATATTR_PRINTF2 -#define MATIO_FORMATATTR_VPRINTF -#endif - -#endif /* MATIO_PUBCONF_H */ diff --git a/third_party/matio/matio_pubconf.h b/third_party/matio/matio_pubconf.h deleted file mode 100644 index bbb8d98..0000000 --- a/third_party/matio/matio_pubconf.h +++ /dev/null @@ -1,183 +0,0 @@ -/* src/matio_pubconf.h. Generated from matio_pubconf.h.in by configure. */ -/* - * Copyright (c) 2015-2024, The matio contributors - * Copyright (c) 2010-2014, Christopher C. Hulbert - * All rights reserved. - * - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, this - * list of conditions and the following disclaimer. - * - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE - * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER - * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, - * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - */ - -#ifndef MATIO_PUBCONF_H -#define MATIO_PUBCONF_H 1 - -/* Matio major version number */ -#define MATIO_MAJOR_VERSION 1 - -/* Matio minor version number */ -#define MATIO_MINOR_VERSION 5 - -/* Matio release level number */ -#define MATIO_RELEASE_LEVEL 27 - -/* Matio version number */ -#define MATIO_VERSION 1527 - -/* Matio version string */ -#define MATIO_VERSION_STR "1.5.27" - -/* Default file format */ -#define MAT_FT_DEFAULT MAT_FT_MAT5 - -/* Define to 1 if you have the header file. */ -#define MATIO_HAVE_STDINT_H 1 - -/* Define to 1 if you have the header file. */ -#define MATIO_HAVE_INTTYPES_H 1 - -/* int16 type */ -#define _mat_int16_t int16_t - -/* int32 type */ -#define _mat_int32_t int32_t - -/* int64 type */ -#define _mat_int64_t int64_t - -/* int8 type */ -#define _mat_int8_t int8_t - -/* uint16 type */ -#define _mat_uint16_t uint16_t - -/* uint32 type */ -#define _mat_uint32_t uint32_t - -/* uint64 type */ -#define _mat_uint64_t uint64_t - -/* uint8 type */ -#define _mat_uint8_t uint8_t - -#if MATIO_HAVE_INTTYPES_H -#include -#endif - -#if MATIO_HAVE_STDINT_H -#include -#endif - -#ifdef _mat_int64_t -typedef _mat_int64_t mat_int64_t; -#endif -#ifdef _mat_uint64_t -typedef _mat_uint64_t mat_uint64_t; -#endif -#ifdef _mat_int32_t -typedef _mat_int32_t mat_int32_t; -#endif -#ifdef _mat_uint32_t -typedef _mat_uint32_t mat_uint32_t; -#endif -#ifdef _mat_int16_t -typedef _mat_int16_t mat_int16_t; -#endif -#ifdef _mat_uint16_t -typedef _mat_uint16_t mat_uint16_t; -#endif -#ifdef _mat_int8_t -typedef _mat_int8_t mat_int8_t; -#endif -#ifdef _mat_uint8_t -typedef _mat_uint8_t mat_uint8_t; -#endif - -/* - The following macros handle noreturn attributes according to the - C11/C++11 standard with fallback to GNU, Clang or MSVC extensions if using - an older compiler. -*/ -#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L -#define MATIO_NORETURN _Noreturn -#define MATIO_NORETURNATTR -#elif __cplusplus >= 201103L -#if (defined(__GNUC__) && __GNUC__ >= 5) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 4 && __GNUC_MINOR__ >= 8) -#define MATIO_NORETURN [[noreturn]] -#define MATIO_NORETURNATTR -#elif (defined(__GNUC__) && __GNUC__ >= 3) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 2 && __GNUC_MINOR__ >= 8) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#elif defined(__GNUC__) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#else -#define MATIO_NORETURN [[noreturn]] -#define MATIO_NORETURNATTR -#endif -#elif defined(__clang__) -#if __has_attribute(noreturn) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#else -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#endif -#elif (defined(__GNUC__) && __GNUC__ >= 3) || \ - (defined(__GNUC__) && defined(__GNUC_MINOR__) && __GNUC__ == 2 && __GNUC_MINOR__ >= 8) || \ - (defined(__SUNPRO_C) && __SUNPRO_C >= 0x5110) -#define MATIO_NORETURN -#define MATIO_NORETURNATTR __attribute__((noreturn)) -#elif (defined(_MSC_VER) && _MSC_VER >= 1200) || defined(__BORLANDC__) -#define MATIO_NORETURN __declspec(noreturn) -#define MATIO_NORETURNATTR -#else -#define MATIO_NORETURN -#define MATIO_NORETURNATTR -#endif - -/* - The following macros handle format attributes for type-checks against a - format string. -*/ - -#if defined(__GNUC__) && __GNUC__ >= 3 -#define MATIO_FORMATATTR_PRINTF1 __attribute__((format(printf, 1, 2))) -#define MATIO_FORMATATTR_PRINTF2 __attribute__((format(printf, 2, 3))) -#define MATIO_FORMATATTR_VPRINTF __attribute__((format(printf, 1, 0))) -#elif defined(__clang__) -#if __has_attribute(format) -#define MATIO_FORMATATTR_PRINTF1 __attribute__((format(printf, 1, 2))) -#define MATIO_FORMATATTR_PRINTF2 __attribute__((format(printf, 2, 3))) -#define MATIO_FORMATATTR_VPRINTF __attribute__((format(printf, 1, 0))) -#else -#define MATIO_FORMATATTR_PRINTF1 -#define MATIO_FORMATATTR_PRINTF2 -#define MATIO_FORMATATTR_VPRINTF -#endif -#else -#define MATIO_FORMATATTR_PRINTF1 -#define MATIO_FORMATATTR_PRINTF2 -#define MATIO_FORMATATTR_VPRINTF -#endif - -#endif /* MATIO_PUBCONF_H */ diff --git a/third_party/openblas.BUILD b/third_party/openblas.BUILD index 937b0b2..a4ce04c 100644 --- a/third_party/openblas.BUILD +++ b/third_party/openblas.BUILD @@ -3,7 +3,8 @@ load("@rules_cc//cc:cc_library.bzl", "cc_library") # homebrew installed local openblas cc_library( name = "openblas", - srcs = ["lib/libopenblas.a"], + # Homebrew includes LAPACKE in OpenBLAS; distro packages may split it out. + srcs = ["lib/libopenblas.a"] + glob(["lib/liblapacke.a"]), hdrs = glob([ "include/*.h", ]), @@ -15,7 +16,10 @@ cc_library( filegroup( name = "install_files", - srcs = ["lib/libopenblas.a"] + glob(["include/*.h"]), + srcs = ["lib/libopenblas.a"] + glob([ + "lib/liblapacke.a", + "include/*.h", + ]), visibility = ["//visibility:public"], ) diff --git a/third_party/pcg.BUILD b/third_party/pcg.BUILD deleted file mode 100644 index 9a5c506..0000000 --- a/third_party/pcg.BUILD +++ /dev/null @@ -1,13 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -cc_library( - name = "pcg", - srcs = glob(["src/*.c"]), - hdrs = ["include/pcg_variants.h"], - includes = ["include"], - copts = [ - "-O2", - "-Wno-macro-redefined" - ], - visibility = ["//visibility:public"], -) \ No newline at end of file diff --git a/third_party/suitesparse.BUILD b/third_party/suitesparse.BUILD deleted file mode 100644 index 6470828..0000000 --- a/third_party/suitesparse.BUILD +++ /dev/null @@ -1,23 +0,0 @@ - -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -# homebrew installed local suitesparse -cc_library( - name = "suitesparse", - srcs = glob([ - "lib/*.a", - ]), - hdrs = glob([ - "include/suitesparse/*.h", - ]), - includes = [ - "include/suitesparse", - ], - visibility = ["//visibility:public"], -) - -filegroup( - name = "install_files", - srcs = glob(["lib/*.a"]) + glob(["include/suitesparse/*.h"]), - visibility = ["//visibility:public"], -) diff --git a/third_party/zlib.BUILD b/third_party/zlib.BUILD deleted file mode 100644 index f5ecdbe..0000000 --- a/third_party/zlib.BUILD +++ /dev/null @@ -1,67 +0,0 @@ -load("@rules_cc//cc:cc_library.bzl", "cc_library") - -licenses(["notice"]) # BSD/MIT-like license (for zlib) - -cc_library( - name = "zlib", - srcs = [ - "adler32.c", - "compress.c", - "crc32.c", - "crc32.h", - "deflate.c", - "deflate.h", - "gzclose.c", - "gzguts.h", - "gzlib.c", - "gzread.c", - "gzwrite.c", - "infback.c", - "inffast.c", - "inffast.h", - "inffixed.h", - "inflate.c", - "inflate.h", - "inftrees.c", - "inftrees.h", - "trees.c", - "trees.h", - "uncompr.c", - "zconf.h", - "zutil.c", - "zutil.h", - ], - hdrs = ["zlib.h"], - copts = [ - "-O", - "-Wno-implicit-function-declaration", - ], - includes = ["."], - visibility = ["//visibility:public"], -) - -# /* zlib.h -- interface of the 'zlib' general purpose compression library -# version 1.3, August 18th, 2023 - -# Copyright (C) 1995-2023 Jean-loup Gailly and Mark Adler - -# This software is provided 'as-is', without any express or implied -# warranty. In no event will the authors be held liable for any damages -# arising from the use of this software. - -# Permission is granted to anyone to use this software for any purpose, -# including commercial applications, and to alter it and redistribute it -# freely, subject to the following restrictions: - -# 1. The origin of this software must not be misrepresented; you must not -# claim that you wrote the original software. If you use this software -# in a product, an acknowledgment in the product documentation would be -# appreciated but is not required. -# 2. Altered source versions must be plainly marked as such, and must not be -# misrepresented as being the original software. -# 3. This notice may not be removed or altered from any source distribution. - -# Jean-loup Gailly Mark Adler -# jloup@gzip.org madler@alumni.caltech.edu - -# */ diff --git a/tools/ci/install_consumer.c b/tools/ci/install_consumer.c new file mode 100644 index 0000000..0a71a64 --- /dev/null +++ b/tools/ci/install_consumer.c @@ -0,0 +1,49 @@ +/* Exercise public APIs using installed headers and archives only. */ +#include +#include +#include +#include "sm.h" +#include "dms.h" +#include "sv_optim.h" +#ifdef __APPLE__ +#include "st.h" +#include "st_convert.h" +#include "st_interp.h" +#include "st_f16.h" +#endif + +int main(void) { + FloatMatrix *a = sm_create(2, 2); + assert(a); + for (size_t i = 0; i < 4; ++i) a->values[i] = (float)(i + 1); + FloatMatrix *product = sm_multiply(a, a); + assert(product && product->values[0] == 7.0f && product->values[3] == 22.0f); + FloatMatrix *rhs = sm_create_with_values(2, 1, (float[]){5.0f, 11.0f}); + assert(rhs); + FloatMatrix *solution = sm_solve_system(a, rhs); + assert(solution && fabsf(solution->values[0] - 1.0f) < 1e-5f && + fabsf(solution->values[1] - 2.0f) < 1e-5f); + DoubleSparseMatrix *sparse = dms_create_identity(2); + assert(sparse && dms_get(sparse, 1, 1) == 1.0); + float parameter = 1.0f, gradient = 2.0f; + assert(sv_optim_sgd(¶meter, &gradient, 1, 0.25f, 1.0f, 0.0f)); + assert(parameter == 0.5f); +#ifdef __APPLE__ + FloatTensor *x = st_from_sm(a); + assert(x); + FloatTensor *sum = st_binary_broadcast(ST_BIN_ADD, x, x); + assert(sum); + FloatMatrix *result = sm_from_st(sum); + assert(result && result->values[3] == 8.0f); + assert(st_f32_to_f16(1.0f) == 0x3c00); + sm_destroy(result); + st_destroy(sum); + st_destroy(x); +#endif + dms_destroy(sparse); + sm_destroy(solution); + sm_destroy(rhs); + sm_destroy(product); + sm_destroy(a); + return 0; +} diff --git a/tools/ci/test_install.sh b/tools/ci/test_install.sh new file mode 100644 index 0000000..e05780e --- /dev/null +++ b/tools/ci/test_install.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +workspace="${BUILD_WORKSPACE_DIRECTORY:-$(pwd)}" +output="$(mktemp -d "${TMPDIR:-/tmp}/mmatrix-install-test.XXXXXX")" +completed_modes=0 +# Older Bash can return success after a nounset expansion error in a loop. +# Require both consumers to finish, including on the EXIT cleanup path. +trap 'status=$?; rm -rf "$output"; [[ "$completed_modes" == 2 ]] || status=1; exit "$status"' EXIT + +case "$(uname -s)" in + Darwin) + compiler=(xcrun clang) + linkopts=(-framework Accelerate -framework Metal -framework MetalPerformanceShaders + -framework MetalPerformanceShadersGraph -framework Foundation -lc++) + ;; + Linux) + # GNU ld resolves static archives from left to right; the dependency + # closure is installed as separate archives rather than one link order. + compiler=(clang -Wl,--start-group) + linkopts=(-Wl,--end-group -lstdc++ -lgfortran -lquadmath -lm -lpthread -ldl) + ;; + *) echo "Unsupported installation test host" >&2; exit 1 ;; +esac + +for mode in standard bundle; do + prefix="$output/$mode" + bazel run //:matrix_installer --color=no --curses=no -- --mode="$mode" "$prefix" + "${compiler[@]}" "$workspace/tools/ci/install_consumer.c" -I"$prefix/include" \ + "$prefix"/lib/*.a "${linkopts[@]}" -o "$output/consumer-$mode" + "$output/consumer-$mode" + echo "$mode installation consumer passed" + completed_modes=$((completed_modes + 1)) +done diff --git a/tools/doxygen/generate_matrix_api_docs.sh b/tools/doxygen/generate_matrix_api_docs.sh index 66dcd16..997547a 100755 --- a/tools/doxygen/generate_matrix_api_docs.sh +++ b/tools/doxygen/generate_matrix_api_docs.sh @@ -35,7 +35,7 @@ cat >"${DOXYFILE}" <"${DOXYFILE}" </dev/null 2>&1; then + if [[ "$(uname -s)" == "Darwin" ]] && command -v libtool >/dev/null 2>&1; then libtool -static -o "${bundle_tmp}" "${archives[@]}" elif command -v ar >/dev/null 2>&1; then local mri="${tmp_dir}/bundle.mri" @@ -140,7 +140,7 @@ function create_bundle_library() { [[ -d "${prefix}/lib" ]] || \ $sudo mkdir -p "${prefix}/lib" echo "Installing bundled static archive → ${prefix}/lib/libmatrix_full.a" - $sudo install -m 644 -T -- "${bundle_tmp}" "${prefix}/lib/libmatrix_full.a" + $sudo install -m 644 -- "${bundle_tmp}" "${prefix}/lib/libmatrix_full.a" rm -rf -- "${tmp_dir}" } @@ -168,7 +168,7 @@ function install_file() { $sudo mkdir -p "${target_dir}" $sudo install -m "${target_mode}" \ - -T -- "${source}" "${target_dir}/${target_name}" + -- "${source}" "${target_dir}/${target_name}" } function main() { diff --git a/tools/unity_test.bzl b/tools/unity_test.bzl index adecfb3..b5b4b7a 100644 --- a/tools/unity_test.bzl +++ b/tools/unity_test.bzl @@ -38,7 +38,7 @@ def unity_test(name, srcs, my_config = None, deps = None, **kwargs): deps = deps, srcs = [runner_file_name(file_name)] + srcs, size = "small", - copts = ["-DUNITY_USE_COMMAND_LINE_ARGS"] + coverage_copts, + copts = coverage_copts, linkopts = coverage_linkopts, visibility = ["//visibility:public"], linkstatic = True, diff --git a/tools/workspace_status.sh b/tools/workspace_status.sh index b691be0..59c3f98 100755 --- a/tools/workspace_status.sh +++ b/tools/workspace_status.sh @@ -10,7 +10,7 @@ # Note: This script is called by Bazel. So it should not depend on # any external tools. echo "CURRENT_TIME $(date +%s)" -echo "STABLE_GIT_COMMIT $(git rev-parse HEAD)" +echo "STABLE_GIT_COMMIT $(git rev-parse HEAD 2>/dev/null || echo 'unknown')" echo "STABLE_USER_NAME $USER" echo "STABLE_HOST_NAME $(hostname)" echo "STABLE_REPOSITORY_STATUS $(git describe --tags --dirty 2>/dev/null || echo 'unknown')"