From 5b969d93d804965bf262f950ef829f6334d91968 Mon Sep 17 00:00:00 2001 From: Izel Nakri Date: Tue, 12 Aug 2025 20:12:13 +0200 Subject: [PATCH 1/4] initial Intel GPU support for whisper.cpp with SYCL and BLAS --- src/ggml/whisper-cpp.nix | 181 +++++++++++++++++++++++++++++---------- src/unified-runtime.nix | 2 +- 2 files changed, 138 insertions(+), 45 deletions(-) diff --git a/src/ggml/whisper-cpp.nix b/src/ggml/whisper-cpp.nix index a509ffd..ddc04a0 100644 --- a/src/ggml/whisper-cpp.nix +++ b/src/ggml/whisper-cpp.nix @@ -1,52 +1,145 @@ -{ - fetchFromGitHub, - llvm, - cmake, - ninja, - oneDNN, - oneMath, - oneTBB, - # mkl, - git, - opencl-headers, - ocl-icd, -}: let - version = "1.7.6"; -in - llvm.stdenv.mkDerivation { - pname = "whisper-cpp"; - inherit version; - - src = fetchFromGitHub { - owner = "ggml-org"; - repo = "whisper.cpp"; - tag = "v${version}"; - hash = "sha256-dppBhiCS4C3ELw/Ckx5W0KOMUvOHUiisdZvkS7gkxj4="; - }; - - nativeBuildInputs = [ - cmake - ninja - git - ]; +{ lib +, fetchFromGitHub +, llvm +, cmake +, makeWrapper +, oneDNN +, oneMath +, oneTBB +, git +, openvino +, llvmPackages +, SDL2 +, opencl-headers +, ocl-icd +, openblas # NOTE: needed for now until whisper.cpp moves to OneMath +}: + +let + oneMathCmakeShim = llvm.stdenv.mkDerivation { + pname = "onemath-cmake-shim"; + version = "0"; buildInputs = [ - oneDNN oneMath - oneTBB - # mkl opencl-headers ocl-icd - llvm.baseLlvm.openmp + llvmPackages.openmp ]; - hardeningDisable = [ - "zerocallusedregs" - "pacret" - # "shadowstack" - ]; + installPhase = '' + mkdir -p $out/lib/cmake/OpenCL + mkdir -p $out/lib/cmake/OpenCLHeaders + mkdir -p $out/lib/cmake/MKL - cmakeFlags = [ - "-DWHISPER_SYCL=ON" - ]; - } + cat > $out/lib/cmake/MKL/MKLConfig.cmake < $out/lib/cmake/OpenCL/OpenCLConfig.cmake < Date: Thu, 14 Aug 2025 07:37:01 +0200 Subject: [PATCH 2/4] blenderfreaky requested changes to whisper.cpp except openvino --- src/ggml/whisper-cpp.nix | 42 ++++++++++++++-------------------------- 1 file changed, 15 insertions(+), 27 deletions(-) diff --git a/src/ggml/whisper-cpp.nix b/src/ggml/whisper-cpp.nix index ddc04a0..639c04d 100644 --- a/src/ggml/whisper-cpp.nix +++ b/src/ggml/whisper-cpp.nix @@ -1,38 +1,34 @@ -{ lib +{ cmake , fetchFromGitHub +, git +, lib , llvm -, cmake +, llvmPackages , makeWrapper , oneDNN , oneMath , oneTBB -, git , openvino -, llvmPackages , SDL2 , opencl-headers , ocl-icd , openblas # NOTE: needed for now until whisper.cpp moves to OneMath +, runCommand }: let - oneMathCmakeShim = llvm.stdenv.mkDerivation { - pname = "onemath-cmake-shim"; - version = "0"; - + oneMathCmakeShim = runCommand "oneMathCmakeShim" { buildInputs = [ oneMath opencl-headers ocl-icd - llvmPackages.openmp + llvm.baseLlvm.openmp ]; + } '' + mkdir -p $out/lib/cmake/OpenCL + mkdir -p $out/lib/cmake/MKL - installPhase = '' - mkdir -p $out/lib/cmake/OpenCL - mkdir -p $out/lib/cmake/OpenCLHeaders - mkdir -p $out/lib/cmake/MKL - - cat > $out/lib/cmake/MKL/MKLConfig.cmake < $out/lib/cmake/MKL/MKLConfig.cmake < $out/lib/cmake/OpenCL/OpenCLConfig.cmake < $out/lib/cmake/OpenCL/OpenCLConfig.cmake < Date: Mon, 18 Aug 2025 10:50:05 +0200 Subject: [PATCH 3/4] Add improved openvino nix package --- src/default.nix | 1 + src/openvino.nix | 218 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 src/openvino.nix diff --git a/src/default.nix b/src/default.nix index e73fec7..8fbc673 100644 --- a/src/default.nix +++ b/src/default.nix @@ -31,6 +31,7 @@ oneTBB = callPackage ./onetbb.nix { #inherit llvm; }; + openvino = callPackage ./openvino.nix {}; spirv-llvm-translator = callPackage ./spirv-llvm-translator.nix {}; spirv-llvm-translator-test = diff --git a/src/openvino.nix b/src/openvino.nix new file mode 100644 index 0000000..52203ba --- /dev/null +++ b/src/openvino.nix @@ -0,0 +1,218 @@ +# TODO: This needs to be compiled with Intel C++ Compiler due to ENABLE_INTEL_OPENMP ? (If openmp is false, it gets the compiler default one) +# TODO: Remove TBB 2.5 when OneTBB released, +# TODO: Add autoAddDriverRunpath when tests are working +# TODO: Run the tests on the packaging, there is a github issue created +# TODO: Manual test openvino plugins +# TODO: Enable ENABLE_OV_JAX_FRONTEND +{ + lib, + stdenv, + fetchFromGitHub, + fetchurl, + cudaSupport ? opencv.cudaSupport or false, + + # build + scons, + addDriverRunpath, + autoPatchelfHook, + cmake, + git, + libarchive, + patchelf, + pkg-config, + python3Packages, + shellcheck, + + # runtime + flatbuffers, + gflags, + level-zero, + libusb1, + libxml2, + ocl-icd, + opencv, + protobuf, + pugixml, + snappy, + tbb_2022, + cudaPackages, + + tree +}: + +let + inherit (lib) + cmakeBool + ; + + # prevent scons from leaking in the default python version + scons' = scons.override { inherit python3Packages; }; + + tbbbind_version = "2_5"; + tbbbind = fetchurl { + url = "https://storage.openvinotoolkit.org/dependencies/thirdparty/linux/tbbbind_${tbbbind_version}_static_lin_v4.tgz"; + hash = "sha256-Tr8wJGUweV8Gb7lhbmcHxrF756ZdKdNRi1eKdp3VTuo="; + }; + + python = python3Packages.python.withPackages ( + ps: with ps; [ + cython + distutils + pybind11 + setuptools + sphinx + wheel + ] + ); + +in + +stdenv.mkDerivation rec { + pname = "openvino"; + version = "2025.2.1"; + + src = fetchFromGitHub { + owner = "openvinotoolkit"; + repo = "openvino"; + tag = version; + fetchSubmodules = true; + hash = "sha256-Bu0m7nGqyHwHsa7FKr4nvUh/poJxMTgwuAU81QBmI4g="; + }; + + outputs = [ + "out" + "python" + ]; + + nativeBuildInputs = [ + addDriverRunpath + autoPatchelfHook + cmake + git + libarchive + patchelf + pkg-config + python + scons' + shellcheck + ] + ++ lib.optionals cudaSupport [ + cudaPackages.cuda_nvcc + ]; + + postPatch = '' + mkdir -p temp/tbbbind_${tbbbind_version} + pushd temp/tbbbind_${tbbbind_version} + bsdtar -xf ${tbbbind} + echo "${tbbbind.url}" > ie_dependency.info + popd + ''; + + dontUseSconsCheck = true; + dontUseSconsBuild = true; + dontUseSconsInstall = true; + + cmakeFlags = [ + "-Wno-dev" + "-DCMAKE_MODULE_PATH:PATH=${placeholder "out"}/lib/cmake" + "-DCMAKE_PREFIX_PATH:PATH=${placeholder "out"}" + + "-DCMAKE_CXX_FLAGS=-Wno-odr" + "-DCMAKE_C_FLAGS=-Wno-odr" + + "-DOpenCV_DIR=${lib.getLib opencv}/lib/cmake/opencv4/" + "-DProtobuf_LIBRARIES=${protobuf}/lib/libprotobuf${stdenv.hostPlatform.extensions.sharedLibrary}" + "-DPython_EXECUTABLE=${python.interpreter}" + + (cmakeBool "CMAKE_VERBOSE_MAKEFILE" true) + (cmakeBool "NCC_STYLE" false) + (cmakeBool "ENABLE_CPPLINT" false) + (cmakeBool "ENABLE_SAMPLES" false) + + # features + (cmakeBool "ENABLE_INTEL_CPU" stdenv.hostPlatform.isx86_64) + (cmakeBool "ENABLE_INTEL_GPU" true) + (cmakeBool "ENABLE_INTEL_NPU" stdenv.hostPlatform.isx86_64) + (cmakeBool "ENABLE_JS" false) # NOTE: What is this?! + (cmakeBool "ENABLE_LTO" true) + (cmakeBool "ENABLE_OPENCV" true) + # This fails regardless: + (cmakeBool "ENABLE_OV_JAX_FRONTEND" false) # on true: auto-patchelf could not satisfy dependency libopenvino_jax_frontend.so.2450 + (cmakeBool "ENABLE_PYTHON" true) + + # system libs + (cmakeBool "ENABLE_SYSTEM_FLATBUFFERS" true) + (cmakeBool "ENABLE_SYSTEM_OPENCL" true) + (cmakeBool "ENABLE_SYSTEM_PROTOBUF" false) # NOTE: Change this? + (cmakeBool "ENABLE_SYSTEM_PUGIXML" true) + (cmakeBool "ENABLE_SYSTEM_SNAPPY" true) + (cmakeBool "ENABLE_SYSTEM_TBB" true) + ]; + + autoPatchelfIgnoreMissingDeps = [ + "libngraph_backend.so" + ]; + + buildInputs = [ + flatbuffers + gflags + level-zero + libusb1 + libxml2 + ocl-icd + opencv + pugixml + snappy + tbb_2022 + tree + ] + ++ lib.optionals cudaSupport [ + cudaPackages.cuda_cudart + ]; + + enableParallelBuilding = true; + + postInstall = '' + mkdir -p $out/lib $out/include $out/lib/cmake/openvino + + for lib in $out/runtime/lib/intel64/*; do + ln -s "$lib" $out/lib/ || cp -a "$lib" $out/lib/ + done + + for hdr in $out/runtime/include/*; do + ln -s "$hdr" $out/include/ || cp -a "$hdr" $out/include/ + done + + for cmakeFile in $out/runtime/cmake/*; do + ln -s "$cmakeFile" $out/lib/cmake/openvino/ || cp -a "$cmakeFile" $out/lib/cmake/openvino/ + done + + mkdir -p $python + mv $out/python/* $python/ + rmdir $out/python + ''; + + postFixup = '' + # Link to OpenCL + find $out -type f \( -name '*.so' -or -name '*.so.*' \) | while read lib; do + addDriverRunpath "$lib" + done + ''; + + meta = with lib; { + changelog = "https://github.com/openvinotoolkit/openvino/releases/tag/${src.tag}"; + description = "OpenVINO™ Toolkit repository"; + longDescription = '' + This toolkit allows developers to deploy pre-trained deep learning models through a high-level C++ Inference Engine API integrated with application logic. + + This open source version includes several components: namely Model Optimizer, nGraph and Inference Engine, as well as CPU, GPU, MYRIAD, + multi device and heterogeneous plugins to accelerate deep learning inferencing on Intel® CPUs and Intel® Processor Graphics. + It supports pre-trained models from the Open Model Zoo, along with 100+ open source and public models in popular formats such as Caffe*, TensorFlow*, MXNet* and ONNX*. + ''; + homepage = "https://docs.openvinotoolkit.org/"; + license = with licenses; [ asl20 ]; + platforms = platforms.all; + broken = stdenv.hostPlatform.isDarwin; # Cannot find macos sdk + maintainers = with maintainers; [ tfmoraes ]; + }; +} From 94daaf847555dc446bebac175b7a2f8cf04d45e1 Mon Sep 17 00:00:00 2001 From: Izel Nakri Date: Mon, 18 Aug 2025 18:27:03 +0200 Subject: [PATCH 4/4] Improved whisper-cpp after openvino improvements --- src/default.nix | 1 + src/ggml/whisper-cpp.nix | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/default.nix b/src/default.nix index 8fbc673..db5146b 100644 --- a/src/default.nix +++ b/src/default.nix @@ -73,6 +73,7 @@ oneDNN oneMath oneTBB + openvino ; }; llama-cpp = callPackage ./ggml/llama-cpp.nix { diff --git a/src/ggml/whisper-cpp.nix b/src/ggml/whisper-cpp.nix index 639c04d..abf9032 100644 --- a/src/ggml/whisper-cpp.nix +++ b/src/ggml/whisper-cpp.nix @@ -1,4 +1,5 @@ -{ cmake +{ autoAddDriverRunpath +, cmake , fetchFromGitHub , git , lib @@ -78,6 +79,7 @@ llvm.stdenv.mkDerivation rec { makeWrapper git oneMathCmakeShim + autoAddDriverRunpath ]; buildInputs = [ @@ -122,12 +124,14 @@ llvm.stdenv.mkDerivation rec { "-DCMAKE_PREFIX_PATH=${oneMathCmakeShim}/lib/cmake" ]; + # NOTE: TODO: Uncommenting/removing this today causes runtime problems later, granted I didnt rebase master yet postInstall = '' for binary in $out/bin/*; do if [[ -f "$binary" && -x "$binary" ]]; then wrapProgram "$binary" \ - --prefix LD_LIBRARY_PATH : "/run/opengl-driver/lib:/run/opengl-driver-32/lib:${openvino}/runtime/lib/intel64" + --prefix LD_LIBRARY_PATH : "/run/opengl-driver/lib:/run/opengl-driver-32/lib" fi done ''; } +