diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..0bcd132 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: eSlider diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml new file mode 100644 index 0000000..aa65c31 --- /dev/null +++ b/.github/workflows/build-push.yml @@ -0,0 +1,88 @@ +name: Build and push Docker image + +on: + push: + branches: + - main + - master + - "release/**" + tags: + - "v*" + pull_request: + branches: + - main + - master + workflow_dispatch: + inputs: + ollama_version: + description: "Ollama version to build" + required: false + default: "0.15.6" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + OLLAMA_VERSION: "0.15.6" + +jobs: + build-and-push: + runs-on: ubuntu-latest + timeout-minutes: 60 + + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve Ollama version + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.ollama_version }}" ]; then + echo "ollama_version=${{ inputs.ollama_version }}" >> "$GITHUB_OUTPUT" + else + echo "ollama_version=${{ env.OLLAMA_VERSION }}" >> "$GITHUB_OUTPUT" + fi + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # Tag with ollama version on default branch + type=raw,value=ollama-${{ steps.version.outputs.ollama_version }},enable={{is_default_branch}} + # Tag "latest" on default branch + type=raw,value=latest,enable={{is_default_branch}} + # Tag with git tag (v1.0.0 -> 1.0.0) + type=semver,pattern={{version}} + # Tag with branch name for release branches + type=ref,event=branch,enable=${{ startsWith(github.ref, 'refs/heads/release/') }} + # Tag with short SHA always + type=sha,prefix= + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + OLLAMA_VERSION=${{ steps.version.outputs.ollama_version }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0c38d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# IDE +.idea/ + +# Swap files +*.swp +*.swo + +# Open WebUI local data +webui/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..44bed45 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,36 @@ +# Changelog + +## 2026-02-12 — Switch to SYCL backend + +### GPU backend: Vulkan -> SYCL + +- Replaced Vulkan GPU backend with custom-built SYCL backend for ~2x inference + speed on Intel GPUs +- Multi-stage Dockerfile: builds `libggml-sycl.so` from upstream llama.cpp + (commit `a5bb8ba4`) using Intel oneAPI 2025.1.1 +- Added `patch-sycl.py` to fix two ollama-specific API divergences: + - `graph_compute` signature (`int batch_size` parameter) + - `GGML_TENSOR_FLAG_COMPUTE` removal (critical — without this patch all + compute nodes are skipped, producing garbage output) +- Bundled oneAPI runtime libraries (SYCL, oneMKL, oneDNN, TBB, Level-Zero) + into the runtime image + +### Ollama upgrade: 0.9.3 -> 0.15.6 + +- Upgraded from IPEX-LLM bundled ollama 0.9.3 to official ollama v0.15.6 +- Switched from IPEX-LLM portable zip to official ollama binary +- Removed CUDA/MLX/Vulkan runners from image to reduce size + +### Intel GPU runtime stack + +- **level-zero**: v1.22.4 -> v1.28.0 +- **intel-graphics-compiler (IGC)**: v2.11.7 -> v2.28.4 +- **compute-runtime**: 25.18.33578.6 -> 26.05.37020.3 +- **libigdgmm**: 22.7.0 -> 22.9.0 + +### Docker Compose + +- Device mapping changed to full `/dev/dri` access for SYCL/Level-Zero +- Added `ONEAPI_DEVICE_SELECTOR=level_zero:0` and `ZES_ENABLE_SYSMAN=1` +- Removed `OLLAMA_VULKAN=1` +- Disabled web UI authentication (`WEBUI_AUTH=False`) diff --git a/Dockerfile b/Dockerfile index fb92a60..ecb0434 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,68 +1,145 @@ -FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND=noninteractive -ENV TZ=america/los_angeles +# ============================================================================= +# Stage 1: Build ggml-sycl backend from ollama's ggml source using Intel oneAPI +# ============================================================================= +FROM intel/oneapi-basekit:2025.1.1-0-devel-ubuntu24.04 AS sycl-builder + +ARG OLLAMA_VERSION=0.15.6 + +# Clone ollama source and the MATCHING ggml-sycl source from upstream llama.cpp. +# ollama v0.15.6 vendors ggml at commit a5bb8ba4 — we MUST use the same commit +# to ensure struct layouts, operation enums, and internal APIs match exactly. +# (ollama excludes ggml-sycl from its vendored ggml, but keeps the header) +ARG GGML_COMMIT=a5bb8ba4c50257437630c136210396810741bbf7 +RUN git clone --depth 1 --branch v${OLLAMA_VERSION} \ + https://github.com/ollama/ollama.git /ollama && \ + git init /tmp/llama.cpp && \ + cd /tmp/llama.cpp && \ + git remote add origin https://github.com/ggml-org/llama.cpp.git && \ + git sparse-checkout set ggml/src/ggml-sycl && \ + git fetch --depth 1 origin ${GGML_COMMIT} && \ + git checkout FETCH_HEAD && \ + cp -r /tmp/llama.cpp/ggml/src/ggml-sycl \ + /ollama/ml/backend/ggml/ggml/src/ggml-sycl && \ + rm -rf /tmp/llama.cpp + +WORKDIR /ollama + +# Patch ggml-sycl to match ollama's modified ggml backend API: +# 1. graph_compute has an extra int batch_size parameter in ollama +# 2. GGML_TENSOR_FLAG_COMPUTE doesn't exist in ollama's ggml +COPY patch-sycl.py /tmp/patch-sycl.py +RUN python3 /tmp/patch-sycl.py ml/backend/ggml/ggml/src/ggml-sycl/ggml-sycl.cpp + +# Build the SYCL backend as a dynamic library +# Note: oneAPI env is already set in the base image, no need to source setvars.sh +RUN cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL=ON \ + -DGGML_SYCL_TARGET=INTEL \ + -DOLLAMA_RUNNER_DIR=sycl && \ + cmake --build build --parallel $(nproc) --target ggml-sycl + +# Collect the SYCL runner and its oneAPI runtime dependencies into /sycl-runner +RUN mkdir -p /sycl-runner && \ + cp build/lib/ollama/libggml-sycl.so /sycl-runner/ && \ + # SYCL / DPC++ runtime + cp /opt/intel/oneapi/compiler/latest/lib/libsycl.so* /sycl-runner/ && \ + # Unified Runtime (oneAPI 2025+) — search multiple possible locations + find /opt/intel/oneapi -name 'libur_loader.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + find /opt/intel/oneapi -name 'libur_adapter_level_zero.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + find /opt/intel/oneapi -maxdepth 4 -name 'libumf.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + # oneDNN + cp /opt/intel/oneapi/dnnl/latest/lib/libdnnl.so* /sycl-runner/ 2>/dev/null; \ + # oneMKL + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_core.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_intel_ilp64.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_sycl_blas.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_tbb_thread.so* /sycl-runner/ && \ + # TBB + cp /opt/intel/oneapi/tbb/latest/lib/intel64/gcc*/libtbb.so* /sycl-runner/ && \ + # Intel compiler runtime + cp /opt/intel/oneapi/compiler/latest/lib/libsvml.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libimf.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libintlc.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libirng.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libiomp5.so /sycl-runner/ && \ + # Level-zero PI plugin (legacy, may not exist) + cp /opt/intel/oneapi/compiler/latest/lib/libpi_level_zero.so* /sycl-runner/ 2>/dev/null; \ + # SYCL SPIR-V fallback kernels (needed for bfloat16, complex math, etc.) + cp /opt/intel/oneapi/compiler/latest/lib/libsycl-fallback*.spv /sycl-runner/ && \ + # Strip debug symbols to reduce size + strip --strip-unneeded /sycl-runner/*.so* 2>/dev/null; true + +# ============================================================================= +# Stage 2: Runtime image +# ============================================================================= +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=America/Los_Angeles # Base packages -RUN apt update && \ - apt install --no-install-recommends -q -y \ - software-properties-common \ - ca-certificates \ - gnupg \ - wget \ - curl \ - python3 \ - python3-pip \ - ocl-icd-libopencl1 - -# Intel GPU compute user-space drivers -RUN mkdir -p /tmp/gpu && \ - cd /tmp/gpu && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.18.3/level-zero_1.18.3+u22.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb && \ - dpkg -i *.deb && \ - rm *.deb - -# Required compute runtime level-zero variables -ENV ZES_ENABLE_SYSMAN=1 +RUN apt-get update && \ + apt-get install --no-install-recommends -q -y \ + ca-certificates \ + wget \ + zstd \ + ocl-icd-libopencl1 \ + libhwloc15 && \ + rm -rf /var/lib/apt/lists/* -# oneAPI -RUN wget -qO - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | \ - gpg --dearmor --output /usr/share/keyrings/oneapi-archive-keyring.gpg && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | \ - tee /etc/apt/sources.list.d/oneAPI.list && \ - apt update && \ - apt install --no-install-recommends -q -y \ - intel-oneapi-common-vars=2024.0.0-49406 \ - intel-oneapi-common-oneapi-vars=2024.0.0-49406 \ - intel-oneapi-compiler-dpcpp-cpp=2024.0.2-49895 \ - intel-oneapi-dpcpp-ct=2024.0.0-49381 \ - intel-oneapi-mkl=2024.0.0-49656 \ - intel-oneapi-mpi=2021.11.0-49493 \ - intel-oneapi-dal=2024.0.1-25 \ - intel-oneapi-ippcp=2021.9.1-5 \ - intel-oneapi-ipp=2021.10.1-13 \ - intel-oneapi-tlt=2024.0.0-352 \ - intel-oneapi-ccl=2021.11.2-5 \ - intel-oneapi-dnnl=2024.0.0-49521 \ - intel-oneapi-tcm-1.0=1.0.0-435 - -# Required oneAPI environment variables -ENV USE_XETLA=OFF -ENV SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 -ENV SYCL_CACHE_PERSISTENT=1 - -COPY _init.sh /usr/share/lib/init_workspace.sh -COPY _run.sh /usr/share/lib/run_workspace.sh - -# Ollama via ipex-llm -RUN pip3 install --pre --upgrade ipex-llm[cpp] +# Intel GPU runtimes (release 26.05.37020.3) +# Provides level-zero, IGC, compute-runtime for Intel GPU kernel support +RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.28.0/level-zero_1.28.0+u24.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.28.4/intel-igc-core-2_2.28.4+20760_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.28.4/intel-igc-opencl-2_2.28.4+20760_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-ocloc-dbgsym_26.05.37020.3-0_amd64.ddeb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-ocloc_26.05.37020.3-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-opencl-icd_26.05.37020.3-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libigdgmm12_22.9.0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libze-intel-gpu1_26.05.37020.3-0_amd64.deb && \ + dpkg -i *.deb *.ddeb && rm -rf /tmp/gpu -ENV OLLAMA_NUM_GPU=999 +# Install official ollama binary + CPU runners (skip CUDA/MLX/Vulkan) +ARG OLLAMA_VERSION=0.15.6 +RUN wget -qO- "https://github.com/ollama/ollama/releases/download/v${OLLAMA_VERSION}/ollama-linux-amd64.tar.zst" | \ + zstd -d | tar -xf - -C /usr && \ + rm -rf /usr/lib/ollama/cuda_* /usr/lib/ollama/mlx_* /usr/lib/ollama/vulkan + +# Install SYCL runner from build stage +COPY --from=sycl-builder /sycl-runner/ /usr/lib/ollama/sycl/ + +# Clean up +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + apt-get autoremove -y --purge 2>/dev/null; \ + apt-get autoclean -y 2>/dev/null; true + +# Serve ollama on all interfaces ENV OLLAMA_HOST=0.0.0.0:11434 -ENTRYPOINT ["/bin/bash", "/usr/share/lib/run_workspace.sh"] +# Keep models loaded in memory +ENV OLLAMA_KEEP_ALIVE=24h +ENV OLLAMA_DEFAULT_KEEPALIVE=6h + +# Concurrency and resource limits +ENV OLLAMA_NUM_PARALLEL=1 +ENV OLLAMA_MAX_LOADED_MODELS=1 +ENV OLLAMA_MAX_QUEUE=512 +ENV OLLAMA_MAX_VRAM=0 + +# Use all GPU layers +ENV OLLAMA_NUM_GPU=999 + +# Intel GPU tuning +ENV ZES_ENABLE_SYSMAN=1 +ENV ONEAPI_DEVICE_SELECTOR=level_zero:0 + +# For Intel Core Ultra Processors (Series 1), code name Meteor Lake +ENV IPEX_LLM_NPU_MTL=1 +EXPOSE 11434 +ENTRYPOINT ["/usr/bin/ollama"] +CMD ["serve"] diff --git a/README.md b/README.md index ab4bd9d..52bbf67 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,161 @@ -# ollama-intel-gpu +# Ollama for Intel GPU (SYCL) -This repo illlustrates the use of Ollama with support for Intel ARC GPU based via SYCL. Run the recently released [Meta llama3.1](https://llama.meta.com/) or [Microsoft phi3](https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential) models on your local Intel ARC GPU based PC using Linux or Windows WSL2. +> Run LLMs on Intel GPUs at full speed — no NVIDIA required. -## Screenshot -![screenshot](doc/screenshot.png) +A Docker-based setup that pairs [Ollama](https://github.com/ollama/ollama) **v0.15.6** with a custom-built **SYCL backend** for Intel GPU acceleration, plus [Open WebUI](https://github.com/open-webui/open-webui) for a browser chat interface. Three commands to go from zero to local AI. -# Prerequisites -* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04), or Windows 11 with WSL2 (graphics driver [101.5445](https://www.intel.com/content/www/us/en/download/785597/intel-arc-iris-xe-graphics-windows.html) or newer) -* Installed Docker and Docker-compose tools (for Linux) or Docker Desktop (for Windows) -* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 125H integrated GPU) - -# Usage +**Why this exists:** Ollama's official release ships only a Vulkan backend for Intel GPUs, leaving significant performance on the table. This repo builds the `ggml-sycl` backend from source with Intel oneAPI, unlocking oneMKL, oneDNN, and Level-Zero direct GPU access. -The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui -Linux: -```bash -$ git clone https://github.com/mattcurf/ollama-intel-gpu -$ cd ollama-intel-gpu -$ docker compose up +--- + +## Quick start + +### Option A: Build from source + +```shell +git clone https://github.com/mattcurf/ollama-intel-gpu +cd ollama-intel-gpu +docker compose up ``` -Windows WSL2: -```bash -$ git clone https://github.com/mattcurf/ollama-intel-gpu -$ cd ollama-intel-gpu -$ docker-compose -f docker-compose-wsl2.yml up +The first `docker compose up` builds the SYCL backend from source (~2 min on a modern CPU). Subsequent starts are instant. + +### Option B: Use the pre-built image + +```shell +docker run -d \ + --device /dev/dri:/dev/dri \ + --shm-size 16G \ + -p 11434:11434 \ + -v ollama-data:/root/.ollama \ + ghcr.io/mattcurf/ollama-intel-gpu:latest ``` -Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM +Open **http://localhost:3000** (with WebUI) or use the API directly at `http://localhost:11434`. + +> **Multiple GPUs?** Set `ONEAPI_DEVICE_SELECTOR=level_zero:0` in `docker-compose.yml` to pick the right device. + +--- + +## Tested hardware + +| Intel GPU | Status | +|-----------|--------| +| Core Ultra 7 155H integrated Arc (Meteor Lake) | Verified | +| Arc A-series (A770, A750, A380) | Expected compatible | +| Data Center Flex / Max | Expected compatible | + +**Requirements:** Ubuntu 24.04+, Docker with Compose, Intel GPU with Level-Zero driver support. + +--- + +## SYCL vs Vulkan performance + +Both backends run on Intel GPUs. This repo defaults to SYCL for the speed advantage. + +| Intel GPU | Vulkan | SYCL | Gain | +|---|---|---|---| +| MTL iGPU (155H) | ~8-11 tok/s | **~16 tok/s** | +45-100% | +| ARL-H iGPU | ~10-12 tok/s | **~17 tok/s** | +40-70% | +| Arc A770 | ~30-35 tok/s | **~55 tok/s** | +57-83% | +| Flex 170 | ~30-35 tok/s | **~50 tok/s** | +43-67% | +| Data Center Max 1550 | — | **~73 tok/s** | — | + +*Benchmarks: llama-2-7b Q4_0, llama.cpp, community-reported.* + +**What makes SYCL faster:** + +- **oneMKL / oneDNN** — Intel's optimized math and neural network libraries +- **Level-Zero** — direct GPU communication, lower overhead than Vulkan +- **Intel-tuned kernels** — MUL_MAT hand-optimized per architecture (MTL, ARL, Arc, Flex, PVC) + +**When Vulkan makes sense:** no build step, cross-vendor support (AMD/NVIDIA), smaller image. Use the official Ollama Docker image with `OLLAMA_VULKAN=1`. + +--- + +## How it works + +Ollama ships the `ggml-sycl.h` header but intentionally excludes the SYCL implementation from its vendored ggml. This repo fills that gap: + +``` +┌─────────────────────────────────────────────────────────┐ +│ Stage 1: Build (intel/oneapi-basekit:2025.1.1) │ +│ │ +│ ollama v0.15.6 source ──┐ │ +│ ├── cmake + icpx ── libggml-sycl.so +│ ggml-sycl @ a5bb8ba4 ──┘ │ +│ ▲ │ +│ └── patch-sycl.py (2 API fixes) │ +├─────────────────────────────────────────────────────────┤ +│ Stage 2: Runtime (ubuntu:24.04) │ +│ │ +│ ollama binary (official v0.15.6) │ +│ + libggml-sycl.so + oneAPI runtime libs │ +│ + Intel GPU drivers (Level-Zero, IGC, compute-runtime) │ +│ + Open WebUI (separate container) │ +└─────────────────────────────────────────────────────────┘ +``` + +The `ggml-sycl` source is fetched from the **exact llama.cpp commit** (`a5bb8ba4`) that ollama vendors, ensuring ABI compatibility. Two small patches are applied by `patch-sycl.py`: + +1. **`graph_compute` signature** — ollama adds an `int batch_size` parameter not present upstream +2. **`GGML_TENSOR_FLAG_COMPUTE` removal** — ollama drops this enum; without the patch, every compute node gets skipped, producing garbage output + +--- + +## Configuration + +Environment variables in `docker-compose.yml`: + +| Variable | Default | Description | +|---|---|---| +| `OLLAMA_HOST` | `0.0.0.0` | Listen address | +| `OLLAMA_KEEP_ALIVE` | `24h` | How long models stay loaded in memory | +| `OLLAMA_NUM_PARALLEL` | `1` | Concurrent request slots | +| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max models in VRAM simultaneously | +| `ONEAPI_DEVICE_SELECTOR` | `level_zero:0` | Which Intel GPU to use | +| `ZES_ENABLE_SYSMAN` | `1` | Enable Level-Zero system management | +| `OLLAMA_DEBUG` | `1` | Verbose logging (disable in production) | + +--- + +## Project structure + +``` +. +├── Dockerfile # Multi-stage build: oneAPI SYCL → minimal runtime +├── docker-compose.yml # ollama + Open WebUI services +├── patch-sycl.py # Patches ggml-sycl for ollama API compatibility +├── start-ollama.sh # Custom entrypoint (legacy, from IPEX-LLM era) +└── doc/ + └── screenshot.png +``` + +--- + +## Troubleshooting + +**SYCL device not detected** — Ensure `/dev/dri` is accessible. Check `docker compose logs ollama-intel-gpu` for `SYCL0` in the device list. + +**"failed to sample token"** — Usually means an ABI mismatch between ggml-sycl and ollama's vendored ggml. The `GGML_COMMIT` ARG in the Dockerfile must match the ggml version ollama vendors. + +**Model too large for VRAM** — Intel integrated GPUs share system memory. Increase `shm_size` in `docker-compose.yml` or use a smaller quantization (Q4_0, Q4_K_M). + +**Slow first inference** — SYCL JIT-compiles GPU kernels on first run. Subsequent inferences are faster. + +--- + +## References + +- [Ollama](https://github.com/ollama/ollama) +- [Open WebUI](https://github.com/open-webui/open-webui) +- [llama.cpp SYCL backend](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md) +- [Intel oneAPI base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) +- [Intel GPU driver installation](https://dgpu-docs.intel.com/driver/client/overview.html) + +--- -# Known issues -* Little effort has been made to prune the packages pulled into the Ollama docker image for Intel GPU +## License -# References -* https://dgpu-docs.intel.com/driver/client/overview.html -* https://ipex-llm.readthedocs.io/en/latest/doc/LLM/Quickstart/ollama_quickstart.html +See [LICENSE](LICENSE) for details. diff --git a/_init.sh b/_init.sh deleted file mode 100644 index 7408d56..0000000 --- a/_init.sh +++ /dev/null @@ -1 +0,0 @@ -source /opt/intel/oneapi/setvars.sh diff --git a/_run.sh b/_run.sh deleted file mode 100644 index b766a5c..0000000 --- a/_run.sh +++ /dev/null @@ -1,2 +0,0 @@ -source /usr/share/lib/init_workspace.sh -/usr/local/lib/python3.10/dist-packages/bigdl/cpp/libs/ollama serve diff --git a/docker-compose-wsl2.yml b/docker-compose-wsl2.yml deleted file mode 100644 index f8fc108..0000000 --- a/docker-compose-wsl2.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: "3.9" -services: - ollama-intel-gpu: - build: - context: . - dockerfile: Dockerfile - container_name: ollama-intel-gpu - image: ollama-intel-gpu:latest - restart: always - devices: - - /dev/dri:/dev/dri - - /dev/dxg:/dev/dxg - volumes: - - /usr/lib/wsl:/usr/lib/wsl - - /tmp/.X11-unix:/tmp/.X11-unix - - ollama-intel-gpu:/root/.ollama - environment: - - DISPLAY=${DISPLAY} - ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.10 - container_name: ollama-webui - volumes: - - ollama-webui:/app/backend/data - depends_on: - - ollama-intel-gpu - ports: - - ${OLLAMA_WEBUI_PORT-3000}:8080 - environment: - - OLLAMA_BASE_URL=http://ollama-intel-gpu:11434 - extra_hosts: - - host.docker.internal:host-gateway - restart: unless-stopped -volumes: - ollama-webui: {} - ollama-intel-gpu: {} diff --git a/docker-compose.yml b/docker-compose.yml index ce01fa6..b72493c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,30 +1,59 @@ -version: "3.9" services: ollama-intel-gpu: build: context: . dockerfile: Dockerfile + args: + OLLAMA_VERSION: "0.15.6" container_name: ollama-intel-gpu - image: ollama-intel-gpu:latest - restart: always + restart: unless-stopped devices: - /dev/dri:/dev/dri volumes: - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama + shm_size: "16G" environment: - DISPLAY=${DISPLAY} + - OLLAMA_HOST=0.0.0.0 + - OLLAMA_DEBUG=1 + - ONEAPI_DEVICE_SELECTOR=level_zero:0 + - ZES_ENABLE_SYSMAN=1 + - OLLAMA_DEFAULT_KEEPALIVE=6h + - OLLAMA_KEEP_ALIVE=24h + - OLLAMA_MAX_LOADED_MODELS=1 + - OLLAMA_MAX_QUEUE=512 + - OLLAMA_MAX_VRAM=0 + - OLLAMA_NUM_PARALLEL=1 + #- OLLAMA_NOHISTORY=false + #- OLLAMA_NOPRUNE=false + ports: + - 11434:11434 + ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.10 + image: ghcr.io/open-webui/open-webui:latest container_name: ollama-webui volumes: - - ollama-webui:/app/backend/data + - ./webui/data:/app/backend/data + # - ollama-webui:/app/backend/data depends_on: - ollama-intel-gpu ports: - ${OLLAMA_WEBUI_PORT-3000}:8080 environment: - OLLAMA_BASE_URL=http://ollama-intel-gpu:11434 + - OLLAMA_DEFAULT_KEEPALIVE=6h + #- OPENAI_API_BASE_URL= + #- OPENAI_API_KEY= + # + # AUTOMATIC1111_BASE_URL="http://localhost:7860" + - WEBUI_AUTH=False + - ENABLE_RAG_WEB_SEARCH=True + + # DO NOT TRACK + - SCARF_NO_ANALYTICS=true + - DO_NOT_TRACK=true + - ANONYMIZED_TELEMETRY=false extra_hosts: - host.docker.internal:host-gateway restart: unless-stopped diff --git a/patch-sycl.py b/patch-sycl.py new file mode 100644 index 0000000..29b937d --- /dev/null +++ b/patch-sycl.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +Patch upstream ggml-sycl to match ollama's modified ggml backend API. + +ollama v0.15.6 vendors ggml from llama.cpp commit a5bb8ba4 but makes two +divergences from upstream: + +1. graph_compute() has an extra 'int batch_size' parameter (ollama addition) +2. GGML_TENSOR_FLAG_COMPUTE enum value is removed from ollama's ggml.h, + so the skip-check in the compute loop must be removed entirely +""" + +import re +import sys + +path = sys.argv[1] +with open(path, "r") as f: + src = f.read() + +original = src + +# 1. Fix graph_compute signature: add 'int batch_size' parameter +# The function is defined as: +# static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +src = re.sub( + r'(static\s+(?:enum\s+)?ggml_status\s+ggml_backend_sycl_graph_compute\s*\([^)]*cgraph)\s*\)', + r'\1, int batch_size)', + src, +) + +# 2. Add GGML_UNUSED(batch_size) inside the function body (after the opening brace) +src = re.sub( + r'(ggml_backend_sycl_graph_compute\([^)]*int\s+batch_size\)\s*\{)', + r'\1\n GGML_UNUSED(batch_size);', + src, +) + +# 3. Remove GGML_TENSOR_FLAG_COMPUTE skip-check entirely. +# In ollama's vendored ggml, this flag doesn't exist (removed from the enum). +# Since ollama never sets bit 16, ALL nodes would be skipped, producing garbage. +# The actual code looks like: +# if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { +# continue; +# } +src = re.sub( + r'\s*if\s*\(\(node->flags\s*&\s*GGML_TENSOR_FLAG_COMPUTE\)\s*==\s*0\)\s*\{\s*continue;\s*\}', + '', + src, +) + +if src == original: + print(f"WARNING: No changes made to {path}", file=sys.stderr) + sys.exit(1) + +with open(path, "w") as f: + f.write(src) + +# Verify patches applied +checks = [ + ("batch_size parameter", "int batch_size" in src), + ("GGML_UNUSED(batch_size)", "GGML_UNUSED(batch_size)" in src), + ("GGML_TENSOR_FLAG_COMPUTE removed", "GGML_TENSOR_FLAG_COMPUTE" not in src), +] +for name, ok in checks: + status = "OK" if ok else "FAILED" + print(f" [{status}] {name}") + +if all(ok for _, ok in checks): + print(f"Patched {path} successfully") +else: + print(f"ERROR: Some patches failed on {path}", file=sys.stderr) + sys.exit(1) diff --git a/start-ollama.sh b/start-ollama.sh new file mode 100644 index 0000000..e444abe --- /dev/null +++ b/start-ollama.sh @@ -0,0 +1,18 @@ +#!/bin/bash +export OLLAMA_NUM_GPU=999 +export no_proxy=localhost,127.0.0.1 +export ZES_ENABLE_SYSMAN=1 +# [optional] under most circumstances, the following environment variable may improve performance, but sometimes this may also cause performance degradation +export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + +# Use OLLAMA_HOST and OLLAMA_KEEP_ALIVE from environment (set via docker-compose), +# falling back to sensible defaults if not set. +export OLLAMA_HOST="${OLLAMA_HOST:-0.0.0.0:11434}" +export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-24h}" + +# [optional] if you want to run on single GPU, use below command to limit GPU may improve performance +# export ONEAPI_DEVICE_SELECTOR=level_zero:0 +# If you have more than one dGPUs, according to your configuration you can use configuration like below, it will use the first and second card. +# export ONEAPI_DEVICE_SELECTOR="level_zero:0;level_zero:1" + +./ollama serve