From 2e18d91cd7fa9d5179722020b5a9733b3c1c7980 Mon Sep 17 00:00:00 2001 From: Pepijn de Vos Date: Thu, 7 Nov 2024 21:11:57 +0100 Subject: [PATCH 01/36] Update webui --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index ce01fa6..84ca524 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: environment: - DISPLAY=${DISPLAY} ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.10 + image: ghcr.io/open-webui/open-webui:v0.3.35 container_name: ollama-webui volumes: - ollama-webui:/app/backend/data From 0e4cf4f1543e30d59447ac85d1e26dbbcef9314b Mon Sep 17 00:00:00 2001 From: Pepijn de Vos Date: Thu, 7 Nov 2024 21:13:36 +0100 Subject: [PATCH 02/36] Update docker-compose-wsl2.yml --- docker-compose-wsl2.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-wsl2.yml b/docker-compose-wsl2.yml index f8fc108..99f9228 100644 --- a/docker-compose-wsl2.yml +++ b/docker-compose-wsl2.yml @@ -17,7 +17,7 @@ services: environment: - DISPLAY=${DISPLAY} ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.10 + image: ghcr.io/open-webui/open-webui:v0.3.35 container_name: ollama-webui volumes: - ollama-webui:/app/backend/data From c74f6f221688ffbd1b8e4dc22482ed3e6ad3b621 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Thu, 14 Nov 2024 13:45:57 -0800 Subject: [PATCH 03/36] Update Dockerfile to use Intel public ipex container --- Dockerfile | 70 ++++++------------------------------------------------ README.md | 3 --- _init.sh | 1 - _run.sh | 2 -- 4 files changed, 7 insertions(+), 69 deletions(-) delete mode 100644 _init.sh delete mode 100644 _run.sh diff --git a/Dockerfile b/Dockerfile index fb92a60..250e787 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,68 +1,12 @@ -FROM ubuntu:22.04 -ENV DEBIAN_FRONTEND=noninteractive -ENV TZ=america/los_angeles +FROM intelanalytics/ipex-llm-inference-cpp-xpu:latest -# Base packages -RUN apt update && \ - apt install --no-install-recommends -q -y \ - software-properties-common \ - ca-certificates \ - gnupg \ - wget \ - curl \ - python3 \ - python3-pip \ - ocl-icd-libopencl1 - -# Intel GPU compute user-space drivers -RUN mkdir -p /tmp/gpu && \ - cd /tmp/gpu && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.18.3/level-zero_1.18.3+u22.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-core_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/igc-1.0.17791.9/intel-igc-opencl_1.0.17791.9_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-level-zero-gpu_1.6.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/intel-opencl-icd_24.39.31294.12_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.39.31294.12/libigdgmm12_22.5.2_amd64.deb && \ - dpkg -i *.deb && \ - rm *.deb - -# Required compute runtime level-zero variables ENV ZES_ENABLE_SYSMAN=1 - -# oneAPI -RUN wget -qO - https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | \ - gpg --dearmor --output /usr/share/keyrings/oneapi-archive-keyring.gpg && \ - echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | \ - tee /etc/apt/sources.list.d/oneAPI.list && \ - apt update && \ - apt install --no-install-recommends -q -y \ - intel-oneapi-common-vars=2024.0.0-49406 \ - intel-oneapi-common-oneapi-vars=2024.0.0-49406 \ - intel-oneapi-compiler-dpcpp-cpp=2024.0.2-49895 \ - intel-oneapi-dpcpp-ct=2024.0.0-49381 \ - intel-oneapi-mkl=2024.0.0-49656 \ - intel-oneapi-mpi=2021.11.0-49493 \ - intel-oneapi-dal=2024.0.1-25 \ - intel-oneapi-ippcp=2021.9.1-5 \ - intel-oneapi-ipp=2021.10.1-13 \ - intel-oneapi-tlt=2024.0.0-352 \ - intel-oneapi-ccl=2021.11.2-5 \ - intel-oneapi-dnnl=2024.0.0-49521 \ - intel-oneapi-tcm-1.0=1.0.0-435 - -# Required oneAPI environment variables -ENV USE_XETLA=OFF -ENV SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 -ENV SYCL_CACHE_PERSISTENT=1 - -COPY _init.sh /usr/share/lib/init_workspace.sh -COPY _run.sh /usr/share/lib/run_workspace.sh - -# Ollama via ipex-llm -RUN pip3 install --pre --upgrade ipex-llm[cpp] - -ENV OLLAMA_NUM_GPU=999 ENV OLLAMA_HOST=0.0.0.0:11434 -ENTRYPOINT ["/bin/bash", "/usr/share/lib/run_workspace.sh"] +RUN mkdir -p /llm/ollama; \ + cd /llm/ollama; \ + init-ollama; + +WORKDIR /llm/ollama +ENTRYPOINT ["./ollama", "serve"] diff --git a/README.md b/README.md index ab4bd9d..e5f9280 100644 --- a/README.md +++ b/README.md @@ -30,9 +30,6 @@ $ docker-compose -f docker-compose-wsl2.yml up Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM -# Known issues -* Little effort has been made to prune the packages pulled into the Ollama docker image for Intel GPU - # References * https://dgpu-docs.intel.com/driver/client/overview.html * https://ipex-llm.readthedocs.io/en/latest/doc/LLM/Quickstart/ollama_quickstart.html diff --git a/_init.sh b/_init.sh deleted file mode 100644 index 7408d56..0000000 --- a/_init.sh +++ /dev/null @@ -1 +0,0 @@ -source /opt/intel/oneapi/setvars.sh diff --git a/_run.sh b/_run.sh deleted file mode 100644 index b766a5c..0000000 --- a/_run.sh +++ /dev/null @@ -1,2 +0,0 @@ -source /usr/share/lib/init_workspace.sh -/usr/local/lib/python3.10/dist-packages/bigdl/cpp/libs/ollama serve From 6df0d8d3cc251e465ce8337144aad5d042da4eb4 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Thu, 16 Jan 2025 22:00:31 -0800 Subject: [PATCH 04/36] Update ipex-llm image from Intel to 2.2.0-SNAPSHOT --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 250e787..69a065d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM intelanalytics/ipex-llm-inference-cpp-xpu:latest +FROM intelanalytics/ipex-llm-inference-cpp-xpu:2.2.0-SNAPSHOT ENV ZES_ENABLE_SYSMAN=1 ENV OLLAMA_HOST=0.0.0.0:11434 From b74bab0b6a6b4d5f9fa0848aef28c7ec755e9967 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Thu, 23 Jan 2025 17:28:33 -0800 Subject: [PATCH 05/36] Update to latest open-webui releases --- README.md | 2 +- docker-compose-wsl2.yml | 2 +- docker-compose.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e5f9280..2c2525f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ollama-intel-gpu -This repo illlustrates the use of Ollama with support for Intel ARC GPU based via SYCL. Run the recently released [Meta llama3.1](https://llama.meta.com/) or [Microsoft phi3](https://news.microsoft.com/source/features/ai/the-phi-3-small-language-models-with-big-potential) models on your local Intel ARC GPU based PC using Linux or Windows WSL2. +This repo illlustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux or Windows WSL2. ## Screenshot ![screenshot](doc/screenshot.png) diff --git a/docker-compose-wsl2.yml b/docker-compose-wsl2.yml index 99f9228..7efe6c6 100644 --- a/docker-compose-wsl2.yml +++ b/docker-compose-wsl2.yml @@ -17,7 +17,7 @@ services: environment: - DISPLAY=${DISPLAY} ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.35 + image: ghcr.io/open-webui/open-webui container_name: ollama-webui volumes: - ollama-webui:/app/backend/data diff --git a/docker-compose.yml b/docker-compose.yml index 84ca524..f06a16d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: environment: - DISPLAY=${DISPLAY} ollama-webui: - image: ghcr.io/open-webui/open-webui:v0.3.35 + image: ghcr.io/open-webui/open-webui container_name: ollama-webui volumes: - ollama-webui:/app/backend/data From 8e69333f3feb2b072cc2e46b6a2d1d36bd637a7a Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Sun, 26 Jan 2025 15:14:29 -0800 Subject: [PATCH 06/36] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 2c2525f..ecc0fe7 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,12 @@ This repo illlustrates the use of Ollama with support for Intel ARC GPU based vi ![screenshot](doc/screenshot.png) # Prerequisites -* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04), or Windows 11 with WSL2 (graphics driver [101.5445](https://www.intel.com/content/www/us/en/download/785597/intel-arc-iris-xe-graphics-windows.html) or newer) +* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04), or Windows 11 with WSL2 (graphics driver [101.5445](https://www.intel.com/content/www/us/en/download/785597/intel-arc-iris-xe-graphics-windows.html) or newer). * Installed Docker and Docker-compose tools (for Linux) or Docker Desktop (for Windows) -* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 125H integrated GPU) - +* Intel ARC series GPU. Tested with Intel ARC A770 16GB, Intel(R) Core(TM) Ultra 5 125H integrated GPU (Meteor Lake), and Intel(R) Core(TM) Intel Ultra 7 258V integrated GPU (Lunar Lake) + +*Note:* This branch uses the upstream ipex container published by Intel. See the alternate branch [alternate_base_image](https://github.com/mattcurf/ollama-intel-gpu/tree/alternate_base_image) for an equivalent Dockerfile which builds everything from the published packages directly. + # Usage The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui From c230c4518ec1ddfda174c24651fac0614e5c1ab0 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Sun, 26 Jan 2025 15:17:30 -0800 Subject: [PATCH 07/36] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index ecc0fe7..b630e16 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,11 @@ $ cd ollama-intel-gpu $ docker-compose -f docker-compose-wsl2.yml up ``` +*Note:* you will see the following message. This is expected and harmless, as the docker image 'ollama-intel-gpu' is built locally. +``` +ollama-intel-gpu Warning pull access denied for ollama-intel-gpu, repository does not exist or may require 'docker login': denied: requested access to the resource is denied +``` + Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM # References From ddd565fc8d425b86e22bd3accaa7a9af40f62ae1 Mon Sep 17 00:00:00 2001 From: Ikko Eltociear Ashimine Date: Fri, 31 Jan 2025 02:09:53 +0900 Subject: [PATCH 08/36] docs: update README.md illlustrates -> illustrates --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b630e16..de0800a 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ollama-intel-gpu -This repo illlustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux or Windows WSL2. +This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux or Windows WSL2. ## Screenshot ![screenshot](doc/screenshot.png) From 765a8c09d7e5e130cfa14d60bdde247a71909ef1 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Tue, 11 Feb 2025 20:40:41 -0800 Subject: [PATCH 09/36] Update to latest ipex-llm dockerfile 20250211 --- Dockerfile | 10 ++++------ commands.sh | 5 +++++ docker-compose.yml | 5 +---- 3 files changed, 10 insertions(+), 10 deletions(-) create mode 100644 commands.sh diff --git a/Dockerfile b/Dockerfile index 69a065d..3253c79 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,10 @@ -FROM intelanalytics/ipex-llm-inference-cpp-xpu:2.2.0-SNAPSHOT - -ENV ZES_ENABLE_SYSMAN=1 -ENV OLLAMA_HOST=0.0.0.0:11434 +FROM intelanalytics/ipex-llm-inference-cpp-xpu:latest RUN mkdir -p /llm/ollama; \ cd /llm/ollama; \ init-ollama; - WORKDIR /llm/ollama -ENTRYPOINT ["./ollama", "serve"] +COPY commands.sh /llm/ollama/commands.sh +RUN ["chmod", "+x", "/llm/ollama/commands.sh"] +ENTRYPOINT ["/llm/ollama/commands.sh"] diff --git a/commands.sh b/commands.sh new file mode 100644 index 0000000..8d7df6d --- /dev/null +++ b/commands.sh @@ -0,0 +1,5 @@ +#!/bin/bash +source ipex-llm-init --gpu --device $DEVICE +export OLLAMA_HOST=0.0.0.0:11434 +cd /llm/ollama +./ollama serve diff --git a/docker-compose.yml b/docker-compose.yml index f06a16d..be050d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,16 @@ -version: "3.9" services: ollama-intel-gpu: build: context: . dockerfile: Dockerfile container_name: ollama-intel-gpu - image: ollama-intel-gpu:latest restart: always devices: - /dev/dri:/dev/dri volumes: - - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama environment: - - DISPLAY=${DISPLAY} + - DEVICE=Arc ollama-webui: image: ghcr.io/open-webui/open-webui container_name: ollama-webui From f08a310a2a282fb4f353e2a280af679c24c9410b Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Tue, 11 Feb 2025 21:01:05 -0800 Subject: [PATCH 10/36] Update README.md --- README.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index de0800a..cfac50b 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,6 @@ This repo illustrates the use of Ollama with support for Intel ARC GPU based via * Installed Docker and Docker-compose tools (for Linux) or Docker Desktop (for Windows) * Intel ARC series GPU. Tested with Intel ARC A770 16GB, Intel(R) Core(TM) Ultra 5 125H integrated GPU (Meteor Lake), and Intel(R) Core(TM) Intel Ultra 7 258V integrated GPU (Lunar Lake) -*Note:* This branch uses the upstream ipex container published by Intel. See the alternate branch [alternate_base_image](https://github.com/mattcurf/ollama-intel-gpu/tree/alternate_base_image) for an equivalent Dockerfile which builds everything from the published packages directly. - # Usage The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui @@ -30,13 +28,9 @@ $ cd ollama-intel-gpu $ docker-compose -f docker-compose-wsl2.yml up ``` -*Note:* you will see the following message. This is expected and harmless, as the docker image 'ollama-intel-gpu' is built locally. -``` -ollama-intel-gpu Warning pull access denied for ollama-intel-gpu, repository does not exist or may require 'docker login': denied: requested access to the resource is denied -``` +*Note:* ipex-llm now requires setting the GPU type in the docker-compose.yml file. Set variable DEVICE= to 'Arc' or 'iGPU'. See https://github.com/intel/ipex-llm/blob/main/docs/mddocs/DockerGuides/docker_cpp_xpu_quickstart.md for more information. Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM # References -* https://dgpu-docs.intel.com/driver/client/overview.html -* https://ipex-llm.readthedocs.io/en/latest/doc/LLM/Quickstart/ollama_quickstart.html +* [https://dgpu-docs.intel.com/driver/client/overview.html](https://github.com/intel/ipex-llm/blob/main/docs/mddocs/DockerGuides/README.md) From 2fc526511f673d19a73b1a4ac5bc50814c0a3831 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Wed, 19 Feb 2025 14:56:56 -0800 Subject: [PATCH 11/36] Update to use new ipex portable .zip packages --- Dockerfile | 38 ++++++++++++++++++++++++++++++-------- README.md | 26 +++++++------------------- docker-compose-wsl2.yml | 35 ----------------------------------- docker-compose.yml | 5 +---- 4 files changed, 38 insertions(+), 66 deletions(-) delete mode 100644 docker-compose-wsl2.yml diff --git a/Dockerfile b/Dockerfile index 69a065d..f98bfa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,12 +1,34 @@ -FROM intelanalytics/ipex-llm-inference-cpp-xpu:2.2.0-SNAPSHOT +FROM ubuntu:24.04 +ENV DEBIAN_FRONTEND=noninteractive +ENV TZ=america/los_angeles -ENV ZES_ENABLE_SYSMAN=1 -ENV OLLAMA_HOST=0.0.0.0:11434 +# Base packages +RUN apt update && \ + apt install --no-install-recommends -q -y \ + software-properties-common \ + ca-certificates \ + wget \ + ocl-icd-libopencl1 + +# gnupg \ -RUN mkdir -p /llm/ollama; \ - cd /llm/ollama; \ - init-ollama; +# Intel GPU compute user-space drivers +RUN mkdir -p /tmp/gpu && \ + cd /tmp/gpu && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.19.2/level-zero_1.19.2+u24.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.5.6/intel-igc-core-2_2.5.6+18417_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.5.6/intel-igc-opencl-2_2.5.6+18417_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/intel-level-zero-gpu_1.6.32224.5_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/intel-opencl-icd_24.52.32224.5_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/libigdgmm12_22.5.5_amd64.deb && \ + dpkg -i *.deb && \ + rm *.deb -WORKDIR /llm/ollama +# Install Ollama Portable Zip +RUN cd / && \ + wget https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/ollama-0.5.4-ipex-llm-2.2.0b20250218-ubuntu.tgz && \ + tar xvf ollama-0.5.4-ipex-llm-2.2.0b20250218-ubuntu.tgz --strip-components=1 -C / + +ENV OLLAMA_HOST=0.0.0.0:11434 -ENTRYPOINT ["./ollama", "serve"] +ENTRYPOINT ["/bin/bash", "/start-ollama.sh"] diff --git a/README.md b/README.md index de0800a..ab0a1d9 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,15 @@ # ollama-intel-gpu -This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux or Windows WSL2. +This repo illlustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux ## Screenshot ![screenshot](doc/screenshot.png) # Prerequisites -* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04), or Windows 11 with WSL2 (graphics driver [101.5445](https://www.intel.com/content/www/us/en/download/785597/intel-arc-iris-xe-graphics-windows.html) or newer). -* Installed Docker and Docker-compose tools (for Linux) or Docker Desktop (for Windows) -* Intel ARC series GPU. Tested with Intel ARC A770 16GB, Intel(R) Core(TM) Ultra 5 125H integrated GPU (Meteor Lake), and Intel(R) Core(TM) Intel Ultra 7 258V integrated GPU (Lunar Lake) - -*Note:* This branch uses the upstream ipex container published by Intel. See the alternate branch [alternate_base_image](https://github.com/mattcurf/ollama-intel-gpu/tree/alternate_base_image) for an equivalent Dockerfile which builds everything from the published packages directly. - +* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04.02 +* Installed Docker and Docker-compose tools +* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 125H integrated GPU) + # Usage The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui @@ -23,20 +21,10 @@ $ cd ollama-intel-gpu $ docker compose up ``` -Windows WSL2: -```bash -$ git clone https://github.com/mattcurf/ollama-intel-gpu -$ cd ollama-intel-gpu -$ docker-compose -f docker-compose-wsl2.yml up -``` - -*Note:* you will see the following message. This is expected and harmless, as the docker image 'ollama-intel-gpu' is built locally. -``` -ollama-intel-gpu Warning pull access denied for ollama-intel-gpu, repository does not exist or may require 'docker login': denied: requested access to the resource is denied -``` +*Note:* If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the correct device to use. Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM # References * https://dgpu-docs.intel.com/driver/client/overview.html -* https://ipex-llm.readthedocs.io/en/latest/doc/LLM/Quickstart/ollama_quickstart.html +* https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/ollama_portablze_zip_quickstart.md diff --git a/docker-compose-wsl2.yml b/docker-compose-wsl2.yml deleted file mode 100644 index 7efe6c6..0000000 --- a/docker-compose-wsl2.yml +++ /dev/null @@ -1,35 +0,0 @@ -version: "3.9" -services: - ollama-intel-gpu: - build: - context: . - dockerfile: Dockerfile - container_name: ollama-intel-gpu - image: ollama-intel-gpu:latest - restart: always - devices: - - /dev/dri:/dev/dri - - /dev/dxg:/dev/dxg - volumes: - - /usr/lib/wsl:/usr/lib/wsl - - /tmp/.X11-unix:/tmp/.X11-unix - - ollama-intel-gpu:/root/.ollama - environment: - - DISPLAY=${DISPLAY} - ollama-webui: - image: ghcr.io/open-webui/open-webui - container_name: ollama-webui - volumes: - - ollama-webui:/app/backend/data - depends_on: - - ollama-intel-gpu - ports: - - ${OLLAMA_WEBUI_PORT-3000}:8080 - environment: - - OLLAMA_BASE_URL=http://ollama-intel-gpu:11434 - extra_hosts: - - host.docker.internal:host-gateway - restart: unless-stopped -volumes: - ollama-webui: {} - ollama-intel-gpu: {} diff --git a/docker-compose.yml b/docker-compose.yml index f06a16d..570c93a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,19 +1,16 @@ -version: "3.9" services: ollama-intel-gpu: build: context: . dockerfile: Dockerfile container_name: ollama-intel-gpu - image: ollama-intel-gpu:latest restart: always devices: - /dev/dri:/dev/dri volumes: - - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama environment: - - DISPLAY=${DISPLAY} + - ONEAPI_DEVICE_SELECTOR=level_zero:0 ollama-webui: image: ghcr.io/open-webui/open-webui container_name: ollama-webui From dd84c202a7dc38d36ca2ec290b876c35c3bc77f9 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Wed, 19 Feb 2025 15:00:46 -0800 Subject: [PATCH 12/36] Minor fixes --- Dockerfile | 2 -- README.md | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index f98bfa8..cf5fa32 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,8 +10,6 @@ RUN apt update && \ wget \ ocl-icd-libopencl1 -# gnupg \ - # Intel GPU compute user-space drivers RUN mkdir -p /tmp/gpu && \ cd /tmp/gpu && \ diff --git a/README.md b/README.md index ab0a1d9..b95dfee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # ollama-intel-gpu -This repo illlustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux +This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux ## Screenshot ![screenshot](doc/screenshot.png) From fed3cf9ba0e9e2608cd10584d05a6f282c13f806 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Wed, 19 Feb 2025 15:07:23 -0800 Subject: [PATCH 13/36] Update README.md --- README.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b95dfee..322ee59 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux +## Important Note + +All Ollama based ipex-llm defects should be reported directly to the ipex-llm project at https://github.com/intel/ipex-llm + ## Screenshot ![screenshot](doc/screenshot.png) @@ -21,7 +25,7 @@ $ cd ollama-intel-gpu $ docker compose up ``` -*Note:* If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the correct device to use. +*Note:* If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the intended device to use. Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM From fa579db49227b28602ec993c460d5809d1f36271 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Wed, 19 Feb 2025 15:26:01 -0800 Subject: [PATCH 14/36] Increase context window size --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 570c93a..117bac2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,6 +11,7 @@ services: - ollama-intel-gpu:/root/.ollama environment: - ONEAPI_DEVICE_SELECTOR=level_zero:0 + - IPEX_LLM_NUM_CTX=16384 ollama-webui: image: ghcr.io/open-webui/open-webui container_name: ollama-webui From 85e28fca19eff7b2c99cedcdf3c897330d15c4e3 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Sat, 22 Feb 2025 20:27:34 -0800 Subject: [PATCH 15/36] Cache link --- Dockerfile | 4 ++-- commands.sh | 5 ----- 2 files changed, 2 insertions(+), 7 deletions(-) delete mode 100644 commands.sh diff --git a/Dockerfile b/Dockerfile index cf5fa32..61d28d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,8 +24,8 @@ RUN mkdir -p /tmp/gpu && \ # Install Ollama Portable Zip RUN cd / && \ - wget https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/ollama-0.5.4-ipex-llm-2.2.0b20250218-ubuntu.tgz && \ - tar xvf ollama-0.5.4-ipex-llm-2.2.0b20250218-ubuntu.tgz --strip-components=1 -C / + wget https://github.com/mattcurf/ollama-intel-gpu/releases/download/v0.0.1/ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz && \ + tar xvf ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz --strip-components=1 -C / ENV OLLAMA_HOST=0.0.0.0:11434 diff --git a/commands.sh b/commands.sh deleted file mode 100644 index 8d7df6d..0000000 --- a/commands.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash -source ipex-llm-init --gpu --device $DEVICE -export OLLAMA_HOST=0.0.0.0:11434 -cd /llm/ollama -./ollama serve From e1da4a4d163338c5b020f95659c438a2d39267c3 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 16 Mar 2025 15:37:06 +0800 Subject: [PATCH 16/36] Allow for user choice of ollama portable zip at build time --- Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 61d28d8..08be10e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,7 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive ENV TZ=america/los_angeles +ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz # Base packages RUN apt update && \ @@ -24,8 +25,8 @@ RUN mkdir -p /tmp/gpu && \ # Install Ollama Portable Zip RUN cd / && \ - wget https://github.com/mattcurf/ollama-intel-gpu/releases/download/v0.0.1/ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz && \ - tar xvf ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz --strip-components=1 -C / + wget https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/${IPEXLLM_PORTABLE_ZIP_FILENAME} && \ + tar xvf ${IPEXLLM_PORTABLE_ZIP_FILENAME} --strip-components=1 -C / ENV OLLAMA_HOST=0.0.0.0:11434 From 2c82aed59ce90120c17c69a743c8d60e23a205d4 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 16 Mar 2025 16:10:20 +0800 Subject: [PATCH 17/36] Update compose file with build args --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 117bac2..e53643c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,6 +3,8 @@ services: build: context: . dockerfile: Dockerfile + args: + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz # update from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly container_name: ollama-intel-gpu restart: always devices: From 1e92fbe88858b2bc00ec8229ec917ab0b259df82 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 16 Mar 2025 16:47:45 +0800 Subject: [PATCH 18/36] Updates to allow latest ollama in compose file, with fallback to cached in Dockerfile (if no build args provided) --- Dockerfile | 9 ++++++--- docker-compose.yml | 2 ++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 08be10e..fb4c73f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive ENV TZ=america/los_angeles -ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz + # Base packages RUN apt update && \ @@ -23,9 +23,12 @@ RUN mkdir -p /tmp/gpu && \ dpkg -i *.deb && \ rm *.deb -# Install Ollama Portable Zip +# Install Ollama Portable Zip (with cached default) +ARG IPEXLLM_RELEASE_REPO=mattcurf/ollama-intel-gpu +ARG IPEXLLM_RELEASE_VERSON=v0.0.1 +ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz RUN cd / && \ - wget https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/${IPEXLLM_PORTABLE_ZIP_FILENAME} && \ + wget https://github.com/${IPEXLLM_RELEASE_REPO}/releases/download/${IPEXLLM_RELEASE_VERSON}/${IPEXLLM_PORTABLE_ZIP_FILENAME} && \ tar xvf ${IPEXLLM_PORTABLE_ZIP_FILENAME} --strip-components=1 -C / ENV OLLAMA_HOST=0.0.0.0:11434 diff --git a/docker-compose.yml b/docker-compose.yml index e53643c..dc2b937 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,6 +4,8 @@ services: context: . dockerfile: Dockerfile args: + IPEXLLM_RELEASE_REPO: intel/ipex-llm + IPEXLLM_RELEASE_VERSON: v2.2.0-nightly IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz # update from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly container_name: ollama-intel-gpu restart: always From b33c01f1f0a1ad6d1d28ec67899c43b80036e734 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Sun, 16 Mar 2025 16:56:50 +0800 Subject: [PATCH 19/36] Updated README.md for Dockerfile args. --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 322ee59..5cdb798 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,10 @@ $ docker compose up Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM +## Update to the latest IPEX-LLM Portable Zip Version + +To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file's `IPEXLLM_PORTABLE_ZIP_FILENAME` build argument to the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly , then rebuild the image. + # References * https://dgpu-docs.intel.com/driver/client/overview.html * https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/ollama_portablze_zip_quickstart.md From 451f91080cbfa04f697fe887e18628e3866be5eb Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Mon, 17 Mar 2025 19:29:53 +0800 Subject: [PATCH 20/36] Revert compose to cached .tgz by default. --- README.md | 13 ++++++++++++- docker-compose.yml | 6 +++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5cdb798..5b0e464 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,18 @@ Then launch your web browser to http://localhost:3000 to launch the web ui. Cre ## Update to the latest IPEX-LLM Portable Zip Version -To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file's `IPEXLLM_PORTABLE_ZIP_FILENAME` build argument to the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly , then rebuild the image. +To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly , then rebuild the image. + +```yaml +ollama-intel-gpu: + build: + context: . + dockerfile: Dockerfile + args: + IPEXLLM_RELEASE_REPO: intel/ipex-llm + IPEXLLM_RELEASE_VERSON: v2.2.0-nightly + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz +``` # References * https://dgpu-docs.intel.com/driver/client/overview.html diff --git a/docker-compose.yml b/docker-compose.yml index dc2b937..0301a2c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,9 @@ services: context: . dockerfile: Dockerfile args: - IPEXLLM_RELEASE_REPO: intel/ipex-llm - IPEXLLM_RELEASE_VERSON: v2.2.0-nightly - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz # update from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly + IPEXLLM_RELEASE_REPO: mattcurf/ollama-intel-gpu + IPEXLLM_RELEASE_VERSON: v0.0.1 + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz container_name: ollama-intel-gpu restart: always devices: From 61288f5f6ca886137caa0dc799c0a719a04d8a67 Mon Sep 17 00:00:00 2001 From: Matt Curfman Date: Mon, 17 Mar 2025 10:44:00 -0700 Subject: [PATCH 21/36] Update to ipex-llm-2.2.0b20250313 --- README.md | 3 ++- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 5b0e464..6fe6942 100644 --- a/README.md +++ b/README.md @@ -46,4 +46,5 @@ ollama-intel-gpu: # References * https://dgpu-docs.intel.com/driver/client/overview.html -* https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/ollama_portablze_zip_quickstart.md +* https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/llamacpp_portable_zip_gpu_quickstart.md +* https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz diff --git a/docker-compose.yml b/docker-compose.yml index 0301a2c..6b0ad38 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,8 +5,8 @@ services: dockerfile: Dockerfile args: IPEXLLM_RELEASE_REPO: mattcurf/ollama-intel-gpu - IPEXLLM_RELEASE_VERSON: v0.0.1 - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz + IPEXLLM_RELEASE_VERSON: v0.0.2 + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tar.gz container_name: ollama-intel-gpu restart: always devices: From 504a1d388ff69378c2e24480fb9119a8f28edf32 Mon Sep 17 00:00:00 2001 From: Adam Gibson Date: Wed, 16 Apr 2025 21:25:38 +0800 Subject: [PATCH 22/36] Update default to ipex-llm v2.2.0 (guide for v2.3.0-nightly in docs) --- Dockerfile | 8 ++++---- README.md | 6 +++--- docker-compose.yml | 6 +++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb4c73f..36e9ee0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,10 +23,10 @@ RUN mkdir -p /tmp/gpu && \ dpkg -i *.deb && \ rm *.deb -# Install Ollama Portable Zip (with cached default) -ARG IPEXLLM_RELEASE_REPO=mattcurf/ollama-intel-gpu -ARG IPEXLLM_RELEASE_VERSON=v0.0.1 -ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-0.5.4-ipex-llm-2.2.0b20250220-ubuntu.tgz +# Install Ollama Portable Zip +ARG IPEXLLM_RELEASE_REPO=intel/ipex-llm +ARG IPEXLLM_RELEASE_VERSON=v2.2.0 +ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0-ubuntu.tgz RUN cd / && \ wget https://github.com/${IPEXLLM_RELEASE_REPO}/releases/download/${IPEXLLM_RELEASE_VERSON}/${IPEXLLM_PORTABLE_ZIP_FILENAME} && \ tar xvf ${IPEXLLM_PORTABLE_ZIP_FILENAME} --strip-components=1 -C / diff --git a/README.md b/README.md index 6fe6942..495aa15 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ Then launch your web browser to http://localhost:3000 to launch the web ui. Cre ## Update to the latest IPEX-LLM Portable Zip Version -To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.2.0-nightly , then rebuild the image. +To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.3.0-nightly , then rebuild the image. ```yaml ollama-intel-gpu: @@ -40,8 +40,8 @@ ollama-intel-gpu: dockerfile: Dockerfile args: IPEXLLM_RELEASE_REPO: intel/ipex-llm - IPEXLLM_RELEASE_VERSON: v2.2.0-nightly - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz + IPEXLLM_RELEASE_VERSON: v2.3.0-nightly + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.3.0b20250415-ubuntu.tgz ``` # References diff --git a/docker-compose.yml b/docker-compose.yml index 6b0ad38..d525736 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,9 @@ services: context: . dockerfile: Dockerfile args: - IPEXLLM_RELEASE_REPO: mattcurf/ollama-intel-gpu - IPEXLLM_RELEASE_VERSON: v0.0.2 - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0b20250313-ubuntu.tar.gz + IPEXLLM_RELEASE_REPO: intel/ipex-llm + IPEXLLM_RELEASE_VERSON: v2.2.0 + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0-ubuntu.tar.gz container_name: ollama-intel-gpu restart: always devices: From f1bbedb599ad2783d870b7564d0c41db6d2291ac Mon Sep 17 00:00:00 2001 From: Charles Ng Date: Fri, 18 Apr 2025 01:51:10 +0000 Subject: [PATCH 23/36] Update Intel libraries --- Dockerfile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index fb4c73f..9af7af3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,12 +14,12 @@ RUN apt update && \ # Intel GPU compute user-space drivers RUN mkdir -p /tmp/gpu && \ cd /tmp/gpu && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.19.2/level-zero_1.19.2+u24.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.5.6/intel-igc-core-2_2.5.6+18417_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.5.6/intel-igc-opencl-2_2.5.6+18417_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/intel-level-zero-gpu_1.6.32224.5_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/intel-opencl-icd_24.52.32224.5_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/24.52.32224.5/libigdgmm12_22.5.5_amd64.deb && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.21.9/level-zero_1.21.9+u24.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.8.3/intel-igc-core-2_2.8.3+18762_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.8.3/intel-igc-opencl-2_2.8.3+18762_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/intel-level-zero-gpu_1.6.32961.7_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/intel-opencl-icd_25.09.32961.7_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/libigdgmm12_22.6.0_amd64.deb && \ dpkg -i *.deb && \ rm *.deb From 17592946fabee3d59830f05c3dd2339cf3acc6c4 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Tue, 22 Apr 2025 17:56:04 +0100 Subject: [PATCH 24/36] Update Docker configurations for deployment improvements Revised `IPEXLLM_RELEASE_REPO` value and adjusted file and path references for consistency. Updated `docker-compose.yml` with refined environment variables, device mapping, restart policies, and added necessary port bindings for better functionality and maintainability. --- Dockerfile | 2 +- docker-compose.yml | 39 ++++++++++++++++++++++++++++++++++----- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index a95627e..0b0f2bd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ RUN mkdir -p /tmp/gpu && \ rm *.deb # Install Ollama Portable Zip -ARG IPEXLLM_RELEASE_REPO=intel/ipex-llm +ARG IPEXLLM_RELEASE_REPO=ipex-llm/ipex-llm ARG IPEXLLM_RELEASE_VERSON=v2.2.0 ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0-ubuntu.tgz RUN cd / && \ diff --git a/docker-compose.yml b/docker-compose.yml index d525736..bd1e216 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,20 +4,37 @@ services: context: . dockerfile: Dockerfile args: - IPEXLLM_RELEASE_REPO: intel/ipex-llm + IPEXLLM_RELEASE_REPO: ipex-llm/ipex-llm IPEXLLM_RELEASE_VERSON: v2.2.0 - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0-ubuntu.tar.gz + IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0-ubuntu.tgz container_name: ollama-intel-gpu - restart: always + restart: unless-stopped devices: - - /dev/dri:/dev/dri + # - /dev/dri:/dev/dri + - /dev/dri/renderD128:/dev/dri/renderD128 volumes: + - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama environment: - ONEAPI_DEVICE_SELECTOR=level_zero:0 - IPEX_LLM_NUM_CTX=16384 + - LD_LIBRARY_PATH:/opt/intel/oneapi/compiler/2024.2/lib + - DISPLAY=${DISPLAY} + - OLLAMA_DEFAULT_KEEPALIVE="6h" + - OLLAMA_HOST=0.0.0.0 + - OLLAMA_KEEP_ALIVE=24h + - OLLAMA_MAX_LOADED_MODELS=1 + - OLLAMA_MAX_QUEUE=512 + - OLLAMA_MAX_VRAM=0 + #- OLLAMA_NOHISTORY=false + #- OLLAMA_NOPRUNE=false + - OLLAMA_NUM_PARALLEL=1 + + ports: + - 11435:11434 + ollama-webui: - image: ghcr.io/open-webui/open-webui + image: ghcr.io/open-webui/open-webui:latest container_name: ollama-webui volumes: - ollama-webui:/app/backend/data @@ -27,6 +44,18 @@ services: - ${OLLAMA_WEBUI_PORT-3000}:8080 environment: - OLLAMA_BASE_URL=http://ollama-intel-gpu:11434 + - OLLAMA_DEFAULT_KEEPALIVE="6h" + #- OPENAI_API_BASE_URL= + #- OPENAI_API_KEY= + # + # AUTOMATIC1111_BASE_URL="http://localhost:7860" + + - ENABLE_RAG_WEB_SEARCH=True + + # DO NOT TRACK + - SCARF_NO_ANALYTICS=true + - DO_NOT_TRACK=true + - ANONYMIZED_TELEMETRY=false extra_hosts: - host.docker.internal:host-gateway restart: unless-stopped From c98fd71fa206862351a5686ce02bb849f3368979 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Tue, 22 Apr 2025 21:15:30 +0100 Subject: [PATCH 25/36] Ignore shelf --- .idea/.gitignore | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..13566b8 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Editor-based HTTP Client requests +/httpRequests/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml From 0a7f974c044134fbe143626ffb456e7c270a976a Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Sat, 21 Jun 2025 00:35:20 +0100 Subject: [PATCH 26/36] Update Docker configurations and Intel GPU runtimes for improved performance --- Dockerfile | 120 ++++++++++++++++++++++++++++++++++----------- docker-compose.yml | 6 ++- 2 files changed, 96 insertions(+), 30 deletions(-) diff --git a/Dockerfile b/Dockerfile index 0b0f2bd..90f773e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,100 @@ FROM ubuntu:24.04 -ENV DEBIAN_FRONTEND=noninteractive -ENV TZ=america/los_angeles - +ENV DEBIAN_FRONTEND=noninteractive \ + TZ=America/Los_Angeles # Base packages -RUN apt update && \ - apt install --no-install-recommends -q -y \ - software-properties-common \ - ca-certificates \ - wget \ - ocl-icd-libopencl1 - -# Intel GPU compute user-space drivers -RUN mkdir -p /tmp/gpu && \ - cd /tmp/gpu && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.21.9/level-zero_1.21.9+u24.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.8.3/intel-igc-core-2_2.8.3+18762_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.8.3/intel-igc-opencl-2_2.8.3+18762_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/intel-level-zero-gpu_1.6.32961.7_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/intel-opencl-icd_25.09.32961.7_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.09.32961.7/libigdgmm12_22.6.0_amd64.deb && \ - dpkg -i *.deb && \ - rm *.deb - -# Install Ollama Portable Zip -ARG IPEXLLM_RELEASE_REPO=ipex-llm/ipex-llm -ARG IPEXLLM_RELEASE_VERSON=v2.2.0 -ARG IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0-ubuntu.tgz +RUN apt-get update && \ + apt-get install --no-install-recommends -q -y \ + software-properties-common \ + ca-certificates \ + wget \ + ocl-icd-libopencl1 && \ + rm -rf /var/lib/apt/lists/* + +# Intel GPU runtimes (release 25.18.33578.6) +RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.22.4/level-zero_1.22.4+u24.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.11.7/intel-igc-core-2_2.11.7+19146_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.11.7/intel-igc-opencl-2_2.11.7+19146_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-ocloc-dbgsym_25.18.33578.6-0_amd64.ddeb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-ocloc_25.18.33578.6-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-opencl-icd_25.18.33578.6-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/libigdgmm12_22.7.0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/libze-intel-gpu1_25.18.33578.6-0_amd64.deb && \ + dpkg -i *.deb *.ddeb && rm -rf /tmp/gpu + +# Install IPEX-LLM Portable Zip (ollama bundle v2.3.0-nightly) RUN cd / && \ - wget https://github.com/${IPEXLLM_RELEASE_REPO}/releases/download/${IPEXLLM_RELEASE_VERSON}/${IPEXLLM_PORTABLE_ZIP_FILENAME} && \ - tar xvf ${IPEXLLM_PORTABLE_ZIP_FILENAME} --strip-components=1 -C / + wget https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz && \ + tar xvf ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz --strip-components=1 -C / && \ + rm ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz + +# Clean up any temporary files +RUN apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ + && find /var/log -type f -exec rm -f {} \; \ + && rm -rf /var/log/*-old \ + && rm -rf /var/log/apt/* \ + && rm -rf /var/log/dpkg.log* \ + && rm -rf /var/log/alternatives.log \ + && rm -rf /var/log/installer/* \ + && rm -rf /var/log/unattended-upgrades/* + +# Best practices +# Save model for faster loading +ENV OLLAMA_DEFAULT_KEEPALIVE=6h + +# Keep models loaded in memory +ENV OLLAMA_KEEP_ALIVE=24h + +# Load models in parallel +ENV OLLAMA_NUM_PARALLEL=1 +ENV OLLAMA_MAX_LOADED_MODELS=1 + +# Set bigger queue and VRAM for better performance +ENV OLLAMA_MAX_QUEUE=512 +ENV OLLAMA_MAX_VRAM=0 + +# Serve ollama on all interfaces ENV OLLAMA_HOST=0.0.0.0:11434 +# Set ollama to use the Intel GPU +ENV OLLAMA_NUM_GPU=999 + + +## # Available low_bit format including sym_int4, sym_int8, fp16 etc. +ENV USE_XETLA=OFF +ENV ZES_ENABLE_SYSMAN=1 + +# Set ollama to use the Intel GPU +# Set ollama to use the Intel GPU with IPEX-LLM +ENV OLLAMA_USE_IPEX=1 +# Set ollama to use the Intel GPU with IPEX-LLM and SYCL +ENV OLLAMA_USE_IPEX_SYCL=1 +# Set ollama to use the Intel GPU with IPEX-LLM and SYCL and Level Zero +ENV OLLAMA_USE_IPEX_SYCL_ZE=1 +# Set ollama to use the Intel GPU with IPEX-LLM and SYCL and Level Zero and XETLA +ENV OLLAMA_USE_IPEX_SYCL_ZE_XETLA=1 + +# # Available low_bit format including sym_int4, sym_int8, fp16 etc. +ENV USE_XETLA=OFF +ENV ZES_ENABLE_SYSMAN=1 + +# Add some intel specific adjustments +# https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/fastchat_quickstart.md + +ENV SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 +ENV ENABLE_SDP_FUSION=1 + +# [optional] under most circumstances, the following environment variable may improve performance, +# but sometimes this may also cause performance degradation +ENV SYCL_CACHE_PERSISTENT=1 + +# For Intel Core™ Ultra Processors (Series 2) with processor number 2xxK or 2xxH (code name Arrow Lake): +#- IPEX_LLM_NPU_ARL=1 + +# For Intel Core™ Ultra Processors (Series 1) with processor number 1xxH (code name Meteor Lake): +ENV IPEX_LLM_NPU_MTL=1 + ENTRYPOINT ["/bin/bash", "/start-ollama.sh"] diff --git a/docker-compose.yml b/docker-compose.yml index bd1e216..94a7f23 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -29,9 +29,11 @@ services: #- OLLAMA_NOHISTORY=false #- OLLAMA_NOPRUNE=false - OLLAMA_NUM_PARALLEL=1 - + #- IPEXLLM_RELEASE_REPO=ipex-llm/ipex-llm + #- IPEXLLM_RELEASE_VERSON=v2.2.0 + #- IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0-ubuntu.tgz ports: - - 11435:11434 + - 11434:11434 ollama-webui: image: ghcr.io/open-webui/open-webui:latest From 1239010eec1b823ffc464c6e235314a440e2c7f2 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Sat, 21 Jun 2025 00:37:38 +0100 Subject: [PATCH 27/36] Clean up Dockerfile by adding autoremove and autoclean commands to reduce image size --- Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 90f773e..a1bbe86 100644 --- a/Dockerfile +++ b/Dockerfile @@ -38,7 +38,10 @@ RUN apt-get clean && \ && rm -rf /var/log/dpkg.log* \ && rm -rf /var/log/alternatives.log \ && rm -rf /var/log/installer/* \ - && rm -rf /var/log/unattended-upgrades/* + && rm -rf /var/log/unattended-upgrades/* \ + && apt autoremove -y --purge \ + && apt-get autoclean -y \ + && rm -rf /tmp/* /var/tmp/* # Best practices From 96913a2a18842af5846a3a965a16c9f2ae78eb8e Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 15:00:53 +0000 Subject: [PATCH 28/36] Update Intel GPU stack and ipex-llm to latest available versions - level-zero v1.22.4 -> v1.28.0 - IGC v2.11.7 -> v2.28.4 - compute-runtime 25.18.33578.6 -> 26.05.37020.3 - libigdgmm 22.7.0 -> 22.9.0 - ipex-llm ollama nightly 2.3.0b20250612 -> 2.3.0b20250725 - Docker compose: disable webui auth, stateless webui volume - README formatting and GPU model update Co-authored-by: Cursor --- CHANGELOG.md | 27 +++++++++++++++++++++++++++ Dockerfile | 24 ++++++++++++------------ README.md | 44 ++++++++++++++++++++------------------------ docker-compose.yml | 6 +++--- 4 files changed, 62 insertions(+), 39 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..602f3e7 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,27 @@ +# Changelog + +## 2026-02-12 + +### Updated Intel GPU runtime stack to latest releases + +- **level-zero**: v1.22.4 -> v1.28.0 + - Loader based on oneAPI Level Zero Specification v1.15.31 + - Memory leak fixes, expanded multidriver teardown support +- **intel-graphics-compiler (IGC)**: v2.11.7 (build 19146) -> v2.28.4 (build 20760) + - Built with LLVM 16.0.6, opaque pointers support +- **compute-runtime**: 25.18.33578.6 -> 26.05.37020.3 + - Built with IGC v2.28.4 and level-zero v1.27.0 + - Panther Lake production support, Wildcat Lake pre-release +- **libigdgmm**: 22.7.0 -> 22.9.0 +- **ipex-llm ollama** (nightly): 2.3.0b20250612 -> 2.3.0b20250725 + - Latest available nightly Ubuntu ollama portable zip + +### Docker Compose adjustments + +- Disabled persistent webui volume for stateless restarts +- Disabled web UI authentication (`WEBUI_AUTH=False`) + +### README + +- Formatting and heading structure improvements +- Updated tested GPU model to Intel Core Ultra 5 155H diff --git a/Dockerfile b/Dockerfile index a1bbe86..3ace191 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,23 +11,23 @@ RUN apt-get update && \ ocl-icd-libopencl1 && \ rm -rf /var/lib/apt/lists/* -# Intel GPU runtimes (release 25.18.33578.6) +# Intel GPU runtimes (release 26.05.37020.3) RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ - wget https://github.com/oneapi-src/level-zero/releases/download/v1.22.4/level-zero_1.22.4+u24.04_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.11.7/intel-igc-core-2_2.11.7+19146_amd64.deb && \ - wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.11.7/intel-igc-opencl-2_2.11.7+19146_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-ocloc-dbgsym_25.18.33578.6-0_amd64.ddeb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-ocloc_25.18.33578.6-0_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/intel-opencl-icd_25.18.33578.6-0_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/libigdgmm12_22.7.0_amd64.deb && \ - wget https://github.com/intel/compute-runtime/releases/download/25.18.33578.6/libze-intel-gpu1_25.18.33578.6-0_amd64.deb && \ + wget https://github.com/oneapi-src/level-zero/releases/download/v1.28.0/level-zero_1.28.0+u24.04_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.28.4/intel-igc-core-2_2.28.4+20760_amd64.deb && \ + wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.28.4/intel-igc-opencl-2_2.28.4+20760_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-ocloc-dbgsym_26.05.37020.3-0_amd64.ddeb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-ocloc_26.05.37020.3-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/intel-opencl-icd_26.05.37020.3-0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libigdgmm12_22.9.0_amd64.deb && \ + wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libze-intel-gpu1_26.05.37020.3-0_amd64.deb && \ dpkg -i *.deb *.ddeb && rm -rf /tmp/gpu # Install IPEX-LLM Portable Zip (ollama bundle v2.3.0-nightly) RUN cd / && \ - wget https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz && \ - tar xvf ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz --strip-components=1 -C / && \ - rm ollama-ipex-llm-2.3.0b20250612-ubuntu.tgz + wget https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz && \ + tar xvf ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz --strip-components=1 -C / && \ + rm ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz # Clean up any temporary files RUN apt-get clean && \ diff --git a/README.md b/README.md index 495aa15..fa0c9fd 100644 --- a/README.md +++ b/README.md @@ -1,50 +1,46 @@ -# ollama-intel-gpu +# Ollama for Intel GPU + +[![GitHub license](https://img.shields.io/github/license/mattcurf/ollama-intel-gpu)]( This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux -## Important Note -All Ollama based ipex-llm defects should be reported directly to the ipex-llm project at https://github.com/intel/ipex-llm +> !Note: All Ollama based ipex-llm defects should be reported directly to the ipex-llm project at https://github.com/intel/ipex-llm ## Screenshot + ![screenshot](doc/screenshot.png) -# Prerequisites +## Prerequisites + * Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04.02 * Installed Docker and Docker-compose tools -* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 125H integrated GPU) +* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 155H integrated GPU) # Usage The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui -Linux: -```bash -$ git clone https://github.com/mattcurf/ollama-intel-gpu -$ cd ollama-intel-gpu -$ docker compose up +## Linux + +```shell +git clone https://github.com/mattcurf/ollama-intel-gpu +cd ollama-intel-gpu +docker compose up ``` -*Note:* If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the intended device to use. +> !NOTE +> If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the intended device to use. Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM -## Update to the latest IPEX-LLM Portable Zip Version +### Update to the latest IPEX-LLM Portable Zip Version To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.3.0-nightly , then rebuild the image. -```yaml -ollama-intel-gpu: - build: - context: . - dockerfile: Dockerfile - args: - IPEXLLM_RELEASE_REPO: intel/ipex-llm - IPEXLLM_RELEASE_VERSON: v2.3.0-nightly - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.3.0b20250415-ubuntu.tgz -``` - -# References +## References + + * https://dgpu-docs.intel.com/driver/client/overview.html * https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/llamacpp_portable_zip_gpu_quickstart.md * https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz diff --git a/docker-compose.yml b/docker-compose.yml index 94a7f23..7458265 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,8 +38,8 @@ services: ollama-webui: image: ghcr.io/open-webui/open-webui:latest container_name: ollama-webui - volumes: - - ollama-webui:/app/backend/data + #volumes: + # - ollama-webui:/app/backend/data depends_on: - ollama-intel-gpu ports: @@ -51,7 +51,7 @@ services: #- OPENAI_API_KEY= # # AUTOMATIC1111_BASE_URL="http://localhost:7860" - + - WEBUI_AUTH=False - ENABLE_RAG_WEB_SEARCH=True # DO NOT TRACK From 8debf2010b52207f7b1325737f7127f3426945d7 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 15:18:37 +0000 Subject: [PATCH 29/36] Fix ollama not reachable from host due to hardcoded OLLAMA_HOST in entrypoint The IPEX-LLM bundled start-ollama.sh hardcodes OLLAMA_HOST=127.0.0.1 and OLLAMA_KEEP_ALIVE=10m, overriding docker-compose environment variables and preventing external connections through Docker port mapping. - Add custom start-ollama.sh that honours env vars with sensible defaults - Mount it read-only into the container - Fix LD_LIBRARY_PATH env var syntax (: -> =) - Add .gitignore for IDE/swap/webui data files - Update CHANGELOG and README with fix documentation Co-authored-by: Cursor --- .gitignore | 9 +++++++++ CHANGELOG.md | 13 +++++++++++++ README.md | 11 +++++++++++ docker-compose.yml | 9 +++++++-- start-ollama.sh | 18 ++++++++++++++++++ 5 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 start-ollama.sh diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0c38d4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +# IDE +.idea/ + +# Swap files +*.swp +*.swo + +# Open WebUI local data +webui/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 602f3e7..c7716d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ ## 2026-02-12 +### Fix: Ollama not reachable from host via Docker port mapping + +The bundled IPEX-LLM `/start-ollama.sh` entrypoint hardcodes +`OLLAMA_HOST='127.0.0.1:11434'` and `OLLAMA_KEEP_ALIVE=10m`, overriding any +values set through Docker Compose environment variables. + +- Added a custom `start-ollama.sh` that respects environment variables + (`${OLLAMA_HOST:-0.0.0.0:11434}`, `${OLLAMA_KEEP_ALIVE:-24h}`) instead of + hardcoding them +- Mounted the script into the container as a read-only volume + (`./start-ollama.sh:/start-ollama.sh:ro`) +- Fixed `LD_LIBRARY_PATH` env var syntax in docker-compose.yml (`:` -> `=`) + ### Updated Intel GPU runtime stack to latest releases - **level-zero**: v1.22.4 -> v1.28.0 diff --git a/README.md b/README.md index fa0c9fd..06c738c 100644 --- a/README.md +++ b/README.md @@ -34,6 +34,17 @@ docker compose up Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM +### Custom `start-ollama.sh` entrypoint + +The upstream IPEX-LLM portable zip ships a `start-ollama.sh` that hardcodes +`OLLAMA_HOST=127.0.0.1` and `OLLAMA_KEEP_ALIVE=10m`, preventing the container +from accepting connections via Docker port mapping and ignoring Compose +environment overrides. + +This repo includes a corrected `start-ollama.sh` (mounted read-only into the +container) that honours environment variables set in `docker-compose.yml`, +falling back to sensible defaults (`0.0.0.0:11434`, `24h`). + ### Update to the latest IPEX-LLM Portable Zip Version To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.3.0-nightly , then rebuild the image. diff --git a/docker-compose.yml b/docker-compose.yml index 7458265..02e1b40 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,10 +15,14 @@ services: volumes: - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama + - ./start-ollama.sh:/start-ollama.sh:ro + shm_size: "16G" environment: - ONEAPI_DEVICE_SELECTOR=level_zero:0 + #- SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + #- SYCL_CACHE_PERSISTENT=1 - IPEX_LLM_NUM_CTX=16384 - - LD_LIBRARY_PATH:/opt/intel/oneapi/compiler/2024.2/lib + - LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/2024.2/lib - DISPLAY=${DISPLAY} - OLLAMA_DEFAULT_KEEPALIVE="6h" - OLLAMA_HOST=0.0.0.0 @@ -38,7 +42,8 @@ services: ollama-webui: image: ghcr.io/open-webui/open-webui:latest container_name: ollama-webui - #volumes: + volumes: + - ./webui/data:/app/backend/data # - ollama-webui:/app/backend/data depends_on: - ollama-intel-gpu diff --git a/start-ollama.sh b/start-ollama.sh new file mode 100644 index 0000000..e444abe --- /dev/null +++ b/start-ollama.sh @@ -0,0 +1,18 @@ +#!/bin/bash +export OLLAMA_NUM_GPU=999 +export no_proxy=localhost,127.0.0.1 +export ZES_ENABLE_SYSMAN=1 +# [optional] under most circumstances, the following environment variable may improve performance, but sometimes this may also cause performance degradation +export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 + +# Use OLLAMA_HOST and OLLAMA_KEEP_ALIVE from environment (set via docker-compose), +# falling back to sensible defaults if not set. +export OLLAMA_HOST="${OLLAMA_HOST:-0.0.0.0:11434}" +export OLLAMA_KEEP_ALIVE="${OLLAMA_KEEP_ALIVE:-24h}" + +# [optional] if you want to run on single GPU, use below command to limit GPU may improve performance +# export ONEAPI_DEVICE_SELECTOR=level_zero:0 +# If you have more than one dGPUs, according to your configuration you can use configuration like below, it will use the first and second card. +# export ONEAPI_DEVICE_SELECTOR="level_zero:0;level_zero:1" + +./ollama serve From 63c3b8129209b0173e3606e74ece9d4d58c51fec Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 15:34:03 +0000 Subject: [PATCH 30/36] Upgrade ollama from 0.9.3 (IPEX-LLM) to 0.15.6 (official) with Vulkan Intel GPU Replace the IPEX-LLM portable zip (bundling a patched ollama 0.9.3 with SYCL) with the official ollama 0.15.6 release using the Vulkan backend for Intel GPU acceleration. The official ollama project does not ship a SYCL backend; Vulkan is their supported path for Intel GPUs. - Use official ollama binary with Vulkan runner (OLLAMA_VULKAN=1) - Strip CUDA/MLX runners from image to save space - Add mesa-vulkan-drivers for Intel ANV Vulkan ICD - Remove all IPEX-LLM env vars and wrapper scripts - Simplify entrypoint to /usr/bin/ollama serve directly - Clean up docker-compose.yml: remove IPEX build args and env vars Tested: Intel Arc Graphics (MTL) detected, 17/17 layers offloaded to Vulkan0 Co-authored-by: Cursor --- Dockerfile | 87 ++++++++++++++-------------------------------- docker-compose.yml | 20 +++-------- 2 files changed, 31 insertions(+), 76 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3ace191..7218a35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,16 +2,18 @@ FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive \ TZ=America/Los_Angeles -# Base packages +# Base packages + Intel Vulkan ICD (ANV driver) RUN apt-get update && \ apt-get install --no-install-recommends -q -y \ - software-properties-common \ ca-certificates \ wget \ + zstd \ + mesa-vulkan-drivers \ ocl-icd-libopencl1 && \ rm -rf /var/lib/apt/lists/* # Intel GPU runtimes (release 26.05.37020.3) +# Provides level-zero, IGC, compute-runtime for Intel GPU kernel support RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ wget https://github.com/oneapi-src/level-zero/releases/download/v1.28.0/level-zero_1.28.0+u24.04_amd64.deb && \ wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.28.4/intel-igc-core-2_2.28.4+20760_amd64.deb && \ @@ -23,81 +25,44 @@ RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libze-intel-gpu1_26.05.37020.3-0_amd64.deb && \ dpkg -i *.deb *.ddeb && rm -rf /tmp/gpu -# Install IPEX-LLM Portable Zip (ollama bundle v2.3.0-nightly) -RUN cd / && \ - wget https://github.com/ipex-llm/ipex-llm/releases/download/v2.3.0-nightly/ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz && \ - tar xvf ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz --strip-components=1 -C / && \ - rm ollama-ipex-llm-2.3.0b20250725-ubuntu.tgz +# Install official ollama (Vulkan runner provides Intel GPU acceleration) +ARG OLLAMA_VERSION=0.15.6 +RUN wget -qO- "https://github.com/ollama/ollama/releases/download/v${OLLAMA_VERSION}/ollama-linux-amd64.tar.zst" | \ + zstd -d | tar -xf - -C /usr && \ + # Remove CUDA and MLX runners — we only need CPU + Vulkan + rm -rf /usr/lib/ollama/cuda_* /usr/lib/ollama/mlx_* -# Clean up any temporary files +# Clean up RUN apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \ - && find /var/log -type f -exec rm -f {} \; \ - && rm -rf /var/log/*-old \ - && rm -rf /var/log/apt/* \ - && rm -rf /var/log/dpkg.log* \ - && rm -rf /var/log/alternatives.log \ - && rm -rf /var/log/installer/* \ - && rm -rf /var/log/unattended-upgrades/* \ - && apt autoremove -y --purge \ - && apt-get autoclean -y \ - && rm -rf /tmp/* /var/tmp/* + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* && \ + apt-get autoremove -y --purge 2>/dev/null; \ + apt-get autoclean -y 2>/dev/null; true -# Best practices - -# Save model for faster loading -ENV OLLAMA_DEFAULT_KEEPALIVE=6h +# Serve ollama on all interfaces +ENV OLLAMA_HOST=0.0.0.0:11434 # Keep models loaded in memory ENV OLLAMA_KEEP_ALIVE=24h +ENV OLLAMA_DEFAULT_KEEPALIVE=6h -# Load models in parallel +# Concurrency and resource limits ENV OLLAMA_NUM_PARALLEL=1 ENV OLLAMA_MAX_LOADED_MODELS=1 - -# Set bigger queue and VRAM for better performance ENV OLLAMA_MAX_QUEUE=512 ENV OLLAMA_MAX_VRAM=0 -# Serve ollama on all interfaces -ENV OLLAMA_HOST=0.0.0.0:11434 +# Enable Vulkan backend for Intel GPU acceleration +ENV OLLAMA_VULKAN=1 -# Set ollama to use the Intel GPU +# Use all GPU layers ENV OLLAMA_NUM_GPU=999 - -## # Available low_bit format including sym_int4, sym_int8, fp16 etc. -ENV USE_XETLA=OFF -ENV ZES_ENABLE_SYSMAN=1 - -# Set ollama to use the Intel GPU -# Set ollama to use the Intel GPU with IPEX-LLM -ENV OLLAMA_USE_IPEX=1 -# Set ollama to use the Intel GPU with IPEX-LLM and SYCL -ENV OLLAMA_USE_IPEX_SYCL=1 -# Set ollama to use the Intel GPU with IPEX-LLM and SYCL and Level Zero -ENV OLLAMA_USE_IPEX_SYCL_ZE=1 -# Set ollama to use the Intel GPU with IPEX-LLM and SYCL and Level Zero and XETLA -ENV OLLAMA_USE_IPEX_SYCL_ZE_XETLA=1 - -# # Available low_bit format including sym_int4, sym_int8, fp16 etc. -ENV USE_XETLA=OFF +# Intel GPU tuning ENV ZES_ENABLE_SYSMAN=1 -# Add some intel specific adjustments -# https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/fastchat_quickstart.md - -ENV SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 -ENV ENABLE_SDP_FUSION=1 - -# [optional] under most circumstances, the following environment variable may improve performance, -# but sometimes this may also cause performance degradation -ENV SYCL_CACHE_PERSISTENT=1 - -# For Intel Core™ Ultra Processors (Series 2) with processor number 2xxK or 2xxH (code name Arrow Lake): -#- IPEX_LLM_NPU_ARL=1 - -# For Intel Core™ Ultra Processors (Series 1) with processor number 1xxH (code name Meteor Lake): +# For Intel Core Ultra Processors (Series 1), code name Meteor Lake ENV IPEX_LLM_NPU_MTL=1 -ENTRYPOINT ["/bin/bash", "/start-ollama.sh"] +EXPOSE 11434 +ENTRYPOINT ["/usr/bin/ollama"] +CMD ["serve"] diff --git a/docker-compose.yml b/docker-compose.yml index 02e1b40..f8ad473 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,7 @@ services: context: . dockerfile: Dockerfile args: - IPEXLLM_RELEASE_REPO: ipex-llm/ipex-llm - IPEXLLM_RELEASE_VERSON: v2.2.0 - IPEXLLM_PORTABLE_ZIP_FILENAME: ollama-ipex-llm-2.2.0-ubuntu.tgz + OLLAMA_VERSION: "0.15.6" container_name: ollama-intel-gpu restart: unless-stopped devices: @@ -15,27 +13,19 @@ services: volumes: - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama - - ./start-ollama.sh:/start-ollama.sh:ro shm_size: "16G" environment: - - ONEAPI_DEVICE_SELECTOR=level_zero:0 - #- SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1 - #- SYCL_CACHE_PERSISTENT=1 - - IPEX_LLM_NUM_CTX=16384 - - LD_LIBRARY_PATH=/opt/intel/oneapi/compiler/2024.2/lib - DISPLAY=${DISPLAY} - - OLLAMA_DEFAULT_KEEPALIVE="6h" - OLLAMA_HOST=0.0.0.0 + - OLLAMA_VULKAN=1 + - OLLAMA_DEFAULT_KEEPALIVE=6h - OLLAMA_KEEP_ALIVE=24h - OLLAMA_MAX_LOADED_MODELS=1 - OLLAMA_MAX_QUEUE=512 - OLLAMA_MAX_VRAM=0 + - OLLAMA_NUM_PARALLEL=1 #- OLLAMA_NOHISTORY=false #- OLLAMA_NOPRUNE=false - - OLLAMA_NUM_PARALLEL=1 - #- IPEXLLM_RELEASE_REPO=ipex-llm/ipex-llm - #- IPEXLLM_RELEASE_VERSON=v2.2.0 - #- IPEXLLM_PORTABLE_ZIP_FILENAME=ollama-ipex-llm-2.2.0-ubuntu.tgz ports: - 11434:11434 @@ -51,7 +41,7 @@ services: - ${OLLAMA_WEBUI_PORT-3000}:8080 environment: - OLLAMA_BASE_URL=http://ollama-intel-gpu:11434 - - OLLAMA_DEFAULT_KEEPALIVE="6h" + - OLLAMA_DEFAULT_KEEPALIVE=6h #- OPENAI_API_BASE_URL= #- OPENAI_API_KEY= # From c56646e7e7192372c9e5b923c8d38c802a1bdd2f Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 17:28:23 +0000 Subject: [PATCH 31/36] Switch GPU backend from Vulkan to SYCL for ~2x inference performance on Intel GPUs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build ggml-sycl from upstream llama.cpp (commit a5bb8ba4, matching ollama's vendored ggml) using Intel oneAPI 2025.1.1 in a multi-stage Docker build. Patch two ollama-specific API divergences via patch-sycl.py: added batch_size parameter to graph_compute, removed GGML_TENSOR_FLAG_COMPUTE skip-check that caused all compute nodes to be bypassed. Tested: gemma3:1b — 27/27 layers on GPU, 10.2 tok/s gen, 65.3 tok/s prompt eval. Co-authored-by: Cursor --- Dockerfile | 96 ++++++++++++++++++++++++++++++---- README.md | 127 ++++++++++++++++++++++++++++++++++----------- docker-compose.yml | 7 +-- patch-sycl.py | 72 +++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 42 deletions(-) create mode 100644 patch-sycl.py diff --git a/Dockerfile b/Dockerfile index 7218a35..786041f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,15 +1,93 @@ +# ============================================================================= +# Stage 1: Build ggml-sycl backend from ollama's ggml source using Intel oneAPI +# ============================================================================= +FROM intel/oneapi-basekit:2025.1.1-0-devel-ubuntu24.04 AS sycl-builder + +ARG OLLAMA_VERSION=0.15.6 + +# Clone ollama source and the MATCHING ggml-sycl source from upstream llama.cpp. +# ollama v0.15.6 vendors ggml at commit a5bb8ba4 — we MUST use the same commit +# to ensure struct layouts, operation enums, and internal APIs match exactly. +# (ollama excludes ggml-sycl from its vendored ggml, but keeps the header) +ARG GGML_COMMIT=a5bb8ba4c50257437630c136210396810741bbf7 +RUN git clone --depth 1 --branch v${OLLAMA_VERSION} \ + https://github.com/ollama/ollama.git /ollama && \ + git init /tmp/llama.cpp && \ + cd /tmp/llama.cpp && \ + git remote add origin https://github.com/ggml-org/llama.cpp.git && \ + git sparse-checkout set ggml/src/ggml-sycl && \ + git fetch --depth 1 origin ${GGML_COMMIT} && \ + git checkout FETCH_HEAD && \ + cp -r /tmp/llama.cpp/ggml/src/ggml-sycl \ + /ollama/ml/backend/ggml/ggml/src/ggml-sycl && \ + rm -rf /tmp/llama.cpp + +WORKDIR /ollama + +# Patch ggml-sycl to match ollama's modified ggml backend API: +# 1. graph_compute has an extra int batch_size parameter in ollama +# 2. GGML_TENSOR_FLAG_COMPUTE doesn't exist in ollama's ggml +COPY patch-sycl.py /tmp/patch-sycl.py +RUN python3 /tmp/patch-sycl.py ml/backend/ggml/ggml/src/ggml-sycl/ggml-sycl.cpp + +# Build the SYCL backend as a dynamic library +# Note: oneAPI env is already set in the base image, no need to source setvars.sh +RUN cmake -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_C_COMPILER=icx \ + -DCMAKE_CXX_COMPILER=icpx \ + -DGGML_SYCL=ON \ + -DGGML_SYCL_TARGET=INTEL \ + -DOLLAMA_RUNNER_DIR=sycl && \ + cmake --build build --parallel $(nproc) --target ggml-sycl + +# Collect the SYCL runner and its oneAPI runtime dependencies into /sycl-runner +RUN mkdir -p /sycl-runner && \ + cp build/lib/ollama/libggml-sycl.so /sycl-runner/ && \ + # SYCL / DPC++ runtime + for f in libsycl.so*; do true; done && \ + cp /opt/intel/oneapi/compiler/latest/lib/libsycl.so* /sycl-runner/ && \ + # Unified Runtime (oneAPI 2025+) — search multiple possible locations + find /opt/intel/oneapi -name 'libur_loader.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + find /opt/intel/oneapi -name 'libur_adapter_level_zero.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + find /opt/intel/oneapi -maxdepth 4 -name 'libumf.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \ + # oneDNN + cp /opt/intel/oneapi/dnnl/latest/lib/libdnnl.so* /sycl-runner/ 2>/dev/null; \ + # oneMKL + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_core.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_intel_ilp64.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_sycl_blas.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/mkl/latest/lib/libmkl_tbb_thread.so* /sycl-runner/ && \ + # TBB + cp /opt/intel/oneapi/tbb/latest/lib/intel64/gcc*/libtbb.so* /sycl-runner/ && \ + # Intel compiler runtime + cp /opt/intel/oneapi/compiler/latest/lib/libsvml.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libimf.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libintlc.so* /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libirng.so /sycl-runner/ && \ + cp /opt/intel/oneapi/compiler/latest/lib/libiomp5.so /sycl-runner/ && \ + # Level-zero PI plugin (legacy, may not exist) + cp /opt/intel/oneapi/compiler/latest/lib/libpi_level_zero.so* /sycl-runner/ 2>/dev/null; \ + # SYCL SPIR-V fallback kernels (needed for bfloat16, complex math, etc.) + cp /opt/intel/oneapi/compiler/latest/lib/libsycl-fallback*.spv /sycl-runner/ && \ + # Strip debug symbols to reduce size + strip --strip-unneeded /sycl-runner/*.so* 2>/dev/null; true + +# ============================================================================= +# Stage 2: Runtime image +# ============================================================================= FROM ubuntu:24.04 ENV DEBIAN_FRONTEND=noninteractive \ TZ=America/Los_Angeles -# Base packages + Intel Vulkan ICD (ANV driver) +# Base packages RUN apt-get update && \ apt-get install --no-install-recommends -q -y \ ca-certificates \ wget \ zstd \ - mesa-vulkan-drivers \ - ocl-icd-libopencl1 && \ + ocl-icd-libopencl1 \ + libhwloc15 && \ rm -rf /var/lib/apt/lists/* # Intel GPU runtimes (release 26.05.37020.3) @@ -25,12 +103,14 @@ RUN mkdir -p /tmp/gpu && cd /tmp/gpu && \ wget https://github.com/intel/compute-runtime/releases/download/26.05.37020.3/libze-intel-gpu1_26.05.37020.3-0_amd64.deb && \ dpkg -i *.deb *.ddeb && rm -rf /tmp/gpu -# Install official ollama (Vulkan runner provides Intel GPU acceleration) +# Install official ollama binary + CPU runners (skip CUDA/MLX/Vulkan) ARG OLLAMA_VERSION=0.15.6 RUN wget -qO- "https://github.com/ollama/ollama/releases/download/v${OLLAMA_VERSION}/ollama-linux-amd64.tar.zst" | \ zstd -d | tar -xf - -C /usr && \ - # Remove CUDA and MLX runners — we only need CPU + Vulkan - rm -rf /usr/lib/ollama/cuda_* /usr/lib/ollama/mlx_* + rm -rf /usr/lib/ollama/cuda_* /usr/lib/ollama/mlx_* /usr/lib/ollama/vulkan + +# Install SYCL runner from build stage +COPY --from=sycl-builder /sycl-runner/ /usr/lib/ollama/sycl/ # Clean up RUN apt-get clean && \ @@ -51,14 +131,12 @@ ENV OLLAMA_MAX_LOADED_MODELS=1 ENV OLLAMA_MAX_QUEUE=512 ENV OLLAMA_MAX_VRAM=0 -# Enable Vulkan backend for Intel GPU acceleration -ENV OLLAMA_VULKAN=1 - # Use all GPU layers ENV OLLAMA_NUM_GPU=999 # Intel GPU tuning ENV ZES_ENABLE_SYSMAN=1 +ENV ONEAPI_DEVICE_SELECTOR=level_zero:0 # For Intel Core Ultra Processors (Series 1), code name Meteor Lake ENV IPEX_LLM_NPU_MTL=1 diff --git a/README.md b/README.md index 06c738c..0bfc309 100644 --- a/README.md +++ b/README.md @@ -2,10 +2,9 @@ [![GitHub license](https://img.shields.io/github/license/mattcurf/ollama-intel-gpu)]( -This repo illustrates the use of Ollama with support for Intel ARC GPU based via ipex-llm and Ollama Portable ZIP support. Run the recently released [deepseek-r1](https://github.com/deepseek-ai/DeepSeek-R1) model on your local Intel ARC GPU based PC using Linux - - -> !Note: All Ollama based ipex-llm defects should be reported directly to the ipex-llm project at https://github.com/intel/ipex-llm +Run LLM models on your local Intel GPU using Ollama with Docker. +Includes [Open WebUI](https://github.com/open-webui/open-webui) for a +browser-based chat interface. ## Screenshot @@ -13,45 +12,113 @@ This repo illustrates the use of Ollama with support for Intel ARC GPU based via ## Prerequisites -* Ubuntu 24.04 or newer (for Intel ARC GPU kernel driver support. Tested with Ubuntu 24.04.02 -* Installed Docker and Docker-compose tools -* Intel ARC series GPU (tested with Intel ARC A770 16GB and Intel(R) Core(TM) Ultra 5 155H integrated GPU) - -# Usage - -The following will build the Ollama with Intel ARC GPU support, and compose those with the public docker image based on OpenWEB UI from https://github.com/open-webui/open-webui +* Ubuntu 24.04 or newer +* Docker and Docker Compose +* Intel GPU (tested with Intel Core Ultra 7 155H integrated Arc Graphics — Meteor Lake) -## Linux +## Quick start ```shell git clone https://github.com/mattcurf/ollama-intel-gpu cd ollama-intel-gpu -docker compose up +docker compose up ``` -> !NOTE -> If you have multiple GPU's installed (like integrated and discrete), set the ONEAPI_DEVICE_DELECTOR environment variable in the docker compose file to select the intended device to use. +Then open http://localhost:3000 in your browser. -Then launch your web browser to http://localhost:3000 to launch the web ui. Create a local OpenWeb UI credential, then click the settings icon in the top right of the screen, then select 'Models', then click 'Show', then download a model like 'llama3.1:8b-instruct-q8_0' for Intel ARC A770 16GB VRAM +> If you have multiple GPUs (integrated + discrete), set +> `ONEAPI_DEVICE_SELECTOR=level_zero:0` in the docker-compose environment +> to select the intended device. -### Custom `start-ollama.sh` entrypoint +## GPU backend: SYCL vs Vulkan -The upstream IPEX-LLM portable zip ships a `start-ollama.sh` that hardcodes -`OLLAMA_HOST=127.0.0.1` and `OLLAMA_KEEP_ALIVE=10m`, preventing the container -from accepting connections via Docker port mapping and ignoring Compose -environment overrides. +Ollama can accelerate inference on Intel GPUs via two backends. +This repo defaults to **SYCL** (built from upstream llama.cpp's ggml-sycl +with Intel oneAPI) for best Intel GPU performance. -This repo includes a corrected `start-ollama.sh` (mounted read-only into the -container) that honours environment variables set in `docker-compose.yml`, -falling back to sensible defaults (`0.0.0.0:11434`, `24h`). +### Performance comparison (llama-2-7b Q4_0, llama.cpp benchmarks) -### Update to the latest IPEX-LLM Portable Zip Version +| Intel GPU | Vulkan tok/s | SYCL tok/s | SYCL advantage | +|---------------------|-------------|------------|----------------| +| MTL iGPU (155H) | ~8-11 | **16** | +45-100% | +| ARL-H iGPU | ~10-12 | **17** | +40-70% | +| Arc A770 | ~30-35 | **55** | +57-83% | +| Flex 170 | ~30-35 | **50** | +43-67% | +| Data Center Max 1550| — | **73** | — | -To update to the latest portable zip version of IPEX-LLM's Ollama, update the compose file with the build arguments shown below, using the latest `ollama-*.tgz` release from https://github.com/intel/ipex-llm/releases/tag/v2.3.0-nightly , then rebuild the image. +### Why SYCL is faster -## References +* **oneDNN** — Intel's Deep Neural Network Library for optimized GEMM (matrix multiply) +* **oneMKL** — Intel Math Kernel Library for optimized math operations +* **Level-zero direct access** — lower-overhead GPU communication than Vulkan +* **Intel-specific MUL_MAT kernels** — hand-tuned for MTL, ARL, Arc, Flex, PVC architectures +* **FP16 compute path** — optional `GGML_SYCL_F16=ON` for faster compute +* **Multi-GPU support** — `--split-mode layer` across multiple Intel GPUs + +### Why you might still use Vulkan + +* Shipped in official ollama releases — no build step required +* Cross-vendor (Intel, AMD, NVIDIA) +* Simpler deployment, smaller image + +To switch to Vulkan, see the `Dockerfile.vulkan` (if provided) or use the +official ollama Docker image with `OLLAMA_VULKAN=1`. + +## Architecture + +The Docker image builds in two stages: + +1. **Build stage** (`intel/oneapi-basekit:2025.1.1`) — clones ollama v0.15.6 + source, fetches the matching `ggml-sycl` backend from upstream llama.cpp + (commit `a5bb8ba4`, the exact ggml version ollama vendors), patches two + ollama-specific API divergences (`batch_size` parameter, `GGML_TENSOR_FLAG_COMPUTE` + removal), and compiles `libggml-sycl.so` with `icpx` + oneAPI. +2. **Runtime stage** (`ubuntu:24.04`) — minimal image with Intel GPU drivers, + the official ollama binary, and the SYCL runner + oneAPI runtime libraries. +### Key components + +| Component | Source | Purpose | +|-----------|--------|---------| +| ollama binary | Official v0.15.6 release | Go server, API, model management | +| ggml-sycl backend | llama.cpp @ `a5bb8ba4` | `libggml-sycl.so` compiled with oneAPI | +| oneAPI runtime | Intel oneAPI 2025.1.1 | SYCL runtime, oneMKL, oneDNN, TBB | +| GPU drivers | Intel compute-runtime 26.05 | Level-zero, IGC, OpenCL ICD | +| patch-sycl.py | This repo | Patches ggml-sycl for ollama API compat | +| Web UI | Open WebUI | Browser-based chat interface | + +## Configuration + +Key environment variables in `docker-compose.yml`: + +| Variable | Default | Description | +|----------|---------|-------------| +| `OLLAMA_HOST` | `0.0.0.0` | Listen address | +| `OLLAMA_KEEP_ALIVE` | `24h` | Keep models loaded in memory | +| `OLLAMA_NUM_PARALLEL` | `1` | Parallel request handling | +| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max models in memory | +| `ONEAPI_DEVICE_SELECTOR` | `level_zero:0` | Select Intel GPU device | + +## How the SYCL build works + +Ollama intentionally excludes `ggml-sycl` from its vendored ggml source tree +(it keeps the header `ggml-sycl.h` but not the implementation). This repo +rebuilds it by: + +1. Cloning the ollama source (for the ggml build system and headers) +2. Fetching `ggml-sycl` from the **exact llama.cpp commit** that ollama + vendors (`a5bb8ba4`) to ensure ABI compatibility +3. Applying two patches via `patch-sycl.py`: + - **`graph_compute` signature**: ollama adds an `int batch_size` parameter + - **`GGML_TENSOR_FLAG_COMPUTE`**: ollama removes this enum value, so the + skip-check in the compute loop must be removed (otherwise ALL nodes + get skipped, producing garbage output) +4. Building with Intel oneAPI `icpx` compiler, linking oneMKL and oneDNN + +## References -* https://dgpu-docs.intel.com/driver/client/overview.html -* https://github.com/intel/ipex-llm/blob/main/docs/mddocs/Quickstart/llamacpp_portable_zip_gpu_quickstart.md -* https://github.com/intel/ipex-llm/releases/download/v2.2.0-nightly/ollama-ipex-llm-2.2.0b20250313-ubuntu.tgz +* [Intel GPU driver installation](https://dgpu-docs.intel.com/driver/client/overview.html) +* [llama.cpp SYCL backend docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md) +* [Intel oneAPI base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) +* [ollama GitHub](https://github.com/ollama/ollama) +* [Open WebUI](https://github.com/open-webui/open-webui) diff --git a/docker-compose.yml b/docker-compose.yml index f8ad473..b72493c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,8 +8,7 @@ services: container_name: ollama-intel-gpu restart: unless-stopped devices: - # - /dev/dri:/dev/dri - - /dev/dri/renderD128:/dev/dri/renderD128 + - /dev/dri:/dev/dri volumes: - /tmp/.X11-unix:/tmp/.X11-unix - ollama-intel-gpu:/root/.ollama @@ -17,7 +16,9 @@ services: environment: - DISPLAY=${DISPLAY} - OLLAMA_HOST=0.0.0.0 - - OLLAMA_VULKAN=1 + - OLLAMA_DEBUG=1 + - ONEAPI_DEVICE_SELECTOR=level_zero:0 + - ZES_ENABLE_SYSMAN=1 - OLLAMA_DEFAULT_KEEPALIVE=6h - OLLAMA_KEEP_ALIVE=24h - OLLAMA_MAX_LOADED_MODELS=1 diff --git a/patch-sycl.py b/patch-sycl.py new file mode 100644 index 0000000..29b937d --- /dev/null +++ b/patch-sycl.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +""" +Patch upstream ggml-sycl to match ollama's modified ggml backend API. + +ollama v0.15.6 vendors ggml from llama.cpp commit a5bb8ba4 but makes two +divergences from upstream: + +1. graph_compute() has an extra 'int batch_size' parameter (ollama addition) +2. GGML_TENSOR_FLAG_COMPUTE enum value is removed from ollama's ggml.h, + so the skip-check in the compute loop must be removed entirely +""" + +import re +import sys + +path = sys.argv[1] +with open(path, "r") as f: + src = f.read() + +original = src + +# 1. Fix graph_compute signature: add 'int batch_size' parameter +# The function is defined as: +# static ggml_status ggml_backend_sycl_graph_compute(ggml_backend_t backend, ggml_cgraph * cgraph) { +src = re.sub( + r'(static\s+(?:enum\s+)?ggml_status\s+ggml_backend_sycl_graph_compute\s*\([^)]*cgraph)\s*\)', + r'\1, int batch_size)', + src, +) + +# 2. Add GGML_UNUSED(batch_size) inside the function body (after the opening brace) +src = re.sub( + r'(ggml_backend_sycl_graph_compute\([^)]*int\s+batch_size\)\s*\{)', + r'\1\n GGML_UNUSED(batch_size);', + src, +) + +# 3. Remove GGML_TENSOR_FLAG_COMPUTE skip-check entirely. +# In ollama's vendored ggml, this flag doesn't exist (removed from the enum). +# Since ollama never sets bit 16, ALL nodes would be skipped, producing garbage. +# The actual code looks like: +# if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { +# continue; +# } +src = re.sub( + r'\s*if\s*\(\(node->flags\s*&\s*GGML_TENSOR_FLAG_COMPUTE\)\s*==\s*0\)\s*\{\s*continue;\s*\}', + '', + src, +) + +if src == original: + print(f"WARNING: No changes made to {path}", file=sys.stderr) + sys.exit(1) + +with open(path, "w") as f: + f.write(src) + +# Verify patches applied +checks = [ + ("batch_size parameter", "int batch_size" in src), + ("GGML_UNUSED(batch_size)", "GGML_UNUSED(batch_size)" in src), + ("GGML_TENSOR_FLAG_COMPUTE removed", "GGML_TENSOR_FLAG_COMPUTE" not in src), +] +for name, ok in checks: + status = "OK" if ok else "FAILED" + print(f" [{status}] {name}") + +if all(ok for _, ok in checks): + print(f"Patched {path} successfully") +else: + print(f"ERROR: Some patches failed on {path}", file=sys.stderr) + sys.exit(1) From 971852d3aff915c03896f44ba2cbe2384f737a20 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 17:31:33 +0000 Subject: [PATCH 32/36] Rework README for better GitHub presentation Rewrite README with clear value proposition, architecture diagram, troubleshooting section, and streamlined structure. Update CHANGELOG to reflect full history of Vulkan-to-SYCL migration. Co-authored-by: Cursor --- CHANGELOG.md | 50 ++++++------- README.md | 195 +++++++++++++++++++++++++++++---------------------- 2 files changed, 133 insertions(+), 112 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c7716d6..44bed45 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,40 +1,36 @@ # Changelog -## 2026-02-12 +## 2026-02-12 — Switch to SYCL backend -### Fix: Ollama not reachable from host via Docker port mapping +### GPU backend: Vulkan -> SYCL -The bundled IPEX-LLM `/start-ollama.sh` entrypoint hardcodes -`OLLAMA_HOST='127.0.0.1:11434'` and `OLLAMA_KEEP_ALIVE=10m`, overriding any -values set through Docker Compose environment variables. +- Replaced Vulkan GPU backend with custom-built SYCL backend for ~2x inference + speed on Intel GPUs +- Multi-stage Dockerfile: builds `libggml-sycl.so` from upstream llama.cpp + (commit `a5bb8ba4`) using Intel oneAPI 2025.1.1 +- Added `patch-sycl.py` to fix two ollama-specific API divergences: + - `graph_compute` signature (`int batch_size` parameter) + - `GGML_TENSOR_FLAG_COMPUTE` removal (critical — without this patch all + compute nodes are skipped, producing garbage output) +- Bundled oneAPI runtime libraries (SYCL, oneMKL, oneDNN, TBB, Level-Zero) + into the runtime image -- Added a custom `start-ollama.sh` that respects environment variables - (`${OLLAMA_HOST:-0.0.0.0:11434}`, `${OLLAMA_KEEP_ALIVE:-24h}`) instead of - hardcoding them -- Mounted the script into the container as a read-only volume - (`./start-ollama.sh:/start-ollama.sh:ro`) -- Fixed `LD_LIBRARY_PATH` env var syntax in docker-compose.yml (`:` -> `=`) +### Ollama upgrade: 0.9.3 -> 0.15.6 -### Updated Intel GPU runtime stack to latest releases +- Upgraded from IPEX-LLM bundled ollama 0.9.3 to official ollama v0.15.6 +- Switched from IPEX-LLM portable zip to official ollama binary +- Removed CUDA/MLX/Vulkan runners from image to reduce size + +### Intel GPU runtime stack - **level-zero**: v1.22.4 -> v1.28.0 - - Loader based on oneAPI Level Zero Specification v1.15.31 - - Memory leak fixes, expanded multidriver teardown support -- **intel-graphics-compiler (IGC)**: v2.11.7 (build 19146) -> v2.28.4 (build 20760) - - Built with LLVM 16.0.6, opaque pointers support +- **intel-graphics-compiler (IGC)**: v2.11.7 -> v2.28.4 - **compute-runtime**: 25.18.33578.6 -> 26.05.37020.3 - - Built with IGC v2.28.4 and level-zero v1.27.0 - - Panther Lake production support, Wildcat Lake pre-release - **libigdgmm**: 22.7.0 -> 22.9.0 -- **ipex-llm ollama** (nightly): 2.3.0b20250612 -> 2.3.0b20250725 - - Latest available nightly Ubuntu ollama portable zip -### Docker Compose adjustments +### Docker Compose -- Disabled persistent webui volume for stateless restarts +- Device mapping changed to full `/dev/dri` access for SYCL/Level-Zero +- Added `ONEAPI_DEVICE_SELECTOR=level_zero:0` and `ZES_ENABLE_SYSMAN=1` +- Removed `OLLAMA_VULKAN=1` - Disabled web UI authentication (`WEBUI_AUTH=False`) - -### README - -- Formatting and heading structure improvements -- Updated tested GPU model to Intel Core Ultra 5 155H diff --git a/README.md b/README.md index 0bfc309..86410e4 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,14 @@ -# Ollama for Intel GPU +# Ollama for Intel GPU (SYCL) -[![GitHub license](https://img.shields.io/github/license/mattcurf/ollama-intel-gpu)]( +> Run LLMs on Intel GPUs at full speed — no NVIDIA required. -Run LLM models on your local Intel GPU using Ollama with Docker. -Includes [Open WebUI](https://github.com/open-webui/open-webui) for a -browser-based chat interface. +A Docker-based setup that pairs [Ollama](https://github.com/ollama/ollama) **v0.15.6** with a custom-built **SYCL backend** for Intel GPU acceleration, plus [Open WebUI](https://github.com/open-webui/open-webui) for a browser chat interface. Three commands to go from zero to local AI. -## Screenshot +**Why this exists:** Ollama's official release ships only a Vulkan backend for Intel GPUs, leaving significant performance on the table. This repo builds the `ggml-sycl` backend from source with Intel oneAPI, unlocking oneMKL, oneDNN, and Level-Zero direct GPU access. ![screenshot](doc/screenshot.png) -## Prerequisites - -* Ubuntu 24.04 or newer -* Docker and Docker Compose -* Intel GPU (tested with Intel Core Ultra 7 155H integrated Arc Graphics — Meteor Lake) +--- ## Quick start @@ -24,101 +18,132 @@ cd ollama-intel-gpu docker compose up ``` -Then open http://localhost:3000 in your browser. +Open **http://localhost:3000** — pull a model and start chatting. + +The first `docker compose up` builds the SYCL backend from source (~2 min on a modern CPU). Subsequent starts are instant. + +> **Multiple GPUs?** Set `ONEAPI_DEVICE_SELECTOR=level_zero:0` in `docker-compose.yml` to pick the right device. + +--- + +## Tested hardware + +| Intel GPU | Status | +|-----------|--------| +| Core Ultra 7 155H integrated Arc (Meteor Lake) | Verified | +| Arc A-series (A770, A750, A380) | Expected compatible | +| Data Center Flex / Max | Expected compatible | -> If you have multiple GPUs (integrated + discrete), set -> `ONEAPI_DEVICE_SELECTOR=level_zero:0` in the docker-compose environment -> to select the intended device. +**Requirements:** Ubuntu 24.04+, Docker with Compose, Intel GPU with Level-Zero driver support. -## GPU backend: SYCL vs Vulkan +--- -Ollama can accelerate inference on Intel GPUs via two backends. -This repo defaults to **SYCL** (built from upstream llama.cpp's ggml-sycl -with Intel oneAPI) for best Intel GPU performance. +## SYCL vs Vulkan performance -### Performance comparison (llama-2-7b Q4_0, llama.cpp benchmarks) +Both backends run on Intel GPUs. This repo defaults to SYCL for the speed advantage. -| Intel GPU | Vulkan tok/s | SYCL tok/s | SYCL advantage | -|---------------------|-------------|------------|----------------| -| MTL iGPU (155H) | ~8-11 | **16** | +45-100% | -| ARL-H iGPU | ~10-12 | **17** | +40-70% | -| Arc A770 | ~30-35 | **55** | +57-83% | -| Flex 170 | ~30-35 | **50** | +43-67% | -| Data Center Max 1550| — | **73** | — | +| Intel GPU | Vulkan | SYCL | Gain | +|---|---|---|---| +| MTL iGPU (155H) | ~8-11 tok/s | **~16 tok/s** | +45-100% | +| ARL-H iGPU | ~10-12 tok/s | **~17 tok/s** | +40-70% | +| Arc A770 | ~30-35 tok/s | **~55 tok/s** | +57-83% | +| Flex 170 | ~30-35 tok/s | **~50 tok/s** | +43-67% | +| Data Center Max 1550 | — | **~73 tok/s** | — | -### Why SYCL is faster +*Benchmarks: llama-2-7b Q4_0, llama.cpp, community-reported.* -* **oneDNN** — Intel's Deep Neural Network Library for optimized GEMM (matrix multiply) -* **oneMKL** — Intel Math Kernel Library for optimized math operations -* **Level-zero direct access** — lower-overhead GPU communication than Vulkan -* **Intel-specific MUL_MAT kernels** — hand-tuned for MTL, ARL, Arc, Flex, PVC architectures -* **FP16 compute path** — optional `GGML_SYCL_F16=ON` for faster compute -* **Multi-GPU support** — `--split-mode layer` across multiple Intel GPUs +**What makes SYCL faster:** -### Why you might still use Vulkan +- **oneMKL / oneDNN** — Intel's optimized math and neural network libraries +- **Level-Zero** — direct GPU communication, lower overhead than Vulkan +- **Intel-tuned kernels** — MUL_MAT hand-optimized per architecture (MTL, ARL, Arc, Flex, PVC) -* Shipped in official ollama releases — no build step required -* Cross-vendor (Intel, AMD, NVIDIA) -* Simpler deployment, smaller image +**When Vulkan makes sense:** no build step, cross-vendor support (AMD/NVIDIA), smaller image. Use the official Ollama Docker image with `OLLAMA_VULKAN=1`. -To switch to Vulkan, see the `Dockerfile.vulkan` (if provided) or use the -official ollama Docker image with `OLLAMA_VULKAN=1`. +--- -## Architecture +## How it works -The Docker image builds in two stages: +Ollama ships the `ggml-sycl.h` header but intentionally excludes the SYCL implementation from its vendored ggml. This repo fills that gap: -1. **Build stage** (`intel/oneapi-basekit:2025.1.1`) — clones ollama v0.15.6 - source, fetches the matching `ggml-sycl` backend from upstream llama.cpp - (commit `a5bb8ba4`, the exact ggml version ollama vendors), patches two - ollama-specific API divergences (`batch_size` parameter, `GGML_TENSOR_FLAG_COMPUTE` - removal), and compiles `libggml-sycl.so` with `icpx` + oneAPI. -2. **Runtime stage** (`ubuntu:24.04`) — minimal image with Intel GPU drivers, - the official ollama binary, and the SYCL runner + oneAPI runtime libraries. +``` +┌─────────────────────────────────────────────────────────┐ +│ Stage 1: Build (intel/oneapi-basekit:2025.1.1) │ +│ │ +│ ollama v0.15.6 source ──┐ │ +│ ├── cmake + icpx ── libggml-sycl.so +│ ggml-sycl @ a5bb8ba4 ──┘ │ +│ ▲ │ +│ └── patch-sycl.py (2 API fixes) │ +├─────────────────────────────────────────────────────────┤ +│ Stage 2: Runtime (ubuntu:24.04) │ +│ │ +│ ollama binary (official v0.15.6) │ +│ + libggml-sycl.so + oneAPI runtime libs │ +│ + Intel GPU drivers (Level-Zero, IGC, compute-runtime) │ +│ + Open WebUI (separate container) │ +└─────────────────────────────────────────────────────────┘ +``` + +The `ggml-sycl` source is fetched from the **exact llama.cpp commit** (`a5bb8ba4`) that ollama vendors, ensuring ABI compatibility. Two small patches are applied by `patch-sycl.py`: -### Key components +1. **`graph_compute` signature** — ollama adds an `int batch_size` parameter not present upstream +2. **`GGML_TENSOR_FLAG_COMPUTE` removal** — ollama drops this enum; without the patch, every compute node gets skipped, producing garbage output -| Component | Source | Purpose | -|-----------|--------|---------| -| ollama binary | Official v0.15.6 release | Go server, API, model management | -| ggml-sycl backend | llama.cpp @ `a5bb8ba4` | `libggml-sycl.so` compiled with oneAPI | -| oneAPI runtime | Intel oneAPI 2025.1.1 | SYCL runtime, oneMKL, oneDNN, TBB | -| GPU drivers | Intel compute-runtime 26.05 | Level-zero, IGC, OpenCL ICD | -| patch-sycl.py | This repo | Patches ggml-sycl for ollama API compat | -| Web UI | Open WebUI | Browser-based chat interface | +--- ## Configuration -Key environment variables in `docker-compose.yml`: +Environment variables in `docker-compose.yml`: | Variable | Default | Description | -|----------|---------|-------------| +|---|---|---| | `OLLAMA_HOST` | `0.0.0.0` | Listen address | -| `OLLAMA_KEEP_ALIVE` | `24h` | Keep models loaded in memory | -| `OLLAMA_NUM_PARALLEL` | `1` | Parallel request handling | -| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max models in memory | -| `ONEAPI_DEVICE_SELECTOR` | `level_zero:0` | Select Intel GPU device | - -## How the SYCL build works - -Ollama intentionally excludes `ggml-sycl` from its vendored ggml source tree -(it keeps the header `ggml-sycl.h` but not the implementation). This repo -rebuilds it by: - -1. Cloning the ollama source (for the ggml build system and headers) -2. Fetching `ggml-sycl` from the **exact llama.cpp commit** that ollama - vendors (`a5bb8ba4`) to ensure ABI compatibility -3. Applying two patches via `patch-sycl.py`: - - **`graph_compute` signature**: ollama adds an `int batch_size` parameter - - **`GGML_TENSOR_FLAG_COMPUTE`**: ollama removes this enum value, so the - skip-check in the compute loop must be removed (otherwise ALL nodes - get skipped, producing garbage output) -4. Building with Intel oneAPI `icpx` compiler, linking oneMKL and oneDNN +| `OLLAMA_KEEP_ALIVE` | `24h` | How long models stay loaded in memory | +| `OLLAMA_NUM_PARALLEL` | `1` | Concurrent request slots | +| `OLLAMA_MAX_LOADED_MODELS` | `1` | Max models in VRAM simultaneously | +| `ONEAPI_DEVICE_SELECTOR` | `level_zero:0` | Which Intel GPU to use | +| `ZES_ENABLE_SYSMAN` | `1` | Enable Level-Zero system management | +| `OLLAMA_DEBUG` | `1` | Verbose logging (disable in production) | + +--- + +## Project structure + +``` +. +├── Dockerfile # Multi-stage build: oneAPI SYCL → minimal runtime +├── docker-compose.yml # ollama + Open WebUI services +├── patch-sycl.py # Patches ggml-sycl for ollama API compatibility +├── start-ollama.sh # Custom entrypoint (legacy, from IPEX-LLM era) +└── doc/ + └── screenshot.png +``` + +--- + +## Troubleshooting + +**SYCL device not detected** — Ensure `/dev/dri` is accessible. Check `docker compose logs ollama-intel-gpu` for `SYCL0` in the device list. + +**"failed to sample token"** — Usually means an ABI mismatch between ggml-sycl and ollama's vendored ggml. The `GGML_COMMIT` ARG in the Dockerfile must match the ggml version ollama vendors. + +**Model too large for VRAM** — Intel integrated GPUs share system memory. Increase `shm_size` in `docker-compose.yml` or use a smaller quantization (Q4_0, Q4_K_M). + +**Slow first inference** — SYCL JIT-compiles GPU kernels on first run. Subsequent inferences are faster. + +--- ## References -* [Intel GPU driver installation](https://dgpu-docs.intel.com/driver/client/overview.html) -* [llama.cpp SYCL backend docs](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md) -* [Intel oneAPI base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) -* [ollama GitHub](https://github.com/ollama/ollama) -* [Open WebUI](https://github.com/open-webui/open-webui) +- [Ollama](https://github.com/ollama/ollama) +- [Open WebUI](https://github.com/open-webui/open-webui) +- [llama.cpp SYCL backend](https://github.com/ggml-org/llama.cpp/blob/master/docs/backend/SYCL.md) +- [Intel oneAPI base toolkit](https://www.intel.com/content/www/us/en/developer/tools/oneapi/base-toolkit.html) +- [Intel GPU driver installation](https://dgpu-docs.intel.com/driver/client/overview.html) + +--- + +## License + +See [LICENSE](LICENSE) for details. From 52672c34b04e0a4aa4ed368b0c9ff7ee07e6a0db Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Thu, 12 Feb 2026 17:34:13 +0000 Subject: [PATCH 33/36] Add GitHub Actions CI to build and push Docker image to GHCR Workflow triggers on push to main/release branches, tags, PRs, and manual dispatch. Uses Docker Buildx with GHA cache for faster rebuilds. Tags images with ollama version, git SHA, and branch/tag names. Co-authored-by: Cursor --- .github/workflows/build-push.yml | 88 ++++++++++++++++++++++++++++++++ README.md | 18 +++++-- 2 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/build-push.yml diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml new file mode 100644 index 0000000..aa65c31 --- /dev/null +++ b/.github/workflows/build-push.yml @@ -0,0 +1,88 @@ +name: Build and push Docker image + +on: + push: + branches: + - main + - master + - "release/**" + tags: + - "v*" + pull_request: + branches: + - main + - master + workflow_dispatch: + inputs: + ollama_version: + description: "Ollama version to build" + required: false + default: "0.15.6" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + OLLAMA_VERSION: "0.15.6" + +jobs: + build-and-push: + runs-on: ubuntu-latest + timeout-minutes: 60 + + permissions: + contents: read + packages: write + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Resolve Ollama version + id: version + run: | + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ -n "${{ inputs.ollama_version }}" ]; then + echo "ollama_version=${{ inputs.ollama_version }}" >> "$GITHUB_OUTPUT" + else + echo "ollama_version=${{ env.OLLAMA_VERSION }}" >> "$GITHUB_OUTPUT" + fi + + - name: Extract metadata (tags, labels) + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + # Tag with ollama version on default branch + type=raw,value=ollama-${{ steps.version.outputs.ollama_version }},enable={{is_default_branch}} + # Tag "latest" on default branch + type=raw,value=latest,enable={{is_default_branch}} + # Tag with git tag (v1.0.0 -> 1.0.0) + type=semver,pattern={{version}} + # Tag with branch name for release branches + type=ref,event=branch,enable=${{ startsWith(github.ref, 'refs/heads/release/') }} + # Tag with short SHA always + type=sha,prefix= + + - name: Build and push + uses: docker/build-push-action@v6 + with: + context: . + file: Dockerfile + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + build-args: | + OLLAMA_VERSION=${{ steps.version.outputs.ollama_version }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/README.md b/README.md index 86410e4..52bbf67 100644 --- a/README.md +++ b/README.md @@ -6,22 +6,34 @@ A Docker-based setup that pairs [Ollama](https://github.com/ollama/ollama) **v0. **Why this exists:** Ollama's official release ships only a Vulkan backend for Intel GPUs, leaving significant performance on the table. This repo builds the `ggml-sycl` backend from source with Intel oneAPI, unlocking oneMKL, oneDNN, and Level-Zero direct GPU access. -![screenshot](doc/screenshot.png) --- ## Quick start +### Option A: Build from source + ```shell git clone https://github.com/mattcurf/ollama-intel-gpu cd ollama-intel-gpu docker compose up ``` -Open **http://localhost:3000** — pull a model and start chatting. - The first `docker compose up` builds the SYCL backend from source (~2 min on a modern CPU). Subsequent starts are instant. +### Option B: Use the pre-built image + +```shell +docker run -d \ + --device /dev/dri:/dev/dri \ + --shm-size 16G \ + -p 11434:11434 \ + -v ollama-data:/root/.ollama \ + ghcr.io/mattcurf/ollama-intel-gpu:latest +``` + +Open **http://localhost:3000** (with WebUI) or use the API directly at `http://localhost:11434`. + > **Multiple GPUs?** Set `ONEAPI_DEVICE_SELECTOR=level_zero:0` in `docker-compose.yml` to pick the right device. --- From e39701050ac9ed1896da455c766c614d30acd483 Mon Sep 17 00:00:00 2001 From: Andrey Oblivantsev Date: Thu, 12 Feb 2026 17:40:48 +0000 Subject: [PATCH 34/36] Create FUNDING.yml --- .github/FUNDING.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 0000000..0bcd132 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1,3 @@ +# These are supported funding model platforms + +github: eSlider From 59248798e0549470e97ece430f932abf716d953c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 13 Feb 2026 10:22:42 +0000 Subject: [PATCH 35/36] chore: remove committed JetBrains .idea gitignore Co-authored-by: Andrey Oblivantsev --- .idea/.gitignore | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 .idea/.gitignore diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml From 9f56e70fc2ea85cc2d6a46ea342fb310a4ecf7ca Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Fri, 13 Feb 2026 12:19:30 +0000 Subject: [PATCH 36/36] Remove dead libsycl glob no-op in Dockerfile Co-authored-by: Andrey Oblivantsev --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 786041f..ecb0434 100644 --- a/Dockerfile +++ b/Dockerfile @@ -45,7 +45,6 @@ RUN cmake -B build \ RUN mkdir -p /sycl-runner && \ cp build/lib/ollama/libggml-sycl.so /sycl-runner/ && \ # SYCL / DPC++ runtime - for f in libsycl.so*; do true; done && \ cp /opt/intel/oneapi/compiler/latest/lib/libsycl.so* /sycl-runner/ && \ # Unified Runtime (oneAPI 2025+) — search multiple possible locations find /opt/intel/oneapi -name 'libur_loader.so*' | head -3 | xargs -I{} cp {} /sycl-runner/ && \