diff --git a/.github/models.csv b/.github/models.csv new file mode 100644 index 00000000..e792b2d9 --- /dev/null +++ b/.github/models.csv @@ -0,0 +1,21 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +# +# Single source of truth for the CI model set: one "filename,url" row per model. +# Consumed by: +# - .github/workflows/publish.yml download-models job (manifest-driven download loop; +# the cache key is gguf-models-, so EDITING THIS FILE automatically +# creates a fresh cache entry - no manual cache deletion needed) +# - .github/validate-models.sh / .bat (the required-model list) +# Filenames must not contain spaces or commas. Lines starting with # are ignored. +codellama-7b.Q2_K.gguf,https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q2_K.gguf +jina-reranker-v1-tiny-en-Q4_0.gguf,https://huggingface.co/gpustack/jina-reranker-v1-tiny-en-GGUF/resolve/main/jina-reranker-v1-tiny-en-Q4_0.gguf +AMD-Llama-135m-code.Q2_K.gguf,https://huggingface.co/QuantFactory/AMD-Llama-135m-code-GGUF/resolve/main/AMD-Llama-135m-code.Q2_K.gguf +Qwen3-0.6B-Q4_K_M.gguf,https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf +Qwen2.5-1.5B-Instruct-Q4_K_M.gguf,https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf +nomic-embed-text-v1.5.f16.gguf,https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf +SmolVLM-500M-Instruct-Q8_0.gguf,https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/SmolVLM-500M-Instruct-Q8_0.gguf +mmproj-SmolVLM-500M-Instruct-Q8_0.gguf,https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/mmproj-SmolVLM-500M-Instruct-Q8_0.gguf +OuteTTS-0.2-500M-Q4_K_M.gguf,https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/OuteTTS-0.2-500M-Q4_K_M.gguf +WavTokenizer-Large-75-F16.gguf,https://huggingface.co/ggml-org/WavTokenizer/resolve/main/WavTokenizer-Large-75-F16.gguf diff --git a/.github/package-fatjars.sh b/.github/package-fatjars.sh new file mode 100755 index 00000000..4d84ed19 --- /dev/null +++ b/.github/package-fatjars.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT + +# Assembles the per-OS "all backends" server fat jars distributed as GitHub Release +# assets (never deployed to Maven Central). +# +# For every OS/arch that has GPU classifier jars, the default jar-with-dependencies +# uber jar (library classes + Java runtime deps + default CPU natives for every +# platform) is copied and each backend's native tree is added under a backend +# subdirectory (net/ladenthin/llama////), together with a +# jllama-backends.txt manifest listing the backends in priority order. LlamaLoader +# reads that manifest at runtime and loads the first backend whose library loads, +# falling back to the default CPU library (see LlamaLoader.BACKEND_MANIFEST_FILE). +# +# Fail-loud invariants (a broken invariant must red the pipeline, never skip): +# * every in llama/pom.xml must be parseable/explicitly excluded, +# * the pom classifier set and the on-disk classifier-jar set must match exactly, +# * every expected backend library must end up inside the combined jar, +# * the Main-Class of the combined jar must survive the zip update. +# +# Usage: package-fatjars.sh [pom] +# jars-dir directory holding the `llama-jars` artifact (llama/target/*.jar) +# out-dir output directory for the combined fat jars + sha256 files +# pom path to llama/pom.xml (default: llama/pom.xml) +set -euo pipefail + +JARS_DIR="${1:?usage: package-fatjars.sh [pom]}" +OUT_DIR="${2:?usage: package-fatjars.sh [pom]}" +POM="${3:-llama/pom.xml}" + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +# Resolve to absolute paths: the zip update below runs from the staging directory. +[ -d "$JARS_DIR" ] || fail "jars dir not found: $JARS_DIR" +JARS_DIR="$(cd "$JARS_DIR" && pwd)" +mkdir -p "$OUT_DIR" +OUT_DIR="$(cd "$OUT_DIR" && pwd)" + +# Classifiers that intentionally get no combined-jar treatment: +# msvc-windows CPU-only build variant, redundant with the default Windows natives +# opencl-android-aarch64 `java -jar` is not applicable on Android (AAR is the delivery path) +EXCLUDED_CLASSIFIERS=("msvc-windows" "opencl-android-aarch64") + +# Backend priority order used for the manifest (first loadable backend wins at runtime). +# A classifier whose backend token is not in this list fails the build: a new backend +# must be consciously ranked here, not silently appended. +BACKEND_PRIORITY=("cuda13" "rocm" "sycl-fp16" "sycl-fp32" "sycl" "vulkan" "opencl" "openvino") + +MAIN_CLASS="net.ladenthin.llama.server.ServerLauncher" + +is_excluded() { + local classifier="$1" excluded + for excluded in "${EXCLUDED_CLASSIFIERS[@]}"; do + [ "$classifier" = "$excluded" ] && return 0 + done + return 1 +} + +backend_priority_index() { + local backend="$1" i + for i in "${!BACKEND_PRIORITY[@]}"; do + [ "${BACKEND_PRIORITY[$i]}" = "$backend" ] && { echo "$i"; return 0; } + done + return 1 +} + +# --- 1. Source of truth #1: the set declared in the pom ------------------- +[ -f "$POM" ] || fail "pom not found: $POM" +mapfile -t POM_CLASSIFIERS < <(grep -oP '\K[^<]+(?=)' "$POM" | sort -u) +[ "${#POM_CLASSIFIERS[@]}" -gt 0 ] || fail "no entries parsed from $POM" +echo "pom classifiers (${#POM_CLASSIFIERS[@]}): ${POM_CLASSIFIERS[*]}" + +# --- 2. The base fat jar (exactly one) + the version derived from its file name -------- +mapfile -t BASE_FAT_JARS < <(find "$JARS_DIR" -maxdepth 1 -name 'llama-*-jar-with-dependencies.jar' | sort) +[ "${#BASE_FAT_JARS[@]}" -eq 1 ] \ + || fail "expected exactly 1 default jar-with-dependencies in $JARS_DIR, got ${#BASE_FAT_JARS[@]}: ${BASE_FAT_JARS[*]:-none}" +BASE_FAT_JAR="${BASE_FAT_JARS[0]}" +VERSION="$(basename "$BASE_FAT_JAR")" +VERSION="${VERSION#llama-}" +VERSION="${VERSION%-jar-with-dependencies.jar}" +echo "base fat jar: $BASE_FAT_JAR (version $VERSION)" + +# --- 3. Source of truth #2: the classifier jars actually built by the package job ------ +DISK_CLASSIFIERS=() +for jar in "$JARS_DIR"/llama-"$VERSION"-*.jar; do + [ -e "$jar" ] || continue + classifier="$(basename "$jar")" + classifier="${classifier#llama-"$VERSION"-}" + classifier="${classifier%.jar}" + case "$classifier" in + sources | javadoc | jar-with-dependencies) continue ;; + esac + DISK_CLASSIFIERS+=("$classifier") +done +[ "${#DISK_CLASSIFIERS[@]}" -gt 0 ] || fail "no classifier jars found in $JARS_DIR for version $VERSION" +mapfile -t DISK_CLASSIFIERS < <(printf '%s\n' "${DISK_CLASSIFIERS[@]}" | sort -u) + +# --- 4. Exact set equality in BOTH directions (no silent gaps, no unknown jars) -------- +if ! diff <(printf '%s\n' "${POM_CLASSIFIERS[@]}") <(printf '%s\n' "${DISK_CLASSIFIERS[@]}"); then + fail "classifier set mismatch between $POM and the jars in $JARS_DIR (see diff above)" +fi +echo "classifier sets match (${#POM_CLASSIFIERS[@]} classifiers)" + +# --- 5. Parse every non-excluded classifier as -- ------------------- +# Associative arrays keyed by "-" in classifier notation (e.g. linux-x86-64). +declare -A TARGET_BACKENDS # target -> space-separated backend names +declare -A BACKEND_CLASSIFIER # "/" -> classifier string +for classifier in "${POM_CLASSIFIERS[@]}"; do + if is_excluded "$classifier"; then + echo "excluded from combined jars: $classifier" + continue + fi + case "$classifier" in + *-linux-x86-64) os="linux" arch="x86-64" backend="${classifier%-linux-x86-64}" ;; + *-linux-aarch64) os="linux" arch="aarch64" backend="${classifier%-linux-aarch64}" ;; + *-windows-x86-64) os="windows" arch="x86-64" backend="${classifier%-windows-x86-64}" ;; + *-windows-aarch64) os="windows" arch="aarch64" backend="${classifier%-windows-aarch64}" ;; + *) fail "unparseable classifier '$classifier': expected -- — add a parse rule or an explicit exclusion" ;; + esac + backend_priority_index "$backend" > /dev/null \ + || fail "backend '$backend' (classifier '$classifier') is not in BACKEND_PRIORITY — rank the new backend consciously" + target="$os-$arch" + TARGET_BACKENDS[$target]="${TARGET_BACKENDS[$target]:-} $backend" + BACKEND_CLASSIFIER["$target/$backend"]="$classifier" +done +[ "${#TARGET_BACKENDS[@]}" -gt 0 ] || fail "no combined-jar targets derived from the classifier set" + +# --- 6. Assemble one combined jar per target -------------------------------------------- +WORK_DIR="$(mktemp -d)" +trap 'rm -rf "$WORK_DIR"' EXIT + +for target in $(printf '%s\n' "${!TARGET_BACKENDS[@]}" | sort); do + case "$target" in + *-x86-64) os="${target%-x86-64}" arch="x86-64" ;; + *-aarch64) os="${target%-aarch64}" arch="aarch64" ;; + *) fail "internal error: unexpected target '$target'" ;; + esac + # Classifier notation -> resource-folder notation (OSInfo folder names). + case "$os" in + linux) os_folder="Linux" main_lib="libjllama.so" ;; + windows) os_folder="Windows" main_lib="jllama.dll" ;; + *) fail "internal error: unexpected os '$os'" ;; + esac + case "$arch" in + x86-64) arch_folder="x86_64" ;; + aarch64) arch_folder="aarch64" ;; + *) fail "internal error: unexpected arch '$arch'" ;; + esac + resource_dir="net/ladenthin/llama/$os_folder/$arch_folder" + + # Order this target's backends by BACKEND_PRIORITY. + ordered_backends=() + for backend in "${BACKEND_PRIORITY[@]}"; do + case " ${TARGET_BACKENDS[$target]} " in + *" $backend "*) ordered_backends+=("$backend") ;; + esac + done + + staging="$WORK_DIR/$target" + mkdir -p "$staging/$resource_dir" + manifest="$staging/$resource_dir/jllama-backends.txt" + { + echo "# Native backends in priority order; first loadable backend wins." + echo "# Extra tokens after a backend name are sibling files loaded before its library." + } > "$manifest" + + for backend in "${ordered_backends[@]}"; do + classifier="${BACKEND_CLASSIFIER["$target/$backend"]}" + classifier_jar="$JARS_DIR/llama-$VERSION-$classifier.jar" + extract_dir="$WORK_DIR/extract-$target-$backend" + mkdir -p "$extract_dir" + unzip -q "$classifier_jar" "$resource_dir/*" -d "$extract_dir" \ + || fail "$classifier_jar contains no native tree at $resource_dir" + [ -f "$extract_dir/$resource_dir/$main_lib" ] \ + || fail "$classifier_jar: expected $resource_dir/$main_lib not found" + mkdir -p "$staging/$resource_dir/$backend" + mv "$extract_dir/$resource_dir/"* "$staging/$resource_dir/$backend/" + # Manifest line: backend name + every sibling file (sorted, deterministic) except + # the main library — the loader extracts and loads the siblings first. + extras="$(cd "$staging/$resource_dir/$backend" && find . -type f ! -name "$main_lib" -printf '%P\n' | sort | tr '\n' ' ')" + extras="${extras% }" + if [ -n "$extras" ]; then + echo "$backend $extras" >> "$manifest" + else + echo "$backend" >> "$manifest" + fi + done + + out_jar="$OUT_DIR/llama-$VERSION-all-$target-jar-with-dependencies.jar" + cp "$BASE_FAT_JAR" "$out_jar" + (cd "$staging" && zip -q -ur "$out_jar" net) + + # --- Verify the combined jar ----------------------------------------------------- + listing="$(unzip -l "$out_jar")" + for backend in "${ordered_backends[@]}"; do + echo "$listing" | grep -qF "$resource_dir/$backend/$main_lib" \ + || fail "$out_jar: missing $resource_dir/$backend/$main_lib after zip update" + done + echo "$listing" | grep -qF "$resource_dir/jllama-backends.txt" \ + || fail "$out_jar: missing $resource_dir/jllama-backends.txt" + unzip -p "$out_jar" META-INF/MANIFEST.MF | grep -qF "Main-Class: $MAIN_CLASS" \ + || fail "$out_jar: Main-Class $MAIN_CLASS did not survive the zip update" + sample_backend="${ordered_backends[0]}" + unzip -p "$out_jar" "$resource_dir/$sample_backend/$main_lib" \ + | cmp -s - "$staging/$resource_dir/$sample_backend/$main_lib" \ + || fail "$out_jar: $resource_dir/$sample_backend/$main_lib differs from its source" + + (cd "$OUT_DIR" && sha256sum "$(basename "$out_jar")" > "$(basename "$out_jar").sha256") + echo "OK: $(basename "$out_jar") ($(du -h "$out_jar" | cut -f1); backends: ${ordered_backends[*]})" +done + +# --- 7. The default (all-platform CPU) fat jar is a release asset too ------------------ +cp "$BASE_FAT_JAR" "$OUT_DIR/" +(cd "$OUT_DIR" && sha256sum "$(basename "$BASE_FAT_JAR")" > "$(basename "$BASE_FAT_JAR").sha256") +echo "OK: $(basename "$BASE_FAT_JAR") (default CPU fat jar, copied as-is)" + +ls -lh "$OUT_DIR" diff --git a/.github/smoke-test-fatjar.ps1 b/.github/smoke-test-fatjar.ps1 new file mode 100644 index 00000000..d846a327 --- /dev/null +++ b/.github/smoke-test-fatjar.ps1 @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT + +# Smoke test for an all-backends server fat jar on a GPU-less Windows runner — +# the PowerShell analogue of smoke-test-fatjar.sh (see there for what it proves). +# +# Usage: smoke-test-fatjar.ps1 -JarDir -JarGlob -Model [-Port

] +# Server output is written to server-out.log / server-err.log in the working dir +# (uploaded by the CI job on failure). +param( + [Parameter(Mandatory = $true)][string]$JarDir, + [Parameter(Mandatory = $true)][string]$JarGlob, + [Parameter(Mandatory = $true)][string]$Model, + [int]$Port = 18080 +) +$ErrorActionPreference = 'Stop' + +function Dump-ServerLogs { + foreach ($log in 'server-out.log', 'server-err.log') { + if (Test-Path $log) { + Write-Host "--- $log (tail) ---" + Get-Content $log -Tail 50 + } + } +} + +$jars = @(Get-ChildItem -Path $JarDir -Filter $JarGlob -File) +if ($jars.Count -ne 1) { + Write-Error "expected exactly 1 jar matching $JarGlob in $JarDir, got $($jars.Count)" +} +$jar = $jars[0].FullName +if (-not (Test-Path $Model)) { Write-Error "model file missing: $Model" } +Write-Host "smoke jar: $jar" + +$proc = Start-Process java -PassThru -NoNewWindow ` + -RedirectStandardOutput server-out.log -RedirectStandardError server-err.log ` + -ArgumentList '-jar', $jar, '-m', $Model, '--host', '127.0.0.1', '--port', "$Port", '--chat-template', 'chatml' +try { + # Poll /health until 200 (model loaded); 100 x 3 s = 5 min budget. An early server + # exit (e.g. an UnsatisfiedLinkError the fallback chain failed to absorb) fails fast. + $healthy = $false + foreach ($i in 1..100) { + if ($proc.HasExited) { + Dump-ServerLogs + Write-Error "server process exited before becoming healthy (exit code $($proc.ExitCode))" + } + try { + $health = Invoke-WebRequest -Uri "http://127.0.0.1:$Port/health" -UseBasicParsing -TimeoutSec 5 + if ($health.StatusCode -eq 200) { $healthy = $true; break } + } catch { + # 503 while loading / connection refused before listening: keep polling. + } + Start-Sleep -Seconds 3 + } + if (-not $healthy) { + Dump-ServerLogs + Write-Error "/health never returned 200" + } + Write-Host "health OK" + + $body = '{"messages":[{"role":"user","content":"Say hello."}],"max_tokens":16,"temperature":0}' + $response = Invoke-RestMethod -Uri "http://127.0.0.1:$Port/v1/chat/completions" ` + -Method Post -ContentType 'application/json' -Body $body -TimeoutSec 300 + if (-not $response.choices -or $response.choices.Count -lt 1 -or -not $response.choices[0].message) { + Write-Error "malformed chat completion response: $($response | ConvertTo-Json -Depth 6 -Compress)" + } + Write-Host "chat completion OK: $($response.choices[0].message.content)" + + # The loader must have reported its backend decision (a chosen backend on a GPU + # machine, the CPU fallback on a GPU-less runner) — this pins that the manifest was + # actually read, i.e. the smoke really ran the multi-backend code path. + $selection = Select-String -Path 'server-out.log', 'server-err.log' ` + -Pattern '\[jllama\] (using native backend|no manifest backend loadable)' + if (-not $selection) { + Dump-ServerLogs + Write-Error "no backend-selection log line found - the backend manifest was not processed" + } + Write-Host "backend selection: $($selection[0].Line)" + Write-Host "smoke test PASSED" +} finally { + if (-not $proc.HasExited) { Stop-Process -Id $proc.Id -Force } +} diff --git a/.github/smoke-test-fatjar.sh b/.github/smoke-test-fatjar.sh new file mode 100755 index 00000000..30a3eb34 --- /dev/null +++ b/.github/smoke-test-fatjar.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash + +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT + +# Smoke test for an all-backends server fat jar on a GPU-less runner: +# `java -jar` must start the embedded server — with every manifest backend failing +# its load cleanly and the loader falling back to the default CPU natives — then +# answer GET /health with 200 and a POST /v1/chat/completions with a valid choice. +# This exercises manifest parsing, per-backend extraction, and the fallback chain +# end-to-end through a real fat-jar launch. +# +# Usage: smoke-test-fatjar.sh [port] +# Server output is written to server-out.log / server-err.log in the working dir +# (uploaded by the CI job on failure). +set -euo pipefail + +JAR_DIR="${1:?usage: smoke-test-fatjar.sh [port]}" +JAR_GLOB="${2:?usage: smoke-test-fatjar.sh [port]}" +MODEL="${3:?usage: smoke-test-fatjar.sh [port]}" +PORT="${4:-18080}" + +fail() { + echo "::error::$*" >&2 + exit 1 +} + +mapfile -t JARS < <(find "$JAR_DIR" -maxdepth 1 -name "$JAR_GLOB" | sort) +[ "${#JARS[@]}" -eq 1 ] || fail "expected exactly 1 jar matching $JAR_GLOB in $JAR_DIR, got ${#JARS[@]}: ${JARS[*]:-none}" +JAR="${JARS[0]}" +[ -f "$MODEL" ] || fail "model file missing: $MODEL" +echo "smoke jar: $JAR" + +java -jar "$JAR" -m "$MODEL" --host 127.0.0.1 --port "$PORT" --chat-template chatml \ + > server-out.log 2> server-err.log & +SERVER_PID=$! +cleanup() { kill "$SERVER_PID" 2> /dev/null || true; } +trap cleanup EXIT + +# Poll /health until 200 (model loaded); 100 x 3 s = 5 min budget. An early server +# exit (e.g. an UnsatisfiedLinkError the fallback chain failed to absorb) fails fast. +CODE="" +for _ in $(seq 1 100); do + if ! kill -0 "$SERVER_PID" 2> /dev/null; then + echo "--- server-out.log ---" && cat server-out.log + echo "--- server-err.log ---" && cat server-err.log + fail "server process exited before becoming healthy" + fi + CODE="$(curl -s -o /dev/null -w '%{http_code}' "http://127.0.0.1:$PORT/health" || true)" + [ "$CODE" = "200" ] && break + sleep 3 +done +if [ "$CODE" != "200" ]; then + echo "--- server-out.log (tail) ---" && tail -50 server-out.log + echo "--- server-err.log (tail) ---" && tail -50 server-err.log + fail "/health never returned 200 (last code: ${CODE:-none})" +fi +echo "health OK" + +RESPONSE="$(curl -sS --fail -X POST "http://127.0.0.1:$PORT/v1/chat/completions" \ + -H 'Content-Type: application/json' \ + -d '{"messages":[{"role":"user","content":"Say hello."}],"max_tokens":16,"temperature":0}')" \ + || fail "chat completion request failed" +echo "$RESPONSE" | python3 -c ' +import json, sys +response = json.load(sys.stdin) +message = response["choices"][0]["message"] +assert message is not None, "choices[0].message missing" +print("chat completion OK:", json.dumps(message)[:200]) +' || fail "malformed chat completion response: $RESPONSE" + +# The loader must have reported its backend decision (a chosen backend on a GPU +# machine, the CPU fallback on a GPU-less runner) — this pins that the manifest was +# actually read, i.e. the smoke really ran the multi-backend code path. +grep -hE '\[jllama\] (using native backend|no manifest backend loadable)' server-out.log server-err.log \ + || fail "no backend-selection log line found — the backend manifest was not processed" + +echo "smoke test PASSED" diff --git a/.github/validate-models.bat b/.github/validate-models.bat index 17886089..2a21c4b5 100644 --- a/.github/validate-models.bat +++ b/.github/validate-models.bat @@ -9,10 +9,28 @@ REM GGUF files start with magic bytes: 0x47 0x47 0x55 0x46 ("GGUF") setlocal enabledelayedexpansion -REM Every CI Java test job (incl. Windows) now downloads the full model set before +REM Every CI Java test job (incl. Windows) restores the shared model cache before REM validating and runs the embedding / vision / TTS integration tests, so all of REM these are REQUIRED (a missing one is a hard failure, not a silent self-skip). -set "MODELS=models\codellama-7b.Q2_K.gguf" "models\jina-reranker-v1-tiny-en-Q4_0.gguf" "models\AMD-Llama-135m-code.Q2_K.gguf" "models\Qwen3-0.6B-Q4_K_M.gguf" "models\Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" "models\nomic-embed-text-v1.5.f16.gguf" "models\SmolVLM-500M-Instruct-Q8_0.gguf" "models\mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" "models\OuteTTS-0.2-500M-Q4_K_M.gguf" "models\WavTokenizer-Large-75-F16.gguf" +REM +REM The required set comes from the single source of truth .github\models.csv +REM (filename,url per line; # comments ignored) so this gate can never drift from +REM what download-models actually fetches. The list is built UNQUOTED (no filename +REM contains spaces): an earlier hardcoded form embedded literal quotes into the +REM variable, which made the for-loop see the whole list as ONE token — `if not +REM exist` then parsed it as path-plus-command, so only a fragment was ever checked +REM and the exit /b 1 never fired (the ERROR printed but the step passed; observed +REM in run 28805360584, where an empty model cache sailed through this "gate"). +if not exist "%~dp0models.csv" ( + echo ERROR: model manifest not found: %~dp0models.csv + exit /b 1 +) +set MODELS= +for /f "usebackq eol=# tokens=1 delims=," %%N in ("%~dp0models.csv") do set "MODELS=!MODELS! models\%%N" +if "!MODELS!"=="" ( + echo ERROR: no models parsed from %~dp0models.csv + exit /b 1 +) REM No optional models remain (the audio-input model has no CI download and its REM test self-skips). Left empty so the optional loop below is a no-op. diff --git a/.github/validate-models.sh b/.github/validate-models.sh index efb081b1..bdfbbd38 100755 --- a/.github/validate-models.sh +++ b/.github/validate-models.sh @@ -10,23 +10,21 @@ set -e -# Every CI Java test job (Linux + all macOS + all Windows) now downloads the full -# model set before validating, and runs the embedding / vision / TTS integration -# tests with their properties set — so all of these are REQUIRED, not optional. A -# missing model is a hard failure here (it would otherwise let an integration test -# silently self-skip). See .github/workflows/publish.yml. -MODELS=( - "models/codellama-7b.Q2_K.gguf" - "models/jina-reranker-v1-tiny-en-Q4_0.gguf" - "models/AMD-Llama-135m-code.Q2_K.gguf" - "models/Qwen3-0.6B-Q4_K_M.gguf" - "models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" - "models/nomic-embed-text-v1.5.f16.gguf" - "models/SmolVLM-500M-Instruct-Q8_0.gguf" - "models/mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" - "models/OuteTTS-0.2-500M-Q4_K_M.gguf" - "models/WavTokenizer-Large-75-F16.gguf" -) +# Every CI Java test job (Linux + all macOS + all Windows) restores the shared model +# cache before validating, and runs the embedding / vision / TTS integration tests +# with their properties set — so all of these are REQUIRED, not optional. A missing +# model is a hard failure here (it would otherwise let an integration test silently +# self-skip). The required set comes from the single source of truth +# .github/models.csv (filename,url per line; # comments ignored) so this gate can +# never drift from what download-models actually fetches. +MODELS_CSV="$(dirname "$0")/models.csv" +[ -f "${MODELS_CSV}" ] || { echo "ERROR: model manifest not found: ${MODELS_CSV}"; exit 1; } +MODELS=() +while IFS=, read -r name _url; do + case "$name" in ''|\#*) continue ;; esac + MODELS+=("models/$name") +done < "${MODELS_CSV}" +[ "${#MODELS[@]}" -gt 0 ] || { echo "ERROR: no models parsed from ${MODELS_CSV}"; exit 1; } # Optional GGUFs validated only when present. The vision test image is committed to # src/test/resources/images/test-image.jpg and is not validated here — its presence diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6477ac03..df51366d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -21,30 +21,23 @@ on: default: true env: JAVA_VERSION: '21' - MODEL_URL: "https://huggingface.co/TheBloke/CodeLlama-7B-GGUF/resolve/main/codellama-7b.Q2_K.gguf" + # Model DOWNLOAD URLS live in .github/models.csv (the single source of truth for the + # CI model set; the model cache key is derived from that file's hash). The *_NAME + # vars below are consumer-side wiring only (-Dnet.ladenthin.llama.* test properties) + # and must match the filename column of models.csv. MODEL_NAME: "codellama-7b.Q2_K.gguf" - RERANKING_MODEL_URL: "https://huggingface.co/gpustack/jina-reranker-v1-tiny-en-GGUF/resolve/main/jina-reranker-v1-tiny-en-Q4_0.gguf" RERANKING_MODEL_NAME: "jina-reranker-v1-tiny-en-Q4_0.gguf" - DRAFT_MODEL_URL: "https://huggingface.co/QuantFactory/AMD-Llama-135m-code-GGUF/resolve/main/AMD-Llama-135m-code.Q2_K.gguf" DRAFT_MODEL_NAME: "AMD-Llama-135m-code.Q2_K.gguf" - REASONING_MODEL_URL: "https://huggingface.co/unsloth/Qwen3-0.6B-GGUF/resolve/main/Qwen3-0.6B-Q4_K_M.gguf" REASONING_MODEL_NAME: "Qwen3-0.6B-Q4_K_M.gguf" - TOOL_MODEL_URL: "https://huggingface.co/bartowski/Qwen2.5-1.5B-Instruct-GGUF/resolve/main/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" TOOL_MODEL_NAME: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf" - NOMIC_EMBED_MODEL_URL: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.f16.gguf" NOMIC_EMBED_MODEL_NAME: "nomic-embed-text-v1.5.f16.gguf" # Vision model + mmproj for MultimodalIntegrationTest. # SmolVLM-500M is the smallest community vision GGUF that loads reliably - # under the upstream mtmd pipeline. Total download ~600 MB across model - # plus mmproj; matches the existing per-test-job download budget. - VISION_MODEL_URL: "https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/SmolVLM-500M-Instruct-Q8_0.gguf" + # under the upstream mtmd pipeline. VISION_MODEL_NAME: "SmolVLM-500M-Instruct-Q8_0.gguf" - VISION_MMPROJ_URL: "https://huggingface.co/ggml-org/SmolVLM-500M-Instruct-GGUF/resolve/main/mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" VISION_MMPROJ_NAME: "mmproj-SmolVLM-500M-Instruct-Q8_0.gguf" # Text-to-speech models for AudioInputIntegrationTest's sibling TtsIntegrationTest (OuteTTS pipeline). - TTS_MODEL_URL: "https://huggingface.co/second-state/OuteTTS-0.2-500M-GGUF/resolve/main/OuteTTS-0.2-500M-Q4_K_M.gguf" TTS_MODEL_NAME: "OuteTTS-0.2-500M-Q4_K_M.gguf" - TTS_VOCODER_URL: "https://huggingface.co/ggml-org/WavTokenizer/resolve/main/WavTokenizer-Large-75-F16.gguf" TTS_VOCODER_NAME: "WavTokenizer-Large-75-F16.gguf" # Test image used by MultimodalIntegrationTest is committed to the repo # at src/test/resources/images/test-image.jpg (see the README in that @@ -71,7 +64,7 @@ jobs: # --------------------------------------------------------------------------- # Download + cache the GGUF test models ONCE, upfront, for the whole pipeline. # Every Java test job (`test-java-*`) and the langchain4j integration job `needs:` this - # job and then only RESTORES the shared cache (key gguf-models-v1) — so the ~5 GB model + # job and then only RESTORES the shared cache (key gguf-models-) — so the ~5 GB model # set is fetched from HuggingFace at most once per cache lifetime instead of racing to # download in each job. GGUF is platform-independent, so this single ubuntu job's cache # is reused by the macOS and Windows jobs too. On a warm cache this job is a no-op @@ -87,37 +80,64 @@ jobs: steps: - uses: actions/checkout@v7 - name: Cache GGUF models (GitHub Actions cache; avoids re-downloading from HuggingFace) + # This job is the ONLY writer of the model cache (every consumer job uses the + # restore-only action). enableCrossOsArchive makes the one ubuntu-built entry + # restorable on macOS AND Windows — without it, cache entries are versioned + # per-OS, and the unreachable Windows-side entry was once found re-saved EMPTY + # (343 B) after an eviction, silently starving the Windows jobs of models. uses: actions/cache@v6 with: path: models/ - # GGUF is platform-independent, so ubuntu + macOS + Windows share one entry; - # bump the suffix when the model set / URLs change. - key: gguf-models-v1 - - name: Download text generation model - run: test -f models/${MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${MODEL_URL} --create-dirs -o models/${MODEL_NAME} - - name: Download reranking model - run: test -f models/${RERANKING_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${RERANKING_MODEL_URL} --create-dirs -o models/${RERANKING_MODEL_NAME} - - name: Download draft model - run: test -f models/${DRAFT_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${DRAFT_MODEL_URL} --create-dirs -o models/${DRAFT_MODEL_NAME} - - name: Download reasoning model - run: test -f models/${REASONING_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${REASONING_MODEL_URL} --create-dirs -o models/${REASONING_MODEL_NAME} - - name: Download tool-calling model - run: test -f models/${TOOL_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${TOOL_MODEL_URL} --create-dirs -o models/${TOOL_MODEL_NAME} - - name: Download nomic embedding model (issue #98 regression) - run: test -f models/${NOMIC_EMBED_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${NOMIC_EMBED_MODEL_URL} --create-dirs -o models/${NOMIC_EMBED_MODEL_NAME} - - name: Download vision model - run: test -f models/${VISION_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${VISION_MODEL_URL} --create-dirs -o models/${VISION_MODEL_NAME} - - name: Download vision mmproj - run: test -f models/${VISION_MMPROJ_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${VISION_MMPROJ_URL} --create-dirs -o models/${VISION_MMPROJ_NAME} - - name: Download TTS model (OuteTTS) - run: test -f models/${TTS_MODEL_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${TTS_MODEL_URL} --create-dirs -o models/${TTS_MODEL_NAME} - - name: Download TTS vocoder (WavTokenizer) - run: test -f models/${TTS_VOCODER_NAME} || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors ${TTS_VOCODER_URL} --create-dirs -o models/${TTS_VOCODER_NAME} + # GGUF is platform-independent, so ubuntu + macOS + Windows share one entry. + # The key is derived from the model manifest, so EDITING models.csv + # automatically creates a fresh complete entry — no manual cache deletion. + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true + - name: Download missing models (manifest-driven, .github/models.csv) + # The manifest is the single source of truth for the model set; this loop is + # the ONLY place in CI that downloads models. Files already restored from the + # cache are skipped, so a warm cache downloads nothing. + run: | + while IFS=, read -r name url; do + case "$name" in ''|\#*) continue ;; esac + test -f "models/$name" || curl -L --proto =https --proto-redir =https --fail --retry 5 --retry-all-errors "$url" --create-dirs -o "models/$name" + done < .github/models.csv - name: List files in models directory run: ls -l models/ - name: Validate model files run: bash .github/validate-models.sh + # Prove the freshly written cache entry is restorable AND complete on every OS the + # pipeline uses BEFORE any model-consuming job starts: the same restore-only action + # the consumers use (fail-on-cache-miss makes an unrestorable entry fail here, not + # deep inside a test job), followed by the full validate gate. Every model-backed + # job `needs:` this instead of download-models directly. + verify-model-cache: + name: "Verify model cache (${{ matrix.os }})" + needs: download-models + strategy: + fail-fast: false + matrix: + include: + - os: ubuntu-latest + validate: bash .github/validate-models.sh + - os: macos-15 + validate: bash .github/validate-models.sh + - os: windows-2025-vs2026 + validate: .github\validate-models.bat + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v7 + - name: Restore shared GGUF model cache (populated by download-models; no re-download) + uses: actions/cache/restore@v6 + with: + path: models/ + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true + fail-on-cache-miss: true + - name: Validate model files + run: ${{ matrix.validate }} + # --------------------------------------------------------------------------- # Cross-compile jobs (Docker / dockcross) — produce release artifacts, no testing # --------------------------------------------------------------------------- @@ -209,7 +229,7 @@ jobs: test-java-llama-langchain4j-integration: name: Integration Test llama-langchain4j (model-backed) - needs: [crosscompile-linux-x86_64, download-models] + needs: [crosscompile-linux-x86_64, verify-model-cache] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -219,10 +239,15 @@ jobs: name: Linux-x86_64-libraries path: ${{ github.workspace }}/llama/src/main/resources/net/ladenthin/llama/ - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - uses: actions/setup-java@v5 with: distribution: 'temurin' @@ -838,7 +863,7 @@ jobs: test-android-emulator: name: Android emulator on-device test (x86_64) - needs: [crosscompile-android-aarch64, crosscompile-android-x86_64, download-models] + needs: [crosscompile-android-aarch64, crosscompile-android-x86_64, verify-model-cache] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -876,10 +901,15 @@ jobs: cp stage/cpu-x86_64/Linux-Android/x86_64/libjllama.so llama-android/natives/cpu/x86_64/ gradle -p llama-android publishLlamaAndroidPublicationToMavenLocal - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Run on-emulator instrumentation (connectedDebugAndroidTest) uses: reactivecircus/android-emulator-runner@v2 with: @@ -1798,7 +1828,7 @@ jobs: test-java-linux-x86_64: name: Java Tests Ubuntu Latest x86_64 - needs: [crosscompile-linux-x86_64, download-models] + needs: [crosscompile-linux-x86_64, verify-model-cache] runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -1817,13 +1847,18 @@ jobs: # GGUF models are downloaded + cached ONCE by the upstream `download-models` job # (this job `needs:` it), so here we only RESTORE the shared cache — no per-job # download logic. GGUF is platform-independent, so ubuntu + macOS + Windows share - # one entry (key gguf-models-v1). validate-models is kept as an integrity guard so a + # one entry (key gguf-models-). validate-models is kept as an integrity guard so a # partial/absent restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: bash .github/validate-models.sh - uses: actions/setup-java@v5 @@ -1916,7 +1951,7 @@ jobs: test-java-macos-arm64-metal: name: Java Tests macOS 14 arm64 (Metal) - needs: [build-macos-arm64-metal, download-models] + needs: [build-macos-arm64-metal, verify-model-cache] runs-on: macos-14 steps: - uses: actions/checkout@v7 @@ -1935,13 +1970,18 @@ jobs: # GGUF models are downloaded + cached ONCE by the upstream `download-models` job # (this job `needs:` it), so here we only RESTORE the shared cache — no per-job # download logic. GGUF is platform-independent, so ubuntu + macOS + Windows share - # one entry (key gguf-models-v1). validate-models is kept as an integrity guard so a + # one entry (key gguf-models-). validate-models is kept as an integrity guard so a # partial/absent restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: bash .github/validate-models.sh - uses: actions/setup-java@v5 @@ -1980,7 +2020,7 @@ jobs: test-java-macos-arm64-no-metal: name: Java Tests macOS 15 arm64 (no Metal) - needs: [build-macos-arm64-no-metal, download-models] + needs: [build-macos-arm64-no-metal, verify-model-cache] runs-on: macos-15 steps: - uses: actions/checkout@v7 @@ -1999,13 +2039,18 @@ jobs: # GGUF models are downloaded + cached ONCE by the upstream `download-models` job # (this job `needs:` it), so here we only RESTORE the shared cache — no per-job # download logic. GGUF is platform-independent, so ubuntu + macOS + Windows share - # one entry (key gguf-models-v1). validate-models is kept as an integrity guard so a + # one entry (key gguf-models-). validate-models is kept as an integrity guard so a # partial/absent restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: bash .github/validate-models.sh - uses: actions/setup-java@v5 @@ -2044,7 +2089,7 @@ jobs: test-java-macos-arm64-metal-15: name: Java Tests macOS 15 arm64 (Metal) - needs: [build-macos-arm64-metal-15, download-models] + needs: [build-macos-arm64-metal-15, verify-model-cache] runs-on: macos-15 steps: - uses: actions/checkout@v7 @@ -2063,13 +2108,18 @@ jobs: # GGUF models are downloaded + cached ONCE by the upstream `download-models` job # (this job `needs:` it), so here we only RESTORE the shared cache — no per-job # download logic. GGUF is platform-independent, so ubuntu + macOS + Windows share - # one entry (key gguf-models-v1). validate-models is kept as an integrity guard so a + # one entry (key gguf-models-). validate-models is kept as an integrity guard so a # partial/absent restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: bash .github/validate-models.sh - uses: actions/setup-java@v5 @@ -2108,7 +2158,7 @@ jobs: test-java-windows-x86_64: name: Java Tests Windows 2025 x86_64 (default / Ninja) - needs: [build-windows-x86_64, download-models] + needs: [build-windows-x86_64, verify-model-cache] runs-on: windows-2025-vs2026 steps: - uses: actions/checkout@v7 @@ -2132,10 +2182,15 @@ jobs: # download logic. validate-models is kept as an integrity guard so a partial/absent # restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: .github\validate-models.bat - uses: actions/setup-java@v5 @@ -2197,7 +2252,7 @@ jobs: # validated end-to-end before the `msvc-windows` classifier JAR ships. test-java-windows-x86_64-msvc: name: Java Tests Windows 2025 x86_64 (MSVC classifier) - needs: [build-windows-x86_64-msvc, download-models] + needs: [build-windows-x86_64-msvc, verify-model-cache] runs-on: windows-2025-vs2026 steps: - uses: actions/checkout@v7 @@ -2221,10 +2276,15 @@ jobs: # download logic. validate-models is kept as an integrity guard so a partial/absent # restore fails loudly instead of silently self-skipping required tests. - name: Restore shared GGUF model cache (populated by download-models; no re-download) - uses: actions/cache@v6 + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 with: path: models/ - key: gguf-models-v1 + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true - name: Validate model files run: .github\validate-models.bat - uses: actions/setup-java@v5 @@ -2408,8 +2468,9 @@ jobs: # `assembly` additionally produces the fat jar-with-dependencies uber JAR # (llama--jar-with-dependencies.jar: library classes + Java runtime deps + # default-platform native libs in one drop-on-classpath JAR, runnable via its - # OpenAiCompatServer Main-Class). It lands in target/ and is uploaded in the `llama-jars` - # artifact below - a CI run artifact only, not a Maven Central / GitHub-Release asset. + # ServerLauncher Main-Class). It lands in target/ and is uploaded in the `llama-jars` + # artifact below - never a Maven Central asset; the `package-fatjars` job downstream + # combines it with the classifier jars into the GitHub-Release fat-jar assets. # Windows classifier JARs: `windows-msvc` (MSVC-built CPU natives) plus the GPU # backends `cuda-windows` / `vulkan-windows` / `opencl-windows`. The default JAR's # Windows natives are the Ninja `*-libraries` merged into src/main/resources/ above. @@ -2420,6 +2481,135 @@ jobs: name: llama-jars path: llama/target/*.jar + package-fatjars: + name: Package all-backends fat jars + needs: [package] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: llama-jars + path: jars/ + # One self-contained multi-backend server fat jar per OS/arch + # (llama--all---jar-with-dependencies.jar): the default + # jar-with-dependencies plus every GPU backend of that OS/arch in backend + # subdirectories, described by the jllama-backends.txt priority manifest that + # LlamaLoader reads at runtime (first loadable backend wins, CPU fallback). + # These are GitHub-Release download assets ONLY - never deployed to Maven + # Central (the deploy jobs run without the `assembly` profile and are untouched). + # The script enumerates the classifiers from llama/pom.xml and fails loud on any + # mismatch with the built classifier jars, so a new classifier cannot be silently + # skipped. + - name: Assemble all-backends fat jars + run: bash .github/package-fatjars.sh jars fatjars llama/pom.xml + - name: Upload fat jars + uses: actions/upload-artifact@v7 + with: + name: llama-fatjars + path: fatjars/ + compression-level: 0 # jars are already deflated + retention-days: 7 # multi-GB artifact; release jobs consume it within the same run + if-no-files-found: error + # Small single-jar artifacts so the smoke jobs don't download the multi-GB set. + - name: Upload Linux smoke jar + uses: actions/upload-artifact@v7 + with: + name: llama-fatjar-smoke-linux + path: fatjars/llama-*-all-linux-x86-64-jar-with-dependencies.jar + compression-level: 0 + retention-days: 7 + if-no-files-found: error + - name: Upload Windows smoke jar + uses: actions/upload-artifact@v7 + with: + name: llama-fatjar-smoke-windows + path: fatjars/llama-*-all-windows-x86-64-jar-with-dependencies.jar + compression-level: 0 + retention-days: 7 + if-no-files-found: error + + # GPU-less runners: every manifest backend must fail its load cleanly (missing vendor + # runtimes) and the server must come up on the default CPU natives — this exercises + # manifest parsing, per-backend extraction, and the fallback chain end-to-end through + # a real `java -jar`, then proves the server serves /health + /v1/chat/completions. + smoke-fatjar-linux: + name: Smoke test all-backends fat jar (Linux) + needs: [package-fatjars, verify-model-cache] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: llama-fatjar-smoke-linux + path: fatjars/ + - name: Restore shared GGUF model cache (populated by download-models; no re-download) + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 + with: + path: models/ + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true + - name: Validate model files + run: .github/validate-models.sh + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ env.JAVA_VERSION }} + - name: Run fat-jar server smoke test + run: .github/smoke-test-fatjar.sh fatjars 'llama-*-all-linux-x86-64-jar-with-dependencies.jar' "models/${DRAFT_MODEL_NAME}" + - name: Upload server logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fatjar-smoke-linux-logs + path: | + server-out.log + server-err.log + if-no-files-found: warn + + smoke-fatjar-windows: + name: Smoke test all-backends fat jar (Windows) + needs: [package-fatjars, verify-model-cache] + runs-on: windows-2025-vs2026 + steps: + - uses: actions/checkout@v7 + - uses: actions/download-artifact@v8 + with: + name: llama-fatjar-smoke-windows + path: fatjars/ + - name: Restore shared GGUF model cache (populated by download-models; no re-download) + # Restore-only: consumer jobs can NEVER write the cache, so a job running on a + # cache miss cannot re-save an empty/partial entry under the immutable key. + # download-models is the single writer; enableCrossOsArchive matches its + # cross-OS entry version so Windows/macOS restore the same ubuntu-built entry. + uses: actions/cache/restore@v6 + with: + path: models/ + key: gguf-models-${{ hashFiles('.github/models.csv') }} + enableCrossOsArchive: true + - name: Validate model files + run: .github\validate-models.bat + - uses: actions/setup-java@v5 + with: + distribution: 'temurin' + java-version: ${{ env.JAVA_VERSION }} + - name: Run fat-jar server smoke test + shell: pwsh + run: .github/smoke-test-fatjar.ps1 -JarDir fatjars -JarGlob 'llama-*-all-windows-x86-64-jar-with-dependencies.jar' -Model "models/$env:DRAFT_MODEL_NAME" + - name: Upload server logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fatjar-smoke-windows-logs + path: | + server-out.log + server-err.log + if-no-files-found: warn + report: name: Report needs: [package] @@ -2470,7 +2660,7 @@ jobs: publish-snapshot: name: Publish Snapshot to Central - needs: [check-snapshot, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin] + needs: [check-snapshot, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin, package-fatjars, smoke-fatjar-linux, smoke-fatjar-windows] if: needs.check-snapshot.result == 'success' && inputs.publish_to_central runs-on: ubuntu-latest environment: maven-central @@ -2618,8 +2808,8 @@ jobs: github-snapshot: name: Update Snapshot Pre-release on GitHub - needs: [publish-snapshot] - if: needs.publish-snapshot.result == 'success' + needs: [publish-snapshot, package-fatjars] + if: needs.publish-snapshot.result == 'success' && needs.package-fatjars.result == 'success' runs-on: ubuntu-latest permissions: contents: write @@ -2628,6 +2818,14 @@ jobs: with: name: signed-snapshot-assets path: snapshot-assets/ + # All-backends server fat jars (+ default CPU fat jar + sha256 files) — GitHub + # download assets only, deliberately NOT deployed to Maven Central. Downloaded + # into the same directory so the one upload glob below picks everything up + # (fat-jar names are disjoint from the signed thin-jar names). + - uses: actions/download-artifact@v8 + with: + name: llama-fatjars + path: snapshot-assets/ - name: Update snapshot pre-release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -2645,7 +2843,7 @@ jobs: publish-release: name: Publish Release to Central if: needs.check-tag.result == 'success' && inputs.publish_to_central - needs: [check-tag, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin] + needs: [check-tag, crosscompile-linux-x86_64-cuda, crosscompile-android-aarch64-opencl, package-android-aar, test-android-emulator, code-style, test-java-llama-langchain4j, test-java-llama-kotlin, package-fatjars, smoke-fatjar-linux, smoke-fatjar-windows] runs-on: ubuntu-latest environment: maven-central permissions: @@ -2793,8 +2991,8 @@ jobs: github-release-signed: name: Attach Signed Binaries to GitHub Release - needs: [publish-release] - if: needs.publish-release.result == 'success' + needs: [publish-release, package-fatjars] + if: needs.publish-release.result == 'success' && needs.package-fatjars.result == 'success' runs-on: ubuntu-latest permissions: contents: write @@ -2803,6 +3001,14 @@ jobs: with: name: signed-release-assets path: release-assets/ + # All-backends server fat jars (+ default CPU fat jar + sha256 files) — GitHub + # download assets only, deliberately NOT deployed to Maven Central. Downloaded + # into the same directory so the one upload glob below picks everything up + # (fat-jar names are disjoint from the signed thin-jar names). + - uses: actions/download-artifact@v8 + with: + name: llama-fatjars + path: release-assets/ - name: Upload release assets uses: softprops/action-gh-release@v3 with: diff --git a/CLAUDE.md b/CLAUDE.md index 3ee87ab2..a5f17094 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -6,7 +6,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co Java bindings for [llama.cpp](https://github.com/ggerganov/llama.cpp) via JNI, providing a high-level API for LLM inference in Java. The Java layer communicates with a native C++ library through JNI. -Current llama.cpp pinned version: **b9886** +Current llama.cpp pinned version: **b9894** ## Upgrading CUDA Version @@ -175,7 +175,7 @@ build is shipped as the **`msvc-windows`** classifier; and three GPU backends sh `CMAKE_{C,CXX}_COMPILER_LAUNCHER`, so only Ninja Multi-Config can front `cl.exe` with sccache over Depot WebDAV. **Both generators use the same MSVC toolchain** (`cl.exe`, static `/MT` CRT via `CMAKE_MSVC_RUNTIME_LIBRARY`, same Release flags, same runner), so the produced -`jllama.dll`/`llama.dll`/`ggml.dll` are **functionally equivalent with identical runtime +`jllama.dll` binaries are **functionally equivalent with identical runtime dependencies** — the only difference is build-system plumbing + caching. Making Ninja the default gives the most-pulled JAR the sccache cache; MSVC stays available as a classifier for anyone who wants the Visual-Studio-generator build. (Upstream llama.cpp also builds its Windows artifacts with @@ -183,8 +183,8 @@ Ninja Multi-Config + MSVC.) Both Windows CPU builds are validated end-to-end wit model-backed Java suite (`test-java-windows-x86_64` = default/Ninja, `test-java-windows-x86_64-msvc` = MSVC classifier). -**GPU runtime libraries are NOT bundled.** The GPU JARs ship only `jllama.dll`/`llama.dll`/`ggml.dll` -(plus the embedded backend). The consumer's driver/toolkit must supply the runtime: CUDA needs the +**GPU runtime libraries are NOT bundled.** The GPU JARs ship only the single monolithic +`jllama.dll` (llama.cpp + ggml + the backend are statically linked in — `BUILD_SHARED_LIBS OFF`). The consumer's driver/toolkit must supply the runtime: CUDA needs the installed CUDA 13 Toolkit (`cudart64_13.dll`/`cublas64_13.dll`/`cublasLt64_13.dll` on `PATH`); Vulkan needs `vulkan-1.dll` (ships with current GPU drivers); OpenCL needs the vendor ICD (`System32\OpenCL.dll`). Not bundling = no NVIDIA-EULA redistribution obligation. **GitHub-hosted @@ -343,6 +343,51 @@ pinned to a specific version): if a URL/version 404s in CI, the job fails loud a `src/main/resources_{linux_rocm,windows_rocm,linux_sycl_fp16,linux_sycl_fp32,windows_sycl,linux_openvino,windows_openvino}/` are all git-ignored (staged by CI, never committed). +## All-backends server fat jars (GitHub Release assets, never Maven Central) + +Every pipeline run assembles **per-OS multi-backend server fat jars** and, on the release +paths, attaches them to GitHub: `llama--all---jar-with-dependencies.jar` +for `linux-x86-64`, `linux-aarch64`, `windows-x86-64`, `windows-aarch64`, plus the default +CPU fat jar — each with a `.sha256` file. They are **download assets only**: the Central +deploy invocations run without the `assembly` profile and are untouched. + +Mechanism (three pieces): + +1. **`.github/package-fatjars.sh`** — run by the `package-fatjars` job (`needs: [package]`, + downloads `llama-jars`). Enumerates the `` set from `llama/pom.xml` (source of + truth) and cross-checks it in **both** directions against the built classifier jars; parses + each classifier as `--`; **fails loud** on unparseable classifiers, + backends missing from its priority table, missing native trees, or zip-update corruption + (entry list, `Main-Class`, sample byte-compare). Excluded by design: `msvc-windows` + (redundant CPU variant) and `opencl-android-aarch64` (no `java -jar` on Android). For each + OS/arch it copies the default fat jar and adds every backend's native tree under + `net/ladenthin/llama////` plus a **`jllama-backends.txt`** manifest + (backends in priority order `cuda13 rocm sycl-fp16 sycl-fp32 sycl vulkan opencl openvino`; + extra tokens per line list sibling files such as openvino-windows' bundled `OpenCL.dll`). + **A new classifier fails this script until it is consciously ranked/excluded** — that is the + no-silent-gaps guarantee. +2. **`LlamaLoader` backend selection** — when (and only when) the manifest resource exists, + the loader tries each backend subdirectory in order: extract into a per-backend temp subdir + (`jllama-backend-/`; backends share file names), load manifest extras first, then the + backend's `jllama` library. A load failure (missing vendor runtime → `UnsatisfiedLinkError`) + moves to the next backend; after the list it falls back to the default CPU natives. System + property `net.ladenthin.llama.backend` forces one backend (fail-loud) or `default`/`cpu`. + Jars without a manifest take the unchanged legacy path. A backend whose extra module is + already resident from a previously failed attempt is skipped (by-name import cross-wiring). +3. **`publish.yml` wiring** — `smoke-fatjar-linux` / `smoke-fatjar-windows` run the + `all--x86-64` jar via real `java -jar` on GPU-less runners (cached draft model, + `--chat-template chatml`): poll `/health` to 200, assert a `/v1/chat/completions` choice, + and require the loader's backend-selection log line. `publish-snapshot`/`publish-release` + `need` `package-fatjars` + both smokes (fail-loud gating); `github-release-signed` and + `github-snapshot` additionally download `llama-fatjars` into their asset directory so the + fat jars land on the tag release and the rolling `snapshot` pre-release. + +A backend loading successfully but finding **zero usable devices** (e.g. CUDA toolkit +installed, no NVIDIA GPU) is benign: ggml's backend registry contributes no devices and +inference runs on CPU inside that library. The known trade-off is that such a host never +reaches a *different* GPU backend later in the list — the `net.ladenthin.llama.backend` +override is the escape hatch (documented in the README table). + ## WebUI (llama.cpp Svelte UI) embedding The llama.cpp WebUI is **built once in CI and shared to every native build**, then @@ -376,7 +421,7 @@ needs no extra step here, `build-webui` re-reads the tag and rebuilds the matchi ships no UI): ```bash # needs node/npm + network; embed.cpp is plain C++17 (no npm) -git clone --depth 1 --branch b9886 https://github.com/ggml-org/llama.cpp /tmp/lc +git clone --depth 1 --branch b9894 https://github.com/ggml-org/llama.cpp /tmp/lc ( cd /tmp/lc/tools/ui && npm ci && npm run build \ && ( cd dist && find . -type f -not -path './_gzip/*' \ | while read -r f; do mkdir -p "_gzip/$(dirname "$f")"; gzip -9 -c "$f" > "_gzip/$f"; done ) \ @@ -416,7 +461,7 @@ cache lives in **Depot Cache** over sccache's **WebDAV** backend: - `SCCACHE_WEBDAV_TOKEN: ${{ secrets.DEPOT_TOKEN }}` — a Depot **organization** token, stored as the repo secret **`DEPOT_TOKEN`**. -Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b9886`), the +Because `sccache` is **content-addressed** and llama.cpp is pinned (`GIT_TAG b9894`), the ~280 upstream object files are byte-identical every run, so a warm cache recompiles only the *changed* files. Depot's cache is **shared across all branches** (unlike GitHub's per-branch `actions/cache`), so every branch builds incrementally; a `b` version bump @@ -728,11 +773,16 @@ folder name is produced on both ends. | Linux aarch64 | `libjllama.so` | `src/main/resources/net/ladenthin/llama/Linux/aarch64/` | | macOS Apple Silicon | `libjllama.dylib` | `src/main/resources/net/ladenthin/llama/Mac/aarch64/` | | macOS Intel | `libjllama.dylib` | `src/main/resources/net/ladenthin/llama/Mac/x86_64/` | -| Windows x86_64 | `jllama.dll` (+ `llama.dll`, `ggml.dll`) | `src/main/resources/net/ladenthin/llama/Windows/x86_64/` | +| Windows x86_64 | `jllama.dll` | `src/main/resources/net/ladenthin/llama/Windows/x86_64/` | -The Windows `RUNTIME_OUTPUT_DIRECTORY_*` properties (`CMakeLists.txt:266-269`) -deposit `jllama.dll` alongside the upstream `llama.dll` / `ggml.dll`; all -three must remain co-located so the loader can resolve transitive imports. +On every platform exactly **one** `jllama` library is produced: `CMakeLists.txt` forces +`BUILD_SHARED_LIBS OFF`, so upstream `llama` and `ggml` are static libraries linked into +`jllama` (the `RUNTIME_OUTPUT_DIRECTORY_*` block that also names the `llama`/`ggml` targets +is a no-op for them — verified against the published 5.0.5 jars, which contain only +`jllama.dll` per Windows arch). `LlamaLoader` accordingly extracts and loads a single file +from the jar (plus `ggml-metal.metal` on macOS). Historical note: upstream kherud once +shipped split `ggml` + `jllama` libraries, which is where stale "three co-located DLLs" +claims came from. End-to-end local workflow for running Java tests: @@ -1111,18 +1161,29 @@ these as **required** (a missing model hard-fails the job before tests run, so a regression can never silently downgrade to a skip). The only model still self-skipping is the audio-input model (`AudioInputIntegrationTest`) — the prompt clip is committed (`src/test/resources/audios/sample.wav`) but the audio model + mmproj have no CI download. -The shared GGUF cache (`actions/cache`, key `gguf-models-v1`, path `models/`) holds the full set -and is populated **once, upfront** by a dedicated **`download-models`** job (`needs: startgate`): -it is the single place the ~5 GB set is fetched from HuggingFace (the ten `curl` steps + the -`validate-models.sh` gate live only there). Every `test-java-*` job — and the langchain4j -integration job — `needs: download-models` and then only **restores** that cache (no per-job -download, no cold-start save race), keeping `validate-models.{sh,bat}` as a per-job integrity -guard. GGUF is platform-independent, so the one ubuntu `download-models` cache is reused by the -macOS and Windows jobs too. `validate-models.{sh,bat}` treats the models as **required** (a -missing model hard-fails the job before tests run). Because the cache key is immutable, changing -the model set means the **existing cache entry must be deleted** (not bumped to `v2`) so -`download-models` rebuilds it complete — locally the model tests still self-skip when a GGUF is -absent (`Assume.assumeTrue`), so a partial local checkout is fine. +The model set has a **single source of truth: `.github/models.csv`** (one `filename,url` row per +model; `#` comments). Everything derives from it: the **`download-models`** job (ubuntu, +`needs: startgate`) is the only place models are fetched from HuggingFace (one manifest-driven +`curl` loop; files already restored from cache are skipped) and the **only writer** of the shared +GGUF cache (path `models/`, key **`gguf-models-`** — so *editing the manifest +automatically creates a fresh complete cache entry*; no manual cache deletion on a model-set +change). The writer sets **`enableCrossOsArchive: true`**, making the one ubuntu-built entry the +same entry macOS and Windows restore. A **`verify-model-cache` matrix job** (ubuntu / macOS / +Windows, `needs: download-models`) then proves the entry is restorable +(`actions/cache/restore` + `fail-on-cache-miss: true`) **and complete** +(`validate-models.{sh,bat}`, which read their required list from the same manifest) on every OS +**before any model-consuming job starts** — all `test-java-*` jobs, the langchain4j integration +job, the Android emulator job and the fat-jar smoke jobs `need: verify-model-cache` and use the +**restore-only** action themselves (no per-job download, no save — a consumer can never re-save +an empty/partial entry), keeping validate as a per-job integrity guard. (This design hardened +after run 28805360584: without the cross-OS flag, cache entries are versioned per-OS and the +unreachable Windows-side entry had been re-saved **empty** (343 B) after an eviction; and +`validate-models.bat`'s quoted `MODELS` list broke cmd's for-tokenization so its `exit /b 1` +never fired — the empty cache sailed through the "gate" and the Windows jobs silently +self-skipped every model-backed test until the fat-jar smoke's hard check caught it.) The +`*_MODEL_NAME` workflow env vars remain consumer-side wiring for the `-Dnet.ladenthin.llama.*` +test properties and must match the manifest's filename column — locally the model tests still +self-skip when a GGUF is absent (`Assume.assumeTrue`), so a partial local checkout is fine. Set the model path via system property or environment variable (see test files for exact property names). @@ -1167,7 +1228,7 @@ ctest --test-dir build --output-on-failure -R "ResultsToJson" #### Upstream source location (in CMake build tree) -llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b9886`. +llama.cpp is fetched via CMake FetchContent, pinned to `GIT_TAG b9894`. **GoogleTest** is a separate `BUILD_TESTING`-only FetchContent (`GIT_TAG v1.17.0`), used solely by the `jllama_test` C++ unit-test binary — not by the shipped library, and not coupled to the diff --git a/README.md b/README.md index fbb9f590..e4200b69 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ **Build:** ![Java 8+](https://img.shields.io/badge/Java-8%2B-informational) ![Platform](https://img.shields.io/badge/Platform-Linux%20%7C%20macOS%20%7C%20Windows%20%7C%20Android-lightgrey) -[![llama.cpp b9886](https://img.shields.io/badge/llama.cpp-%23b9886-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b9886) +[![llama.cpp b9894](https://img.shields.io/badge/llama.cpp-%23b9894-informational)](https://github.com/ggml-org/llama.cpp/releases/tag/b9894) [![JPMS](https://img.shields.io/badge/JPMS-modular%20JAR-25A162)](https://openjdk.org/projects/jigsaw/) ![JUnit](https://img.shields.io/badge/tested%20with-JUnit6-25A162) [![JSpecify](https://img.shields.io/badge/JSpecify-1.0.0%20%40NullMarked-25A162)](https://jspecify.dev) @@ -254,6 +254,40 @@ there. Pick **at most one** classifier (they are mutually exclusive): > The minimum required Android version is **API 28 (Android 9.0 Pie)**. > Devices running Android 8.1 (API 27) or earlier are not supported. +### Standalone server fat jars (GitHub Releases) + +For running the [embedded server](#native-server-with-the-built-in-webui-nativeserver) +without any Maven setup, every tagged [GitHub Release](https://github.com/bernardladenthin/java-llama.cpp/releases) +(and the rolling `snapshot` pre-release from `main`) attaches self-contained +**all-backends fat jars** — one download per OS/arch, runnable directly: + +```bash +java -jar llama--all-linux-x86-64-jar-with-dependencies.jar -m model.gguf --port 8080 +``` + +| Release asset | Bundled GPU backends | CPU fallback | +|---|---|---| +| `llama--all-linux-x86-64-jar-with-dependencies.jar` | CUDA 13, ROCm, SYCL fp16/fp32, Vulkan, OpenVINO | yes | +| `llama--all-linux-aarch64-jar-with-dependencies.jar` | Vulkan | yes | +| `llama--all-windows-x86-64-jar-with-dependencies.jar` | CUDA 13, ROCm, SYCL, Vulkan, OpenCL, OpenVINO | yes | +| `llama--all-windows-aarch64-jar-with-dependencies.jar` | OpenCL (Adreno / Snapdragon X) | yes | +| `llama--jar-with-dependencies.jar` | none (CPU only, incl. macOS Metal) | — | + +Each all-backends jar contains the library classes, all Java runtime +dependencies, the default CPU natives for **every** platform, and every GPU +backend for its named OS/arch. At startup the loader tries the bundled backends +in priority order (CUDA → ROCm → SYCL → Vulkan → OpenCL → OpenVINO) and uses +the **first one whose native library loads**; if none loads — e.g. no GPU +driver/toolkit installed — it falls back to the CPU natives, so the jar starts +everywhere. The usual GPU policy applies: vendor runtimes are **not** bundled +(see the classifier table above for what each backend needs on the host). +Force a specific backend with `-Dnet.ladenthin.llama.backend=` +(e.g. `vulkan`; fails loud instead of falling back) or force CPU with +`-Dnet.ladenthin.llama.backend=default`. A `.sha256` checksum file accompanies +every jar. These fat jars are GitHub download assets only — they are **not** +published to Maven Central (Maven users combine the thin classifier jars +instead, see above). + ### Setup required If none of the above listed platforms matches yours, currently you have to compile the library yourself (also if you @@ -308,6 +342,7 @@ Every `net.ladenthin.llama.*` system property recognised by the library, deep-sc | `net.ladenthin.llama.lib.path` | unset (falls back to `java.library.path`) | runtime | `LlamaLoader` | Directory containing the native `jllama` shared library. Checked first, before `java.library.path`. Set with `-Dnet.ladenthin.llama.lib.path=/path/to/dir`. | | `net.ladenthin.llama.tmpdir` | unset (falls back to `java.io.tmpdir`) | runtime | `LlamaLoader` | Custom temporary directory used when extracting the native library from the JAR. | | `net.ladenthin.llama.osinfo.architecture` | unset (uses `os.arch`) | runtime | `OSInfo` | Override for the architecture string used to locate the bundled library inside the JAR. Useful when `os.arch` reports an unexpected value (e.g. inside dockcross / chrooted environments). | +| `net.ladenthin.llama.backend` | unset (auto: first loadable backend, then CPU) | runtime | `LlamaLoader` | Backend override for the [all-backends fat jars](#standalone-server-fat-jars-github-releases) (jars carrying a `jllama-backends.txt` manifest). Names one bundled backend (e.g. `cuda13`, `vulkan`) to load exclusively — failure is then fatal instead of falling back — or `default`/`cpu` to skip all GPU backends. Ignored by jars without a backend manifest. | | `net.ladenthin.llama.test.ngl` | `43` for the general suite; `0` for `ToolCallingIntegrationTest` | test | Model-backed integration tests | Number of GPU layers used during testing. Pin to `0` on CPU-only hosts: `mvn test -Dnet.ladenthin.llama.test.ngl=0`. The tool test also selects device `none` at zero layers so Metal/CUDA is not initialized. | | `net.ladenthin.llama.tool.model` | `models/Qwen2.5-1.5B-Instruct-Q4_K_M.gguf` (test self-skips if missing) | test | `ToolCallingIntegrationTest` | Path to a tool-capable GGUF used to verify required blocking and streaming tool calls. The default matches the Qwen2.5 model in upstream llama.cpp's tool-call test matrix. | | `net.ladenthin.llama.nomic.path` | unset (test self-skips) | test | `LlamaEmbeddingsTest#testNomicEmbedLoads` | Path to a Nomic embedding model (`nomic-embed-text-v1.5.f16.gguf` or a compatible BERT-family encoder). Regression test for upstream issue #98 (BERT-encoder `result_output` assertion). | diff --git a/REUSE.toml b/REUSE.toml index 56bccff6..494c010c 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -98,6 +98,17 @@ path = "llama/src/test/resources/audios/sample.wav" SPDX-FileCopyrightText = "2026 Bernard Ladenthin " SPDX-License-Identifier = "MIT" +# Backend-manifest loader test fixtures (backendtest/ + backendtest-ok/): +# deliberately unloadable fake *.so placeholders (one-line text files) plus two +# trivial real x86-64 ELF dummies in backendtest-ok/ (built from a one-line C +# TU, no upstream code) — .so files have no inline-comment syntax for SPDX +# headers; the manifest .txt files carry theirs inline. Consumed by +# BackendManifestLoadTest. +[[annotations]] +path = "llama/src/test/resources/net/ladenthin/llama/Linux/backendtest*/**" +SPDX-FileCopyrightText = "2026 Bernard Ladenthin " +SPDX-License-Identifier = "MIT" + # JNI header files from Oracle (GPL-2.0-only WITH Classpath-exception-2.0) [[annotations]] path = [ diff --git a/docs/history/llama-cpp-breaking-changes.md b/docs/history/llama-cpp-breaking-changes.md index 9aea11ab..fecd6fa9 100644 --- a/docs/history/llama-cpp-breaking-changes.md +++ b/docs/history/llama-cpp-breaking-changes.md @@ -431,3 +431,7 @@ Used during `llama.cpp` version bumps: when upgrading, scan this file from the r | b9876–b9878 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9878: applied in filename order onto a clean b9878 checkout, all clean. The range touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged). Full build + `ctest` to be confirmed by the CI pipeline. | | b9878–b9886 | `ggml/src/ggml-cpu/{arch/arm/quants.c,llamafile/sgemm.cpp,simd-mappings.h}` + `ggml/src/ggml-cuda/conv-transpose-1d.cu` + `ggml/src/ggml-hip/CMakeLists.txt` + `ggml/src/ggml-vulkan/ggml-vulkan.cpp` + `scripts/ui-assets.cmake` + `tools/ui/**` | Internal-only, no API surface (12 files, ~12.4 KiB), ggml + WebUI only. **(1)** ARM NVFP4 dot product switches to the shared UE4M3 LUT (`quants.c`/`simd-mappings.h`); **(2)** tiled matmul enabled on AIX (`sgemm.cpp` — irrelevant to our targets); **(3)** CUDA `conv_transpose_1d` indexing optimized (`.cu`, CUDA classifiers only); **(4)** Vulkan `CEIL_DIV` 32-bit overflow fix; **(5)** HIP builds add `-ffast-math` (ROCm classifiers only); **(6)** `scripts/ui-assets.cmake` uses `HF_TOKEN` when downloading UI assets — not used by this repo's `build-webui` (it builds the Svelte UI from source); **(7)** `tools/ui/**` WebUI fixes (Ctrl+B sidebar shortcut, MCP proxy DELETE handling — auto-followed by `build-webui`). All inside upstream-compiled TUs or non-shipped tooling; no project source changes required. | | b9878–b9886 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9886: applied in filename order onto a clean b9886 checkout, all clean. The range touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged). Full build + `ctest` to be confirmed by the CI pipeline. | +| b9886–b9888 | `ggml/src/ggml-cuda/fattn.cu` + `tools/server/tests/unit/test_router.py` | Internal-only, no API surface (2 files, ~2.4 KiB). **(1)** CUDA flash attention extends its K-type validation to V-types (upstream #24403) — inside the upstream-compiled CUDA TU, CUDA classifiers only; **(2)** an upstream router python test gains two assertions (not compiled/shipped here). All eight priority headers byte-identical across the range; no project source changes required. | +| b9886–b9888 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9888: applied in filename order onto a clean b9888 checkout, all clean. The range touches **no** patch-target file and **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged). Full build + `ctest` to be confirmed by the CI pipeline. | +| b9888–b9894 | `ggml/src/ggml-cuda/**` + `ggml/src/ggml-opencl/**` + `ggml/src/ggml-metal/**` + `ggml/src/ggml-vulkan/ggml-vulkan.cpp` + `common/common.cpp` + `src/llama-model.cpp` + `tools/server/server-models.cpp` | Internal-only, no API surface (26 files, ~340 KiB — over the 100 KiB chunk threshold, but the bulk is GPU-backend kernel code inside upstream-compiled TUs: a large CUDA dequantize/convert refactor, new/extended OpenCL flash-attention + `mul_mv_f16_f32_l4` kernels with Adreno tuning, new Metal ops, a Vulkan `GGML_OP_SET_ROWS` f16 guard). CPU-side: `common_cpu_get_num_physical_cores()` gains AIX/PowerPC detection (`common/common.cpp`, irrelevant to our targets); the router fixes a download-thread join deadlock in `server_models::load_models` (`server-models.cpp` — different region than patch `0008`, which still applies). All **eight** priority headers byte-identical across the range; no project source changes required. | +| b9888–b9894 | upstream verification (sandbox) | All **eight** patches (`0001`–`0008`) re-verified against b9894: applied in filename order onto a clean b9894 checkout, all clean (incl. `0008` despite the `server-models.cpp` change in the range — different function). The range touches **no** OuteTTS generator anchor (`tools/tts/tts.cpp` unchanged) and no `test-arg-parser.cpp`. Full build + `ctest` to be confirmed by the CI pipeline. | diff --git a/llama/CMakeLists.txt b/llama/CMakeLists.txt index 602b9931..6a126458 100644 --- a/llama/CMakeLists.txt +++ b/llama/CMakeLists.txt @@ -174,7 +174,7 @@ set(LLAMA_BUILD_APP OFF CACHE BOOL "" FORCE) FetchContent_Declare( llama.cpp GIT_REPOSITORY https://github.com/ggerganov/llama.cpp.git - GIT_TAG b9886 + GIT_TAG b9894 PATCH_COMMAND ${CMAKE_COMMAND} -DPATCH_DIR=${CMAKE_CURRENT_SOURCE_DIR}/patches -DLLAMA_SRC= @@ -197,7 +197,7 @@ execute_process( COMMAND ${CMAKE_COMMAND} -DTTS_SRC=${llama.cpp_SOURCE_DIR}/tools/tts/tts.cpp -DOUT_CPP=${JLLAMA_TTS_GEN_CPP} - -DLLAMA_TAG=b9886 + -DLLAMA_TAG=b9894 -P ${CMAKE_CURRENT_SOURCE_DIR}/cmake/generate-tts-upstream.cmake RESULT_VARIABLE JLLAMA_TTS_GEN_RESULT ) diff --git a/llama/spotbugs-exclude.xml b/llama/spotbugs-exclude.xml index 2fc0dffa..02e3dcbe 100644 --- a/llama/spotbugs-exclude.xml +++ b/llama/spotbugs-exclude.xml @@ -767,4 +767,30 @@ SPDX-License-Identifier: MIT + + + + + + + + + + + + + + diff --git a/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java b/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java index 96121e19..f9ca7d21 100644 --- a/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java +++ b/llama/src/main/java/net/ladenthin/llama/loader/LlamaLoader.java @@ -6,15 +6,23 @@ package net.ladenthin.llama.loader; import java.io.BufferedInputStream; +import java.io.BufferedReader; import java.io.File; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.LinkOption; import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.StandardCopyOption; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Stream; import lombok.ToString; import org.jspecify.annotations.Nullable; @@ -67,6 +75,47 @@ public class LlamaLoader { */ private static final String NATIVE_RESOURCE_BASE = "/net/ladenthin/llama"; + /** + * File name of the optional multi-backend manifest located next to the native libraries + * ({@code net/ladenthin/llama///jllama-backends.txt}). Only the multi-backend + * ("all") fat jars assembled by the release pipeline carry it; jars without it behave + * exactly as before. Format: one backend per line in priority order — the backend + * subdirectory name optionally followed by whitespace-separated extra files to extract and + * load before the main library; blank lines and {@code #} comments are skipped. + */ + static final String BACKEND_MANIFEST_FILE = "jllama-backends.txt"; + + /** + * Prefix of the per-backend extraction subdirectory below the temp dir. Deliberately starts + * with {@code jllama} so {@link #shouldCleanPath(Path)} matches it during cleanup. + */ + static final String BACKEND_TEMP_DIR_PREFIX = "jllama-backend-"; + + /** Special {@code net.ladenthin.llama.backend} value selecting the default (CPU) library. */ + static final String BACKEND_DEFAULT = "default"; + + /** Alias of {@link #BACKEND_DEFAULT}. */ + static final String BACKEND_CPU = "cpu"; + + /** + * One entry of the multi-backend manifest: a backend subdirectory below the OS/arch native + * resource folder, plus the extra files (e.g. a bundled ICD loader) to extract and load + * before the main {@code jllama} library, in listed order. + */ + static final class BackendEntry { + + /** Backend subdirectory name below the OS/arch native resource folder. */ + final String name; + + /** Extra files to extract and load before the main library, in order; may be empty. */ + final List extraFiles; + + BackendEntry(String name, List extraFiles) { + this.name = name; + this.extraFiles = Collections.unmodifiableList(new ArrayList<>(extraFiles)); + } + } + /** Static utility holder; not instantiable. */ private LlamaLoader() {} @@ -110,13 +159,22 @@ static boolean shouldCleanPath(Path path) { return false; } String fileName = fileNamePath.toString(); - return fileName.startsWith("jllama") || fileName.startsWith("llama"); + // "ggml" covers the ggml-metal.metal file that initialize() extracts on macOS — it was + // never matched here, so a stale extracted copy accumulated in the temp dir forever. + return fileName.startsWith("jllama") || fileName.startsWith("llama") || fileName.startsWith("ggml"); } private static void cleanPath(Path path) { try { + // Backend extractions live in per-backend subdirectories (BACKEND_TEMP_DIR_PREFIX), + // so directories are cleaned recursively; each delete stays individually best-effort. + if (Files.isDirectory(path, LinkOption.NOFOLLOW_LINKS)) { + try (Stream entries = Files.list(path)) { + entries.forEach(LlamaLoader::cleanPath); + } + } Files.delete(path); - } catch (Exception e) { + } catch (IOException | RuntimeException e) { System.err.println("Failed to delete old native lib: " + e.getMessage()); } } @@ -168,8 +226,38 @@ private static void loadNativeLibrary(String name) { } } + // Multi-backend ("all") fat jars carry a backend manifest next to their native + // libraries. Try each listed backend subdirectory in priority order; the first one + // whose library loads wins (a missing vendor runtime fails its load cleanly, and the + // next backend is tried). Jars without a manifest skip this block entirely. + String backendOverride = systemProperties.getBackend(); + List backendCandidates = selectBackendCandidates(readBackendManifest(), backendOverride); + String nativeResourcePath = getNativeResourcePath(); + Set residentExtraFiles = new HashSet<>(); + for (BackendEntry backend : backendCandidates) { + if (tryLoadBackend(nativeResourcePath, backend, residentExtraFiles)) { + System.out.println("[jllama] using native backend '" + backend.name + "'"); + return; + } + triedPaths.add(nativeResourcePath + "/" + backend.name); + } + if (isForcedBackend(backendOverride) && !backendCandidates.isEmpty()) { + // An explicitly requested backend must fail loud instead of silently falling back. + throw new UnsatisfiedLinkError(String.format( + "Forced native backend '%s' (%s.backend) could not be loaded for os.name=%s, os.arch=%s," + + " paths=[%s]", + backendOverride, + LlamaSystemProperties.PREFIX, + OSInfo.getOSName(), + OSInfo.getArchName(), + String.join(File.pathSeparator, triedPaths))); + } + if (!backendCandidates.isEmpty()) { + System.out.println("[jllama] no manifest backend loadable, using default (CPU) native library"); + } + // As a last resort try load the os-dependent library from the jar file - nativeLibPath = getNativeResourcePath(); + nativeLibPath = nativeResourcePath; if (hasNativeLib(nativeLibPath, nativeLibName)) { // temporary library folder String tempFolder = getTempDir().getAbsolutePath(); @@ -288,6 +376,131 @@ static boolean resourceMatchesFile(String resourcePath, Path file) throws IOExce } } + /** + * Parses the multi-backend manifest (see {@link #BACKEND_MANIFEST_FILE} for the format). + * + * @param reader the manifest content + * @return the listed backends in manifest (priority) order; empty for an empty manifest + * @throws IOException when reading fails + */ + static List parseBackendManifest(BufferedReader reader) throws IOException { + List entries = new ArrayList<>(); + String line; + while ((line = reader.readLine()) != null) { + String trimmed = line.trim(); + if (trimmed.isEmpty() || trimmed.startsWith("#")) { + continue; + } + String[] tokens = trimmed.split("\\s+"); + entries.add(new BackendEntry(tokens[0], Arrays.asList(tokens).subList(1, tokens.length))); + } + return entries; + } + + /** + * Selects which backends to attempt, honoring the {@code net.ladenthin.llama.backend} + * override. + * + * @param manifest the parsed manifest entries in priority order + * @param override the override property value, or {@code null} if unset + * @return the manifest as-is without an override; an empty list for + * {@link #BACKEND_DEFAULT}/{@link #BACKEND_CPU}; otherwise exactly the named backend + * (synthesized without extra files when the manifest does not list it) + */ + static List selectBackendCandidates(List manifest, @Nullable String override) { + if (override == null) { + return manifest; + } + if (BACKEND_DEFAULT.equals(override) || BACKEND_CPU.equals(override)) { + return Collections.emptyList(); + } + for (BackendEntry entry : manifest) { + if (entry.name.equals(override)) { + return Collections.singletonList(entry); + } + } + return Collections.singletonList(new BackendEntry(override, Collections.emptyList())); + } + + /** + * Whether the override names a specific backend (as opposed to being unset or selecting the + * default library), which makes a load failure fatal instead of falling back. + * + * @param override the override property value, or {@code null} if unset + * @return {@code true} when a specific backend is forced + */ + static boolean isForcedBackend(@Nullable String override) { + return override != null && !BACKEND_DEFAULT.equals(override) && !BACKEND_CPU.equals(override); + } + + /** + * Reads the multi-backend manifest from the classpath, if present. + * + * @return the parsed entries, or an empty list when no manifest ships in this jar (the + * normal case for every artifact except the multi-backend fat jars) + */ + private static List readBackendManifest() { + String manifestResource = getNativeResourcePath() + "/" + BACKEND_MANIFEST_FILE; + InputStream stream = LlamaLoader.class.getResourceAsStream(manifestResource); + if (stream == null) { + return Collections.emptyList(); + } + try (BufferedReader reader = new BufferedReader(new InputStreamReader(stream, StandardCharsets.UTF_8))) { + return parseBackendManifest(reader); + } catch (IOException e) { + System.err.println("Failed to read backend manifest " + manifestResource + ": " + e.getMessage()); + return Collections.emptyList(); + } + } + + /** + * Attempts to extract and load one manifest backend from its resource subdirectory into a + * per-backend temp subdirectory (backends share file names, so they must not overwrite each + * other's extractions). + * + * @param baseResourcePath the OS/arch native resource folder + * @param entry the backend to attempt + * @param residentExtraFiles file names of extra modules already loaded by earlier failed + * attempts; updated with this attempt's loaded extras. A native + * module cannot be unloaded, and imports bind by module name, so a + * backend declaring an extra file that is already resident from + * another backend must be skipped to avoid cross-wiring. + * @return whether the backend's main library was successfully loaded + */ + private static boolean tryLoadBackend(String baseResourcePath, BackendEntry entry, Set residentExtraFiles) { + String backendResourcePath = baseResourcePath + "/" + entry.name; + String mainLibraryFileName = System.mapLibraryName("jllama"); + if (!hasNativeLib(backendResourcePath, mainLibraryFileName)) { + return false; + } + for (String extraFile : entry.extraFiles) { + if (residentExtraFiles.contains(extraFile)) { + System.err.println("[jllama] skipping backend '" + entry.name + "': module '" + extraFile + + "' is already resident from a previously failed backend attempt"); + return false; + } + } + Path targetDirPath = getTempDir().toPath().resolve(BACKEND_TEMP_DIR_PREFIX + entry.name); + try { + Files.createDirectories(targetDirPath); + } catch (IOException e) { + System.err.println("Failed to create backend temp directory " + targetDirPath + ": " + e.getMessage()); + return false; + } + // Registered before the contained files: File.deleteOnExit processing is LIFO, so the + // files registered afterwards by extractFile are deleted first, then this directory. + targetDirPath.toFile().deleteOnExit(); + String targetFolder = targetDirPath.toAbsolutePath().toString(); + for (String extraFile : entry.extraFiles) { + Path extraPath = extractFile(backendResourcePath, extraFile, targetFolder); + if (extraPath == null || !loadNativeLibrary(extraPath)) { + return false; + } + residentExtraFiles.add(extraFile); + } + return extractAndLoadLibraryFile(backendResourcePath, mainLibraryFileName, targetFolder); + } + /** * Extracts and loads the specified library file to the target folder * diff --git a/llama/src/main/java/net/ladenthin/llama/loader/LlamaSystemProperties.java b/llama/src/main/java/net/ladenthin/llama/loader/LlamaSystemProperties.java index 52067d88..a1952125 100644 --- a/llama/src/main/java/net/ladenthin/llama/loader/LlamaSystemProperties.java +++ b/llama/src/main/java/net/ladenthin/llama/loader/LlamaSystemProperties.java @@ -60,4 +60,18 @@ public LlamaSystemProperties() {} public @Nullable String getTestNgl() { return getProperty(".test.ngl"); } + + /** + * Native-backend override for multi-backend ("all") fat jars that carry a + * {@code jllama-backends.txt} manifest next to their native libraries. Names one backend + * subdirectory (e.g. {@code cuda13}, {@code vulkan}) to load exclusively — loading then + * fails loud instead of falling back — or the special value {@code default} (alias + * {@code cpu}) to skip all manifest backends and load the default library directly. Ignored + * by jars without a backend manifest. + * + * @return the configured backend name, or {@code null} if unset + */ + public @Nullable String getBackend() { + return getProperty(".backend"); + } } diff --git a/llama/src/test/java/net/ladenthin/llama/loader/BackendManifestLoadTest.java b/llama/src/test/java/net/ladenthin/llama/loader/BackendManifestLoadTest.java new file mode 100644 index 00000000..4784cabd --- /dev/null +++ b/llama/src/test/java/net/ladenthin/llama/loader/BackendManifestLoadTest.java @@ -0,0 +1,175 @@ +// SPDX-FileCopyrightText: 2026 Bernard Ladenthin +// +// SPDX-License-Identifier: MIT + +package net.ladenthin.llama.loader; + +import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import net.ladenthin.llama.ClaudeGenerated; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +@ClaudeGenerated( + purpose = "Drive LlamaLoader.initialize() end-to-end against the committed backend-manifest " + + "fixture trees (src/test/resources/net/ladenthin/llama/Linux/backendtest*/) by " + + "redirecting the arch component via the osinfo.architecture override. backendtest/ " + + "holds only unloadable fake libraries, exercising every failure branch (manifest " + + "read, per-backend temp-dir extraction, extra-file handling present and missing, " + + "clean load-failure fallback, forced-backend fail-loud, default/cpu manifest skip, " + + "no-manifest legacy path); backendtest-ok/ adds two trivial real x86-64 ELF dummies " + + "so the success path, the resident-extra bookkeeping, and the same-extra clash skip " + + "execute too. Linux-only (the trees are committed under the Linux OS folder); " + + "self-skips elsewhere.") +public class BackendManifestLoadTest { + + /** Arch-folder override selecting the committed fixture tree {@code Linux/backendtest/}. */ + private static final String FIXTURE_ARCH = "backendtest"; + + private static final String ARCH_PROP = LlamaSystemProperties.PREFIX + ".osinfo.architecture"; + private static final String TMPDIR_PROP = LlamaSystemProperties.PREFIX + ".tmpdir"; + private static final String BACKEND_PROP = LlamaSystemProperties.PREFIX + ".backend"; + + private String previousArch; + private String previousTmpDir; + private String previousBackend; + + @TempDir + Path tempDir; + + @BeforeEach + public void redirectLoaderToFixtures() { + previousArch = System.getProperty(ARCH_PROP); + previousTmpDir = System.getProperty(TMPDIR_PROP); + previousBackend = System.getProperty(BACKEND_PROP); + System.setProperty(ARCH_PROP, FIXTURE_ARCH); + System.setProperty(TMPDIR_PROP, tempDir.toString()); + System.clearProperty(BACKEND_PROP); + } + + @AfterEach + public void restoreProperties() { + restore(ARCH_PROP, previousArch); + restore(TMPDIR_PROP, previousTmpDir); + restore(BACKEND_PROP, previousBackend); + } + + private static void restore(String key, String value) { + if (value == null) { + System.clearProperty(key); + } else { + System.setProperty(key, value); + } + } + + private static void assumeLinuxFixtureTree() { + // The fixture tree is committed under the Linux OS folder; the OS path component + // cannot be overridden, so these tests are meaningful only on a Linux JVM (which is + // where the CI coverage run executes). + assumeTrue("Linux".equals(OSInfo.getOSName()), "backend-manifest fixtures are committed for Linux only"); + } + + private static void assumeX86_64() { + // The loadable dummy libraries in backendtest-ok/ are real ELF shared objects + // compiled for x86-64 Linux (see their fixture README); loading them anywhere else + // fails for the wrong reason. + String osArch = System.getProperty("os.arch", ""); + assumeTrue( + "amd64".equals(osArch) || "x86_64".equals(osArch), + "loadable dummy backend libraries are built for x86-64 only"); + } + + @Test + public void autoModeTriesEveryManifestBackendThenFailsWithoutDefaultLibrary() { + assumeLinuxFixtureTree(); + UnsatisfiedLinkError error = assertThrows(UnsatisfiedLinkError.class, LlamaLoader::initialize); + // Every manifest backend must have been attempted (and failed cleanly): the final + // error lists each backend resource path plus the default path as tried. + String message = error.getMessage(); + String base = "/net/ladenthin/llama/Linux/" + FIXTURE_ARCH; + assertTrue(message.contains(base + "/fakegpu"), message); + assertTrue(message.contains(base + "/missingextra"), message); + assertTrue(message.contains(base + "/nodir"), message); + assertTrue(message.contains(base + "/fallbackgpu"), message); + assertTrue(message.contains(base), message); + // fakegpu: the extra file is extracted into the per-backend temp subdir before its + // load fails; the main library is never reached for that backend. + assertTrue(Files.isRegularFile( + tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "fakegpu").resolve("libextra.so"))); + assertFalse(Files.exists( + tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "fakegpu").resolve("libjllama.so"))); + // fallbackgpu (no extras): its main library is extracted, then fails to load. + assertTrue(Files.isRegularFile(tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "fallbackgpu") + .resolve("libjllama.so"))); + } + + @Test + public void autoModeLoadsFirstWorkingBackendAndSkipsResidentClash() throws java.io.IOException { + assumeLinuxFixtureTree(); + assumeX86_64(); + System.setProperty(ARCH_PROP, "backendtest-ok"); + // Pre-create stale extraction artifacts so cleanup()'s recursive directory branch + // executes on this initialize() call (deletion is not asserted: cleanup is skipped + // when an earlier test class in the same JVM already loaded a native library). + Path stale = + tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "stale").resolve("nested"); + Files.createDirectories(stale); + Files.write(stale.resolve("libjllama.so"), new byte[] {1}); + // Must succeed: extrafail's real extra library loads but its fake main library + // fails; clash declares the same extra file name and is skipped (already resident, + // so its temp dir is never even created); workinggpu's real dummy library loads. + LlamaLoader.initialize(); + assertTrue(Files.isRegularFile(tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "extrafail") + .resolve("libextra.so"))); + assertFalse(Files.exists(tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "clash"))); + assertTrue(Files.isRegularFile(tempDir.resolve(LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "workinggpu") + .resolve("libjllama.so"))); + } + + @Test + public void withoutManifestNoBackendIsAttempted() { + assumeLinuxFixtureTree(); + System.setProperty(ARCH_PROP, "backendtest-none"); + UnsatisfiedLinkError error = assertThrows(UnsatisfiedLinkError.class, LlamaLoader::initialize); + // No manifest resource exists for this arch folder: the loader must take the + // unchanged legacy path with zero backend attempts. + String message = error.getMessage(); + assertFalse(message.contains("backendtest-none/"), message); + assertTrue(message.contains("os.arch=backendtest-none"), message); + } + + @Test + public void forcedBackendFailsLoudInsteadOfFallingBack() { + assumeLinuxFixtureTree(); + System.setProperty(BACKEND_PROP, "fakegpu"); + UnsatisfiedLinkError error = assertThrows(UnsatisfiedLinkError.class, LlamaLoader::initialize); + assertTrue(error.getMessage().contains("Forced native backend 'fakegpu'"), error.getMessage()); + } + + @Test + public void forcedUnknownBackendFailsLoud() { + assumeLinuxFixtureTree(); + System.setProperty(BACKEND_PROP, "does-not-exist"); + UnsatisfiedLinkError error = assertThrows(UnsatisfiedLinkError.class, LlamaLoader::initialize); + assertTrue(error.getMessage().contains("Forced native backend 'does-not-exist'"), error.getMessage()); + } + + @Test + public void forcedDefaultSkipsAllManifestBackends() { + assumeLinuxFixtureTree(); + System.setProperty(BACKEND_PROP, "default"); + UnsatisfiedLinkError error = assertThrows(UnsatisfiedLinkError.class, LlamaLoader::initialize); + // The default library is also missing from the fixture tree (its resource path is + // then not even listed as tried), so loading still fails — but without any backend + // attempt: no backend path may appear in the tried list. + String message = error.getMessage(); + assertFalse(message.contains("/fakegpu"), message); + assertFalse(message.contains("/fallbackgpu"), message); + assertTrue(message.contains("os.arch=" + FIXTURE_ARCH), message); + } +} diff --git a/llama/src/test/java/net/ladenthin/llama/loader/LlamaLoaderTest.java b/llama/src/test/java/net/ladenthin/llama/loader/LlamaLoaderTest.java index 72d424ed..07494c5c 100644 --- a/llama/src/test/java/net/ladenthin/llama/loader/LlamaLoaderTest.java +++ b/llama/src/test/java/net/ladenthin/llama/loader/LlamaLoaderTest.java @@ -19,8 +19,9 @@ @ClaudeGenerated( purpose = "Verify the helper statics extracted from LlamaLoader without requiring any " - + "native library: shouldCleanPath detects jllama/llama-prefixed files for " - + "cleanup; contentsEquals performs a correct byte-level stream comparison " + + "native library: shouldCleanPath detects jllama/llama/ggml-prefixed files for " + + "cleanup (ggml covers the extracted macOS ggml-metal.metal); " + + "contentsEquals performs a correct byte-level stream comparison " + "including BufferedInputStream wrapping and length mismatches; getTempDir " + "honours the 'net.ladenthin.llama.tmpdir' system-property override; and " + "getNativeResourcePath produces the expected classpath resource prefix; and " @@ -94,6 +95,135 @@ public void testShouldCleanPathCaseSensitive() { assertFalse(LlamaLoader.shouldCleanPath(Paths.get("/tmp/Jllama.so"))); } + @Test + public void testShouldCleanPathGgmlMetalFile() { + // Regression: initialize() extracts ggml-metal.metal on macOS, but cleanup() never + // matched the "ggml" prefix, so a stale extracted copy was left in the temp dir forever. + assertTrue(LlamaLoader.shouldCleanPath(Paths.get("/tmp/ggml-metal.metal"))); + } + + @Test + public void testShouldCleanPathGgmlPrefix() { + assertTrue(LlamaLoader.shouldCleanPath(Paths.get("/tmp/ggml.tmp"))); + } + + // ------------------------------------------------------------------------- + // parseBackendManifest + // ------------------------------------------------------------------------- + + private static java.util.List parseManifest(String content) throws IOException { + return LlamaLoader.parseBackendManifest(new java.io.BufferedReader(new java.io.StringReader(content))); + } + + @Test + public void testParseBackendManifestPreservesPriorityOrder() throws IOException { + java.util.List entries = parseManifest("cuda13\nvulkan\nopencl\n"); + assertEquals(3, entries.size()); + assertEquals("cuda13", entries.get(0).name); + assertEquals("vulkan", entries.get(1).name); + assertEquals("opencl", entries.get(2).name); + } + + @Test + public void testParseBackendManifestTokenizesExtraFiles() throws IOException { + java.util.List entries = parseManifest("openvino OpenCL.dll second.dll\n"); + assertEquals(1, entries.size()); + assertEquals("openvino", entries.get(0).name); + assertEquals(java.util.Arrays.asList("OpenCL.dll", "second.dll"), entries.get(0).extraFiles); + } + + @Test + public void testParseBackendManifestNoExtraFiles() throws IOException { + java.util.List entries = parseManifest("vulkan\n"); + assertTrue(entries.get(0).extraFiles.isEmpty()); + } + + @Test + public void testParseBackendManifestSkipsCommentsAndBlankLines() throws IOException { + java.util.List entries = + parseManifest("# priority order\n\n \ncuda13\n# tail comment\n"); + assertEquals(1, entries.size()); + assertEquals("cuda13", entries.get(0).name); + } + + @Test + public void testParseBackendManifestToleratesCrLfAndIndentation() throws IOException { + // BufferedReader.readLine strips \r\n; leading/trailing whitespace is trimmed per line. + java.util.List entries = parseManifest(" cuda13\r\n\tvulkan \r\n"); + assertEquals(2, entries.size()); + assertEquals("cuda13", entries.get(0).name); + assertEquals("vulkan", entries.get(1).name); + } + + @Test + public void testParseBackendManifestEmptyContent() throws IOException { + assertTrue(parseManifest("").isEmpty()); + } + + // ------------------------------------------------------------------------- + // selectBackendCandidates / isForcedBackend + // ------------------------------------------------------------------------- + + @Test + public void testSelectBackendCandidatesWithoutOverrideReturnsManifestOrder() throws IOException { + java.util.List manifest = parseManifest("cuda13\nvulkan\n"); + assertEquals(manifest, LlamaLoader.selectBackendCandidates(manifest, null)); + } + + @Test + public void testSelectBackendCandidatesDefaultSkipsAllBackends() throws IOException { + java.util.List manifest = parseManifest("cuda13\n"); + assertTrue(LlamaLoader.selectBackendCandidates(manifest, "default").isEmpty()); + } + + @Test + public void testSelectBackendCandidatesCpuAliasSkipsAllBackends() throws IOException { + java.util.List manifest = parseManifest("cuda13\n"); + assertTrue(LlamaLoader.selectBackendCandidates(manifest, "cpu").isEmpty()); + } + + @Test + public void testSelectBackendCandidatesForcedKnownBackendKeepsItsExtraFiles() throws IOException { + java.util.List manifest = parseManifest("cuda13\nopenvino OpenCL.dll\n"); + java.util.List selected = LlamaLoader.selectBackendCandidates(manifest, "openvino"); + assertEquals(1, selected.size()); + assertEquals("openvino", selected.get(0).name); + assertEquals(java.util.Collections.singletonList("OpenCL.dll"), selected.get(0).extraFiles); + } + + @Test + public void testSelectBackendCandidatesForcedUnknownBackendIsSynthesized() throws IOException { + // A backend name absent from the manifest is still attempted (stale-manifest override), + // just without extra files; the loader fails loud if its library is missing. + java.util.List manifest = parseManifest("cuda13\n"); + java.util.List selected = LlamaLoader.selectBackendCandidates(manifest, "rocm"); + assertEquals(1, selected.size()); + assertEquals("rocm", selected.get(0).name); + assertTrue(selected.get(0).extraFiles.isEmpty()); + } + + @Test + public void testIsForcedBackendUnsetIsNotForced() { + assertFalse(LlamaLoader.isForcedBackend(null)); + } + + @Test + public void testIsForcedBackendDefaultAndCpuAreNotForced() { + assertFalse(LlamaLoader.isForcedBackend("default")); + assertFalse(LlamaLoader.isForcedBackend("cpu")); + } + + @Test + public void testIsForcedBackendSpecificBackendIsForced() { + assertTrue(LlamaLoader.isForcedBackend("cuda13")); + } + + @Test + public void testBackendTempDirPrefixMatchesCleanup() { + // The per-backend extraction directories must be picked up by the temp-dir cleanup. + assertTrue(LlamaLoader.shouldCleanPath(Paths.get("/tmp/" + LlamaLoader.BACKEND_TEMP_DIR_PREFIX + "cuda13"))); + } + // ------------------------------------------------------------------------- // contentsEquals // ------------------------------------------------------------------------- diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libextra.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libextra.so new file mode 100755 index 00000000..3d67a2d8 Binary files /dev/null and b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libextra.so differ diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libjllama.so new file mode 100644 index 00000000..606ca36e --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/clash/libjllama.so @@ -0,0 +1 @@ +not a real shared library (clash main) diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libextra.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libextra.so new file mode 100755 index 00000000..3d67a2d8 Binary files /dev/null and b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libextra.so differ diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libjllama.so new file mode 100644 index 00000000..7666833b --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/extrafail/libjllama.so @@ -0,0 +1 @@ +not a real shared library (extrafail main) diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/jllama-backends.txt b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/jllama-backends.txt new file mode 100644 index 00000000..5cf2840b --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/jllama-backends.txt @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +# +# Test fixture for BackendManifestLoadTest's success scenario: extrafail's real +# extra library loads but its fake main library fails; clash declares the same +# extra file name and must be skipped (already resident); workinggpu's real +# dummy library loads successfully. +extrafail libextra.so +clash libextra.so +workinggpu diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/workinggpu/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/workinggpu/libjllama.so new file mode 100755 index 00000000..3516f65d Binary files /dev/null and b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest-ok/workinggpu/libjllama.so differ diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libextra.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libextra.so new file mode 100644 index 00000000..a2d73f01 --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libextra.so @@ -0,0 +1 @@ +not a real shared library (fakegpu extra) diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libjllama.so new file mode 100644 index 00000000..71ed1879 --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fakegpu/libjllama.so @@ -0,0 +1 @@ +not a real shared library (fakegpu main) diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fallbackgpu/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fallbackgpu/libjllama.so new file mode 100644 index 00000000..f67d9e33 --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/fallbackgpu/libjllama.so @@ -0,0 +1 @@ +not a real shared library (fallbackgpu) diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/jllama-backends.txt b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/jllama-backends.txt new file mode 100644 index 00000000..85863212 --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/jllama-backends.txt @@ -0,0 +1,10 @@ +# SPDX-FileCopyrightText: 2026 Bernard Ladenthin +# +# SPDX-License-Identifier: MIT +# +# Test fixture for BackendManifestLoadTest: a backend manifest whose fake libraries +# can never actually load, exercising every failure branch of the fallback chain. +fakegpu libextra.so +missingextra libmissing.so +nodir +fallbackgpu diff --git a/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/missingextra/libjllama.so b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/missingextra/libjllama.so new file mode 100644 index 00000000..0d843892 --- /dev/null +++ b/llama/src/test/resources/net/ladenthin/llama/Linux/backendtest/missingextra/libjllama.so @@ -0,0 +1 @@ +not a real shared library (missingextra)